Implement machine readable summary files, fixes #144.

It's now possible to have pgloader print out its summary in one of
several formats: human-readable (default), csv, copy or json. The
choice of format is made depending on the extension of the summary
filename picked on the command line with the option --summary.

pgloader.1.md

@@ -72,7 +72,12 @@ Those options are meant to tweak `pgloader` behavior when loading data.
 
   * `-S`, `--summary`:
     A filename where to copy the summary output. When relative, the filename
-    is expanded into `*root-dir`.
+    is expanded into `*root-dir*`.
+    
+    The format of the filename defaults to being *human readable*. It is
+    possible to have the output in machine friendly formats such as *CSV*,
+    *COPY* (PostgreSQL's own COPY format) or *JSON* by specifying a filename
+    with the extension resp. `.csv`, `.copy` or `.json`.
 
   * `-l <file>`, `--load-lisp-file <file>`:
     Specify a lisp <file> to compile and load into the pgloader image before

src/main.lisp

@@ -5,7 +5,7 @@
   (cond ((and debug verbose) :data)
         (debug   :debug)
 	(verbose :notice)
-	(quiet   :warning)
+	(quiet   :error)
 	(t       (or (find-symbol (string-upcase min-message) "KEYWORD")
 		     :notice))))
 
@@ -146,6 +146,13 @@
       (mkdir-or-die summary-dir debug)
       summary-pathname)))
 
+(defun parse-summary-type (&optional (pathname *summary-pathname*))
+  "Return the summary type we want: human-readable, csv, json."
+  (cond ((string= "csv"  (pathname-type pathname)) :csv)
+        ((string= "json" (pathname-type pathname)) :json)
+        ((string= "copy" (pathname-type pathname)) :copy)
+        (t :human-readable)))
+
 (defvar *--load-list-file-extension-whitelist* '("lisp" "lsp" "cl" "asd")
   "White list of file extensions allowed with the --load option.")
 
@@ -256,8 +263,15 @@
 	;; Now process the arguments
 	(when arguments
 	  ;; Start the logs system
-	  (let ((*log-filename*      (log-file-name logfile))
-                (*summary-pathname*  (parse-summary-filename summary debug)))
+	  (let* ((*log-filename*      (log-file-name logfile))
+                 (*summary-pathname*  (parse-summary-filename summary debug))
+                 (stype               (parse-summary-type *summary-pathname*))
+                 (*footer*              (get-format-for stype :footer))
+                 (*header-line*         (get-format-for stype :header-line))
+                 (*header-tname-format* (get-format-for stype :header-tname-format))
+                 (*header-stats-format* (get-format-for stype :header-stats-format))
+                 (*header-cols-format*  (get-format-for stype :header-cols-format))
+                 (*header-cols-names*   (get-format-for stype :header-cols-names)))
 
             (with-monitor ()
               ;; tell the user where to look for interesting things

src/package.lisp

@@ -38,14 +38,36 @@
                 #:with-monitor
                 #:*monitoring-queue*
                 #:log-message)
-  (:export #:with-monitor
+  (:export #:with-monitor               ; monitor
+
+           ;; logs
            #:log-message
+
+           ;; report
+           #:*header-line*
+           #:*footer*
+           #:*header-tname-format*
+           #:*header-stats-format*
+           #:*header-cols-format*
+           #:*header-cols-names*
+           #:*header-format-strings*
+           #:get-format-for
+
            #:report-header
 	   #:report-table-name
 	   #:report-results
 	   #:report-footer
+	   #:report-summary
+	   #:report-full-summary
+	   #:with-stats-collection
+
+           ;; utils
 	   #:format-interval
 	   #:timing
+	   #:camelCase-to-colname
+           #:unquote
+
+           ;; state
 	   #:make-pgstate
 	   #:pgstate-get-table
 	   #:pgstate-add-table
@@ -56,12 +78,11 @@
 	   #:pgtable-reject-logs
 	   #:report-pgtable-stats
 	   #:report-pgstate-stats
-	   #:report-summary
-	   #:report-full-summary
-	   #:with-stats-collection
-	   #:camelCase-to-colname
-           #:unquote
+
+           ;; threads
            #:make-kernel
+
+           ;; charsets
            #:list-encodings-and-aliases
            #:show-encodings
            #:make-external-format))

src/utils/report.lisp

@@ -7,34 +7,69 @@
 (defvar *header-line*
   "~&------------------------------  ---------  ---------  ---------  --------------")
 
+(defvar *footer* "~&")
 (defvar *header-tname-format* "~&~30@a")
 (defvar *header-stats-format* "  ~9@a  ~9@a  ~9@a  ~14@a")
 (defvar *header-cols-format* (concatenate 'string *header-tname-format*
 					  *header-stats-format*))
 (defvar *header-cols-names* '("table name" "read" "imported" "errors" "time"))
 
+(defvar *header-format-strings*
+  '((:human-readable
+     (:header-line
+      "~&------------------------------  ---------  ---------  ---------  --------------"
+      :header-tname-format "~&~30@a"
+      :header-stats-format "  ~9@a  ~9@a  ~9@a  ~14@a"
+      :header-cols-format  "~&~30@a  ~9@a  ~9@a  ~9@a  ~14@a"
+      :header-cols-names  ("table name" "read" "imported" "errors" "time")
+      :footer              "~%"))
+
+    (:csv
+     (:header-line         ""
+      :header-tname-format "~&~s;"
+      :header-stats-format "~s;~s;~s;~s"
+      :header-cols-format  "~&~s;~s;~s;~s;~s"
+      :header-cols-names  ("table name" "read" "imported" "errors" "time")
+      :footer              "~%"))
+
+    (:copy
+     (:header-line         "~&"
+      :header-tname-format "~&~a	"
+      :header-stats-format "~s	~s	~s	~s"
+      :header-cols-format  "~*~*~*~*~*" ; skip it
+      :header-cols-names  ("table name" "read" "imported" "errors" "time")
+      :footer              "~%"))
+
+    (:json
+     (:header-line         "~&"
+      :header-tname-format "~& {\"table-name\": ~s,"
+      :header-stats-format "\"read\":~s,\"imported\":~s,\"errors\":~s,\"time\":~s}"
+      :header-cols-format  "~*~*~*~*~*" ; skip it
+      :header-cols-names   ("table name" "read" "imported" "errors" "time")
+      :footer              "~%"))))
+
+(defun get-format-for (type key)
+  "Return the format string to use for a given TYPE of output and KEY."
+  (getf (cadr (assoc type *header-format-strings*)) key))
+
 (defun report-header ()
   ;; (apply #'format *report-stream* *header-cols-format* *header-cols-names*)
   (format *report-stream* "~{~}" *header-cols-format* *header-cols-names*)
-  (terpri)
-  (format *report-stream* *header-line*)
-  (terpri))
+  (format *report-stream* *header-line*))
 
 (defun report-table-name (table-name)
   (format *report-stream* *header-tname-format* table-name))
 
 (defun report-results (read rows errors seconds)
   (format *report-stream* *header-stats-format*
-          read rows errors (format-interval seconds nil))
-  (terpri))
+          read rows errors (format-interval seconds nil)))
 
 (defun report-footer (legend read rows errors seconds)
-  (terpri)
   (format *report-stream* *header-line*)
-  (format *report-stream* "~{~}" *header-cols-format*
-          (list legend read rows errors (format-interval seconds nil)))
-  (format *report-stream* "~&")
-  (terpri))
+  (format *report-stream* "~{~}" *header-tname-format* (list legend))
+  (format *report-stream* "~{~}" *header-stats-format*
+          (list read rows errors (format-interval seconds nil)))
+  (format *report-stream* *footer*))
 
 ;;;
 ;;; Pretty print a report from a pgtable and pgstats counters
@@ -58,8 +93,9 @@
      for pgtable = (gethash table-name (pgstate-tables pgstate))
      do
        (with-slots (read rows errs secs) pgtable
-	 (format *report-stream* *header-cols-format*
-		 table-name read rows errs (format-interval secs nil)))
+	 (format *report-stream* *header-tname-format* table-name)
+         (format *report-stream* *header-stats-format*
+                 read rows errs (format-interval secs nil)))
      finally (when footer
 	       (report-pgstate-stats pgstate footer))))
 
@@ -95,8 +131,6 @@
 			    &key before finally parallel)
   "Report the full story when given three different sections of reporting."
 
-  (terpri)
-
   ;; BEFORE
   (if before
       (progn