diff --git a/pgloader.1.md b/pgloader.1.md index cde67d2..ebf032c 100644 --- a/pgloader.1.md +++ b/pgloader.1.md @@ -72,7 +72,12 @@ Those options are meant to tweak `pgloader` behavior when loading data. * `-S`, `--summary`: A filename where to copy the summary output. When relative, the filename - is expanded into `*root-dir`. + is expanded into `*root-dir*`. + + The format of the filename defaults to being *human readable*. It is + possible to have the output in machine friendly formats such as *CSV*, + *COPY* (PostgreSQL's own COPY format) or *JSON* by specifying a filename + with the extension resp. `.csv`, `.copy` or `.json`. * `-l `, `--load-lisp-file `: Specify a lisp to compile and load into the pgloader image before diff --git a/src/main.lisp b/src/main.lisp index c10f0a6..1ac7a1a 100644 --- a/src/main.lisp +++ b/src/main.lisp @@ -5,7 +5,7 @@ (cond ((and debug verbose) :data) (debug :debug) (verbose :notice) - (quiet :warning) + (quiet :error) (t (or (find-symbol (string-upcase min-message) "KEYWORD") :notice)))) @@ -146,6 +146,13 @@ (mkdir-or-die summary-dir debug) summary-pathname))) +(defun parse-summary-type (&optional (pathname *summary-pathname*)) + "Return the summary type we want: human-readable, csv, json." + (cond ((string= "csv" (pathname-type pathname)) :csv) + ((string= "json" (pathname-type pathname)) :json) + ((string= "copy" (pathname-type pathname)) :copy) + (t :human-readable))) + (defvar *--load-list-file-extension-whitelist* '("lisp" "lsp" "cl" "asd") "White list of file extensions allowed with the --load option.") @@ -256,8 +263,15 @@ ;; Now process the arguments (when arguments ;; Start the logs system - (let ((*log-filename* (log-file-name logfile)) - (*summary-pathname* (parse-summary-filename summary debug))) + (let* ((*log-filename* (log-file-name logfile)) + (*summary-pathname* (parse-summary-filename summary debug)) + (stype (parse-summary-type *summary-pathname*)) + (*footer* (get-format-for stype :footer)) + (*header-line* (get-format-for stype :header-line)) + (*header-tname-format* (get-format-for stype :header-tname-format)) + (*header-stats-format* (get-format-for stype :header-stats-format)) + (*header-cols-format* (get-format-for stype :header-cols-format)) + (*header-cols-names* (get-format-for stype :header-cols-names))) (with-monitor () ;; tell the user where to look for interesting things diff --git a/src/package.lisp b/src/package.lisp index 64be8ec..074703b 100644 --- a/src/package.lisp +++ b/src/package.lisp @@ -38,14 +38,36 @@ #:with-monitor #:*monitoring-queue* #:log-message) - (:export #:with-monitor + (:export #:with-monitor ; monitor + + ;; logs #:log-message + + ;; report + #:*header-line* + #:*footer* + #:*header-tname-format* + #:*header-stats-format* + #:*header-cols-format* + #:*header-cols-names* + #:*header-format-strings* + #:get-format-for + #:report-header #:report-table-name #:report-results #:report-footer + #:report-summary + #:report-full-summary + #:with-stats-collection + + ;; utils #:format-interval #:timing + #:camelCase-to-colname + #:unquote + + ;; state #:make-pgstate #:pgstate-get-table #:pgstate-add-table @@ -56,12 +78,11 @@ #:pgtable-reject-logs #:report-pgtable-stats #:report-pgstate-stats - #:report-summary - #:report-full-summary - #:with-stats-collection - #:camelCase-to-colname - #:unquote + + ;; threads #:make-kernel + + ;; charsets #:list-encodings-and-aliases #:show-encodings #:make-external-format)) diff --git a/src/utils/report.lisp b/src/utils/report.lisp index bc3e64f..2431ef5 100644 --- a/src/utils/report.lisp +++ b/src/utils/report.lisp @@ -7,34 +7,69 @@ (defvar *header-line* "~&------------------------------ --------- --------- --------- --------------") +(defvar *footer* "~&") (defvar *header-tname-format* "~&~30@a") (defvar *header-stats-format* " ~9@a ~9@a ~9@a ~14@a") (defvar *header-cols-format* (concatenate 'string *header-tname-format* *header-stats-format*)) (defvar *header-cols-names* '("table name" "read" "imported" "errors" "time")) +(defvar *header-format-strings* + '((:human-readable + (:header-line + "~&------------------------------ --------- --------- --------- --------------" + :header-tname-format "~&~30@a" + :header-stats-format " ~9@a ~9@a ~9@a ~14@a" + :header-cols-format "~&~30@a ~9@a ~9@a ~9@a ~14@a" + :header-cols-names ("table name" "read" "imported" "errors" "time") + :footer "~%")) + + (:csv + (:header-line "" + :header-tname-format "~&~s;" + :header-stats-format "~s;~s;~s;~s" + :header-cols-format "~&~s;~s;~s;~s;~s" + :header-cols-names ("table name" "read" "imported" "errors" "time") + :footer "~%")) + + (:copy + (:header-line "~&" + :header-tname-format "~&~a " + :header-stats-format "~s ~s ~s ~s" + :header-cols-format "~*~*~*~*~*" ; skip it + :header-cols-names ("table name" "read" "imported" "errors" "time") + :footer "~%")) + + (:json + (:header-line "~&" + :header-tname-format "~& {\"table-name\": ~s," + :header-stats-format "\"read\":~s,\"imported\":~s,\"errors\":~s,\"time\":~s}" + :header-cols-format "~*~*~*~*~*" ; skip it + :header-cols-names ("table name" "read" "imported" "errors" "time") + :footer "~%")))) + +(defun get-format-for (type key) + "Return the format string to use for a given TYPE of output and KEY." + (getf (cadr (assoc type *header-format-strings*)) key)) + (defun report-header () ;; (apply #'format *report-stream* *header-cols-format* *header-cols-names*) (format *report-stream* "~{~}" *header-cols-format* *header-cols-names*) - (terpri) - (format *report-stream* *header-line*) - (terpri)) + (format *report-stream* *header-line*)) (defun report-table-name (table-name) (format *report-stream* *header-tname-format* table-name)) (defun report-results (read rows errors seconds) (format *report-stream* *header-stats-format* - read rows errors (format-interval seconds nil)) - (terpri)) + read rows errors (format-interval seconds nil))) (defun report-footer (legend read rows errors seconds) - (terpri) (format *report-stream* *header-line*) - (format *report-stream* "~{~}" *header-cols-format* - (list legend read rows errors (format-interval seconds nil))) - (format *report-stream* "~&") - (terpri)) + (format *report-stream* "~{~}" *header-tname-format* (list legend)) + (format *report-stream* "~{~}" *header-stats-format* + (list read rows errors (format-interval seconds nil))) + (format *report-stream* *footer*)) ;;; ;;; Pretty print a report from a pgtable and pgstats counters @@ -58,8 +93,9 @@ for pgtable = (gethash table-name (pgstate-tables pgstate)) do (with-slots (read rows errs secs) pgtable - (format *report-stream* *header-cols-format* - table-name read rows errs (format-interval secs nil))) + (format *report-stream* *header-tname-format* table-name) + (format *report-stream* *header-stats-format* + read rows errs (format-interval secs nil))) finally (when footer (report-pgstate-stats pgstate footer)))) @@ -95,8 +131,6 @@ &key before finally parallel) "Report the full story when given three different sections of reporting." - (terpri) - ;; BEFORE (if before (progn