From 03a8d57a5043ce61ececfb9d61792677314d93af Mon Sep 17 00:00:00 2001
From: Dimitri Fontaine <dim@tapoueh.org>
Date: Mon, 21 Aug 2017 15:27:13 +0200
Subject: [PATCH] Review --verbose log message.

The verbosity is not that easy to adjust. Remove useless messages and add a
new one telling when the COPY of a table is done. As we might have to wait
for some time for indexes being built. keep the CREATE INDEX lines. Also
keep the ALTER TABLE both for primary keys and foreign keys, again because
the user might have to wait for quite some time.
---
 src/pgsql/connection.lisp          | 14 ++++++++++----
 src/pgsql/pgsql-create-schema.lisp |  4 ++--
 src/sources/common/db-methods.lisp |  9 +++++++--
 3 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/src/pgsql/connection.lisp b/src/pgsql/connection.lisp
index ca1a8f3..c1a966a 100644
--- a/src/pgsql/connection.lisp
+++ b/src/pgsql/connection.lisp
@@ -263,7 +263,7 @@
   (handler-case
       (with-pgsql-connection (pgconn)
         (pomo:with-transaction ()
-          (pgsql-execute-with-timing section label sql)))
+          (pgsql-execute-with-timing section label sql :log-level :notice)))
 
     (postgresql-unavailable (condition)
 
@@ -278,6 +278,7 @@
 
 (defun pgsql-execute-with-timing (section label sql-list
                                   &key
+                                    (log-level :sql)
                                     on-error-stop
                                     client-min-messages)
   "Execute given SQL and resgister its timing into STATE."
@@ -286,13 +287,18 @@
         (timing
           (multiple-value-bind (nb-ok nb-errors)
               (pgsql-execute sql-list
+                             :log-level log-level
                              :on-error-stop on-error-stop
                              :client-min-messages client-min-messages)
             (update-stats section label :rows nb-ok :errs nb-errors)))
       (declare (ignore res))
       (update-stats section label :read (length sql-list) :secs secs))))
 
-(defun pgsql-execute (sql &key client-min-messages (on-error-stop t))
+(defun pgsql-execute (sql
+                      &key
+                        (log-level :sql)
+                        client-min-messages
+                        (on-error-stop t))
   "Execute given SQL list of statements in current transaction.
 
    When ON-ERROR-STOP is non-nil (the default), we stop at the first sql
@@ -309,7 +315,7 @@
     (if on-error-stop
         (loop :for sql :in (alexandria::ensure-list sql)
            :do (progn
-                 (log-message :notice "~a" sql)
+                 (log-message log-level "~a" sql)
                  (pomo:execute sql))
            ;; never executed in case of error, which signals out of here
            :finally (incf nb-ok (length sql)))
@@ -320,7 +326,7 @@
                  (pomo:execute "savepoint pgloader;")
                  (handler-case
                      (progn
-                       (log-message :notice "~a" sql)
+                       (log-message log-level "~a" sql)
                        (pomo:execute sql)
                        (pomo:execute "release savepoint pgloader;")
                        (incf nb-ok))
diff --git a/src/pgsql/pgsql-create-schema.lisp b/src/pgsql/pgsql-create-schema.lisp
index ad53a1f..c813633 100644
--- a/src/pgsql/pgsql-create-schema.lisp
+++ b/src/pgsql/pgsql-create-schema.lisp
@@ -188,7 +188,7 @@
                            (pgsql-execute sql))
                      :count t))))
 
-(defun create-pgsql-fkeys (catalog &key (section :post) label)
+(defun create-pgsql-fkeys (catalog &key (section :post) label log-level)
   "Actually create the Foreign Key References that where declared in the
    MySQL database"
   (let ((fk-sql-list
@@ -202,7 +202,7 @@
                               :do (log-message :debug "EXTRA FK DEPS! ~a" sql)
                               :collect sql)))))
     ;; and now execute our list
-    (pgsql-execute-with-timing section label fk-sql-list)))
+    (pgsql-execute-with-timing section label fk-sql-list :log-level log-level)))
 
 
 
diff --git a/src/sources/common/db-methods.lisp b/src/sources/common/db-methods.lisp
index 0524a82..1d2e3d7 100644
--- a/src/sources/common/db-methods.lisp
+++ b/src/sources/common/db-methods.lisp
@@ -146,7 +146,8 @@
         ;; Turn UNIQUE indexes into PRIMARY KEYS now
         ;;
         (when create-indexes
-          (pgsql-execute-with-timing :post "Primary Keys" pkeys)
+          (pgsql-execute-with-timing :post "Primary Keys" pkeys
+                                     :log-level :notice)
 
           ;;
           ;; Foreign Key Constraints
@@ -158,7 +159,8 @@
           (when foreign-keys
             (create-pgsql-fkeys catalog
                                 :section :post
-                                :label "Create Foreign Keys"))
+                                :label "Create Foreign Keys"
+                                :log-level :notice))
 
           ;;
           ;; Triggers and stored procedures -- includes special default values
@@ -400,6 +402,8 @@
 
                      (when (and create-indexes
                                 (zerop (gethash table writers-count)))
+                       (log-message :notice "DONE copying ~a"
+                                    (format-table-name table))
                        (alexandria:appendf
                         pkeys
                         (create-indexes-in-kernel (target-db copy)
@@ -422,6 +426,7 @@
           (loop :for count :below (count-indexes catalog)
              :do (lp:receive-result idx-channel))
           (lp:end-kernel :wait t)
+          (log-message :info "Done waiting for indexes")
           (count-indexes catalog))))
 
     ;;