Refactor error handling in complete-pgsql-database.

Given new SQLite test case from issue #563 we see that pgloader doesn't
handle errors gracefully in post-copy stage. That's because the API were not
properly defined, we should use pgsql-execute-with-timing rather than other
construct here, because it allows the "on error resume next" behavior we
want with after load DDL statements.

See #563.

src/pgsql/connection.lisp

@@ -239,12 +239,15 @@
     (pomo:with-transaction ()
       (pgsql-execute-with-timing section label sql :count count))))
 
-(defun pgsql-execute-with-timing (section label sql &key (count 1))
+(defun pgsql-execute-with-timing (section label sql
+                                  &key
+                                    client-min-messages
+                                    (count 1))
   "Execute given SQL and resgister its timing into STATE."
   (multiple-value-bind (res secs)
       (timing
         (handler-case
-            (pgsql-execute sql)
+            (pgsql-execute sql :client-min-messages client-min-messages)
          (cl-postgres:database-error (e)
            (log-message :error "~a" e)
            (update-stats section label :errs 1 :rows (- count)))))

src/pgsql/pgsql-create-schema.lisp

@@ -104,7 +104,11 @@
                      :include-drop include-drop
                      :client-min-messages client-min-messages))
 
-(defun create-triggers (catalog &key (client-min-messages :notice))
+(defun create-triggers (catalog
+                        &key
+                          label
+                          (section :post)
+                          (client-min-messages :notice))
   "Create the catalog objects that come after the data has been loaded."
   (let ((sql-list
          (loop :for table :in (table-list catalog)
@@ -113,9 +117,9 @@
             :append (loop :for trigger :in (table-trigger-list table)
                        :collect (format-create-sql (trigger-procedure trigger))
                        :collect (format-create-sql trigger)))))
-    (loop :for sql :in sql-list
+    (pgsql-execute-with-timing section label sql-list
-       :do (pgsql-execute sql :client-min-messages client-min-messages)
+                               :count (length sql-list)
-       :count t)))
+                               :client-min-messages client-min-messages)))
 
 
 ;;;
@@ -183,20 +187,22 @@
                            (pgsql-execute sql))
                      :count t))))
 
-(defun create-pgsql-fkeys (catalog)
+(defun create-pgsql-fkeys (catalog &key (section :post) label)
   "Actually create the Foreign Key References that where declared in the
    MySQL database"
-  (loop :for table :in (table-list catalog)
+  (let ((fk-sql-list
-     :sum (loop :for fkey :in (table-fkey-list table)
+         (loop :for table :in (table-list catalog)
-             :for sql := (format-create-sql fkey)
+            :append (loop :for fkey :in (table-fkey-list table)
-             :do (pgsql-execute sql)
+                       :for sql := (format-create-sql fkey)
-             :count t)
+                       :collect sql)
-     :sum (loop :for index :in (table-index-list table)
+            :append (loop :for index :in (table-index-list table)
-             :sum (loop :for fkey :in (index-fk-deps index)
+                       :do (loop :for fkey :in (index-fk-deps index)
-                     :for sql := (format-create-sql fkey)
+                              :for sql := (format-create-sql fkey)
-                     :do (log-message :debug "EXTRA FK DEPS!")
+                              :do (log-message :debug "EXTRA FK DEPS! ~a" sql)
-                     :do (pgsql-execute sql)
+                              :collect sql)))))
-                     :count t))))
+    ;; and now execute our list
+    (pgsql-execute-with-timing section label fk-sql-list
+                               :count (length fk-sql-list))))
 
 
 
@@ -384,7 +390,7 @@ $$; " tables)))
 ;;;
 ;;; Comments
 ;;;
-(defun comment-on-tables-and-columns (catalog)
+(defun comment-on-tables-and-columns (catalog &key label (section :post))
   "Install comments on tables and columns from CATALOG."
   (let* ((quote
           ;; just something improbably found in a table comment, to use as
@@ -400,20 +406,24 @@ $$; " tables)))
                        "_"
                        (map 'string #'code-char
                             (loop :repeat 5
-                               :collect (+ (random 26) (char-code #\A)))))))
+                               :collect (+ (random 26) (char-code #\A))))))
-    (loop :for table :in (table-list catalog)
-       :for sql := (when (table-comment table)
-                     (format nil "comment on table ~a is $~a$~a$~a$"
-                             (table-name table)
-                             quote (table-comment table) quote))
-       :count (when sql
-                (pgsql-execute-with-timing :post "Comments" sql))
 
-       :sum (loop :for column :in (table-column-list table)
+         (sql-list
-               :for sql := (when (column-comment column)
+          ;; table level comments
-                             (format nil "comment on column ~a.~a is $~a$~a$~a$"
+          (loop :for table :in (table-list catalog)
-                                     (table-name table)
+             :when (table-comment table)
-                                     (column-name column)
+             :collect (format nil "comment on table ~a is $~a$~a$~a$"
-                                     quote (column-comment column) quote))
+                              (table-name table)
-               :count (when sql
+                              quote (table-comment table) quote)
-                        (pgsql-execute-with-timing :post "Comments" sql))))))
+
+             ;; for each table, append column level comments
+             :append
+             (loop :for column :in (table-column-list table)
+                :when (column-comment column)
+                :collect (format nil
+                                 "comment on column ~a.~a is $~a$~a$~a$"
+                                 (table-name table)
+                                 (column-name column)
+                                 quote (column-comment column) quote)))))
+    (pgsql-execute-with-timing section label sql-list
+                               :count (length sql-list))))

src/sources/common/db-methods.lisp

@@ -147,28 +147,25 @@
       ;; and indexes are imported before doing that.
       ;;
       (when foreign-keys
-        (with-stats-collection ("Create Foreign Keys" :section :post
+        (create-pgsql-fkeys catalog
-                                                      :use-result-as-read t
+                            :section :post
-                                                      :use-result-as-rows t)
+                            :label "Create Foreign Keys"))
-          (create-pgsql-fkeys catalog)))
 
       ;;
       ;; Triggers and stored procedures -- includes special default values
       ;;
       (when create-triggers
-        (with-stats-collection ("Create Triggers" :section :post
+        (with-pgsql-transaction (:pgconn (target-db copy))
-                                                  :use-result-as-read t
+          (create-triggers catalog
-                                                  :use-result-as-rows t)
+                           :section :post
-          (with-pgsql-transaction (:pgconn (target-db copy))
+                           :label "Create Triggers"))))
-            (create-triggers catalog)))))
 
     ;;
     ;; And now, comments on tables and columns.
     ;;
-    (with-stats-collection ("Install Comments" :section :post
+    (comment-on-tables-and-columns catalog
-                                               :use-result-as-read t
+                                   :section :post
-                                               :use-result-as-rows t)
+                                   :label "Install Comments")))
-      (comment-on-tables-and-columns catalog))))
 
 (defmethod instanciate-table-copy-object ((copy db-copy) (table table))
   "Create an new instance for copying TABLE data."

test/sqlite-testpk.load

@@ -0,0 +1,9 @@
+load database
+     from 'sqlite/test_pk.db'
+     into postgresql:///pgloader
+
+  before load do
+    $$ drop schema if exists sqlite cascade; $$,
+    $$ create schema if not exists sqlite; $$
+
+  set search_path to 'sqlite';

test/sqlite/test_pk.sql

@@ -0,0 +1,14 @@
+--
+-- sqlite3 -batch test_pk.db < sqlite_pk.sql
+--
+
+PRAGMA foreign_keys = ON;
+
+CREATE TABLE division_kind (
+    division_kind_id INTEGER PRIMARY KEY
+);
+
+CREATE TABLE division (
+    division_id INTEGER PRIMARY KEY,
+    division_kind_id INTEGER REFERENCES division_kind(division_kind_id)
+);