Truncate all tables in a single command, fix #61.

The truncate command is only sent to PostgreSQL when we didn't just
CREATE TABLE before. Some refactoring would be necessary to fit the
TRUNCATE command within the same transaction as the CREATE TABLE
command, for PostgreSQL performances.

This patch has been testing with MySQL and SQLite sources, the trick is
that to be able to test it, it's needed to first make a full
import (creating the target tables), so the test are not modified yet.

src/package.lisp

@@ -70,7 +70,7 @@
 	   #:with-pgsql-connection
 	   #:pgsql-execute
 	   #:pgsql-execute-with-timing
-	   #:truncate-table
+	   #:truncate-tables
 	   #:copy-from-file
 	   #:copy-from-queue
 	   #:list-databases
@@ -212,6 +212,7 @@
 		#:list-tables-and-fkeys
 		#:list-table-oids
 		#:create-tables
+                #:truncate-tables
 		#:format-pgsql-column
 		#:format-extra-type
 		#:make-pgsql-fkey
@@ -244,6 +245,7 @@
 		#:pgsql-execute-with-timing
 		#:apply-identifier-case
 		#:create-tables
+                #:truncate-tables
 		#:format-pgsql-column
 		#:make-pgsql-index
 		#:index-table-name

src/pgsql/queries.lisp

@@ -110,11 +110,6 @@
 ;;;
 ;;; PostgreSQL Utility Queries
 ;;;
-(defun truncate-table (dbname table-name)
-  "Truncate given TABLE-NAME in database DBNAME"
-  (pomo:with-connection (get-connection-spec dbname)
-    (set-session-gucs *pg-settings*)
-    (pomo:execute (format nil "truncate ~a;" table-name))))
 
 (defun list-databases (&optional (username "postgres"))
   "Connect to a local database and get the database list"

src/pgsql/schema.lisp

@@ -176,6 +176,18 @@
        (pgsql-execute sql :client-min-messages client-min-messages)
      finally (return nb-tables)))
 
+(defun truncate-tables (dbname table-name-list
+                        &key identifier-case)
+  "Truncate given TABLE-NAME in database DBNAME"
+  (pomo:with-connection (get-connection-spec dbname)
+    (set-session-gucs *pg-settings*)
+    (let ((sql (format nil "TRUNCATE ~{~s~^,~};"
+                       (loop :for table-name :in table-name-list
+                          :collect (apply-identifier-case table-name
+                                                          identifier-case)))))
+      (log-message :notice "~a" sql)
+      (pomo:execute sql))))
+
 
 ;;;
 ;;; Index support

src/sources/mysql.lisp

@@ -351,26 +351,31 @@
                                 (lp:make-channel)))))
 
       ;; if asked, first drop/create the tables on the PostgreSQL side
-      (when (and (or create-tables schema-only) (not data-only))
+      (handler-case
-        (handler-case
+          (cond ((and (or create-tables schema-only) (not data-only))
-            (prepare-pgsql-database all-columns
+                 (prepare-pgsql-database all-columns
-                                    all-indexes
+                                         all-indexes
-                                    all-fkeys
+                                         all-fkeys
-                                    materialize-views
+                                         materialize-views
-                                    view-columns
+                                         view-columns
-                                    :state state-before
+                                         :state state-before
-                                    :foreign-keys foreign-keys
+                                         :foreign-keys foreign-keys
-                                    :identifier-case identifier-case
+                                         :identifier-case identifier-case
-                                    :include-drop include-drop)
+                                         :include-drop include-drop))
-          ;;
+                (t
-          ;; In case some error happens in the preparatory transaction, we
+                 (when truncate
-          ;; need to stop now and refrain from trying to load the data into
+                   (truncate-tables *pg-dbname*
-          ;; an incomplete schema.
+                                    (mapcar #'car all-columns)
-          ;;
+                                    :identifier-case identifier-case))))
-          (cl-postgres:database-error (e)
+        ;;
-            (declare (ignore e))		; a log has already been printed
+        ;; In case some error happens in the preparatory transaction, we
-            (log-message :fatal "Failed to create the schema, see above.")
+        ;; need to stop now and refrain from trying to load the data into
-            (return-from copy-database))))
+        ;; an incomplete schema.
+        ;;
+        (cl-postgres:database-error (e)
+          (declare (ignore e))		; a log has already been printed
+          (log-message :fatal "Failed to create the schema, see above.")
+          (return-from copy-database)))
 
       (loop
          for (table-name . columns) in (append all-columns view-columns)
@@ -401,7 +406,7 @@
 
              ;; first COPY the data from MySQL to PostgreSQL, using copy-kernel
              (unless schema-only
-               (copy-from table-source :kernel copy-kernel :truncate truncate))
+               (copy-from table-source :kernel copy-kernel))
 
              ;; Create the indexes for that table in parallel with the next
              ;; COPY, and all at once in concurrent threads to benefit from

src/sources/sqlite.lisp

@@ -221,7 +221,8 @@
 			    reset-sequences
 			    only-tables
 			    including
-			    excluding)
+			    excluding
+                            (identifier-case :downcase))
   "Stream the given SQLite database down to PostgreSQL."
   (let* ((summary       (null *state*))
 	 (*state*       (or *state* (make-pgstate)))
@@ -247,13 +248,19 @@
 	 (pg-dbname     (target-db sqlite)))
 
     ;; if asked, first drop/create the tables on the PostgreSQL side
-    (when (and (or create-tables schema-only) (not data-only))
+    (cond ((and (or create-tables schema-only) (not data-only))
-      (log-message :notice "~:[~;DROP then ~]CREATE TABLES" include-drop)
+           (log-message :notice "~:[~;DROP then ~]CREATE TABLES" include-drop)
-      (with-stats-collection ("create, truncate"
+           (with-stats-collection ("create, truncate"
-                              :state state-before
+                                   :state state-before
-                              :summary summary)
+                                   :summary summary)
-	(with-pgsql-transaction ()
+             (with-pgsql-transaction ()
-	  (create-tables all-columns :include-drop include-drop))))
+               (create-tables all-columns
+                              :include-drop include-drop
+                              :identifier-case identifier-case))))
+
+          (truncate
+           (truncate-tables *pg-dbname* (mapcar #'car all-columns)
+                            :identifier-case identifier-case)))
 
     (loop
        for (table-name . columns) in all-columns
@@ -268,7 +275,7 @@
 			       :fields     columns)))
 	   ;; first COPY the data from SQLite to PostgreSQL, using copy-kernel
 	   (unless schema-only
-	     (copy-from table-source :kernel copy-kernel :truncate truncate))
+	     (copy-from table-source :kernel copy-kernel))
 
 	   ;; Create the indexes for that table in parallel with the next
 	   ;; COPY, and all at once in concurrent threads to benefit from