From e710cacad15be81f8b3ce470e6c22d5b947b96e7 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Mon, 19 May 2014 18:07:35 +0200 Subject: [PATCH] Truncate all tables in a single command, fix #61. The truncate command is only sent to PostgreSQL when we didn't just CREATE TABLE before. Some refactoring would be necessary to fit the TRUNCATE command within the same transaction as the CREATE TABLE command, for PostgreSQL performances. This patch has been testing with MySQL and SQLite sources, the trick is that to be able to test it, it's needed to first make a full import (creating the target tables), so the test are not modified yet. --- src/package.lisp | 4 +++- src/pgsql/queries.lisp | 5 ----- src/pgsql/schema.lisp | 12 +++++++++++ src/sources/mysql.lisp | 47 +++++++++++++++++++++++------------------ src/sources/sqlite.lisp | 25 ++++++++++++++-------- 5 files changed, 57 insertions(+), 36 deletions(-) diff --git a/src/package.lisp b/src/package.lisp index 46b52d1..c4d64b8 100644 --- a/src/package.lisp +++ b/src/package.lisp @@ -70,7 +70,7 @@ #:with-pgsql-connection #:pgsql-execute #:pgsql-execute-with-timing - #:truncate-table + #:truncate-tables #:copy-from-file #:copy-from-queue #:list-databases @@ -212,6 +212,7 @@ #:list-tables-and-fkeys #:list-table-oids #:create-tables + #:truncate-tables #:format-pgsql-column #:format-extra-type #:make-pgsql-fkey @@ -244,6 +245,7 @@ #:pgsql-execute-with-timing #:apply-identifier-case #:create-tables + #:truncate-tables #:format-pgsql-column #:make-pgsql-index #:index-table-name diff --git a/src/pgsql/queries.lisp b/src/pgsql/queries.lisp index f5ecc11..5322ea7 100644 --- a/src/pgsql/queries.lisp +++ b/src/pgsql/queries.lisp @@ -110,11 +110,6 @@ ;;; ;;; PostgreSQL Utility Queries ;;; -(defun truncate-table (dbname table-name) - "Truncate given TABLE-NAME in database DBNAME" - (pomo:with-connection (get-connection-spec dbname) - (set-session-gucs *pg-settings*) - (pomo:execute (format nil "truncate ~a;" table-name)))) (defun list-databases (&optional (username "postgres")) "Connect to a local database and get the database list" diff --git a/src/pgsql/schema.lisp b/src/pgsql/schema.lisp index 6a3e5a3..c0e8198 100644 --- a/src/pgsql/schema.lisp +++ b/src/pgsql/schema.lisp @@ -176,6 +176,18 @@ (pgsql-execute sql :client-min-messages client-min-messages) finally (return nb-tables))) +(defun truncate-tables (dbname table-name-list + &key identifier-case) + "Truncate given TABLE-NAME in database DBNAME" + (pomo:with-connection (get-connection-spec dbname) + (set-session-gucs *pg-settings*) + (let ((sql (format nil "TRUNCATE ~{~s~^,~};" + (loop :for table-name :in table-name-list + :collect (apply-identifier-case table-name + identifier-case))))) + (log-message :notice "~a" sql) + (pomo:execute sql)))) + ;;; ;;; Index support diff --git a/src/sources/mysql.lisp b/src/sources/mysql.lisp index 270749c..2990f96 100644 --- a/src/sources/mysql.lisp +++ b/src/sources/mysql.lisp @@ -351,26 +351,31 @@ (lp:make-channel))))) ;; if asked, first drop/create the tables on the PostgreSQL side - (when (and (or create-tables schema-only) (not data-only)) - (handler-case - (prepare-pgsql-database all-columns - all-indexes - all-fkeys - materialize-views - view-columns - :state state-before - :foreign-keys foreign-keys - :identifier-case identifier-case - :include-drop include-drop) - ;; - ;; In case some error happens in the preparatory transaction, we - ;; need to stop now and refrain from trying to load the data into - ;; an incomplete schema. - ;; - (cl-postgres:database-error (e) - (declare (ignore e)) ; a log has already been printed - (log-message :fatal "Failed to create the schema, see above.") - (return-from copy-database)))) + (handler-case + (cond ((and (or create-tables schema-only) (not data-only)) + (prepare-pgsql-database all-columns + all-indexes + all-fkeys + materialize-views + view-columns + :state state-before + :foreign-keys foreign-keys + :identifier-case identifier-case + :include-drop include-drop)) + (t + (when truncate + (truncate-tables *pg-dbname* + (mapcar #'car all-columns) + :identifier-case identifier-case)))) + ;; + ;; In case some error happens in the preparatory transaction, we + ;; need to stop now and refrain from trying to load the data into + ;; an incomplete schema. + ;; + (cl-postgres:database-error (e) + (declare (ignore e)) ; a log has already been printed + (log-message :fatal "Failed to create the schema, see above.") + (return-from copy-database))) (loop for (table-name . columns) in (append all-columns view-columns) @@ -401,7 +406,7 @@ ;; first COPY the data from MySQL to PostgreSQL, using copy-kernel (unless schema-only - (copy-from table-source :kernel copy-kernel :truncate truncate)) + (copy-from table-source :kernel copy-kernel)) ;; Create the indexes for that table in parallel with the next ;; COPY, and all at once in concurrent threads to benefit from diff --git a/src/sources/sqlite.lisp b/src/sources/sqlite.lisp index 81e2a79..d38593a 100644 --- a/src/sources/sqlite.lisp +++ b/src/sources/sqlite.lisp @@ -221,7 +221,8 @@ reset-sequences only-tables including - excluding) + excluding + (identifier-case :downcase)) "Stream the given SQLite database down to PostgreSQL." (let* ((summary (null *state*)) (*state* (or *state* (make-pgstate))) @@ -247,13 +248,19 @@ (pg-dbname (target-db sqlite))) ;; if asked, first drop/create the tables on the PostgreSQL side - (when (and (or create-tables schema-only) (not data-only)) - (log-message :notice "~:[~;DROP then ~]CREATE TABLES" include-drop) - (with-stats-collection ("create, truncate" - :state state-before - :summary summary) - (with-pgsql-transaction () - (create-tables all-columns :include-drop include-drop)))) + (cond ((and (or create-tables schema-only) (not data-only)) + (log-message :notice "~:[~;DROP then ~]CREATE TABLES" include-drop) + (with-stats-collection ("create, truncate" + :state state-before + :summary summary) + (with-pgsql-transaction () + (create-tables all-columns + :include-drop include-drop + :identifier-case identifier-case)))) + + (truncate + (truncate-tables *pg-dbname* (mapcar #'car all-columns) + :identifier-case identifier-case))) (loop for (table-name . columns) in all-columns @@ -268,7 +275,7 @@ :fields columns))) ;; first COPY the data from SQLite to PostgreSQL, using copy-kernel (unless schema-only - (copy-from table-source :kernel copy-kernel :truncate truncate)) + (copy-from table-source :kernel copy-kernel)) ;; Create the indexes for that table in parallel with the next ;; COPY, and all at once in concurrent threads to benefit from