mirrors/pgloader
1
0
Fork 0
mirror of https://github.com/dimitri/pgloader.git synced 2025-08-09 15:56:58 +02:00
Code Issues Projects Releases Wiki Activity

Truncate all tables in a single command, fix #61.

Browse Source 
The truncate command is only sent to PostgreSQL when we didn't just
CREATE TABLE before. Some refactoring would be necessary to fit the
TRUNCATE command within the same transaction as the CREATE TABLE
command, for PostgreSQL performances.

This patch has been testing with MySQL and SQLite sources, the trick is
that to be able to test it, it's needed to first make a full
import (creating the target tables), so the test are not modified yet.
This commit is contained in:
Dimitri Fontaine 2014-05-19 18:07:35 +02:00
parent c3742a9410
commit e710cacad1
5 changed files with 57 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/package.lisp
View File

@ -70,7 +70,7 @@
#:with-pgsql-connection
#:pgsql-execute
#:pgsql-execute-with-timing
#:truncate-table
#:truncate-tables
#:copy-from-file
#:copy-from-queue
#:list-databases
@ -212,6 +212,7 @@
#:list-tables-and-fkeys
#:list-table-oids
#:create-tables
#:truncate-tables
#:format-pgsql-column
#:format-extra-type
#:make-pgsql-fkey
@ -244,6 +245,7 @@
#:pgsql-execute-with-timing
#:apply-identifier-case
#:create-tables
#:truncate-tables
#:format-pgsql-column
#:make-pgsql-index
#:index-table-name

5
src/pgsql/queries.lisp
View File

@ -110,11 +110,6 @@
;;;
;;; PostgreSQL Utility Queries
;;;
(defun truncate-table (dbname table-name)
"Truncate given TABLE-NAME in database DBNAME"
(pomo:with-connection (get-connection-spec dbname)
(set-session-gucs *pg-settings*)
(pomo:execute (format nil "truncate ~a;" table-name))))
(defun list-databases (&optional (username "postgres"))
"Connect to a local database and get the database list"

12
src/pgsql/schema.lisp
View File

@ -176,6 +176,18 @@
(pgsql-execute sql :client-min-messages client-min-messages)
finally (return nb-tables)))
(defun truncate-tables (dbname table-name-list
&key identifier-case)
"Truncate given TABLE-NAME in database DBNAME"
(pomo:with-connection (get-connection-spec dbname)
(set-session-gucs *pg-settings*)
(let ((sql (format nil "TRUNCATE ~{~s~^,~};"
(loop :for table-name :in table-name-list
:collect (apply-identifier-case table-name
identifier-case)))))
(log-message :notice "~a" sql)
(pomo:execute sql))))
;;;
;;; Index support

47
src/sources/mysql.lisp
View File

@ -351,26 +351,31 @@
(lp:make-channel)))))
;; if asked, first drop/create the tables on the PostgreSQL side
(when (and (or create-tables schema-only) (not data-only))
(handler-case
(prepare-pgsql-database all-columns
all-indexes
all-fkeys
materialize-views
view-columns
:state state-before
:foreign-keys foreign-keys
:identifier-case identifier-case
:include-drop include-drop)
;;
;; In case some error happens in the preparatory transaction, we
;; need to stop now and refrain from trying to load the data into
;; an incomplete schema.
;;
(cl-postgres:database-error (e)
(declare (ignore e)) ; a log has already been printed
(log-message :fatal "Failed to create the schema, see above.")
(return-from copy-database))))
(handler-case
(cond ((and (or create-tables schema-only) (not data-only))
(prepare-pgsql-database all-columns
all-indexes
all-fkeys
materialize-views
view-columns
:state state-before
:foreign-keys foreign-keys
:identifier-case identifier-case
:include-drop include-drop))
(t
(when truncate
(truncate-tables *pg-dbname*
(mapcar #'car all-columns)
:identifier-case identifier-case))))
;;
;; In case some error happens in the preparatory transaction, we
;; need to stop now and refrain from trying to load the data into
;; an incomplete schema.
;;
(cl-postgres:database-error (e)
(declare (ignore e)) ; a log has already been printed
(log-message :fatal "Failed to create the schema, see above.")
(return-from copy-database)))
(loop
for (table-name . columns) in (append all-columns view-columns)
@ -401,7 +406,7 @@
;; first COPY the data from MySQL to PostgreSQL, using copy-kernel
(unless schema-only
(copy-from table-source :kernel copy-kernel :truncate truncate))
(copy-from table-source :kernel copy-kernel))
;; Create the indexes for that table in parallel with the next
;; COPY, and all at once in concurrent threads to benefit from

25
src/sources/sqlite.lisp
View File

@ -221,7 +221,8 @@
reset-sequences
only-tables
including
excluding)
excluding
(identifier-case :downcase))
"Stream the given SQLite database down to PostgreSQL."
(let* ((summary (null *state*))
(*state* (or *state* (make-pgstate)))
@ -247,13 +248,19 @@
(pg-dbname (target-db sqlite)))
;; if asked, first drop/create the tables on the PostgreSQL side
(when (and (or create-tables schema-only) (not data-only))
(log-message :notice "~:[~;DROP then ~]CREATE TABLES" include-drop)
(with-stats-collection ("create, truncate"
:state state-before
:summary summary)
(with-pgsql-transaction ()
(create-tables all-columns :include-drop include-drop))))
(cond ((and (or create-tables schema-only) (not data-only))
(log-message :notice "~:[~;DROP then ~]CREATE TABLES" include-drop)
(with-stats-collection ("create, truncate"
:state state-before
:summary summary)
(with-pgsql-transaction ()
(create-tables all-columns
:include-drop include-drop
:identifier-case identifier-case))))
(truncate
(truncate-tables *pg-dbname* (mapcar #'car all-columns)
:identifier-case identifier-case)))
(loop
for (table-name . columns) in all-columns
@ -268,7 +275,7 @@
:fields columns)))
;; first COPY the data from SQLite to PostgreSQL, using copy-kernel
(unless schema-only
(copy-from table-source :kernel copy-kernel :truncate truncate))
(copy-from table-source :kernel copy-kernel))
;; Create the indexes for that table in parallel with the next
;; COPY, and all at once in concurrent threads to benefit from