From 48f451bdbc50dba53d2aa0bdf5fa0bebbaeccbba Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 19 Feb 2015 15:04:41 +0100 Subject: [PATCH] Implement the option to disable triggers when loading data. This option is dangerous and allows to skip ALL triggers when loading data against PostgreSQL. This includes foreign key constraints definitions and will allow loading data out of order. When using both the options "create no table" and "disable triggers" it will be possible to load data into a schema prepared by your favorite external tool, at the cost of not validating FK constraints. Use with care. Fix #167. --- pgloader.1 | 65 ++++++++++++++++++++++- pgloader.1.md | 86 ++++++++++++++++++++++++++++++- src/parsers/command-copy.lisp | 6 ++- src/parsers/command-csv.lisp | 9 +++- src/parsers/command-dbf.lisp | 1 + src/parsers/command-fixed.lisp | 7 ++- src/parsers/command-keywords.lisp | 2 + src/parsers/command-options.lisp | 14 ++--- src/parsers/command-sqlite.lisp | 1 + src/pgsql/pgsql.lisp | 13 +++-- src/pgsql/schema.lisp | 16 ++++++ src/sources/copy.lisp | 5 +- src/sources/csv/csv.lisp | 5 +- src/sources/db3/db3.lisp | 19 ++++--- src/sources/fixed.lisp | 5 +- src/sources/ixf/ixf.lisp | 19 ++++--- src/sources/mssql/mssql.lisp | 27 ++++++---- src/sources/mysql/mysql.lisp | 27 ++++++---- src/sources/sqlite/sqlite.lisp | 21 +++++--- test/csv-districts.load | 1 + test/dbf.load | 5 +- test/parse/hans.goeuro.load | 8 ++- test/regress/expected/dbf.out | 12 ++--- 23 files changed, 296 insertions(+), 78 deletions(-) diff --git a/pgloader.1 b/pgloader.1 index 4ae5e3b..0d140c3 100644 --- a/pgloader.1 +++ b/pgloader.1 @@ -967,6 +967,15 @@ When loading from a \fBCSV\fR file, the following options are supported: When this option is listed, pgloader issues a \fBTRUNCATE\fR command against the PostgreSQL target table before reading the data file\. . .IP "\(bu" 4 +\fIdisable triggers\fR +. +.IP +When this option is listed, pgloader issues an \fBALTER TABLE \.\.\. DISABLE TRIGGER ALL\fR command against the PostgreSQL target table before copying the data, then the command \fBALTER TABLE \.\.\. ENABLE TRIGGER ALL\fR once the \fBCOPY\fR is done\. +. +.IP +This option allows loading data into a pre\-existing table ignoring the \fIforeign key constraints\fR and user defined triggers and may result in invalid \fIforeign key constraints\fR once the data is loaded\. Use with care\. +. +.IP "\(bu" 4 \fIskip header\fR . .IP @@ -1217,7 +1226,7 @@ This option allows to trim whitespaces in the read data, either from both sides \fIWITH\fR . .IP -When loading from a \fBCSV\fR file, the following options are supported: +When loading from a \fBFIXED\fR file, the following options are supported: . .IP "\(bu" 4 \fItruncate\fR @@ -1226,6 +1235,15 @@ When loading from a \fBCSV\fR file, the following options are supported: When this option is listed, pgloader issues a \fBTRUNCATE\fR command against the PostgreSQL target table before reading the data file\. . .IP "\(bu" 4 +\fIdisable triggers\fR +. +.IP +When this option is listed, pgloader issues an \fBALTER TABLE \.\.\. DISABLE TRIGGER ALL\fR command against the PostgreSQL target table before copying the data, then the command \fBALTER TABLE \.\.\. ENABLE TRIGGER ALL\fR once the \fBCOPY\fR is done\. +. +.IP +This option allows loading data into a pre\-existing table ignoring the \fIforeign key constraints\fR and user defined triggers and may result in invalid \fIforeign key constraints\fR once the data is loaded\. Use with care\. +. +.IP "\(bu" 4 \fIskip header\fR . .IP @@ -1298,6 +1316,15 @@ When loading from a \fBCOPY\fR file, the following options are supported: When this option is listed, pgloader issues a \fBTRUNCATE\fR command against the PostgreSQL target table before reading the data file\. . .IP "\(bu" 4 +\fIdisable triggers\fR +. +.IP +When this option is listed, pgloader issues an \fBALTER TABLE \.\.\. DISABLE TRIGGER ALL\fR command against the PostgreSQL target table before copying the data, then the command \fBALTER TABLE \.\.\. ENABLE TRIGGER ALL\fR once the \fBCOPY\fR is done\. +. +.IP +This option allows loading data into a pre\-existing table ignoring the \fIforeign key constraints\fR and user defined triggers and may result in invalid \fIforeign key constraints\fR once the data is loaded\. Use with care\. +. +.IP "\(bu" 4 \fIskip header\fR . .IP @@ -1346,6 +1373,15 @@ When loading from a \fBDBF\fR file, the following options are supported: When this option is listed, pgloader issues a \fBTRUNCATE\fR command against the PostgreSQL target table before reading the data file\. . .IP "\(bu" 4 +\fIdisable triggers\fR +. +.IP +When this option is listed, pgloader issues an \fBALTER TABLE \.\.\. DISABLE TRIGGER ALL\fR command against the PostgreSQL target table before copying the data, then the command \fBALTER TABLE \.\.\. ENABLE TRIGGER ALL\fR once the \fBCOPY\fR is done\. +. +.IP +This option allows loading data into a pre\-existing table ignoring the \fIforeign key constraints\fR and user defined triggers and may result in invalid \fIforeign key constraints\fR once the data is loaded\. Use with care\. +. +.IP "\(bu" 4 \fIcreate table\fR . .IP @@ -1404,6 +1440,15 @@ When loading from a \fBIXF\fR file, the following options are supported: When this option is listed, pgloader issues a \fBTRUNCATE\fR command against the PostgreSQL target table before reading the data file\. . .IP "\(bu" 4 +\fIdisable triggers\fR +. +.IP +When this option is listed, pgloader issues an \fBALTER TABLE \.\.\. DISABLE TRIGGER ALL\fR command against the PostgreSQL target table before copying the data, then the command \fBALTER TABLE \.\.\. ENABLE TRIGGER ALL\fR once the \fBCOPY\fR is done\. +. +.IP +This option allows loading data into a pre\-existing table ignoring the \fIforeign key constraints\fR and user defined triggers and may result in invalid \fIforeign key constraints\fR once the data is loaded\. Use with care\. +. +.IP "\(bu" 4 \fIcreate table\fR . .IP @@ -1642,6 +1687,15 @@ When this option is listed, pgloader issue the \fBTRUNCATE\fR command against ea When this option is listed, pgloader issues no \fBTRUNCATE\fR command\. . .IP "\(bu" 4 +\fIdisable triggers\fR +. +.IP +When this option is listed, pgloader issues an \fBALTER TABLE \.\.\. DISABLE TRIGGER ALL\fR command against the PostgreSQL target table before copying the data, then the command \fBALTER TABLE \.\.\. ENABLE TRIGGER ALL\fR once the \fBCOPY\fR is done\. +. +.IP +This option allows loading data into a pre\-existing table ignoring the \fIforeign key constraints\fR and user defined triggers and may result in invalid \fIforeign key constraints\fR once the data is loaded\. Use with care\. +. +.IP "\(bu" 4 \fIcreate tables\fR . .IP @@ -2190,6 +2244,15 @@ When this option is listed, pgloader issue the \fBTRUNCATE\fR command against ea When this option is listed, pgloader issues no \fBTRUNCATE\fR command\. . .IP "\(bu" 4 +\fIdisable triggers\fR +. +.IP +When this option is listed, pgloader issues an \fBALTER TABLE \.\.\. DISABLE TRIGGER ALL\fR command against the PostgreSQL target table before copying the data, then the command \fBALTER TABLE \.\.\. ENABLE TRIGGER ALL\fR once the \fBCOPY\fR is done\. +. +.IP +This option allows loading data into a pre\-existing table ignoring the \fIforeign key constraints\fR and user defined triggers and may result in invalid \fIforeign key constraints\fR once the data is loaded\. Use with care\. +. +.IP "\(bu" 4 \fIcreate tables\fR . .IP diff --git a/pgloader.1.md b/pgloader.1.md index beade9b..9135672 100644 --- a/pgloader.1.md +++ b/pgloader.1.md @@ -824,6 +824,18 @@ The `csv` format command accepts the following clauses and options: When this option is listed, pgloader issues a `TRUNCATE` command against the PostgreSQL target table before reading the data file. + - *disable triggers* + + When this option is listed, pgloader issues an `ALTER TABLE ... + DISABLE TRIGGER ALL` command against the PostgreSQL target table + before copying the data, then the command `ALTER TABLE ... ENABLE + TRIGGER ALL` once the `COPY` is done. + + This option allows loading data into a pre-existing table ignoring + the *foreign key constraints* and user defined triggers and may + result in invalid *foreign key constraints* once the data is loaded. + Use with care. + - *skip header* Takes a numeric value as argument. Instruct pgloader to skip that @@ -1020,13 +1032,25 @@ The `fixed` format command accepts the following clauses and options: - *WITH* - When loading from a `CSV` file, the following options are supported: + When loading from a `FIXED` file, the following options are supported: - *truncate* When this option is listed, pgloader issues a `TRUNCATE` command against the PostgreSQL target table before reading the data file. + - *disable triggers* + + When this option is listed, pgloader issues an `ALTER TABLE ... + DISABLE TRIGGER ALL` command against the PostgreSQL target table + before copying the data, then the command `ALTER TABLE ... ENABLE + TRIGGER ALL` once the `COPY` is done. + + This option allows loading data into a pre-existing table ignoring + the *foreign key constraints* and user defined triggers and may + result in invalid *foreign key constraints* once the data is loaded. + Use with care. + - *skip header* Takes a numeric value as argument. Instruct pgloader to skip that @@ -1084,6 +1108,18 @@ The `COPY` format command accepts the following clauses and options: When this option is listed, pgloader issues a `TRUNCATE` command against the PostgreSQL target table before reading the data file. + - *disable triggers* + + When this option is listed, pgloader issues an `ALTER TABLE ... + DISABLE TRIGGER ALL` command against the PostgreSQL target table + before copying the data, then the command `ALTER TABLE ... ENABLE + TRIGGER ALL` once the `COPY` is done. + + This option allows loading data into a pre-existing table ignoring + the *foreign key constraints* and user defined triggers and may + result in invalid *foreign key constraints* once the data is loaded. + Use with care. + - *skip header* Takes a numeric value as argument. Instruct pgloader to skip that @@ -1116,6 +1152,18 @@ The `dbf` format command accepts the following clauses and options: When this option is listed, pgloader issues a `TRUNCATE` command against the PostgreSQL target table before reading the data file. + - *disable triggers* + + When this option is listed, pgloader issues an `ALTER TABLE ... + DISABLE TRIGGER ALL` command against the PostgreSQL target table + before copying the data, then the command `ALTER TABLE ... ENABLE + TRIGGER ALL` once the `COPY` is done. + + This option allows loading data into a pre-existing table ignoring + the *foreign key constraints* and user defined triggers and may + result in invalid *foreign key constraints* once the data is loaded. + Use with care. + - *create table* When this option is listed, pgloader creates the table using the @@ -1159,6 +1207,18 @@ The `ixf` format command accepts the following clauses and options: When this option is listed, pgloader issues a `TRUNCATE` command against the PostgreSQL target table before reading the data file. + - *disable triggers* + + When this option is listed, pgloader issues an `ALTER TABLE ... + DISABLE TRIGGER ALL` command against the PostgreSQL target table + before copying the data, then the command `ALTER TABLE ... ENABLE + TRIGGER ALL` once the `COPY` is done. + + This option allows loading data into a pre-existing table ignoring + the *foreign key constraints* and user defined triggers and may + result in invalid *foreign key constraints* once the data is loaded. + Use with care. + - *create table* When this option is listed, pgloader creates the table using the @@ -1362,6 +1422,18 @@ The `database` command accepts the following clauses and options: When this option is listed, pgloader issues no `TRUNCATE` command. + - *disable triggers* + + When this option is listed, pgloader issues an `ALTER TABLE ... + DISABLE TRIGGER ALL` command against the PostgreSQL target table + before copying the data, then the command `ALTER TABLE ... ENABLE + TRIGGER ALL` once the `COPY` is done. + + This option allows loading data into a pre-existing table ignoring + the *foreign key constraints* and user defined triggers and may + result in invalid *foreign key constraints* once the data is loaded. + Use with care. + - *create tables* When this option is listed, pgloader creates the table using the @@ -1760,6 +1832,18 @@ The `sqlite` command accepts the following clauses and options: When this option is listed, pgloader issues no `TRUNCATE` command. + - *disable triggers* + + When this option is listed, pgloader issues an `ALTER TABLE ... + DISABLE TRIGGER ALL` command against the PostgreSQL target table + before copying the data, then the command `ALTER TABLE ... ENABLE + TRIGGER ALL` once the `COPY` is done. + + This option allows loading data into a pre-existing table ignoring + the *foreign key constraints* and user defined triggers and may + result in invalid *foreign key constraints* once the data is loaded. + Use with care. + - *create tables* When this option is listed, pgloader creates the table using the diff --git a/src/parsers/command-copy.lisp b/src/parsers/command-copy.lisp index 251864f..a68510d 100644 --- a/src/parsers/command-copy.lisp +++ b/src/parsers/command-copy.lisp @@ -28,6 +28,7 @@ option-batch-size option-batch-concurrency option-truncate + option-disable-triggers option-skip-header)) (defrule another-copy-option (and comma copy-option) @@ -115,6 +116,7 @@ ,(sql-code-block pg-db-conn 'state-before before "before load") (let ((truncate ,(getf options :truncate)) + (disable-triggers (getf ',options :disable-triggers)) (source (make-instance 'pgloader.copy:copy-copy :target-db ,pg-db-conn @@ -124,7 +126,9 @@ :fields ',fields :columns ',columns :skip-lines ,(or (getf options :skip-line) 0)))) - (pgloader.sources:copy-from source :truncate truncate)) + (pgloader.sources:copy-from source + :truncate truncate + :disable-triggers disable-triggers)) ,(sql-code-block pg-db-conn 'state-after after "after load") diff --git a/src/parsers/command-csv.lisp b/src/parsers/command-csv.lisp index 161bee3..a0024f8 100644 --- a/src/parsers/command-csv.lisp +++ b/src/parsers/command-csv.lisp @@ -93,6 +93,7 @@ option-batch-size option-batch-concurrency option-truncate + option-disable-triggers option-skip-header option-lines-terminated-by option-fields-not-enclosed @@ -434,6 +435,7 @@ ,(sql-code-block pg-db-conn 'state-before before "before load") (let ((truncate (getf ',options :truncate)) + (disable-triggers (getf ',options :disable-triggers)) (source (make-instance 'pgloader.csv:copy-csv :target-db ,pg-db-conn @@ -443,8 +445,11 @@ :fields ',fields :columns ',columns ,@(remove-batch-control-option - options :extras '(:truncate))))) - (pgloader.sources:copy-from source :truncate truncate)) + options :extras '(:truncate + :disable-triggers))))) + (pgloader.sources:copy-from source + :truncate truncate + :disable-triggers disable-triggers)) ,(sql-code-block pg-db-conn 'state-after after "after load") diff --git a/src/parsers/command-dbf.lisp b/src/parsers/command-dbf.lisp index 50ae02e..3dc958a 100644 --- a/src/parsers/command-dbf.lisp +++ b/src/parsers/command-dbf.lisp @@ -22,6 +22,7 @@ option-batch-size option-batch-concurrency option-truncate + option-disable-triggers option-data-only option-schema-only option-include-drop diff --git a/src/parsers/command-fixed.lisp b/src/parsers/command-fixed.lisp index 6b73c1e..88d11b3 100644 --- a/src/parsers/command-fixed.lisp +++ b/src/parsers/command-fixed.lisp @@ -47,6 +47,7 @@ option-batch-size option-batch-concurrency option-truncate + option-disable-triggers option-skip-header)) (defrule another-fixed-option (and comma fixed-option) @@ -134,6 +135,7 @@ ,(sql-code-block pg-db-conn 'state-before before "before load") (let ((truncate ,(getf options :truncate)) + (disable-triggers ,(getf options :disable-triggers)) (source (make-instance 'pgloader.fixed:copy-fixed :target-db ,pg-db-conn @@ -143,7 +145,10 @@ :fields ',fields :columns ',columns :skip-lines ,(or (getf options :skip-line) 0)))) - (pgloader.sources:copy-from source :truncate truncate)) + + (pgloader.sources:copy-from source + :truncate truncate + :disable-triggers disable-triggers)) ,(sql-code-block pg-db-conn 'state-after after "after load") diff --git a/src/parsers/command-keywords.lisp b/src/parsers/command-keywords.lisp index 210c60a..953a8e9 100644 --- a/src/parsers/command-keywords.lisp +++ b/src/parsers/command-keywords.lisp @@ -61,6 +61,8 @@ (def-keyword-rule "encoding") (def-keyword-rule "decoding") (def-keyword-rule "truncate") + (def-keyword-rule "disable") + (def-keyword-rule "triggers") (def-keyword-rule "lines") (def-keyword-rule "having") (def-keyword-rule "fields") diff --git a/src/parsers/command-options.lisp b/src/parsers/command-options.lisp index 338125e..ef616af 100644 --- a/src/parsers/command-options.lisp +++ b/src/parsers/command-options.lisp @@ -104,12 +104,13 @@ (declare (ignore ,@ignore)) (cons ,option (null no)))))) -(make-option-rule include-drop (and kw-include (? kw-no) kw-drop)) -(make-option-rule truncate (and (? kw-no) kw-truncate)) -(make-option-rule create-tables (and kw-create (? kw-no) kw-tables)) -(make-option-rule create-indexes (and kw-create (? kw-no) kw-indexes)) -(make-option-rule reset-sequences (and kw-reset (? kw-no) kw-sequences)) -(make-option-rule foreign-keys (and (? kw-no) kw-foreign kw-keys)) +(make-option-rule include-drop (and kw-include (? kw-no) kw-drop)) +(make-option-rule truncate (and (? kw-no) kw-truncate)) +(make-option-rule disable-triggers (and kw-disable (? kw-no) kw-triggers)) +(make-option-rule create-tables (and kw-create (? kw-no) kw-tables)) +(make-option-rule create-indexes (and kw-create (? kw-no) kw-indexes)) +(make-option-rule reset-sequences (and kw-reset (? kw-no) kw-sequences)) +(make-option-rule foreign-keys (and (? kw-no) kw-foreign kw-keys)) (defrule option-schema-only (and kw-schema kw-only) (:constant (cons :schema-only t))) @@ -127,6 +128,7 @@ option-batch-size option-batch-concurrency option-truncate + option-disable-triggers option-data-only option-schema-only option-include-drop diff --git a/src/parsers/command-sqlite.lisp b/src/parsers/command-sqlite.lisp index 5da7a19..2052247 100644 --- a/src/parsers/command-sqlite.lisp +++ b/src/parsers/command-sqlite.lisp @@ -26,6 +26,7 @@ load database option-batch-size option-batch-concurrency option-truncate + option-disable-triggers option-data-only option-schema-only option-include-drop diff --git a/src/pgsql/pgsql.lisp b/src/pgsql/pgsql.lisp index 2c75fb3..0895a5e 100644 --- a/src/pgsql/pgsql.lisp +++ b/src/pgsql/pgsql.lisp @@ -42,16 +42,20 @@ ;;; content down to PostgreSQL, handling any data related errors in the way. ;;; (defun copy-from-queue (pgconn table-name queue - &key columns (truncate t) ((:state *state*) *state*)) + &key + columns + (truncate t) + disable-triggers + ((:state *state*) *state*)) "Fetch from the QUEUE messages containing how many rows are in the *writer-batch* for us to send down to PostgreSQL, and when that's done update *state*." (when truncate (truncate-tables pgconn (list table-name))) - (log-message :debug "pgsql:copy-from-queue: ~a ~a" table-name columns) - (with-pgsql-connection (pgconn) + (when disable-triggers (disable-triggers table-name)) + (log-message :info "pgsql:copy-from-queue: ~a ~a" table-name columns) (loop for (mesg batch read oversized?) = (lq:pop-queue queue) until (eq mesg :end-of-data) @@ -62,7 +66,8 @@ #+sbcl (when oversized? (sb-ext:gc :full t)) (log-message :debug "copy-batch ~a ~d row~:p~:[~; [oversized]~]" table-name rows oversized?) - (pgstate-incf *state* table-name :rows rows))))) + (pgstate-incf *state* table-name :rows rows))) + (when disable-triggers (enable-triggers table-name)))) ;;; ;;; When a batch has been refused by PostgreSQL with a data-exception, that diff --git a/src/pgsql/schema.lisp b/src/pgsql/schema.lisp index 0d1f440..3e4033d 100644 --- a/src/pgsql/schema.lisp +++ b/src/pgsql/schema.lisp @@ -236,6 +236,22 @@ (log-message :notice "~a" sql) (pomo:execute sql)))) +(defun disable-triggers (table-name) + "Disable triggers on TABLE-NAME. Needs to be called with a PostgreSQL + connection already opened." + (let ((sql (format nil "ALTER TABLE ~a DISABLE TRIGGER ALL;" + (apply-identifier-case table-name)))) + (log-message :info "~a" sql) + (pomo:execute sql))) + +(defun enable-triggers (table-name) + "Disable triggers on TABLE-NAME. Needs to be called with a PostgreSQL + connection already opened." + (let ((sql (format nil "ALTER TABLE ~a ENABLE TRIGGER ALL;" + (apply-identifier-case table-name)))) + (log-message :info "~a" sql) + (pomo:execute sql))) + ;;; ;;; Index support diff --git a/src/sources/copy.lisp b/src/sources/copy.lisp index 39908fa..7c08f48 100644 --- a/src/sources/copy.lisp +++ b/src/sources/copy.lisp @@ -98,7 +98,7 @@ "Copy data from given COPY definition into lparallel.queue DATAQ" (pgloader.queue:map-push-queue copy queue)) -(defmethod copy-from ((copy copy-copy) &key truncate) +(defmethod copy-from ((copy copy-copy) &key truncate disable-triggers) "Copy data from given COPY file definition into its PostgreSQL target table." (let* ((summary (null *state*)) (*state* (or *state* (pgloader.utils:make-pgstate))) @@ -124,7 +124,8 @@ ;; always double quote column names (format nil "~s" (car col))) (columns copy)) - :truncate truncate) + :truncate truncate + :disable-triggers disable-triggers) ;; now wait until both the tasks are over (loop for tasks below 2 do (lp:receive-result channel) diff --git a/src/sources/csv/csv.lisp b/src/sources/csv/csv.lisp index 031fff8..dcd8f69 100644 --- a/src/sources/csv/csv.lisp +++ b/src/sources/csv/csv.lisp @@ -142,7 +142,7 @@ "Copy data from given CSV definition into lparallel.queue DATAQ" (map-push-queue csv queue)) -(defmethod copy-from ((csv copy-csv) &key truncate) +(defmethod copy-from ((csv copy-csv) &key truncate disable-triggers) "Copy data from given CSV file definition into its PostgreSQL target table." (let* ((summary (null *state*)) (*state* (or *state* (pgloader.utils:make-pgstate))) @@ -167,7 +167,8 @@ ;; always double quote column names (format nil "~s" (car col))) (columns csv)) - :truncate truncate) + :truncate truncate + :disable-triggers disable-triggers) ;; now wait until both the tasks are over (loop for tasks below 2 do (lp:receive-result channel) diff --git a/src/sources/db3/db3.lisp b/src/sources/db3/db3.lisp index dda1738..fe889ab 100644 --- a/src/sources/db3/db3.lisp +++ b/src/sources/db3/db3.lisp @@ -59,7 +59,8 @@ (let ((read (pgloader.queue:map-push-queue db3 queue))) (pgstate-incf *state* (target db3) :read read))) -(defmethod copy-from ((db3 copy-db3) &key (kernel nil k-s-p) truncate) +(defmethod copy-from ((db3 copy-db3) + &key (kernel nil k-s-p) truncate disable-triggers) (let* ((summary (null *state*)) (*state* (or *state* (pgloader.utils:make-pgstate))) (lp:*kernel* (or kernel (make-kernel 2))) @@ -78,7 +79,8 @@ (lp:submit-task channel #'pgloader.pgsql:copy-from-queue (target-db db3) (target db3) queue - :truncate truncate) + :truncate truncate + :disable-triggers disable-triggers) ;; now wait until both the tasks are over, and kill the kernel (loop for tasks below 2 do (lp:receive-result channel) @@ -92,11 +94,12 @@ state-before data-only schema-only - (truncate t) - (create-tables t) - (include-drop t) - (create-indexes t) - (reset-sequences t)) + (truncate t) + (disable-triggers nil) + (create-tables t) + (include-drop t) + (create-indexes t) + (reset-sequences t)) "Open the DB3 and stream its content to a PostgreSQL database." (declare (ignore create-indexes reset-sequences)) (let* ((summary (null *state*)) @@ -126,7 +129,7 @@ (return-from copy-database))) (unless schema-only - (copy-from db3 :truncate truncate)) + (copy-from db3 :truncate truncate :disable-triggers disable-triggers)) ;; and report the total time spent on the operation (when summary diff --git a/src/sources/fixed.lisp b/src/sources/fixed.lisp index 0e7f3bb..965289a 100644 --- a/src/sources/fixed.lisp +++ b/src/sources/fixed.lisp @@ -103,7 +103,7 @@ "Copy data from given FIXED definition into lparallel.queue DATAQ" (pgloader.queue:map-push-queue fixed queue)) -(defmethod copy-from ((fixed copy-fixed) &key truncate) +(defmethod copy-from ((fixed copy-fixed) &key truncate disable-triggers) "Copy data from given FIXED file definition into its PostgreSQL target table." (let* ((summary (null *state*)) (*state* (or *state* (pgloader.utils:make-pgstate))) @@ -129,7 +129,8 @@ ;; always double quote column names (format nil "~s" (car col))) (columns fixed)) - :truncate truncate) + :truncate truncate + :disable-triggers disable-triggers) ;; now wait until both the tasks are over (loop for tasks below 2 do (lp:receive-result channel) diff --git a/src/sources/ixf/ixf.lisp b/src/sources/ixf/ixf.lisp index 372bf30..bd3bc57 100644 --- a/src/sources/ixf/ixf.lisp +++ b/src/sources/ixf/ixf.lisp @@ -67,7 +67,8 @@ (let ((read (pgloader.queue:map-push-queue ixf queue))) (pgstate-incf *state* (target ixf) :read read))) -(defmethod copy-from ((ixf copy-ixf) &key (kernel nil k-s-p) truncate) +(defmethod copy-from ((ixf copy-ixf) + &key (kernel nil k-s-p) truncate disable-triggers) (let* ((summary (null *state*)) (*state* (or *state* (pgloader.utils:make-pgstate))) (lp:*kernel* (or kernel (make-kernel 2))) @@ -86,7 +87,8 @@ (lp:submit-task channel #'pgloader.pgsql:copy-from-queue (target-db ixf) (target ixf) queue - :truncate truncate) + :truncate truncate + :disable-triggers disable-triggers) ;; now wait until both the tasks are over, and kill the kernel (loop for tasks below 2 do (lp:receive-result channel) @@ -100,11 +102,12 @@ state-before data-only schema-only - (truncate t) - (create-tables t) - (include-drop t) - (create-indexes t) - (reset-sequences t)) + (truncate t) + (disable-triggers nil) + (create-tables t) + (include-drop t) + (create-indexes t) + (reset-sequences t)) "Open the IXF and stream its content to a PostgreSQL database." (declare (ignore create-indexes reset-sequences)) (let* ((summary (null *state*)) @@ -134,7 +137,7 @@ (return-from copy-database))) (unless schema-only - (copy-from ixf :truncate truncate)) + (copy-from ixf :truncate truncate :disable-triggers disable-triggers)) ;; and report the total time spent on the operation (when summary diff --git a/src/sources/mssql/mssql.lisp b/src/sources/mssql/mssql.lisp index da89fc9..1cb2e92 100644 --- a/src/sources/mssql/mssql.lisp +++ b/src/sources/mssql/mssql.lisp @@ -59,7 +59,8 @@ "Copy data from MSSQL table DBNAME.TABLE-NAME into queue DATAQ" (map-push-queue mssql queue)) -(defmethod copy-from ((mssql copy-mssql) &key (kernel nil k-s-p) truncate) +(defmethod copy-from ((mssql copy-mssql) + &key (kernel nil k-s-p) truncate disable-triggers) "Connect in parallel to MSSQL and PostgreSQL and stream the data." (let* ((summary (null *state*)) (*state* (or *state* (pgloader.utils:make-pgstate))) @@ -82,7 +83,8 @@ ;; and start another task to push that data from the queue to PostgreSQL (lp:submit-task channel #'pgloader.pgsql:copy-from-queue (target-db mssql) (target mssql) queue - :truncate truncate) + :truncate truncate + :disable-triggers disable-triggers) ;; now wait until both the tasks are over (loop for tasks below 2 do (lp:receive-result channel) @@ -173,14 +175,15 @@ state-before state-after state-indexes - (truncate nil) - (data-only nil) - (schema-only nil) - (create-tables t) - (include-drop t) - (create-indexes t) - (reset-sequences t) - (foreign-keys t) + (truncate nil) + (disable-triggers nil) + (data-only nil) + (schema-only nil) + (create-tables t) + (include-drop t) + (create-indexes t) + (reset-sequences t) + (foreign-keys t) (encoding :utf-8) including excluding) @@ -278,7 +281,9 @@ ;; COPY the data to PostgreSQL, using copy-kernel (unless schema-only - (copy-from table-source :kernel copy-kernel)) + (copy-from table-source + :kernel copy-kernel + :disable-triggers disable-triggers)) ;; Create the indexes for that table in parallel with the next ;; COPY, and all at once in concurrent threads to benefit from diff --git a/src/sources/mysql/mysql.lisp b/src/sources/mysql/mysql.lisp index 2a18ae9..023f9cc 100644 --- a/src/sources/mysql/mysql.lisp +++ b/src/sources/mysql/mysql.lisp @@ -103,7 +103,8 @@ ;;; Direct "stream" in between mysql fetching of results and PostgreSQL COPY ;;; protocol ;;; -(defmethod copy-from ((mysql copy-mysql) &key (kernel nil k-s-p) truncate) +(defmethod copy-from ((mysql copy-mysql) + &key (kernel nil k-s-p) truncate disable-triggers) "Connect in parallel to MySQL and PostgreSQL and stream the data." (let* ((summary (null *state*)) (*state* (or *state* (pgloader.utils:make-pgstate))) @@ -129,7 +130,8 @@ :columns (mapcar #'apply-identifier-case (mapcar #'mysql-column-name (fields mysql))) - :truncate truncate) + :truncate truncate + :disable-triggers disable-triggers) ;; now wait until both the tasks are over (loop for tasks below 2 do (lp:receive-result channel) @@ -347,14 +349,15 @@ state-before state-after state-indexes - (truncate nil) - (data-only nil) - (schema-only nil) - (create-tables t) - (include-drop t) - (create-indexes t) - (reset-sequences t) - (foreign-keys t) + (truncate nil) + (disable-triggers nil) + (data-only nil) + (schema-only nil) + (create-tables t) + (include-drop t) + (create-indexes t) + (reset-sequences t) + (foreign-keys t) only-tables including excluding @@ -452,7 +455,9 @@ ;; first COPY the data from MySQL to PostgreSQL, using copy-kernel (unless schema-only - (copy-from table-source :kernel copy-kernel)) + (copy-from table-source + :kernel copy-kernel + :disable-triggers disable-triggers)) ;; Create the indexes for that table in parallel with the next ;; COPY, and all at once in concurrent threads to benefit from diff --git a/src/sources/sqlite/sqlite.lisp b/src/sources/sqlite/sqlite.lisp index 407a7db..6a2c333 100644 --- a/src/sources/sqlite/sqlite.lisp +++ b/src/sources/sqlite/sqlite.lisp @@ -82,7 +82,8 @@ (let ((read (pgloader.queue:map-push-queue sqlite queue))) (pgstate-incf *state* (target sqlite) :read read))) -(defmethod copy-from ((sqlite copy-sqlite) &key (kernel nil k-s-p) truncate) +(defmethod copy-from ((sqlite copy-sqlite) + &key (kernel nil k-s-p) truncate disable-triggers) "Stream the contents from a SQLite database table down to PostgreSQL." (let* ((summary (null *state*)) (*state* (or *state* (pgloader.utils:make-pgstate))) @@ -103,7 +104,8 @@ (lp:submit-task channel #'pgloader.pgsql:copy-from-queue (target-db sqlite) (target sqlite) queue - :truncate truncate) + :truncate truncate + :disable-triggers disable-triggers) ;; now wait until both the tasks are over (loop for tasks below 2 do (lp:receive-result channel) @@ -145,11 +147,12 @@ state-before data-only schema-only - (truncate nil) - (create-tables t) - (include-drop t) - (create-indexes t) - (reset-sequences t) + (truncate nil) + (disable-triggers nil) + (create-tables t) + (include-drop t) + (create-indexes t) + (reset-sequences t) only-tables including excluding @@ -212,7 +215,9 @@ :fields columns))) ;; first COPY the data from SQLite to PostgreSQL, using copy-kernel (unless schema-only - (copy-from table-source :kernel copy-kernel)) + (copy-from table-source + :kernel copy-kernel + :disable-triggers disable-triggers)) ;; Create the indexes for that table in parallel with the next ;; COPY, and all at once in concurrent threads to benefit from diff --git a/test/csv-districts.load b/test/csv-districts.load index 3787f32..2bde239 100644 --- a/test/csv-districts.load +++ b/test/csv-districts.load @@ -29,6 +29,7 @@ LOAD CSV ) WITH truncate, + disable triggers, skip header = 1, batch rows = 200, batch size = 1024 kB, diff --git a/test/dbf.load b/test/dbf.load index 7958105..e4c1f15 100644 --- a/test/dbf.load +++ b/test/dbf.load @@ -4,7 +4,6 @@ * http://www.insee.fr/fr/methodes/nomenclatures/cog/telechargement/2013/dbf/reg2013.dbf */ LOAD DBF - FROM data/reg2013.dbf + FROM data/reg2013.dbf with encoding cp850 INTO postgresql:///pgloader?reg2013 - WITH truncate, create table - SET client_encoding TO 'latin1'; + WITH truncate, create table, disable triggers; diff --git a/test/parse/hans.goeuro.load b/test/parse/hans.goeuro.load index 95cead4..d610e10 100644 --- a/test/parse/hans.goeuro.load +++ b/test/parse/hans.goeuro.load @@ -7,6 +7,8 @@ LOAD DATABASE DECODING TABLE NAMES MATCHING ~/messed/, ~/encoding/ AS utf-8 + SET maintenance_work_mem to '1 GB' + CAST type datetime to timestamptz drop default drop not null using zero-dates-to-null, type date drop not null drop default using zero-dates-to-null, type timestamp to timestamptz drop not null using zero-dates-to-null, @@ -32,4 +34,8 @@ LOAD DATABASE group by cast(d as date); $$ - WITH include drop, create tables, create indexes, reset sequences; + WITH include drop, + create tables, + create indexes, + reset sequences, + disable triggers; diff --git a/test/regress/expected/dbf.out b/test/regress/expected/dbf.out index 3cbc9b0..8c86d97 100644 --- a/test/regress/expected/dbf.out +++ b/test/regress/expected/dbf.out @@ -1,9 +1,9 @@ 01 97105 3 GUADELOUPE Guadeloupe 02 97209 3 MARTINIQUE Martinique 03 97302 3 GUYANE Guyane -04 97411 0 LA REUNION La R‚union +04 97411 0 LA REUNION La Réunion 06 97608 0 MAYOTTE Mayotte -11 75056 1 ILE-DE-FRANCE ×le-de-France +11 75056 1 ILE-DE-FRANCE Île-de-France 21 51108 0 CHAMPAGNE-ARDENNE Champagne-Ardenne 22 80021 0 PICARDIE Picardie 23 76540 0 HAUTE-NORMANDIE Haute-Normandie @@ -13,15 +13,15 @@ 31 59350 2 NORD-PAS-DE-CALAIS Nord-Pas-de-Calais 41 57463 0 LORRAINE Lorraine 42 67482 1 ALSACE Alsace -43 25056 0 FRANCHE-COMTE Franche-Comt‚ +43 25056 0 FRANCHE-COMTE Franche-Comté 52 44109 4 PAYS DE LA LOIRE Pays de la Loire 53 35238 0 BRETAGNE Bretagne 54 86194 2 POITOU-CHARENTES Poitou-Charentes 72 33063 1 AQUITAINE Aquitaine -73 31555 0 MIDI-PYRENEES Midi-Pyr‚n‚es +73 31555 0 MIDI-PYRENEES Midi-Pyrénées 74 87085 2 LIMOUSIN Limousin -82 69123 2 RHONE-ALPES Rh“ne-Alpes +82 69123 2 RHONE-ALPES Rhône-Alpes 83 63113 1 AUVERGNE Auvergne 91 34172 2 LANGUEDOC-ROUSSILLON Languedoc-Roussillon -93 13055 0 PROVENCE-ALPES-COTE D'AZUR Provence-Alpes-C“te d'Azur +93 13055 0 PROVENCE-ALPES-COTE D'AZUR Provence-Alpes-Côte d'Azur 94 2A004 0 CORSE Corse