diff --git a/src/package.lisp b/src/package.lisp index 6f00fa1..83faf4a 100644 --- a/src/package.lisp +++ b/src/package.lisp @@ -271,6 +271,9 @@ #:csv-connection #:specs #:csv-specs) + (:import-from #:pgloader.pgsql + #:maybe-drop-indexes + #:create-indexes-again) (:export #:fixed-connection #:copy-fixed #:copy-to-queue @@ -284,6 +287,9 @@ #:csv-connection #:specs #:csv-specs) + (:import-from #:pgloader.pgsql + #:maybe-drop-indexes + #:create-indexes-again) (:export #:copy-connection #:copy-copy #:copy-to-queue diff --git a/src/parsers/command-copy.lisp b/src/parsers/command-copy.lisp index 660a222..b6eab68 100644 --- a/src/parsers/command-copy.lisp +++ b/src/parsers/command-copy.lisp @@ -37,6 +37,7 @@ option-batch-size option-batch-concurrency option-truncate + option-drop-indexes option-disable-triggers option-skip-header option-delimiter @@ -117,7 +118,8 @@ (let* ((state-before (pgloader.utils:make-pgstate)) (summary (null *state*)) (*state* (or *state* (pgloader.utils:make-pgstate))) - (state-after ,(when after `(pgloader.utils:make-pgstate))) + (state-after ,(when (or after (getf options :drop-indexes)) + `(pgloader.utils:make-pgstate))) ,@(pgsql-connection-bindings pg-db-conn gucs) ,@(batch-control-bindings options) (source-db (with-stats-collection ("fetch" :state state-before) @@ -128,6 +130,7 @@ (let ((truncate ,(getf options :truncate)) (disable-triggers (getf ',options :disable-triggers)) + (drop-indexes (getf ',options :drop-indexes)) (source (make-instance 'pgloader.copy:copy-copy :target-db ,pg-db-conn @@ -138,9 +141,13 @@ :columns ',columns ,@(remove-batch-control-option options :extras '(:truncate + :drop-indexes :disable-triggers))))) (pgloader.sources:copy-from source + :state-before state-before + :state-after state-after :truncate truncate + :drop-indexes drop-indexes :disable-triggers disable-triggers)) ,(sql-code-block pg-db-conn 'state-after after "after load") diff --git a/src/parsers/command-fixed.lisp b/src/parsers/command-fixed.lisp index dd2aa08..9c1f316 100644 --- a/src/parsers/command-fixed.lisp +++ b/src/parsers/command-fixed.lisp @@ -47,6 +47,7 @@ option-batch-size option-batch-concurrency option-truncate + option-drop-indexes option-disable-triggers option-skip-header)) @@ -125,7 +126,8 @@ (let* ((state-before (pgloader.utils:make-pgstate)) (summary (null *state*)) (*state* (or *state* (pgloader.utils:make-pgstate))) - (state-after ,(when after `(pgloader.utils:make-pgstate))) + (state-after ,(when (or after (getf options :drop-indexes)) + `(pgloader.utils:make-pgstate))) ,@(pgsql-connection-bindings pg-db-conn gucs) ,@(batch-control-bindings options) (source-db (with-stats-collection ("fetch" :state state-before) @@ -136,6 +138,7 @@ (let ((truncate ,(getf options :truncate)) (disable-triggers ,(getf options :disable-triggers)) + (drop-indexes ,(getf options :drop-indexes)) (source (make-instance 'pgloader.fixed:copy-fixed :target-db ,pg-db-conn @@ -147,7 +150,10 @@ :skip-lines ,(or (getf options :skip-line) 0)))) (pgloader.sources:copy-from source + :state-before state-before + :state-after state-after :truncate truncate + :drop-indexes drop-indexes :disable-triggers disable-triggers)) ,(sql-code-block pg-db-conn 'state-after after "after load") diff --git a/src/sources/copy.lisp b/src/sources/copy.lisp index 7c9c757..30db993 100644 --- a/src/sources/copy.lisp +++ b/src/sources/copy.lisp @@ -116,13 +116,23 @@ "Copy data from given COPY definition into lparallel.queue DATAQ" (pgloader.queue:map-push-queue copy queue 'pre-formatted)) -(defmethod copy-from ((copy copy-copy) &key truncate disable-triggers) +(defmethod copy-from ((copy copy-copy) + &key + state-before + state-after + truncate + disable-triggers + drop-indexes) "Copy data from given COPY file definition into its PostgreSQL target table." (let* ((summary (null *state*)) (*state* (or *state* (pgloader.utils:make-pgstate))) (lp:*kernel* (make-kernel 2)) (channel (lp:make-channel)) - (queue (lq:make-queue :fixed-capacity *concurrent-batches*))) + (queue (lq:make-queue :fixed-capacity *concurrent-batches*)) + (indexes (maybe-drop-indexes (target-db copy) + (target copy) + state-before + :drop-indexes drop-indexes))) (with-stats-collection ((target copy) :dbname (db-name (target-db copy)) @@ -147,5 +157,9 @@ ;; now wait until both the tasks are over (loop for tasks below 2 do (lp:receive-result channel) - finally (lp:end-kernel)))))) + finally (lp:end-kernel)))) + + ;; re-create the indexes + (create-indexes-again (target-db copy) indexes state-after + :drop-indexes drop-indexes))) diff --git a/src/sources/fixed.lisp b/src/sources/fixed.lisp index 965289a..04152a8 100644 --- a/src/sources/fixed.lisp +++ b/src/sources/fixed.lisp @@ -103,13 +103,23 @@ "Copy data from given FIXED definition into lparallel.queue DATAQ" (pgloader.queue:map-push-queue fixed queue)) -(defmethod copy-from ((fixed copy-fixed) &key truncate disable-triggers) +(defmethod copy-from ((fixed copy-fixed) + &key + state-before + state-after + truncate + disable-triggers + drop-indexes) "Copy data from given FIXED file definition into its PostgreSQL target table." (let* ((summary (null *state*)) (*state* (or *state* (pgloader.utils:make-pgstate))) (lp:*kernel* (make-kernel 2)) (channel (lp:make-channel)) - (queue (lq:make-queue :fixed-capacity *concurrent-batches*))) + (queue (lq:make-queue :fixed-capacity *concurrent-batches*)) + (indexes (maybe-drop-indexes (target-db fixed) + (target fixed) + state-before + :drop-indexes drop-indexes))) (with-stats-collection ((target fixed) :dbname (db-name (target-db fixed)) @@ -134,5 +144,9 @@ ;; now wait until both the tasks are over (loop for tasks below 2 do (lp:receive-result channel) - finally (lp:end-kernel)))))) + finally (lp:end-kernel)))) + + ;; re-create the indexes + (create-indexes-again (target-db fixed) indexes state-after + :drop-indexes drop-indexes))) diff --git a/test/fixed.load b/test/fixed.load index a77db2c..9ee13b2 100644 --- a/test/fixed.load +++ b/test/fixed.load @@ -26,7 +26,7 @@ LOAD FIXED d ) - WITH truncate + WITH truncate, drop indexes SET client_encoding to 'latin1', work_mem to '14MB', @@ -35,7 +35,7 @@ LOAD FIXED BEFORE LOAD DO $$ drop table if exists fixed; $$, $$ create table fixed ( - a integer, + a integer primary key, b date, c time, d text