mirrors/pgloader
1
0
Fork 0
mirror of https://github.com/dimitri/pgloader.git synced 2026-05-04 18:36:12 +02:00
Code Issues Projects Releases Wiki Activity

Implement drop indexes option for copy and fixed.

Browse Source 
The option doesn't seem relevant to the db3 source type which contains a
table definition: pgloader will create the table from scratch and no
indexes are going to be found.
This commit is contained in:
Dimitri Fontaine 2015-07-16 21:39:06 +02:00
parent 4f2385fa4c
commit a98788b670
6 changed files with 57 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/package.lisp
View File

@ -271,6 +271,9 @@
#:csv-connection
#:specs
#:csv-specs)
(:import-from #:pgloader.pgsql
#:maybe-drop-indexes
#:create-indexes-again)
(:export #:fixed-connection
#:copy-fixed
#:copy-to-queue
@ -284,6 +287,9 @@
#:csv-connection
#:specs
#:csv-specs)
(:import-from #:pgloader.pgsql
#:maybe-drop-indexes
#:create-indexes-again)
(:export #:copy-connection
#:copy-copy
#:copy-to-queue

9
src/parsers/command-copy.lisp
View File

@ -37,6 +37,7 @@
option-batch-size
option-batch-concurrency
option-truncate
option-drop-indexes
option-disable-triggers
option-skip-header
option-delimiter
@ -117,7 +118,8 @@
(let* ((state-before (pgloader.utils:make-pgstate))
(summary (null *state*))
(*state* (or *state* (pgloader.utils:make-pgstate)))
(state-after ,(when after `(pgloader.utils:make-pgstate)))
(state-after ,(when (or after (getf options :drop-indexes))
`(pgloader.utils:make-pgstate)))
,@(pgsql-connection-bindings pg-db-conn gucs)
,@(batch-control-bindings options)
(source-db (with-stats-collection ("fetch" :state state-before)
@ -128,6 +130,7 @@
(let ((truncate ,(getf options :truncate))
(disable-triggers (getf ',options :disable-triggers))
(drop-indexes (getf ',options :drop-indexes))
(source
(make-instance 'pgloader.copy:copy-copy
:target-db ,pg-db-conn
@ -138,9 +141,13 @@
:columns ',columns
,@(remove-batch-control-option
options :extras '(:truncate
:drop-indexes
:disable-triggers)))))
(pgloader.sources:copy-from source
:state-before state-before
:state-after state-after
:truncate truncate
:drop-indexes drop-indexes
:disable-triggers disable-triggers))
,(sql-code-block pg-db-conn 'state-after after "after load")

8
src/parsers/command-fixed.lisp
View File

@ -47,6 +47,7 @@
option-batch-size
option-batch-concurrency
option-truncate
option-drop-indexes
option-disable-triggers
option-skip-header))
@ -125,7 +126,8 @@
(let* ((state-before (pgloader.utils:make-pgstate))
(summary (null *state*))
(*state* (or *state* (pgloader.utils:make-pgstate)))
(state-after ,(when after `(pgloader.utils:make-pgstate)))
(state-after ,(when (or after (getf options :drop-indexes))
`(pgloader.utils:make-pgstate)))
,@(pgsql-connection-bindings pg-db-conn gucs)
,@(batch-control-bindings options)
(source-db (with-stats-collection ("fetch" :state state-before)
@ -136,6 +138,7 @@
(let ((truncate ,(getf options :truncate))
(disable-triggers ,(getf options :disable-triggers))
(drop-indexes ,(getf options :drop-indexes))
(source
(make-instance 'pgloader.fixed:copy-fixed
:target-db ,pg-db-conn
@ -147,7 +150,10 @@
:skip-lines ,(or (getf options :skip-line) 0))))
(pgloader.sources:copy-from source
:state-before state-before
:state-after state-after
:truncate truncate
:drop-indexes drop-indexes
:disable-triggers disable-triggers))
,(sql-code-block pg-db-conn 'state-after after "after load")

20
src/sources/copy.lisp
View File

@ -116,13 +116,23 @@
"Copy data from given COPY definition into lparallel.queue DATAQ"
(pgloader.queue:map-push-queue copy queue 'pre-formatted))
(defmethod copy-from ((copy copy-copy) &key truncate disable-triggers)
(defmethod copy-from ((copy copy-copy)
&key
state-before
state-after
truncate
disable-triggers
drop-indexes)
"Copy data from given COPY file definition into its PostgreSQL target table."
(let* ((summary (null *state*))
(*state* (or *state* (pgloader.utils:make-pgstate)))
(lp:*kernel* (make-kernel 2))
(channel (lp:make-channel))
(queue (lq:make-queue :fixed-capacity *concurrent-batches*)))
(queue (lq:make-queue :fixed-capacity *concurrent-batches*))
(indexes (maybe-drop-indexes (target-db copy)
(target copy)
state-before
:drop-indexes drop-indexes)))
(with-stats-collection ((target copy)
:dbname (db-name (target-db copy))
@ -147,5 +157,9 @@
;; now wait until both the tasks are over
(loop for tasks below 2 do (lp:receive-result channel)
finally (lp:end-kernel))))))
finally (lp:end-kernel))))
;; re-create the indexes
(create-indexes-again (target-db copy) indexes state-after
:drop-indexes drop-indexes)))

20
src/sources/fixed.lisp
View File

@ -103,13 +103,23 @@
"Copy data from given FIXED definition into lparallel.queue DATAQ"
(pgloader.queue:map-push-queue fixed queue))
(defmethod copy-from ((fixed copy-fixed) &key truncate disable-triggers)
(defmethod copy-from ((fixed copy-fixed)
&key
state-before
state-after
truncate
disable-triggers
drop-indexes)
"Copy data from given FIXED file definition into its PostgreSQL target table."
(let* ((summary (null *state*))
(*state* (or *state* (pgloader.utils:make-pgstate)))
(lp:*kernel* (make-kernel 2))
(channel (lp:make-channel))
(queue (lq:make-queue :fixed-capacity *concurrent-batches*)))
(queue (lq:make-queue :fixed-capacity *concurrent-batches*))
(indexes (maybe-drop-indexes (target-db fixed)
(target fixed)
state-before
:drop-indexes drop-indexes)))
(with-stats-collection ((target fixed)
:dbname (db-name (target-db fixed))
@ -134,5 +144,9 @@
;; now wait until both the tasks are over
(loop for tasks below 2 do (lp:receive-result channel)
finally (lp:end-kernel))))))
finally (lp:end-kernel))))
;; re-create the indexes
(create-indexes-again (target-db fixed) indexes state-after
:drop-indexes drop-indexes)))

4
test/fixed.load
View File

@ -26,7 +26,7 @@ LOAD FIXED
d
)
WITH truncate
WITH truncate, drop indexes
SET client_encoding to 'latin1',
work_mem to '14MB',
@ -35,7 +35,7 @@ LOAD FIXED
BEFORE LOAD DO
$$ drop table if exists fixed; $$,
$$ create table fixed (
a integer,
a integer primary key,
b date,
c time,
d text