mirror of
https://github.com/dimitri/pgloader.git
synced 2026-05-05 02:46:10 +02:00
Provide "on error stop" as a WITH option.
As seen in #546 it would be easier to be able to specify the option in the load command directly rather than only at the command line. Here we go!
This commit is contained in:
Dimitri Fontaine
parent
45719645da
commit
c6b634caad
@ -1,7 +1,7 @@
|
||||
.\" generated with Ronn/v0.7.3
|
||||
.\" http://github.com/rtomayko/ronn/tree/0.7.3
|
||||
.
|
||||
.TH "PGLOADER" "1" "February 2017" "ff" ""
|
||||
.TH "PGLOADER" "1" "June 2017" "ff" ""
|
||||
.
|
||||
.SH "NAME"
|
||||
\fBpgloader\fR \- PostgreSQL data loader
|
||||
@ -633,6 +633,9 @@ See each specific command for details\.
|
||||
All data sources specific commands support the following options:
|
||||
.
|
||||
.IP "\(bu" 4
|
||||
\fIon error stop\fR
|
||||
.
|
||||
.IP "\(bu" 4
|
||||
\fIbatch rows = R\fR
|
||||
.
|
||||
.IP "\(bu" 4
|
||||
|
||||
@ -578,6 +578,7 @@ Some clauses are common to all commands:
|
||||
|
||||
All data sources specific commands support the following options:
|
||||
|
||||
- *on error stop*
|
||||
- *batch rows = R*
|
||||
- *batch size = ... MB*
|
||||
- *batch concurrency = ...*
|
||||
|
||||
@ -33,7 +33,8 @@
|
||||
(defrule option-null (and kw-null quoted-string)
|
||||
(:destructure (kw null) (declare (ignore kw)) (cons :null-as null)))
|
||||
|
||||
(defrule copy-option (or option-workers
|
||||
(defrule copy-option (or option-on-error-stop
|
||||
option-workers
|
||||
option-concurrency
|
||||
option-batch-rows
|
||||
option-batch-size
|
||||
@ -120,7 +121,8 @@
|
||||
(progn
|
||||
,(sql-code-block pg-db-conn :pre before "before load")
|
||||
|
||||
(let ((truncate ,(getf options :truncate))
|
||||
(let ((on-error-stop (getf ',options :on-error-stop))
|
||||
(truncate (getf ',options :truncate))
|
||||
(disable-triggers (getf ',options :disable-triggers))
|
||||
(drop-indexes (getf ',options :drop-indexes))
|
||||
(max-parallel-create-index (getf ',options :max-parallel-create-index))
|
||||
@ -134,7 +136,8 @@
|
||||
:fields ',fields
|
||||
:columns ',columns
|
||||
,@(remove-batch-control-option
|
||||
options :extras '(:worker-count
|
||||
options :extras '(:on-error-stop
|
||||
:worker-count
|
||||
:concurrency
|
||||
:truncate
|
||||
:drop-indexes
|
||||
@ -145,6 +148,7 @@
|
||||
(list :worker-count worker-count))
|
||||
,@ (when concurrency
|
||||
(list :concurrency concurrency))
|
||||
:on-error-stop on-error-stop
|
||||
:truncate truncate
|
||||
:drop-indexes drop-indexes
|
||||
:disable-triggers disable-triggers
|
||||
|
||||
@ -103,7 +103,8 @@
|
||||
(bind (((_ _ _ escape-mode) term))
|
||||
(cons :escape-mode escape-mode))))
|
||||
|
||||
(defrule csv-option (or option-workers
|
||||
(defrule csv-option (or option-on-error-stop
|
||||
option-workers
|
||||
option-concurrency
|
||||
option-batch-rows
|
||||
option-batch-size
|
||||
@ -414,7 +415,8 @@
|
||||
(progn
|
||||
,(sql-code-block pg-db-conn :pre before "before load")
|
||||
|
||||
(let ((truncate (getf ',options :truncate))
|
||||
(let ((on-error-stop (getf ',options :on-error-stop))
|
||||
(truncate (getf ',options :truncate))
|
||||
(disable-triggers (getf ',options :disable-triggers))
|
||||
(drop-indexes (getf ',options :drop-indexes))
|
||||
(max-parallel-create-index (getf ',options :max-parallel-create-index))
|
||||
@ -428,7 +430,8 @@
|
||||
:fields ',fields
|
||||
:columns ',columns
|
||||
,@(remove-batch-control-option
|
||||
options :extras '(:worker-count
|
||||
options :extras '(:on-error-stop
|
||||
:worker-count
|
||||
:concurrency
|
||||
:truncate
|
||||
:drop-indexes
|
||||
@ -439,6 +442,7 @@
|
||||
(list :worker-count worker-count))
|
||||
,@ (when concurrency
|
||||
(list :concurrency concurrency))
|
||||
:on-error-stop on-error-stop
|
||||
:truncate truncate
|
||||
:drop-indexes drop-indexes
|
||||
:disable-triggers disable-triggers
|
||||
|
||||
@ -18,7 +18,8 @@
|
||||
(bind (((_ _ _ table-name) tn))
|
||||
(cons :table-name (text table-name)))))
|
||||
|
||||
(defrule dbf-option (or option-workers
|
||||
(defrule dbf-option (or option-on-error-stop
|
||||
option-workers
|
||||
option-concurrency
|
||||
option-batch-rows
|
||||
option-batch-size
|
||||
|
||||
@ -43,7 +43,8 @@
|
||||
(:lambda (source)
|
||||
(bind (((_ field-defs _) source)) field-defs)))
|
||||
|
||||
(defrule fixed-option (or option-workers
|
||||
(defrule fixed-option (or option-on-error-stop
|
||||
option-workers
|
||||
option-concurrency
|
||||
option-batch-rows
|
||||
option-batch-size
|
||||
@ -128,7 +129,8 @@
|
||||
(progn
|
||||
,(sql-code-block pg-db-conn :pre before "before load")
|
||||
|
||||
(let ((truncate ,(getf options :truncate))
|
||||
(let ((on-error-stop ,(getf options :on-error-stop))
|
||||
(truncate ,(getf options :truncate))
|
||||
(disable-triggers ,(getf options :disable-triggers))
|
||||
(drop-indexes ,(getf options :drop-indexes))
|
||||
(max-parallel-create-index ,(getf options :max-parallel-create-index))
|
||||
@ -148,6 +150,7 @@
|
||||
(list :worker-count worker-count))
|
||||
,@ (when concurrency
|
||||
(list :concurrency concurrency))
|
||||
:on-error-stop on-error-stop
|
||||
:truncate truncate
|
||||
:drop-indexes drop-indexes
|
||||
:disable-triggers disable-triggers
|
||||
|
||||
@ -18,7 +18,8 @@
|
||||
(:lambda (tzopt)
|
||||
(bind (((_ tz) tzopt)) (cons :timezone tz))))
|
||||
|
||||
(defrule ixf-option (or option-workers
|
||||
(defrule ixf-option (or option-on-error-stop
|
||||
option-workers
|
||||
option-concurrency
|
||||
option-batch-rows
|
||||
option-batch-size
|
||||
|
||||
@ -48,6 +48,9 @@
|
||||
(def-keyword-rule "typemod")
|
||||
(def-keyword-rule "using")
|
||||
(def-keyword-rule "getenv")
|
||||
(def-keyword-rule "on")
|
||||
(def-keyword-rule "error")
|
||||
(def-keyword-rule "stop")
|
||||
;; option for loading from a file
|
||||
(def-keyword-rule "workers")
|
||||
(def-keyword-rule "batch")
|
||||
|
||||
@ -14,7 +14,8 @@
|
||||
;;;
|
||||
(make-option-rule create-schemas (and kw-create (? kw-no) kw-schemas))
|
||||
|
||||
(defrule mssql-option (or option-workers
|
||||
(defrule mssql-option (or option-on-error-stop
|
||||
option-workers
|
||||
option-concurrency
|
||||
option-batch-rows
|
||||
option-batch-size
|
||||
|
||||
@ -7,7 +7,8 @@
|
||||
;;;
|
||||
;;; MySQL options
|
||||
;;;
|
||||
(defrule mysql-option (or option-workers
|
||||
(defrule mysql-option (or option-on-error-stop
|
||||
option-workers
|
||||
option-concurrency
|
||||
option-batch-rows
|
||||
option-batch-size
|
||||
|
||||
@ -132,6 +132,9 @@
|
||||
(defrule option-data-only (and kw-data kw-only)
|
||||
(:constant (cons :data-only t)))
|
||||
|
||||
(defrule option-on-error-stop (and kw-on kw-error kw-stop)
|
||||
(:constant (cons :on-error-stop t)))
|
||||
|
||||
(defrule option-identifiers-case (and (or kw-downcase kw-quote) kw-identifiers)
|
||||
(:lambda (id-case)
|
||||
(bind (((action _) id-case))
|
||||
|
||||
@ -13,7 +13,8 @@ load database
|
||||
|
||||
set work_mem to '16MB', maintenance_work_mem to '512 MB';
|
||||
|#
|
||||
(defrule sqlite-option (or option-workers
|
||||
(defrule sqlite-option (or option-on-error-stop
|
||||
option-workers
|
||||
option-concurrency
|
||||
option-batch-rows
|
||||
option-batch-size
|
||||
|
||||
@ -231,7 +231,8 @@
|
||||
alter-schema
|
||||
materialize-views)
|
||||
"Export database source data and Import it into PostgreSQL"
|
||||
(let* ((copy-data (or data-only (not schema-only)))
|
||||
(let* ((*on-error-stop* on-error-stop)
|
||||
(copy-data (or data-only (not schema-only)))
|
||||
(create-ddl (or schema-only (not data-only)))
|
||||
(create-tables (and create-tables create-ddl))
|
||||
(create-schemas (and create-schemas create-ddl))
|
||||
|
||||
@ -100,7 +100,8 @@
|
||||
create-indexes reset-sequences materialize-views
|
||||
set-table-oids including excluding))
|
||||
|
||||
(let* ((pgconn (target-db copy))
|
||||
(let* ((*on-error-stop* on-error-stop)
|
||||
(pgconn (target-db copy))
|
||||
pgsql-catalog)
|
||||
|
||||
(handler-case
|
||||
|
||||
@ -2,7 +2,7 @@ LOAD IXF
|
||||
FROM data/nsitra.test1.ixf
|
||||
INTO postgresql:///pgloader?nsitra.test1
|
||||
|
||||
WITH truncate, create table, timezone UTC
|
||||
WITH on error stop, truncate, create table, timezone UTC
|
||||
|
||||
BEFORE LOAD DO
|
||||
$$ drop schema if exists nsitra cascade; $$,
|
||||
|
||||
@ -7,7 +7,7 @@ load database
|
||||
|
||||
-- WITH batch rows = 10000
|
||||
|
||||
WITH concurrency = 1, workers = 6,
|
||||
WITH on error stop, concurrency = 1, workers = 6,
|
||||
max parallel create index = 4-- ,
|
||||
-- quote identifiers
|
||||
|
||||
|
||||
@ -6,6 +6,7 @@ load database
|
||||
|
||||
with workers = 4,
|
||||
concurrency = 2,
|
||||
on error stop,
|
||||
include drop, create tables, create indexes, reset sequences, foreign keys
|
||||
|
||||
-- alter table names matching ~/./ set schema 'chinook'
|
||||
|
||||
@ -11,7 +11,9 @@ load database
|
||||
$$ create schema if not exists sqlite; $$
|
||||
|
||||
cast column character.f1 to text drop typemod,
|
||||
column appointments.time to timestamptz drop default,
|
||||
column appointments.time
|
||||
to timestamptz drop default
|
||||
using sqlite-timestamp-to-timestamp,
|
||||
type intege to integer
|
||||
|
||||
set work_mem to '16MB', maintenance_work_mem to '512 MB', search_path to 'sqlite';
|
||||
Loading…
x
Reference in New Issue
Block a user