diff --git a/pgloader.1 b/pgloader.1 index 006498d..0777fc4 100644 --- a/pgloader.1 +++ b/pgloader.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "PGLOADER" "1" "February 2017" "ff" "" +.TH "PGLOADER" "1" "June 2017" "ff" "" . .SH "NAME" \fBpgloader\fR \- PostgreSQL data loader @@ -633,6 +633,9 @@ See each specific command for details\. All data sources specific commands support the following options: . .IP "\(bu" 4 +\fIon error stop\fR +. +.IP "\(bu" 4 \fIbatch rows = R\fR . .IP "\(bu" 4 diff --git a/pgloader.1.md b/pgloader.1.md index 22806ba..ab64553 100644 --- a/pgloader.1.md +++ b/pgloader.1.md @@ -578,6 +578,7 @@ Some clauses are common to all commands: All data sources specific commands support the following options: + - *on error stop* - *batch rows = R* - *batch size = ... MB* - *batch concurrency = ...* diff --git a/src/parsers/command-copy.lisp b/src/parsers/command-copy.lisp index 75055e4..3feae34 100644 --- a/src/parsers/command-copy.lisp +++ b/src/parsers/command-copy.lisp @@ -33,7 +33,8 @@ (defrule option-null (and kw-null quoted-string) (:destructure (kw null) (declare (ignore kw)) (cons :null-as null))) -(defrule copy-option (or option-workers +(defrule copy-option (or option-on-error-stop + option-workers option-concurrency option-batch-rows option-batch-size @@ -120,7 +121,8 @@ (progn ,(sql-code-block pg-db-conn :pre before "before load") - (let ((truncate ,(getf options :truncate)) + (let ((on-error-stop (getf ',options :on-error-stop)) + (truncate (getf ',options :truncate)) (disable-triggers (getf ',options :disable-triggers)) (drop-indexes (getf ',options :drop-indexes)) (max-parallel-create-index (getf ',options :max-parallel-create-index)) @@ -134,7 +136,8 @@ :fields ',fields :columns ',columns ,@(remove-batch-control-option - options :extras '(:worker-count + options :extras '(:on-error-stop + :worker-count :concurrency :truncate :drop-indexes @@ -145,6 +148,7 @@ (list :worker-count worker-count)) ,@ (when concurrency (list :concurrency concurrency)) + :on-error-stop on-error-stop :truncate truncate :drop-indexes drop-indexes :disable-triggers disable-triggers diff --git a/src/parsers/command-csv.lisp b/src/parsers/command-csv.lisp index d35ca93..02fb8f9 100644 --- a/src/parsers/command-csv.lisp +++ b/src/parsers/command-csv.lisp @@ -103,7 +103,8 @@ (bind (((_ _ _ escape-mode) term)) (cons :escape-mode escape-mode)))) -(defrule csv-option (or option-workers +(defrule csv-option (or option-on-error-stop + option-workers option-concurrency option-batch-rows option-batch-size @@ -414,7 +415,8 @@ (progn ,(sql-code-block pg-db-conn :pre before "before load") - (let ((truncate (getf ',options :truncate)) + (let ((on-error-stop (getf ',options :on-error-stop)) + (truncate (getf ',options :truncate)) (disable-triggers (getf ',options :disable-triggers)) (drop-indexes (getf ',options :drop-indexes)) (max-parallel-create-index (getf ',options :max-parallel-create-index)) @@ -428,7 +430,8 @@ :fields ',fields :columns ',columns ,@(remove-batch-control-option - options :extras '(:worker-count + options :extras '(:on-error-stop + :worker-count :concurrency :truncate :drop-indexes @@ -439,6 +442,7 @@ (list :worker-count worker-count)) ,@ (when concurrency (list :concurrency concurrency)) + :on-error-stop on-error-stop :truncate truncate :drop-indexes drop-indexes :disable-triggers disable-triggers diff --git a/src/parsers/command-dbf.lisp b/src/parsers/command-dbf.lisp index b0ec0d3..7e9dbe4 100644 --- a/src/parsers/command-dbf.lisp +++ b/src/parsers/command-dbf.lisp @@ -18,7 +18,8 @@ (bind (((_ _ _ table-name) tn)) (cons :table-name (text table-name))))) -(defrule dbf-option (or option-workers +(defrule dbf-option (or option-on-error-stop + option-workers option-concurrency option-batch-rows option-batch-size diff --git a/src/parsers/command-fixed.lisp b/src/parsers/command-fixed.lisp index a3be78d..2b3ccc3 100644 --- a/src/parsers/command-fixed.lisp +++ b/src/parsers/command-fixed.lisp @@ -43,7 +43,8 @@ (:lambda (source) (bind (((_ field-defs _) source)) field-defs))) -(defrule fixed-option (or option-workers +(defrule fixed-option (or option-on-error-stop + option-workers option-concurrency option-batch-rows option-batch-size @@ -128,7 +129,8 @@ (progn ,(sql-code-block pg-db-conn :pre before "before load") - (let ((truncate ,(getf options :truncate)) + (let ((on-error-stop ,(getf options :on-error-stop)) + (truncate ,(getf options :truncate)) (disable-triggers ,(getf options :disable-triggers)) (drop-indexes ,(getf options :drop-indexes)) (max-parallel-create-index ,(getf options :max-parallel-create-index)) @@ -148,6 +150,7 @@ (list :worker-count worker-count)) ,@ (when concurrency (list :concurrency concurrency)) + :on-error-stop on-error-stop :truncate truncate :drop-indexes drop-indexes :disable-triggers disable-triggers diff --git a/src/parsers/command-ixf.lisp b/src/parsers/command-ixf.lisp index c66eaee..1a51c49 100644 --- a/src/parsers/command-ixf.lisp +++ b/src/parsers/command-ixf.lisp @@ -18,7 +18,8 @@ (:lambda (tzopt) (bind (((_ tz) tzopt)) (cons :timezone tz)))) -(defrule ixf-option (or option-workers +(defrule ixf-option (or option-on-error-stop + option-workers option-concurrency option-batch-rows option-batch-size diff --git a/src/parsers/command-keywords.lisp b/src/parsers/command-keywords.lisp index a4c2e2a..3aae57d 100644 --- a/src/parsers/command-keywords.lisp +++ b/src/parsers/command-keywords.lisp @@ -48,6 +48,9 @@ (def-keyword-rule "typemod") (def-keyword-rule "using") (def-keyword-rule "getenv") + (def-keyword-rule "on") + (def-keyword-rule "error") + (def-keyword-rule "stop") ;; option for loading from a file (def-keyword-rule "workers") (def-keyword-rule "batch") diff --git a/src/parsers/command-mssql.lisp b/src/parsers/command-mssql.lisp index a9688d0..b5e24ab 100644 --- a/src/parsers/command-mssql.lisp +++ b/src/parsers/command-mssql.lisp @@ -14,7 +14,8 @@ ;;; (make-option-rule create-schemas (and kw-create (? kw-no) kw-schemas)) -(defrule mssql-option (or option-workers +(defrule mssql-option (or option-on-error-stop + option-workers option-concurrency option-batch-rows option-batch-size diff --git a/src/parsers/command-mysql.lisp b/src/parsers/command-mysql.lisp index c8f0bd7..c1e761d 100644 --- a/src/parsers/command-mysql.lisp +++ b/src/parsers/command-mysql.lisp @@ -7,7 +7,8 @@ ;;; ;;; MySQL options ;;; -(defrule mysql-option (or option-workers +(defrule mysql-option (or option-on-error-stop + option-workers option-concurrency option-batch-rows option-batch-size diff --git a/src/parsers/command-options.lisp b/src/parsers/command-options.lisp index 54d6b25..8a4a47e 100644 --- a/src/parsers/command-options.lisp +++ b/src/parsers/command-options.lisp @@ -132,6 +132,9 @@ (defrule option-data-only (and kw-data kw-only) (:constant (cons :data-only t))) +(defrule option-on-error-stop (and kw-on kw-error kw-stop) + (:constant (cons :on-error-stop t))) + (defrule option-identifiers-case (and (or kw-downcase kw-quote) kw-identifiers) (:lambda (id-case) (bind (((action _) id-case)) diff --git a/src/parsers/command-sqlite.lisp b/src/parsers/command-sqlite.lisp index f684ec6..4fb7143 100644 --- a/src/parsers/command-sqlite.lisp +++ b/src/parsers/command-sqlite.lisp @@ -13,7 +13,8 @@ load database set work_mem to '16MB', maintenance_work_mem to '512 MB'; |# -(defrule sqlite-option (or option-workers +(defrule sqlite-option (or option-on-error-stop + option-workers option-concurrency option-batch-rows option-batch-size diff --git a/src/sources/common/db-methods.lisp b/src/sources/common/db-methods.lisp index 646afc8..7d91097 100644 --- a/src/sources/common/db-methods.lisp +++ b/src/sources/common/db-methods.lisp @@ -231,7 +231,8 @@ alter-schema materialize-views) "Export database source data and Import it into PostgreSQL" - (let* ((copy-data (or data-only (not schema-only))) + (let* ((*on-error-stop* on-error-stop) + (copy-data (or data-only (not schema-only))) (create-ddl (or schema-only (not data-only))) (create-tables (and create-tables create-ddl)) (create-schemas (and create-schemas create-ddl)) diff --git a/src/sources/common/md-methods.lisp b/src/sources/common/md-methods.lisp index 8bf168c..762429d 100644 --- a/src/sources/common/md-methods.lisp +++ b/src/sources/common/md-methods.lisp @@ -100,7 +100,8 @@ create-indexes reset-sequences materialize-views set-table-oids including excluding)) - (let* ((pgconn (target-db copy)) + (let* ((*on-error-stop* on-error-stop) + (pgconn (target-db copy)) pgsql-catalog) (handler-case diff --git a/test/ixf.load b/test/ixf.load index 54caaf4..b1e12b1 100644 --- a/test/ixf.load +++ b/test/ixf.load @@ -2,7 +2,7 @@ LOAD IXF FROM data/nsitra.test1.ixf INTO postgresql:///pgloader?nsitra.test1 - WITH truncate, create table, timezone UTC + WITH on error stop, truncate, create table, timezone UTC BEFORE LOAD DO $$ drop schema if exists nsitra cascade; $$, diff --git a/test/sakila.load b/test/sakila.load index 45d1f94..3f3a2f1 100644 --- a/test/sakila.load +++ b/test/sakila.load @@ -7,7 +7,7 @@ load database -- WITH batch rows = 10000 - WITH concurrency = 1, workers = 6, + WITH on error stop, concurrency = 1, workers = 6, max parallel create index = 4-- , -- quote identifiers diff --git a/test/sqlite-chinook.load b/test/sqlite-chinook.load index 5bc1631..124166f 100644 --- a/test/sqlite-chinook.load +++ b/test/sqlite-chinook.load @@ -6,6 +6,7 @@ load database with workers = 4, concurrency = 2, + on error stop, include drop, create tables, create indexes, reset sequences, foreign keys -- alter table names matching ~/./ set schema 'chinook' diff --git a/test/sqlite.load b/test/sqlite.load index 38c7e15..2a9ffc0 100644 --- a/test/sqlite.load +++ b/test/sqlite.load @@ -11,7 +11,9 @@ load database $$ create schema if not exists sqlite; $$ cast column character.f1 to text drop typemod, - column appointments.time to timestamptz drop default, + column appointments.time + to timestamptz drop default + using sqlite-timestamp-to-timestamp, type intege to integer set work_mem to '16MB', maintenance_work_mem to '512 MB', search_path to 'sqlite'; \ No newline at end of file