From 1ed07057fd64e33d1454689434b36ddb1cd33f42 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Mon, 21 Mar 2016 20:52:50 +0100 Subject: [PATCH] Implement --on-error-stop command line option. The implementation uses the dynamic binding *on-error-stop* so it's also available when pgloader is used as Common Lisp librairy. The (not-all-that-) recent changes made to the error handling make that implementation straightforward enough, so let's finally do it! Fix #85. --- src/main.lisp | 11 +++++++++-- src/package.lisp | 2 ++ src/params.lisp | 4 ++++ src/pgsql/pgsql.lisp | 24 +++++++++++++++++++----- src/sources/common/db-methods.lisp | 2 ++ src/sources/common/md-methods.lisp | 2 ++ src/sources/common/methods.lisp | 2 ++ 7 files changed, 40 insertions(+), 7 deletions(-) diff --git a/src/main.lisp b/src/main.lisp index 39f5848..c20477e 100644 --- a/src/main.lisp +++ b/src/main.lisp @@ -55,7 +55,10 @@ :documentation "Read user code from files") ("dry-run" :type boolean - :documentation "Only check database connections, don't load anything.") + :documentation "Only check database connections, don't load anything.") + + ("on-error-stop" :type boolean + :documentation "Refrain from handling errors properly.") (("with") :type string :list t :optional t :documentation "Load options") @@ -195,7 +198,8 @@ (usage argv :quit t))) (destructuring-bind (&key help version quiet verbose debug logfile - list-encodings upgrade-config dry-run + list-encodings upgrade-config + dry-run on-error-stop ((:load-lisp-file load)) client-min-messages log-min-messages summary root-dir self-upgrade @@ -268,6 +272,9 @@ ;; Should we run in dry-run mode? (setf *dry-run* dry-run) + ;; Should we stop at first error? + (setf *on-error-stop* on-error-stop) + ;; Now process the arguments (when arguments ;; Start the logs system diff --git a/src/package.lisp b/src/package.lisp index 8d15316..0631394 100644 --- a/src/package.lisp +++ b/src/package.lisp @@ -793,6 +793,8 @@ #:*fd-path-root* #:*root-dir* #:*pg-settings* + #:*dry-run* + #:*on-error-stop* #:load-data #:parse-source-string diff --git a/src/params.lisp b/src/params.lisp index 88fcbed..c889e8f 100644 --- a/src/params.lisp +++ b/src/params.lisp @@ -7,6 +7,7 @@ (:use #:cl) (:export #:*version-string* #:*dry-run* + #:*on-error-stop* #:*self-upgrade-immutable-systems* #:*fd-path-root* #:*root-dir* @@ -68,6 +69,9 @@ (defparameter *dry-run* nil "Set to non-nil to only run checks about the load setup.") +(defparameter *on-error-stop* nil + "Set to non-nil to for pgloader to refrain from handling errors, quitting instead.") + (defparameter *fd-path-root* nil "Where to load files from, when loading from an archive or expanding regexps.") diff --git a/src/pgsql/pgsql.lisp b/src/pgsql/pgsql.lisp index 7f8045c..fc87e46 100644 --- a/src/pgsql/pgsql.lisp +++ b/src/pgsql/pgsql.lisp @@ -25,7 +25,9 @@ (incf (cl-postgres::copier-count copier))) (defun copy-batch (table columns batch batch-rows - &key (db pomo:*database*)) + &key + (db pomo:*database*) + on-error-stop) "Copy current *writer-batch* into TABLE-NAME." ;; We need to keep a copy of the rows we send through the COPY ;; protocol to PostgreSQL to be able to process them again in case @@ -70,9 +72,19 @@ cl-postgres-error::internal-error cl-postgres-error::insufficient-resources cl-postgres-error::program-limit-exceeded) (condition) - ;; clean the current transaction before retrying new ones + (pomo:execute "ROLLBACK") - (retry-batch table columns batch batch-rows condition)) + + (if on-error-stop + ;; re-signal the condition to upper level + (progn + (log-message :fatal "~a" condition) + (error "Stop loading data for table ~s on first error." + (format-table-name table))) + + ;; normal behavior, on-error-stop being nil + ;; clean the current transaction before retrying new ones + (retry-batch table columns batch batch-rows condition))) (condition (c) ;; non retryable failures @@ -86,7 +98,8 @@ (defun copy-from-queue (pgconn table queue &key columns - disable-triggers) + disable-triggers + on-error-stop) "Fetch from the QUEUE messages containing how many rows are in the *writer-batch* for us to send down to PostgreSQL, and when that's done update stats." @@ -105,7 +118,8 @@ :until (eq :end-of-data mesg) :for (rows batch-seconds) := (let ((start-time (get-internal-real-time))) - (list (copy-batch table columns batch read) + (list (copy-batch table columns batch read + :on-error-stop on-error-stop) (elapsed-time-since start-time))) :do (progn ;; The SBCL implementation needs some Garbage Collection diff --git a/src/sources/common/db-methods.lisp b/src/sources/common/db-methods.lisp index abe8033..105d177 100644 --- a/src/sources/common/db-methods.lisp +++ b/src/sources/common/db-methods.lisp @@ -120,6 +120,7 @@ ;;; (defmethod copy-database ((copy db-copy) &key + (on-error-stop *on-error-stop*) (worker-count 4) (concurrency 1) (truncate nil) @@ -216,6 +217,7 @@ :concurrency concurrency :kernel copy-kernel :channel copy-channel + :on-error-stop on-error-stop :disable-triggers disable-triggers)))) ;; now end the kernels diff --git a/src/sources/common/md-methods.lisp b/src/sources/common/md-methods.lisp index a6288ae..5ec8e8b 100644 --- a/src/sources/common/md-methods.lisp +++ b/src/sources/common/md-methods.lisp @@ -71,6 +71,7 @@ (defmethod copy-database ((copy md-copy) &key + (on-error-stop *on-error-stop*) truncate disable-triggers drop-indexes @@ -116,6 +117,7 @@ :concurrency concurrency :kernel lp:*kernel* :channel channel + :on-error-stop on-error-stop :truncate nil :disable-triggers disable-triggers))) diff --git a/src/sources/common/methods.lisp b/src/sources/common/methods.lisp index 39a8288..3d716ab 100644 --- a/src/sources/common/methods.lisp +++ b/src/sources/common/methods.lisp @@ -125,6 +125,7 @@ (worker-count 8) (concurrency 2) truncate + (on-error-stop *on-error-stop*) disable-triggers) "Copy data from COPY source into PostgreSQL. @@ -191,6 +192,7 @@ (target copy) fmtq :columns (copy-column-list copy) + :on-error-stop on-error-stop :disable-triggers disable-triggers)) ;; now wait until both the tasks are over, and kill the kernel