diff --git a/pgloader.asd b/pgloader.asd index 434a024..f689b30 100644 --- a/pgloader.asd +++ b/pgloader.asd @@ -55,7 +55,6 @@ :components ((:file "charsets") (:file "batch") - (:file "threads") (:file "logs") (:file "utils") (:file "state") @@ -82,6 +81,7 @@ "state" "reject" "report")) + (:file "threads" :depends-on ("monitor")) (:file "archive" :depends-on ("logs")) ;; generic connection api @@ -108,45 +108,13 @@ "pgsql-create-schema" "pgsql-schema")))) - (:module "parsers" - :depends-on ("params" "package" "utils" "pgsql" "monkey") - :serial t - :components - ((:file "parse-ini") - (:file "template") - (:file "command-utils") - (:file "command-keywords") - (:file "command-regexp") - (:file "parse-pgpass") - (:file "command-db-uri") - (:file "command-source") - (:file "command-options") - (:file "command-sql-block") - (:file "command-sexp") - (:file "command-csv") - (:file "command-ixf") - (:file "command-fixed") - (:file "command-copy") - (:file "command-dbf") - (:file "command-cast-rules") - (:file "command-materialize-views") - (:file "command-alter-table") - (:file "command-mysql") - (:file "command-including-like") - (:file "command-mssql") - (:file "command-sqlite") - (:file "command-archive") - (:file "command-parser") - (:file "date-format"))) - - ;; Source format specific implementations + ;; Source format specific implementations (:module sources :depends-on ("monkey" ; mssql driver patches "params" "package" "pgsql" - "utils" - "parsers") + "utils") :components ((:module "common" :components @@ -221,6 +189,42 @@ :depends-on ("mysql-cast-rules" "mysql-schema")))))) + (:module "parsers" + :depends-on ("params" + "package" + "utils" + "pgsql" + "sources" + "monkey") + :serial t + :components + ((:file "parse-ini") + (:file "template") + (:file "command-utils") + (:file "command-keywords") + (:file "command-regexp") + (:file "parse-pgpass") + (:file "command-db-uri") + (:file "command-source") + (:file "command-options") + (:file "command-sql-block") + (:file "command-sexp") + (:file "command-csv") + (:file "command-ixf") + (:file "command-fixed") + (:file "command-copy") + (:file "command-dbf") + (:file "command-cast-rules") + (:file "command-materialize-views") + (:file "command-alter-table") + (:file "command-mysql") + (:file "command-including-like") + (:file "command-mssql") + (:file "command-sqlite") + (:file "command-archive") + (:file "command-parser") + (:file "date-format"))) + (:module "regress" :depends-on ("params" "package" "utils" "pgsql") :components ((:file "regress"))) diff --git a/src/utils/monitor.lisp b/src/utils/monitor.lisp index f3bacf6..e90f0db 100644 --- a/src/utils/monitor.lisp +++ b/src/utils/monitor.lisp @@ -286,9 +286,13 @@ secs))))) (bad-row - (%process-bad-row (bad-row-label event) - (bad-row-condition event) - (bad-row-data event)))) + (let* ((pgstate (get-state-section *sections* :data)) + (label (bad-row-label event)) + (table (pgstate-get-label pgstate label))) + (pgstate-incf pgstate label :errs 1) + (%process-bad-row table + (bad-row-condition event) + (bad-row-data event))))) :until (typep event 'stop))) diff --git a/src/utils/reject.lisp b/src/utils/reject.lisp index a65bba1..4b78006 100644 --- a/src/utils/reject.lisp +++ b/src/utils/reject.lisp @@ -9,32 +9,26 @@ ;;; file and the erroneous data in a rejected data file for further ;;; processing. ;;; -(defun %process-bad-row (table-name condition row) +(defun %process-bad-row (table condition row) "Add the row to the reject file, in PostgreSQL COPY TEXT format" - ;; first, update the stats. - (let ((state (get-state-section *sections* :data))) - (pgstate-incf state table-name :errs 1) + (let* ((data (pgtable-reject-data table)) + (logs (pgtable-reject-logs table))) - ;; now, the bad row processing - (let* ((table (pgstate-get-label state table-name)) - (data (pgtable-reject-data table)) - (logs (pgtable-reject-logs table))) - - ;; first log the rejected data - (with-open-file (reject-data-stream data + ;; first log the rejected data + (with-open-file (reject-data-stream data :direction :output :element-type '(unsigned-byte 8) :if-exists :append :if-does-not-exist :create) - ;; the row has already been processed when we get here - (write-sequence row reject-data-stream) - (write-byte #. (char-code #\Newline) reject-data-stream)) + ;; the row has already been processed when we get here + (write-sequence row reject-data-stream) + (write-byte #. (char-code #\Newline) reject-data-stream)) - ;; now log the condition signaled to reject the data - (with-open-file (reject-log-stream logs - :direction :output - :if-exists :append - :if-does-not-exist :create - :external-format :utf-8) - ;; the row has already been processed when we get here - (format reject-log-stream "~a~%" condition))))) + ;; now log the condition signaled to reject the data + (with-open-file (reject-log-stream logs + :direction :output + :if-exists :append + :if-does-not-exist :create + :external-format :utf-8) + ;; the row has already been processed when we get here + (format reject-log-stream "~a~%" condition))))