From 632f7f5b4ea31ff13c39aae57a351aa18e28ed94 Mon Sep 17 00:00:00 2001
From: Dimitri Fontaine <dim@tapoueh.org>
Date: Thu, 14 Feb 2019 23:56:32 +0100
Subject: [PATCH] Implement COPY error handling for non-parsable error
 messages.

pgloarder parses the COPY error messages to find out the line number where
we have a problem in the batch, allowing for a quite efficient recovery
mechanism where it's easy enough to just skip the known faulty input.

Now, some error messages do not contain a COPY line number, such as fkey
violation messages:

  Database error 23503: insert or update on table "produtos" violates
  foreign key constraint "produtos_categorias_produtos_fk"

In that case rather than failing the whole batch at once (thanks to the
previous commit, we used to just badly fail before that), we can retry the
batch one row at a time until we find our culprit, and then continue one
input row at a time.

Fixes #836.
---
 src/pg-copy/copy-retry-batch.lisp | 29 ++++++++++++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/src/pg-copy/copy-retry-batch.lisp b/src/pg-copy/copy-retry-batch.lisp
index 693d897..2491aad 100644
--- a/src/pg-copy/copy-retry-batch.lisp
+++ b/src/pg-copy/copy-retry-batch.lisp
@@ -62,6 +62,33 @@
 
   (log-message :info "Entering error recovery.")
 
+  ;; Not all COPY errors produce a COPY error message. Foreign key violation
+  ;; produce a detailed message containing the data that we can't insert. In
+  ;; that case we're going to insert every single row of the batch, one at a
+  ;; time, and handle the error(s) individually.
+  ;;
+  (unless (parse-copy-error-context (database-error-context condition))
+    (let ((table-name  (format-table-name table))
+          (first-error t))
+      (loop :repeat (batch-count batch)
+         :for pos :from 0
+         :do (handler-case
+                 (incf pos
+                       (copy-partial-batch table-name columns batch 1 pos))
+               (postgresql-retryable (condition)
+                 (pomo:execute "ROLLBACK")
+                 (if first-error
+                     ;; the first error has been logged about already
+                     (setf first-error nil)
+                     (log-message :error "PostgreSQL [~s] ~a"
+                                  table-name condition))
+                 (incf nb-errors)))))
+
+    ;; that's all folks, we're done.
+    (return-from retry-batch nb-errors))
+
+  ;; now deal with the COPY error case where we have a line number and have
+  ;; the opportunity to be smart about it.
   (loop
      :with table-name := (format-table-name table)
      :with next-error := (parse-copy-error-context
@@ -70,7 +97,7 @@
      :while (< current-batch-pos (batch-count batch))
 
      :do
-     (progn                           ; indenting helper
+     (progn                             ; indenting helper
        (log-message :debug "pos: ~s ; err: ~a" current-batch-pos next-error)
        (when (= current-batch-pos next-error)
          (log-message :info "error recovery at ~d/~d, processing bad row"