Skip empty lines when reading from files.

src/sources/sources.lisp

@@ -277,14 +277,20 @@
   (let ((projection (project-fields :fields fields :columns columns)))
     (lambda (row)
       (pgstate-incf *state* target :read 1)
-      (let ((projected-vector
-	     (handler-case
-		 (funcall projection row)
-	       (condition (e)
-		 (pgstate-incf *state* target :errs 1)
-		 (log-message :error "Could not read line ~d: ~a"
-			      (pgloader.utils::pgtable-read
-			       (pgstate-get-table *state* target))
-			      e)))))
-	(when projected-vector
-	  (funcall process-row-fn projected-vector))))))
+      ;; cl-csv returns (nil) for an empty line
+      (if (or (null row)
+              (and (null (car row)) (null (cdr row))))
+          (log-message :notice "Skipping empty line ~d."
+                       (pgloader.utils::pgtable-read
+                        (pgstate-get-table *state* target)))
+          (let ((projected-vector
+                 (handler-case
+                     (funcall projection row)
+                   (condition (e)
+                     (pgstate-incf *state* target :errs 1)
+                     (log-message :error "Could not read line ~d: ~a"
+                                  (pgloader.utils::pgtable-read
+                                   (pgstate-get-table *state* target))
+                                  e)))))
+            (when projected-vector
+              (funcall process-row-fn projected-vector)))))))

test/errors.load

@@ -9,6 +9,9 @@
  * trailing_sep = True
  * columns      = a:1, b:3, c:2
  *
+ *
+ * Note that we added ragged lines, empty lines, and lines with extra
+ * columns.
  */
 
 LOAD CSV
@@ -42,9 +45,12 @@ LOAD CSV
 2|expected error, month 13|2006-13-11|
 3|\ |2006-16-4|
 4|month should be may, ok|2006-5-12|
+
+
 5|another month 13, stress retry path|2006-13-10|
 6|some null date to play with||
 7|and a ragged line|
 8|and a line with extra columns|2014-01-23|hello|there|
 9|and another line|2014-01-22|
 
+