diff --git a/src/parsers/command-csv.lisp b/src/parsers/command-csv.lisp index c7e4e11..0df8dd3 100644 --- a/src/parsers/command-csv.lisp +++ b/src/parsers/command-csv.lisp @@ -134,7 +134,8 @@ option-fields-terminated-by option-trim-unquoted-blanks option-keep-unquoted-blanks - option-csv-escape-mode)) + option-csv-escape-mode + option-null-if)) (defrule csv-options (and kw-with (and csv-option (* (and comma csv-option)))) @@ -429,26 +430,35 @@ (progn ,(sql-code-block pg-db-conn :pre before "before load") - (let ((on-error-stop (getf ',options :on-error-stop)) - (truncate (getf ',options :truncate)) - (disable-triggers (getf ',options :disable-triggers)) - (drop-indexes (getf ',options :drop-indexes)) - (max-parallel-create-index (getf ',options :max-parallel-create-index)) - (source - (make-instance 'copy-csv - :target-db ,pg-db-conn - :source source-db - :target (create-table ',target-table-name) - :encoding ,encoding - :fields ',fields - :columns ',columns - ,@(remove-batch-control-option - options :extras '(:worker-count - :concurrency - :truncate - :drop-indexes - :disable-triggers - :max-parallel-create-index))))) + (let* ((on-error-stop (getf ',options :on-error-stop)) + (truncate (getf ',options :truncate)) + (disable-triggers (getf ',options :disable-triggers)) + (drop-indexes (getf ',options :drop-indexes)) + (max-parallel-create-index (getf ',options :max-parallel-create-index)) + (fields + ',(let ((null-as (getf options :null-as))) + (if null-as + (mapcar (lambda (field) + (if (member :null-as field) field + (append field (list :null-as null-as)))) + fields) + fields))) + (source + (make-instance 'copy-csv + :target-db ,pg-db-conn + :source source-db + :target (create-table ',target-table-name) + :encoding ,encoding + :fields fields + :columns ',columns + ,@(remove-batch-control-option + options :extras '(:null-as + :worker-count + :concurrency + :truncate + :drop-indexes + :disable-triggers + :max-parallel-create-index))))) (copy-database source ,@ (when worker-count (list :worker-count worker-count)) diff --git a/src/sources/common/project-fields.lisp b/src/sources/common/project-fields.lisp index f82d46c..dc47197 100644 --- a/src/sources/common/project-fields.lisp +++ b/src/sources/common/project-fields.lisp @@ -115,12 +115,11 @@ sexp)) (t sexp))))) `(lambda (row) - (declare (optimize speed) (type list row)) + (declare (type list row)) (destructuring-bind (&optional ,@args &rest extra) row (declare (ignorable ,@args) (ignore extra)) (let ,values - (declare (ignorable ,@args) - (type vector ,@args)) + (declare (ignorable ,@args)) (vector ,@newrow))))))))) ;; allow for some debugging (if compile (compile nil projection) projection)))) diff --git a/test/csv-null-if.load b/test/csv-null-if.load new file mode 100644 index 0000000..c35d24b --- /dev/null +++ b/test/csv-null-if.load @@ -0,0 +1,22 @@ +LOAD CSV + FROM INLINE (id, number, data) + INTO postgresql:///pgloader?nullif + + BEFORE LOAD DO + $$ drop table if exists nullif; $$, + $$ CREATE TABLE nullif + ( + id serial primary key, + number integer, + data text + ); + $$ + + WITH null if '\N', + fields terminated by ',', + fields enclosed by '"', + fields escaped by backslash-quote; + + +"1",\N,"testing nulls" +"2","2","another test" \ No newline at end of file