mirrors/pgloader
1
0
Fork 0
mirror of https://github.com/dimitri/pgloader.git synced 2025-08-10 00:07:00 +02:00
Code Issues Projects Releases Wiki Activity

Implement null if support as a WITH option.

Browse Source 
This gives a default "null if" option to all the input columns at once, and
it's still possible to override the default per column.

In passing, fix project-fields declarations that SBCL now complains about
when they're not true, such as declaring a vector when we might have :null
or nil. As a result, remove the (declare (optimize speed)) in the generated
field processing code.
This commit is contained in:
Dimitri Fontaine 2018-11-13 21:35:48 +01:00
parent a6ef7a56a9
commit 5ecf04acb9
3 changed files with 55 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
src/parsers/command-csv.lisp
View File

@ -134,7 +134,8 @@
option-fields-terminated-by option-fields-terminated-by
option-trim-unquoted-blanks option-trim-unquoted-blanks
option-keep-unquoted-blanks option-keep-unquoted-blanks
option-csv-escape-mode)) option-csv-escape-mode
option-null-if))
(defrule csv-options (and kw-with (defrule csv-options (and kw-with
(and csv-option (* (and comma csv-option)))) (and csv-option (* (and comma csv-option))))
@ -429,26 +430,35 @@
(progn (progn
,(sql-code-block pg-db-conn :pre before "before load") ,(sql-code-block pg-db-conn :pre before "before load")
(let ((on-error-stop (getf ',options :on-error-stop)) (let* ((on-error-stop (getf ',options :on-error-stop))
(truncate (getf ',options :truncate)) (truncate (getf ',options :truncate))
(disable-triggers (getf ',options :disable-triggers)) (disable-triggers (getf ',options :disable-triggers))
(drop-indexes (getf ',options :drop-indexes)) (drop-indexes (getf ',options :drop-indexes))
(max-parallel-create-index (getf ',options :max-parallel-create-index)) (max-parallel-create-index (getf ',options :max-parallel-create-index))
(source (fields
(make-instance 'copy-csv ',(let ((null-as (getf options :null-as)))
:target-db ,pg-db-conn (if null-as
:source source-db (mapcar (lambda (field)
:target (create-table ',target-table-name) (if (member :null-as field) field
:encoding ,encoding (append field (list :null-as null-as))))
:fields ',fields fields)
:columns ',columns fields)))
,@(remove-batch-control-option (source
options :extras '(:worker-count (make-instance 'copy-csv
:concurrency :target-db ,pg-db-conn
:truncate :source source-db
:drop-indexes :target (create-table ',target-table-name)
:disable-triggers :encoding ,encoding
:max-parallel-create-index))))) :fields fields
:columns ',columns
,@(remove-batch-control-option
options :extras '(:null-as
:worker-count
:concurrency
:truncate
:drop-indexes
:disable-triggers
:max-parallel-create-index)))))
(copy-database source (copy-database source
,@ (when worker-count ,@ (when worker-count
(list :worker-count worker-count)) (list :worker-count worker-count))

5
src/sources/common/project-fields.lisp
View File

@ -115,12 +115,11 @@
sexp)) sexp))
(t sexp))))) (t sexp)))))
`(lambda (row) `(lambda (row)
(declare (optimize speed) (type list row)) (declare (type list row))
(destructuring-bind (&optional ,@args &rest extra) row (destructuring-bind (&optional ,@args &rest extra) row
(declare (ignorable ,@args) (ignore extra)) (declare (ignorable ,@args) (ignore extra))
(let ,values (let ,values
(declare (ignorable ,@args) (declare (ignorable ,@args))
(type vector ,@args))
(vector ,@newrow))))))))) (vector ,@newrow)))))))))
;; allow for some debugging ;; allow for some debugging
(if compile (compile nil projection) projection)))) (if compile (compile nil projection) projection))))

22
test/csv-null-if.load Normal file
View File

@ -0,0 +1,22 @@
LOAD CSV
FROM INLINE (id, number, data)
INTO postgresql:///pgloader?nullif
BEFORE LOAD DO
$$ drop table if exists nullif; $$,
$$ CREATE TABLE nullif
(
id serial primary key,
number integer,
data text
);
$$
WITH null if '\N',
fields terminated by ',',
fields enclosed by '"',
fields escaped by backslash-quote;
"1",\N,"testing nulls"
"2","2","another test"