mirrors/pgloader
1
0
Fork 0
mirror of https://github.com/dimitri/pgloader.git synced 2025-08-07 06:47:00 +02:00
Code Issues Projects Releases Wiki Activity

Implement null if support as a WITH option.

Browse Source 
This gives a default "null if" option to all the input columns at once, and
it's still possible to override the default per column.

In passing, fix project-fields declarations that SBCL now complains about
when they're not true, such as declaring a vector when we might have :null
or nil. As a result, remove the (declare (optimize speed)) in the generated
field processing code.
This commit is contained in:
Dimitri Fontaine 2018-11-13 21:35:48 +01:00
parent a6ef7a56a9
commit 5ecf04acb9
3 changed files with 55 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
src/parsers/command-csv.lisp
View File

@ -134,7 +134,8 @@
option-fields-terminated-by
option-trim-unquoted-blanks
option-keep-unquoted-blanks
option-csv-escape-mode))
option-csv-escape-mode
option-null-if))
(defrule csv-options (and kw-with
(and csv-option (* (and comma csv-option))))
@ -429,26 +430,35 @@
(progn
,(sql-code-block pg-db-conn :pre before "before load")
(let ((on-error-stop (getf ',options :on-error-stop))
(truncate (getf ',options :truncate))
(disable-triggers (getf ',options :disable-triggers))
(drop-indexes (getf ',options :drop-indexes))
(max-parallel-create-index (getf ',options :max-parallel-create-index))
(source
(make-instance 'copy-csv
:target-db ,pg-db-conn
:source source-db
:target (create-table ',target-table-name)
:encoding ,encoding
:fields ',fields
:columns ',columns
,@(remove-batch-control-option
options :extras '(:worker-count
:concurrency
:truncate
:drop-indexes
:disable-triggers
:max-parallel-create-index)))))
(let* ((on-error-stop (getf ',options :on-error-stop))
(truncate (getf ',options :truncate))
(disable-triggers (getf ',options :disable-triggers))
(drop-indexes (getf ',options :drop-indexes))
(max-parallel-create-index (getf ',options :max-parallel-create-index))
(fields
',(let ((null-as (getf options :null-as)))
(if null-as
(mapcar (lambda (field)
(if (member :null-as field) field
(append field (list :null-as null-as))))
fields)
fields)))
(source
(make-instance 'copy-csv
:target-db ,pg-db-conn
:source source-db
:target (create-table ',target-table-name)
:encoding ,encoding
:fields fields
:columns ',columns
,@(remove-batch-control-option
options :extras '(:null-as
:worker-count
:concurrency
:truncate
:drop-indexes
:disable-triggers
:max-parallel-create-index)))))
(copy-database source
,@ (when worker-count
(list :worker-count worker-count))

5
src/sources/common/project-fields.lisp
View File

@ -115,12 +115,11 @@
sexp))
(t sexp)))))
`(lambda (row)
(declare (optimize speed) (type list row))
(declare (type list row))
(destructuring-bind (&optional ,@args &rest extra) row
(declare (ignorable ,@args) (ignore extra))
(let ,values
(declare (ignorable ,@args)
(type vector ,@args))
(declare (ignorable ,@args))
(vector ,@newrow)))))))))
;; allow for some debugging
(if compile (compile nil projection) projection))))

22
test/csv-null-if.load Normal file
View File

@ -0,0 +1,22 @@
LOAD CSV
FROM INLINE (id, number, data)
INTO postgresql:///pgloader?nullif
BEFORE LOAD DO
$$ drop table if exists nullif; $$,
$$ CREATE TABLE nullif
(
id serial primary key,
number integer,
data text
);
$$
WITH null if '\N',
fields terminated by ',',
fields enclosed by '"',
fields escaped by backslash-quote;
"1",\N,"testing nulls"
"2","2","another test"