diff --git a/src/parsers/command-dbf.lisp b/src/parsers/command-dbf.lisp index 6e1749b..5f8269d 100644 --- a/src/parsers/command-dbf.lisp +++ b/src/parsers/command-dbf.lisp @@ -66,9 +66,8 @@ ;;; dbf defaults to ascii rather than utf-8 (defrule dbf-file-encoding (? (and kw-with kw-encoding encoding)) (:lambda (enc) - (if enc - (bind (((_ _ encoding) enc)) encoding) - :ascii))) + (when enc + (bind (((_ _ encoding) enc)) encoding)))) (defrule load-dbf-command (and dbf-source (? dbf-file-encoding) @@ -93,7 +92,7 @@ (defun lisp-code-for-loading-from-dbf (dbf-db-conn pg-db-conn &key target-table-name - (encoding :ascii) + encoding gucs casts before after options &allow-other-keys) `(lambda () diff --git a/src/sources/db3/db3-cast-rules.lisp b/src/sources/db3/db3-cast-rules.lisp index 17dec83..0366a19 100644 --- a/src/sources/db3/db3-cast-rules.lisp +++ b/src/sources/db3/db3-cast-rules.lisp @@ -20,6 +20,18 @@ :target (:type "integer") :using pgloader.transforms::db3-numeric-to-pgsql-integer) + (:source (:type "Y") + :target (:type "bigint") + :using pgloader.transforms::db3-numeric-to-pgsql-integer) + + (:source (:type "+") + :target (:type "serial") + :using pgloader.transforms::db3-numeric-to-pgsql-integer) + + (:source (:type "F") + :target (:type "double precision") + :using pgloader.transforms::float-to-string) + (:source (:type "L") :target (:type "boolean") :using pgloader.transforms::logical-to-boolean) @@ -30,7 +42,11 @@ (:source (:type "M") :target (:type "text") - :using pgloader.transforms::db3-trim-string)) + :using pgloader.transforms::db3-trim-string) + + (:source (:type "0") + :target (:type "bit(8)") + :using pgloader.transforms::bits-to-hex-bitstring)) "Data Type Casting rules to migrate from DB3 to PostgreSQL") (defstruct (db3-coldef diff --git a/src/sources/db3/db3.lisp b/src/sources/db3/db3.lisp index b303931..fb0e611 100644 --- a/src/sources/db3/db3.lisp +++ b/src/sources/db3/db3.lisp @@ -24,8 +24,17 @@ argument (a list of column values) for each row." (with-connection (conn (source-db copy-db3)) (let ((stream (conn-handle (source-db copy-db3))) - (db3 (fd-db3 (source-db copy-db3))) - (db3:*external-format* (encoding copy-db3))) + (db3 (fd-db3 (source-db copy-db3)))) + + ;; when the pgloader command has an ENCODING clause, it takes + ;; precedence to the encoding embedded in the db3 file, if any. + (when (and (encoding copy-db3) + (db3::encoding db3) + (not (eq (encoding copy-db3) (db3::encoding db3)))) + (log-message :warning "Forcing encoding to ~a, db3 file has ~a" + (encoding copy-db3) (db3::encoding db3)) + (setf (db3::encoding db3) (encoding copy-db3))) + (loop :with count := (db3:record-count db3) :repeat count diff --git a/src/utils/transforms.lisp b/src/utils/transforms.lisp index 3756946..7cf0946 100644 --- a/src/utils/transforms.lisp +++ b/src/utils/transforms.lisp @@ -498,7 +498,7 @@ (defun logical-to-boolean (value) "Convert a DB3 logical value to a PostgreSQL boolean." - (if (string= value "?") nil value)) + (if (member value '("?" " ") :test #'string=) nil value)) (defun db3-trim-string (value) "DB3 Strings a right padded with spaces, fix that." diff --git a/test/data/README.md b/test/data/README.md new file mode 100644 index 0000000..f193ec8 --- /dev/null +++ b/test/data/README.md @@ -0,0 +1,14 @@ +# Test data files. + +Most of the files have been contributed by pgloader users in the context of +an issue where it was helpful to have a test case to reproduce and fix a +bug. + +The following DBF test files come from the Open Source repository at +https://github.com/infused/dbf/tree/master/spec/fixtures + + - dbase_31.dbf + - dbase_31_summary.txt + - dbase_8b.dbf + - dbase_8b.dbt + - dbase_8b_summary.txt diff --git a/test/data/dbase_31.dbf b/test/data/dbase_31.dbf new file mode 100644 index 0000000..057c46b Binary files /dev/null and b/test/data/dbase_31.dbf differ diff --git a/test/data/dbase_31_summary.txt b/test/data/dbase_31_summary.txt new file mode 100644 index 0000000..35e98ae --- /dev/null +++ b/test/data/dbase_31_summary.txt @@ -0,0 +1,20 @@ + +Database: dbase_31.dbf +Type: (31) Visual FoxPro with AutoIncrement field +Memo File: false +Records: 77 + +Fields: +Name Type Length Decimal +------------------------------------------------------------------------------ +PRODUCTID I 4 0 +PRODUCTNAM C 40 0 +SUPPLIERID I 4 0 +CATEGORYID I 4 0 +QUANTITYPE C 20 0 +UNITPRICE Y 8 4 +UNITSINSTO I 4 0 +UNITSONORD I 4 0 +REORDERLEV I 4 0 +DISCONTINU L 1 0 +_NullFlags 0 1 0 diff --git a/test/data/dbase_8b.dbf b/test/data/dbase_8b.dbf new file mode 100644 index 0000000..9e0ec13 Binary files /dev/null and b/test/data/dbase_8b.dbf differ diff --git a/test/data/dbase_8b.dbt b/test/data/dbase_8b.dbt new file mode 100644 index 0000000..527663a Binary files /dev/null and b/test/data/dbase_8b.dbt differ diff --git a/test/data/dbase_8b_summary.txt b/test/data/dbase_8b_summary.txt new file mode 100644 index 0000000..7be9219 --- /dev/null +++ b/test/data/dbase_8b_summary.txt @@ -0,0 +1,15 @@ + +Database: dbase_8b.dbf +Type: (8b) dBase IV with memo file +Memo File: true +Records: 10 + +Fields: +Name Type Length Decimal +------------------------------------------------------------------------------ +CHARACTER C 100 0 +NUMERICAL N 20 2 +DATE D 8 0 +LOGICAL L 1 0 +FLOAT F 20 18 +MEMO M 10 0 diff --git a/test/dbf-31.load b/test/dbf-31.load new file mode 100644 index 0000000..af8ab94 --- /dev/null +++ b/test/dbf-31.load @@ -0,0 +1,5 @@ +LOAD DBF + FROM data/dbase_31.dbf + INTO postgresql:///pgloader + TARGET TABLE dbf.dbase_31 + WITH truncate, create table, disable triggers; diff --git a/test/dbf-8b.load b/test/dbf-8b.load new file mode 100644 index 0000000..46063c3 --- /dev/null +++ b/test/dbf-8b.load @@ -0,0 +1,5 @@ +LOAD DBF + FROM data/dbase_8b.dbf + INTO postgresql:///pgloader + TARGET TABLE dbf.dbase_8b + WITH truncate, create table, disable triggers;