Improve DBF support.

The cl-db3 lib just got improvements for new dbase file types and field
types, reflect those in pgloader.

Also, cl-db3 now can read the encoding of the file (language driver)
directly in the header, meaning we can rely on that metadata by default, and
only override it when the users tells us to.

See #961.

src/parsers/command-dbf.lisp

@@ -66,9 +66,8 @@
 ;;; dbf defaults to ascii rather than utf-8
 (defrule dbf-file-encoding (? (and kw-with kw-encoding encoding))
   (:lambda (enc)
-    (if enc
-        (bind (((_ _ encoding) enc)) encoding)
-	:ascii)))
+    (when enc
+      (bind (((_ _ encoding) enc)) encoding))))
 
 (defrule load-dbf-command (and dbf-source
                                (? dbf-file-encoding)
@@ -93,7 +92,7 @@
 (defun lisp-code-for-loading-from-dbf (dbf-db-conn pg-db-conn
                                        &key
                                          target-table-name
-                                         (encoding :ascii)
+                                         encoding
                                          gucs casts before after options
                                          &allow-other-keys)
   `(lambda ()

src/sources/db3/db3-cast-rules.lisp

@@ -20,6 +20,18 @@
      :target (:type "integer")
      :using pgloader.transforms::db3-numeric-to-pgsql-integer)
 
+    (:source (:type "Y")
+     :target (:type "bigint")
+     :using pgloader.transforms::db3-numeric-to-pgsql-integer)
+
+    (:source (:type "+")
+     :target (:type "serial")
+     :using pgloader.transforms::db3-numeric-to-pgsql-integer)
+
+    (:source (:type "F")
+     :target (:type "double precision")
+     :using pgloader.transforms::float-to-string)
+
     (:source (:type "L")
      :target (:type "boolean")
      :using pgloader.transforms::logical-to-boolean)
@@ -30,7 +42,11 @@
 
     (:source (:type "M")
      :target (:type "text")
-     :using pgloader.transforms::db3-trim-string))
+     :using pgloader.transforms::db3-trim-string)
+
+    (:source (:type "0")
+     :target (:type "bit(8)")
+     :using pgloader.transforms::bits-to-hex-bitstring))
   "Data Type Casting rules to migrate from DB3 to PostgreSQL")
 
 (defstruct (db3-coldef

src/sources/db3/db3.lisp

@@ -24,8 +24,17 @@
    argument (a list of column values) for each row."
   (with-connection (conn (source-db copy-db3))
     (let ((stream (conn-handle (source-db copy-db3)))
-          (db3    (fd-db3 (source-db copy-db3)))
-          (db3:*external-format* (encoding copy-db3)))
+          (db3    (fd-db3 (source-db copy-db3))))
+
+      ;; when the pgloader command has an ENCODING clause, it takes
+      ;; precedence to the encoding embedded in the db3 file, if any.
+      (when (and (encoding copy-db3)
+                 (db3::encoding db3)
+                 (not (eq (encoding copy-db3) (db3::encoding db3))))
+        (log-message :warning "Forcing encoding to ~a, db3 file has ~a"
+                     (encoding copy-db3) (db3::encoding db3))
+        (setf (db3::encoding db3) (encoding copy-db3)))
+
       (loop
          :with count := (db3:record-count db3)
          :repeat count

src/utils/transforms.lisp

@@ -498,7 +498,7 @@
 
 (defun logical-to-boolean (value)
   "Convert a DB3 logical value to a PostgreSQL boolean."
-  (if (string= value "?") nil value))
+  (if (member value '("?" " ") :test #'string=) nil value))
 
 (defun db3-trim-string (value)
   "DB3 Strings a right padded with spaces, fix that."

test/data/README.md

@@ -0,0 +1,14 @@
+# Test data files.
+
+Most of the files have been contributed by pgloader users in the context of
+an issue where it was helpful to have a test case to reproduce and fix a
+bug.
+
+The following DBF test files come from the Open Source repository at
+https://github.com/infused/dbf/tree/master/spec/fixtures
+  
+  - dbase_31.dbf
+  - dbase_31_summary.txt
+  - dbase_8b.dbf
+  - dbase_8b.dbt
+  - dbase_8b_summary.txt

test/data/dbase_31_summary.txt

@@ -0,0 +1,20 @@
+
+Database: dbase_31.dbf
+Type: (31) Visual FoxPro with AutoIncrement field
+Memo File: false
+Records: 77
+
+Fields:
+Name             Type       Length     Decimal
+------------------------------------------------------------------------------
+PRODUCTID        I          4          0         
+PRODUCTNAM       C          40         0         
+SUPPLIERID       I          4          0         
+CATEGORYID       I          4          0         
+QUANTITYPE       C          20         0         
+UNITPRICE        Y          8          4         
+UNITSINSTO       I          4          0         
+UNITSONORD       I          4          0         
+REORDERLEV       I          4          0         
+DISCONTINU       L          1          0         
+_NullFlags       0          1          0         

test/data/dbase_8b_summary.txt

@@ -0,0 +1,15 @@
+
+Database: dbase_8b.dbf
+Type: (8b) dBase IV with memo file
+Memo File: true
+Records: 10
+
+Fields:
+Name             Type       Length     Decimal
+------------------------------------------------------------------------------
+CHARACTER        C          100        0         
+NUMERICAL        N          20         2         
+DATE             D          8          0         
+LOGICAL          L          1          0         
+FLOAT            F          20         18        
+MEMO             M          10         0         

test/dbf-31.load

@@ -0,0 +1,5 @@
+LOAD DBF
+    FROM data/dbase_31.dbf
+    INTO postgresql:///pgloader
+    TARGET TABLE dbf.dbase_31
+    WITH truncate, create table, disable triggers;

test/dbf-8b.load

@@ -0,0 +1,5 @@
+LOAD DBF
+    FROM data/dbase_8b.dbf
+    INTO postgresql:///pgloader
+    TARGET TABLE dbf.dbase_8b
+    WITH truncate, create table, disable triggers;