Add encoding support for db3 sources, fix #176.

It appears that db3 files are not limited to the ASCII character
encoding that they were designed with, so let's clue pgloader about
that.

This commit build
770cbe3526
and the pgloader Makefile has been updated to momentarily fetch cl-db3
from github rather than Quicklisp so that it's possible to enjoy the new
feature immediately.

Makefile

@@ -74,6 +74,9 @@ $(QLDIR)/local-projects/qmynd:
 $(QLDIR)/local-projects/cl-ixf:
 	git clone https://github.com/dimitri/cl-ixf.git $@
 
+$(QLDIR)/local-projects/cl-db3:
+	git clone https://github.com/dimitri/cl-db3.git $@
+
 $(QLDIR)/local-projects/cl-csv:
 	git clone https://github.com/AccelerationNet/cl-csv.git $@
 
@@ -91,6 +94,7 @@ $(QLDIR)/setup.lisp:
 quicklisp: $(QLDIR)/setup.lisp ;
 
 clones: $(QLDIR)/local-projects/cl-ixf \
+        $(QLDIR)/local-projects/cl-db3 \
         $(QLDIR)/local-projects/cl-csv \
         $(QLDIR)/local-projects/qmynd  \
         $(QLDIR)/local-projects/esrap ;

pgloader.asd

@@ -135,7 +135,7 @@
                                 :depends-on ("common" "csv"))
 
 			 (:module "db3"
-                                  :depends-on ("common")
+                                  :depends-on ("common" "csv")
                                   :components
                                   ((:file "db3-schema")
                                    (:file "db3" :depends-on ("db3-schema"))))

src/parsers/command-dbf.lisp

@@ -67,13 +67,15 @@
   (:lambda (clauses-list)
     (alexandria:alist-plist clauses-list)))
 
-(defrule load-dbf-command (and dbf-source target load-dbf-optional-clauses)
+(defrule load-dbf-command (and dbf-source (? file-encoding)
+                               target load-dbf-optional-clauses)
   (:lambda (command)
-    (destructuring-bind (source target clauses) command
-      `(,source ,target ,@clauses))))
+    (destructuring-bind (source encoding target clauses) command
+      `(,source ,encoding ,target ,@clauses))))
 
 (defun lisp-code-for-loading-from-dbf (dbf-db-conn pg-db-conn
                                        &key
+                                         (encoding :ascii)
                                          gucs before after
                                          ((:dbf-options options)))
   `(lambda ()
@@ -91,6 +93,7 @@
             (source
              (make-instance 'pgloader.db3:copy-db3
                             :target-db ,pg-db-conn
+                            :encoding ,encoding
                             :source-db source-db
                             :target table-name)))
 
@@ -110,9 +113,10 @@
 
 (defrule load-dbf-file load-dbf-command
   (:lambda (command)
-    (bind (((source pg-db-uri
+    (bind (((source encoding pg-db-uri
                     &key ((:dbf-options options)) gucs before after) command))
       (lisp-code-for-loading-from-dbf source pg-db-uri
+                                      :encoding encoding
                                       :gucs gucs
                                       :before before
                                       :after after

src/sources/db3/db3.lisp

@@ -7,7 +7,9 @@
 ;;;
 ;;; Integration with pgloader
 ;;;
-(defclass copy-db3 (copy) ()
+(defclass copy-db3 (copy)
+  ((encoding    :accessor encoding	  ; file encoding
+	        :initarg :encoding))
   (:documentation "pgloader DBF Data Source"))
 
 (defmethod initialize-instance :after ((db3 copy-db3) &key)
@@ -31,7 +33,8 @@
    argument (a list of column values) for each row."
   (with-connection (conn (source-db copy-db3))
     (let ((stream (conn-handle (source-db copy-db3)))
-          (db3    (fd-db3 (source-db copy-db3))))
+          (db3    (fd-db3 (source-db copy-db3)))
+          (db3:*external-format* (encoding copy-db3)))
       (loop
          :with count := (db3:record-count db3)
          :repeat count