From 55584406fa054bf2c9897f26e761aa72177fc8a3 Mon Sep 17 00:00:00 2001
From: Dimitri Fontaine <dim@tapoueh.org>
Date: Wed, 18 Feb 2015 22:40:03 +0100
Subject: [PATCH] Add encoding support for db3 sources, fix #176.

It appears that db3 files are not limited to the ASCII character
encoding that they were designed with, so let's clue pgloader about
that.

This commit build
https://github.com/dimitri/cl-db3/commit/770cbe35261f55dbdcbf3681b9d9dbaea6ea216d
and the pgloader Makefile has been updated to momentarily fetch cl-db3
from github rather than Quicklisp so that it's possible to enjoy the new
feature immediately.
---
 Makefile                     |  4 ++++
 pgloader.asd                 |  2 +-
 src/parsers/command-dbf.lisp | 12 ++++++++----
 src/sources/db3/db3.lisp     |  7 +++++--
 4 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/Makefile b/Makefile
index 8c89e29..4902744 100644
--- a/Makefile
+++ b/Makefile
@@ -74,6 +74,9 @@ $(QLDIR)/local-projects/qmynd:
 $(QLDIR)/local-projects/cl-ixf:
 	git clone https://github.com/dimitri/cl-ixf.git $@
 
+$(QLDIR)/local-projects/cl-db3:
+	git clone https://github.com/dimitri/cl-db3.git $@
+
 $(QLDIR)/local-projects/cl-csv:
 	git clone https://github.com/AccelerationNet/cl-csv.git $@
 
@@ -91,6 +94,7 @@ $(QLDIR)/setup.lisp:
 quicklisp: $(QLDIR)/setup.lisp ;
 
 clones: $(QLDIR)/local-projects/cl-ixf \
+        $(QLDIR)/local-projects/cl-db3 \
         $(QLDIR)/local-projects/cl-csv \
         $(QLDIR)/local-projects/qmynd  \
         $(QLDIR)/local-projects/esrap ;
diff --git a/pgloader.asd b/pgloader.asd
index 402c91c..63678f2 100644
--- a/pgloader.asd
+++ b/pgloader.asd
@@ -135,7 +135,7 @@
                                 :depends-on ("common" "csv"))
 
 			 (:module "db3"
-                                  :depends-on ("common")
+                                  :depends-on ("common" "csv")
                                   :components
                                   ((:file "db3-schema")
                                    (:file "db3" :depends-on ("db3-schema"))))
diff --git a/src/parsers/command-dbf.lisp b/src/parsers/command-dbf.lisp
index 2765941..e7b3aaa 100644
--- a/src/parsers/command-dbf.lisp
+++ b/src/parsers/command-dbf.lisp
@@ -67,13 +67,15 @@
   (:lambda (clauses-list)
     (alexandria:alist-plist clauses-list)))
 
-(defrule load-dbf-command (and dbf-source target load-dbf-optional-clauses)
+(defrule load-dbf-command (and dbf-source (? file-encoding)
+                               target load-dbf-optional-clauses)
   (:lambda (command)
-    (destructuring-bind (source target clauses) command
-      `(,source ,target ,@clauses))))
+    (destructuring-bind (source encoding target clauses) command
+      `(,source ,encoding ,target ,@clauses))))
 
 (defun lisp-code-for-loading-from-dbf (dbf-db-conn pg-db-conn
                                        &key
+                                         (encoding :ascii)
                                          gucs before after
                                          ((:dbf-options options)))
   `(lambda ()
@@ -91,6 +93,7 @@
             (source
              (make-instance 'pgloader.db3:copy-db3
                             :target-db ,pg-db-conn
+                            :encoding ,encoding
                             :source-db source-db
                             :target table-name)))
 
@@ -110,9 +113,10 @@
 
 (defrule load-dbf-file load-dbf-command
   (:lambda (command)
-    (bind (((source pg-db-uri
+    (bind (((source encoding pg-db-uri
                     &key ((:dbf-options options)) gucs before after) command))
       (lisp-code-for-loading-from-dbf source pg-db-uri
+                                      :encoding encoding
                                       :gucs gucs
                                       :before before
                                       :after after
diff --git a/src/sources/db3/db3.lisp b/src/sources/db3/db3.lisp
index 843dc3d..dda1738 100644
--- a/src/sources/db3/db3.lisp
+++ b/src/sources/db3/db3.lisp
@@ -7,7 +7,9 @@
 ;;;
 ;;; Integration with pgloader
 ;;;
-(defclass copy-db3 (copy) ()
+(defclass copy-db3 (copy)
+  ((encoding    :accessor encoding	  ; file encoding
+	        :initarg :encoding))
   (:documentation "pgloader DBF Data Source"))
 
 (defmethod initialize-instance :after ((db3 copy-db3) &key)
@@ -31,7 +33,8 @@
    argument (a list of column values) for each row."
   (with-connection (conn (source-db copy-db3))
     (let ((stream (conn-handle (source-db copy-db3)))
-          (db3    (fd-db3 (source-db copy-db3))))
+          (db3    (fd-db3 (source-db copy-db3)))
+          (db3:*external-format* (encoding copy-db3)))
       (loop
          :with count := (db3:record-count db3)
          :repeat count