From 79f622234fc1c5b9fa4f3357d4bc05f8ffddda66 Mon Sep 17 00:00:00 2001
From: Dimitri Fontaine <dim@tapoueh.org>
Date: Fri, 25 Jul 2014 19:01:15 +0200
Subject: [PATCH] Allow specifying the SQLite text encoding, fix #99.

---
 pgloader.1              |  6 ++++++
 pgloader.1.md           |  5 +++++
 src/parser.lisp         | 12 +++++++++++-
 src/sources/sqlite.lisp |  4 +++-
 src/utils.lisp          |  7 ++++++-
 5 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/pgloader.1 b/pgloader.1
index a6a0b94..d9586ba 100644
--- a/pgloader.1
+++ b/pgloader.1
@@ -1659,6 +1659,12 @@ When this option is listed pgloader will refrain from migrating the data over\.
 .IP
 When this option is listed pgloader only issues the \fBCOPY\fR statements, without doing any other processing\.
 .
+.IP "\(bu" 4
+\fIencoding\fR
+.
+.IP
+This option allows to control which encoding to parse the SQLite text data with\. Defaults to UTF\-8\.
+.
 .IP "" 0
 
 .
diff --git a/pgloader.1.md b/pgloader.1.md
index 9ad667f..93b41c2 100644
--- a/pgloader.1.md
+++ b/pgloader.1.md
@@ -1364,6 +1364,11 @@ The `sqlite` command accepts the following clauses and options:
 
 	    When this option is listed pgloader only issues the `COPY`
 	    statements, without doing any other processing.
+        
+      - *encoding*
+      
+        This option allows to control which encoding to parse the SQLite
+        text data with. Defaults to UTF-8.
 
   - *INCLUDING ONLY TABLE NAMES MATCHING*
 
diff --git a/src/parser.lisp b/src/parser.lisp
index 5d52939..2d2eb68 100644
--- a/src/parser.lisp
+++ b/src/parser.lisp
@@ -1090,6 +1090,15 @@ load database
 
   set work_mem to '16MB', maintenance_work_mem to '512 MB';
 |#
+(defrule option-encoding (and kw-encoding encoding)
+  (:lambda (enc)
+    (cons :encoding
+          (if enc
+              (destructuring-bind (kw-encoding encoding) enc
+                (declare (ignore kw-encoding))
+                encoding)
+              :utf-8))))
+
 (defrule sqlite-option (or option-batch-rows
                            option-batch-size
                            option-batch-concurrency
@@ -1099,7 +1108,8 @@ load database
 			   option-include-drop
 			   option-create-tables
 			   option-create-indexes
-			   option-reset-sequences))
+			   option-reset-sequences
+                           option-encoding))
 
 (defrule another-sqlite-option (and comma sqlite-option)
   (:lambda (source)
diff --git a/src/sources/sqlite.lisp b/src/sources/sqlite.lisp
index 41f94c7..bc49a41 100644
--- a/src/sources/sqlite.lisp
+++ b/src/sources/sqlite.lisp
@@ -235,13 +235,15 @@
 			    only-tables
 			    including
 			    excluding
-                            (identifier-case :downcase))
+                            (identifier-case :downcase)
+                            (encoding :utf-8))
   "Stream the given SQLite database down to PostgreSQL."
   (let* ((summary       (null *state*))
 	 (*state*       (or *state* (make-pgstate)))
 	 (state-before  (or state-before (make-pgstate)))
 	 (idx-state     (make-pgstate))
 	 (seq-state     (make-pgstate))
+         (cffi:*default-foreign-encoding* encoding)
          (copy-kernel   (make-kernel 2))
          (all-columns   (filter-column-list (list-all-columns (db sqlite))
 					    :only-tables only-tables
diff --git a/src/utils.lisp b/src/utils.lisp
index de8b76c..3750465 100644
--- a/src/utils.lisp
+++ b/src/utils.lisp
@@ -335,6 +335,11 @@
 			    (*myconn-pass*        . ,*myconn-pass*)
 			    (*state*              . ,*state*)
                             (*client-min-messages* . ,*client-min-messages*)
-                            (*log-min-messages*    . ,*log-min-messages*))))
+                            (*log-min-messages*    . ,*log-min-messages*)
+
+                            ;; bindings updates for libs
+                            ;; CFFI is used by the SQLite lib
+                            (cffi:*default-foreign-encoding*
+                             . ,cffi:*default-foreign-encoding*))))
   "Wrapper around lparallel:make-kernel that sets our usual bindings."
   (lp:make-kernel worker-count :bindings bindings))