Allow specifying the SQLite text encoding, fix #99.

pgloader.1

@@ -1659,6 +1659,12 @@ When this option is listed pgloader will refrain from migrating the data over\.
 .IP
 When this option is listed pgloader only issues the \fBCOPY\fR statements, without doing any other processing\.
 .
+.IP "\(bu" 4
+\fIencoding\fR
+.
+.IP
+This option allows to control which encoding to parse the SQLite text data with\. Defaults to UTF\-8\.
+.
 .IP "" 0
 
 .

pgloader.1.md

@@ -1364,6 +1364,11 @@ The `sqlite` command accepts the following clauses and options:
 
 	    When this option is listed pgloader only issues the `COPY`
 	    statements, without doing any other processing.
+        
+      - *encoding*
+      
+        This option allows to control which encoding to parse the SQLite
+        text data with. Defaults to UTF-8.
 
   - *INCLUDING ONLY TABLE NAMES MATCHING*
 

src/parser.lisp

@@ -1090,6 +1090,15 @@ load database
 
   set work_mem to '16MB', maintenance_work_mem to '512 MB';
 |#
+(defrule option-encoding (and kw-encoding encoding)
+  (:lambda (enc)
+    (cons :encoding
+          (if enc
+              (destructuring-bind (kw-encoding encoding) enc
+                (declare (ignore kw-encoding))
+                encoding)
+              :utf-8))))
+
 (defrule sqlite-option (or option-batch-rows
                            option-batch-size
                            option-batch-concurrency
@@ -1099,7 +1108,8 @@ load database
 			   option-include-drop
 			   option-create-tables
 			   option-create-indexes
-			   option-reset-sequences))
+			   option-reset-sequences
+                           option-encoding))
 
 (defrule another-sqlite-option (and comma sqlite-option)
   (:lambda (source)

src/sources/sqlite.lisp

@@ -235,13 +235,15 @@
 			    only-tables
 			    including
 			    excluding
-                            (identifier-case :downcase))
+                            (identifier-case :downcase)
+                            (encoding :utf-8))
   "Stream the given SQLite database down to PostgreSQL."
   (let* ((summary       (null *state*))
 	 (*state*       (or *state* (make-pgstate)))
 	 (state-before  (or state-before (make-pgstate)))
 	 (idx-state     (make-pgstate))
 	 (seq-state     (make-pgstate))
+         (cffi:*default-foreign-encoding* encoding)
          (copy-kernel   (make-kernel 2))
          (all-columns   (filter-column-list (list-all-columns (db sqlite))
 					    :only-tables only-tables

src/utils.lisp

@@ -335,6 +335,11 @@
 			    (*myconn-pass*        . ,*myconn-pass*)
 			    (*state*              . ,*state*)
                             (*client-min-messages* . ,*client-min-messages*)
-                            (*log-min-messages*    . ,*log-min-messages*))))
+                            (*log-min-messages*    . ,*log-min-messages*)
+
+                            ;; bindings updates for libs
+                            ;; CFFI is used by the SQLite lib
+                            (cffi:*default-foreign-encoding*
+                             . ,cffi:*default-foreign-encoding*))))
   "Wrapper around lparallel:make-kernel that sets our usual bindings."
   (lp:make-kernel worker-count :bindings bindings))