Review SQLite blob types in light of "manifest typing", fix #60.

When using SQLite 3, a blob column might return either string of byte
vector values dynamically depending on the data itself, or maybe some
more complex parameters controlled at data insert time.

Hard-code the rule that a blob column returned as a string is in fact
base64 encoded (which looks like common practice) and decode it
automatically when needed, before sending to byte-vector-to-bytea. It
might be a tad slow but at least the data is properly converted.

In future, that decision might come and byte us in the back again, at
which point it'll be necessary to consider full casting options as in
the MySQL CAST rules. It seems like a big enough win for now if we can
avoid that.

pgloader.asd

@@ -27,6 +27,7 @@
 		 #:db3			; DBF version 3 file reader
 		 #:py-configparser	; Read old-style INI config files
 		 #:sqlite		; Query a SQLite file
+                 #:base64               ; Decode base64 data
 		 #:trivial-backtrace  	; For --debug cli usage
                  #:cl-markdown          ; To produce the website
 		 )

src/sources/sqlite.lisp

@@ -151,7 +151,12 @@
 (defmethod map-rows ((sqlite copy-sqlite) &key process-row-fn)
   "Extract SQLite data and call PROCESS-ROW-FN function with a single
    argument (a list of column values) for each row"
-  (let ((sql (format nil "SELECT * FROM ~a" (source sqlite))))
+  (let ((sql      (format nil "SELECT * FROM ~a" (source sqlite)))
+        (blobs-p
+         (coerce (mapcar (lambda (field)
+                           (string-equal "bytea" (cast (coldef-type field))))
+                         (fields sqlite))
+                 'vector)))
    (loop
       with statement = (sqlite:prepare-statement (db sqlite) sql)
       with len = (loop :for name :in (sqlite:statement-column-names statement)
@@ -159,8 +164,11 @@
       while (sqlite:step-statement statement)
       for row = (let ((v (make-array len)))
                   (loop :for x :below len
-                     :do (setf (aref v x)
-                               (sqlite:statement-column-value statement x)))
+                     :for raw := (sqlite:statement-column-value statement x)
+                     :for val := (if (and (aref blobs-p x) (stringp raw))
+                                     (base64:base64-string-to-usb8-array raw)
+                                     raw)
+                     :do (setf (aref v x) val))
                   v)
       counting t into rows
       do (funcall process-row-fn row)

test/sqlite-base64.load

@@ -0,0 +1,7 @@
+load database
+     from 'sqlite/storage.sqlite'
+     into postgresql:///storage
+
+ with include drop, create tables, create indexes, reset sequences
+
+  set work_mem to '16MB', maintenance_work_mem to '512 MB';