diff --git a/pgloader.asd b/pgloader.asd index a045dff..65199b0 100644 --- a/pgloader.asd +++ b/pgloader.asd @@ -27,6 +27,7 @@ #:db3 ; DBF version 3 file reader #:py-configparser ; Read old-style INI config files #:sqlite ; Query a SQLite file + #:base64 ; Decode base64 data #:trivial-backtrace ; For --debug cli usage #:cl-markdown ; To produce the website ) diff --git a/src/sources/sqlite.lisp b/src/sources/sqlite.lisp index 46e584e..81e2a79 100644 --- a/src/sources/sqlite.lisp +++ b/src/sources/sqlite.lisp @@ -151,7 +151,12 @@ (defmethod map-rows ((sqlite copy-sqlite) &key process-row-fn) "Extract SQLite data and call PROCESS-ROW-FN function with a single argument (a list of column values) for each row" - (let ((sql (format nil "SELECT * FROM ~a" (source sqlite)))) + (let ((sql (format nil "SELECT * FROM ~a" (source sqlite))) + (blobs-p + (coerce (mapcar (lambda (field) + (string-equal "bytea" (cast (coldef-type field)))) + (fields sqlite)) + 'vector))) (loop with statement = (sqlite:prepare-statement (db sqlite) sql) with len = (loop :for name :in (sqlite:statement-column-names statement) @@ -159,8 +164,11 @@ while (sqlite:step-statement statement) for row = (let ((v (make-array len))) (loop :for x :below len - :do (setf (aref v x) - (sqlite:statement-column-value statement x))) + :for raw := (sqlite:statement-column-value statement x) + :for val := (if (and (aref blobs-p x) (stringp raw)) + (base64:base64-string-to-usb8-array raw) + raw) + :do (setf (aref v x) val)) v) counting t into rows do (funcall process-row-fn row) diff --git a/test/sqlite-base64.load b/test/sqlite-base64.load new file mode 100644 index 0000000..88a5fec --- /dev/null +++ b/test/sqlite-base64.load @@ -0,0 +1,7 @@ +load database + from 'sqlite/storage.sqlite' + into postgresql:///storage + + with include drop, create tables, create indexes, reset sequences + + set work_mem to '16MB', maintenance_work_mem to '512 MB'; \ No newline at end of file diff --git a/test/sqlite/storage.sqlite b/test/sqlite/storage.sqlite new file mode 100644 index 0000000..7eba89f Binary files /dev/null and b/test/sqlite/storage.sqlite differ