Assorted fixes for SQLite.

First review the `sqlite_sequence` support so that we can still work with
databases that don't have this catalog, which doesn't always exists -- it
might depend on the SQLite version though.

Then while at it use the sql macro to host the SQLite “queries” in their own
files, enhancing the hackability of the system to some degrees. Not that
much, because we have to use a lot of PGRAMA command and then the column
output isn't documented with the query text itself.

src/sources/sqlite/sql/find-sequence.sql

@@ -0,0 +1,4 @@
+-- params: table-name
+select seq
+  from sqlite_sequence
+ where name = '~a';

src/sources/sqlite/sql/list-columns.sql

@@ -0,0 +1 @@
+PRAGMA table_info(`~a`)

src/sources/sqlite/sql/list-fkeys.sql

@@ -0,0 +1 @@
+PRAGMA foreign_key_list(`~a`)

src/sources/sqlite/sql/list-index-cols.sql

@@ -0,0 +1 @@
+PRAGMA index_info(`~a`)

src/sources/sqlite/sql/list-table-indexes.sql

@@ -0,0 +1,2 @@
+-- params: table-name
+PRAGMA index_list(`~a`)

src/sources/sqlite/sql/list-tables.sql

@@ -0,0 +1,6 @@
+SELECT tbl_name
+  FROM sqlite_master
+ WHERE type='table'
+       AND tbl_name <> 'sqlite_sequence'
+       ~:[~*~;AND (~{~a~^~&~10t or ~})~]
+       ~:[~*~;AND (~{~a~^~&~10t and ~})~]"

src/sources/sqlite/sql/sqlite-sequence.sql

@@ -0,0 +1,3 @@
+SELECT tbl_name
+  FROM sqlite_master
+ WHERE tbl_name = 'sqlite_sequence'

src/sources/sqlite/sqlite-schema.lisp

@@ -10,7 +10,7 @@
 ;;; Integration with the pgloader Source API
 ;;;
 (defclass sqlite-connection (fd-connection)
-  ((has-sequences :accessor has-sequences)))
+  ((has-sequences :initform nil :accessor has-sequences)))
 
 (defmethod initialize-instance :after ((slconn sqlite-connection) &key)
   "Assign the type slot to sqlite."
@@ -21,10 +21,8 @@
         (sqlite:connect (fd-path slconn)))
   (log-message :debug "CONNECTED TO ~a" (fd-path slconn))
   (when check-has-sequences
-    (let ((sql (format nil "SELECT tbl_name
+    (let ((sql (format nil (sql "/sqlite/sqlite-sequence.sql"))))
-                            FROM sqlite_master
+      (log-message :sql "SQLite: ~a" sql)
-                           WHERE tbl_name = 'sqlite_sequence'")))
-      (log-message :info "SQLite: ~a" sql)
       (when (sqlite:execute-single (conn-handle slconn) sql)
         (setf (has-sequences slconn) t))))
   slconn)
@@ -60,24 +58,19 @@
                       including
                       excluding)
   "Return the list of tables found in SQLITE-DB."
-  (let ((sql (format nil "SELECT tbl_name
+  (let ((sql (format nil (sql "/sqlite/list-tables.sql")
-                FROM sqlite_master
-               WHERE type='table'
-                     AND tbl_name <> 'sqlite_sequence'
-                     ~:[~*~;AND (~{~a~^~&~10t or ~})~]
-                     ~:[~*~;AND (~{~a~^~&~10t and ~})~]"
                      including          ; do we print the clause?
                      (filter-list-to-where-clause including nil)
                      excluding          ; do we print the clause?
                      (filter-list-to-where-clause excluding t))))
-    (log-message :info "~a" sql)
+    (log-message :sql "~a" sql)
     (loop for (name) in (sqlite:execute-to-list db sql)
        collect name)))
 
 (defun list-columns (table &key db-has-sequences (db *sqlite-db*) )
   "Return the list of columns found in TABLE-NAME."
   (let* ((table-name (table-source-name table))
-         (sql        (format nil "PRAGMA table_info(`~a`)" table-name)))
+         (sql        (format nil (sql "/sqlite/list-columns.sql") table-name)))
     (loop :for (ctid name type nullable default pk-id)
        :in (sqlite:execute-to-list db sql)
        :do (let* ((ctype (normalize type))
@@ -95,8 +88,8 @@
                         (string-equal (coldef-ctype field) "integer"))
                ;; then it might be an auto_increment, which we know by
                ;; looking at the sqlite_sequence catalog
-               (let* ((sql (format nil "select seq from sqlite_sequence
+               (let* ((sql
-                                         where name = '~a';" table-name))
+                       (format nil (sql "/sqlite/find-sequence.sql") table-name))
                       (seq (sqlite:execute-single db sql)))
                  (when (and seq (not (zerop seq)))
                    ;; magic marker for `apply-casting-rules'
@@ -155,16 +148,18 @@
 
 (defun list-index-cols (index-name &optional (db *sqlite-db*))
   "Return the list of columns in INDEX-NAME."
-  (let ((sql (format nil "PRAGMA index_info(`~a`)" index-name)))
+  (let ((sql (format nil (sql "/sqlite/list-index-cols.sql") index-name)))
     (loop :for (index-pos table-pos col-name) :in (sqlite:execute-to-list db sql)
        :collect col-name)))
 
 (defun list-indexes (table &optional (db *sqlite-db*))
   "Return the list of indexes attached to TABLE."
   (let* ((table-name (table-source-name table))
-         (sql        (format nil "PRAGMA index_list(`~a`)" table-name)))
+         (sql
+          (format nil (sql "/sqlite/list-table-indexes.sql") table-name)))
     (loop
-       :for (seq index-name unique origin partial) :in (sqlite:execute-to-list db sql)
+       :for (seq index-name unique origin partial)
+       :in (sqlite:execute-to-list db sql)
        :do (let* ((cols  (list-index-cols index-name db))
                   (index (make-index :name index-name
                                      :table table
@@ -192,7 +187,8 @@
 (defun list-fkeys (table &optional (db *sqlite-db*))
   "Return the list of indexes attached to TABLE."
   (let* ((table-name (table-source-name table))
-         (sql        (format nil "PRAGMA foreign_key_list(`~a`)" table-name)))
+         (sql
+          (format nil (sql "/sqlite/list-fkeys.sql") table-name)))
     (loop
        :with fkey-table := (make-hash-table)
        :for (id seq ftable-name from to on-update on-delete match)

test/sqlite-chinook-noseq.load

@@ -0,0 +1,10 @@
+load database
+     from 'sqlite/Chinook_Sqlite.sqlite'
+     into postgresql:///pgloader
+
+  set work_mem to '16MB',
+      maintenance_work_mem to '512 MB',
+      search_path to 'chinook'
+
+  before load do
+  $$ create schema if not exists chinook; $$;