Desultory improve the SQLite importer.

src/parser.lisp

@@ -913,7 +913,7 @@ load database
 	(declare (ignore type))
 	(list :sqlite path)))))
 
-(defrule sqlite-uri (or sqlite-db-uri http-uri))
+(defrule sqlite-uri (or sqlite-db-uri http-uri maybe-quoted-filename))
 (defrule sqlite-source (and kw-load kw-database kw-from sqlite-uri)
   (:destructure (l d f u)
 		(declare (ignore l d f))
@@ -938,7 +938,8 @@ load database
 		       (:http     `(with-stats-collection
 				       (,dbname "download" :state state-before)
 				     (pgloader.archive:http-fetch-file ,url)))
-		       (:sqlite url))))
+		       (:sqlite url)
+		       (:filename url))))
 		  (db
 		   (if (string= "zip" (pathname-type db))
 		       (progn
@@ -1808,44 +1809,60 @@ load database
 		      (inject-inline-data-position s-exp position)
 		      s-exp)))
 
+(defun process-relative-pathnames (filename command)
+  "Walk the COMMAND to replace relative pathname with absolute ones, merging
+   them within the directory where we found the command FILENAME."
+  (loop
+     for s-exp in command
+     when (pathnamep s-exp)
+     collect (if (fad:pathname-relative-p s-exp)
+		 (merge-pathnames s-exp (directory-namestring filename))
+		 s-exp)
+     else
+     collect (if (and (consp s-exp) (listp (cdr s-exp)))
+		 (process-relative-pathnames filename s-exp)
+		 s-exp)))
+
 (defun parse-commands-from-file (filename)
   "The command could be using from :inline, in which case we want to parse
    as much as possible then use the command against an already opened stream
    where we moved at the beginning of the data."
   (log-message :log "Parsing commands from file ~s~%" filename)
 
-  (let ((*data-expected-inline* nil)
-	(content (slurp-file-into-string filename)))
-    (multiple-value-bind (commands end-commands-position)
-	(parse 'commands content :junk-allowed t)
+  (process-relative-pathnames
+   filename
+   (let ((*data-expected-inline* nil)
+	 (content (slurp-file-into-string filename)))
+     (multiple-value-bind (commands end-commands-position)
+	 (parse 'commands content :junk-allowed t)
 
-      ;; INLINE is only allowed where we have a single command in the file
-      (if *data-expected-inline*
-	  (progn
-	    (when (= 0 end-commands-position)
-	      ;; didn't find any command, leave error reporting to esrap
-	      (parse 'commands content))
+       ;; INLINE is only allowed where we have a single command in the file
+       (if *data-expected-inline*
+	   (progn
+	     (when (= 0 end-commands-position)
+	       ;; didn't find any command, leave error reporting to esrap
+	       (parse 'commands content))
 
-	    (when (and *data-expected-inline*
-		       (null end-commands-position))
-	      (error "Inline data not found in '~a'." filename))
+	     (when (and *data-expected-inline*
+			(null end-commands-position))
+	       (error "Inline data not found in '~a'." filename))
 
-	    (when (and *data-expected-inline* (not (= 1 (length commands))))
-	      (error (concatenate 'string
-				  "Too many commands found in '~a'.~%"
-				  "To use inline data, use a single command.")
-		     filename))
+	     (when (and *data-expected-inline* (not (= 1 (length commands))))
+	       (error (concatenate 'string
+				   "Too many commands found in '~a'.~%"
+				   "To use inline data, use a single command.")
+		      filename))
 
-	    ;; now we should have a single command and inline data after that
-	    ;; replace the (:inline nil) found in the first (and only) command
-	    ;; with a (:inline position) instead
-	    (list
-	     (inject-inline-data-position
-	      (first commands) (cons filename end-commands-position))))
+	     ;; now we should have a single command and inline data after that
+	     ;; replace the (:inline nil) found in the first (and only) command
+	     ;; with a (:inline position) instead
+	     (list
+	      (inject-inline-data-position
+	       (first commands) (cons filename end-commands-position))))
 
-	  ;; There was no INLINE magic found in the file, reparse it so that
-	  ;; normal error processing happen
-	  (parse 'commands content)))))
+	   ;; There was no INLINE magic found in the file, reparse it so that
+	   ;; normal error processing happen
+	   (parse 'commands content))))))
 
 (defun run-commands (source
 		     &key

src/pgsql/schema.lisp

@@ -238,7 +238,8 @@
        do
 	 (let ((sql
 		(format-pgsql-create-index index :identifier-case identifier-case)))
-	  (log-message :notice "~a" sql)
-	  (lp:submit-task channel
-			  #'pgsql-execute-with-timing
-			  dbname label sql state)))))
+	   (when sql
+	     (log-message :notice "~a" sql)
+	     (lp:submit-task channel
+			     #'pgsql-execute-with-timing
+			     dbname label sql state))))))

src/sources/sqlite.lisp

@@ -14,6 +14,15 @@
 	     (:constructor make-coldef (seq name type nullable default pk-id)))
   seq name type nullable default pk-id)
 
+(defun cast (sqlite-type-name)
+  "Return the PostgreSQL type name for a given SQLite type name."
+  (cond ((and (<= 8 (length sqlite-type-name))
+	      (string-equal sqlite-type-name "nvarchar" :end1 8)) "text")
+
+	((string-equal sqlite-type-name "datetime") "timestamptz")
+
+	(t sqlite-type-name)))
+
 (defmethod format-pgsql-column ((col coldef) &key identifier-case)
   "Return a string representing the PostgreSQL column definition."
   (let* ((column-name
@@ -21,14 +30,16 @@
 	 (type-definition
 	  (format nil
 		  "~a~:[~; not null~]~@[ default ~a~]"
-		  (coldef-type col)
+		  (cast (coldef-type col))
 		  (coldef-nullable col)
 		  (coldef-default col))))
     (format nil "~a ~22t ~a" column-name type-definition)))
 
 (defun list-tables (&optional (db *sqlite-db*))
   "Return the list of tables found in SQLITE-DB."
-  (let ((sql "SELECT tbl_name FROM sqlite_master WHERE type='table'"))
+  (let ((sql "SELECT tbl_name
+                FROM sqlite_master
+               WHERE type='table' AND tbl_name <> 'sqlite_sequence'"))
     (loop for (name) in (sqlite:execute-to-list db sql)
        collect name)))
 
@@ -56,7 +67,9 @@
 
 (defun list-all-indexes (&optional (db *sqlite-db*))
   "Get the list of SQLite index definitions per table."
-  (let ((sql "SELECT name, tbl_name, sql FROM sqlite_master WHERE type='index'"))
+  (let ((sql "SELECT name, tbl_name, replace(replace(sql, '[', ''), ']', '')
+                FROM sqlite_master
+               WHERE type='index'"))
     (loop with schema = nil
        for (index-name table-name sql) in (sqlite:execute-to-list db sql)
        do (let ((entry  (assoc table-name schema :test 'equal))
@@ -106,14 +119,26 @@
       (unless transforms
 	(setf (slot-value source 'transforms)
 	      (loop for field in fields
-		 if (string-equal "float" (coldef-type field))
-		 collect #'pgloader.transforms::float-to-string
-		 else
-		 collect nil))))))
+		 collect
+		   (let ((coltype (cast (coldef-type field))))
+		     ;;
+		     ;; The SQLite drive we use maps the CFFI data type
+		     ;; mapping functions and gets back proper CL typed
+		     ;; objects, where we only want to deal with text.
+		     ;;
+		     (cond ((or (string-equal "float" coltype)
+				(and (<= 7 (length coltype))
+				     (string-equal "numeric" coltype :end2 7)))
+			    #'pgloader.transforms::float-to-string)
 
+			   ((string-equal "text" coltype)
+			    nil)
 
+			   (t
+			    (compile nil (lambda (c)
+					   (when c
+					     (format nil "~a" c)))))))))))))
 
-;;;
 ;;; Map a function to each row extracted from SQLite
 ;;;
 (defmethod map-rows ((sqlite copy-sqlite) &key process-row-fn)

test/sqlite.load

@@ -1,6 +1,6 @@
 load database
-     from sqlite:///Users/dim/Downloads/lastfm_tags.db
-     into postgresql://127.0.0.1:54393/tags
+     from 'sqlite/Chinook_Sqlite_AutoIncrementPKs.sqlite'
+     into postgresql:///pgloader
 
  with include drop, create tables, create indexes, reset sequences
 

test/sqlite/CreateSqlite.bat

@@ -0,0 +1,37 @@
+@echo off
+echo Chinook Database Version 1.4
+echo.
+
+if "%1"=="" goto MENU
+if not exist %1 goto ERROR
+
+set SQLFILE=%1
+goto RUNSQL
+
+:ERROR
+echo The file %1 does not exist.
+echo.
+goto END
+
+:MENU
+echo Options:
+echo.
+echo 1. Run Chinook_Sqlite.sql
+echo 2. Run Chinook_Sqlite_AutoIncrementPKs.sql
+echo 3. Exit
+echo.
+choice /c 123
+if (%ERRORLEVEL%)==(1) set SQLFILE=Chinook_Sqlite.sql
+if (%ERRORLEVEL%)==(2) set SQLFILE=Chinook_Sqlite_AutoIncrementPKs.sql
+if (%ERRORLEVEL%)==(3) goto END
+
+:RUNSQL
+echo.
+echo Running %SQLFILE%...
+if exist %SQLFILE%ite del %SQLFILE%ite
+sqlite3 -init %SQLFILE% %SQLFILE%ite
+
+:END
+echo.
+set SQLFILE=
+