diff --git a/src/parser.lisp b/src/parser.lisp index cda8df2..477e794 100644 --- a/src/parser.lisp +++ b/src/parser.lisp @@ -371,11 +371,10 @@ (*pgconn-pass* ,password)) (pgloader.pgsql:copy-from-file ,dbname ,table-name ',source))))) -(defrule database-source (and ignore-whitespace - kw-load kw-database kw-from +(defrule database-source (and kw-load kw-database kw-from db-connection-uri) (:lambda (source) - (destructuring-bind (nil l d f uri) source + (destructuring-bind (l d f uri) source (declare (ignore l d f)) uri))) @@ -589,10 +588,10 @@ casts))) ;;; LOAD DATABASE FROM mysql:// -(defrule load-database (and database-source target - (? mysql-options) - (? gucs) - (? casts)) +(defrule load-mysql-database (and database-source target + (? mysql-options) + (? gucs) + (? casts)) (:lambda (source) (destructuring-bind (my-db-uri pg-db-uri options gucs casts) source (destructuring-bind (&key ((:host myhost)) @@ -632,6 +631,104 @@ `(:only-tables ',(list table-name))) ,@options)))))))) + +;;; +;;; LOAD DATABASE FROM SQLite +;;; +#| +load database + from sqlite:///Users/dim/Downloads/lastfm_tags.db + into postgresql:///tags + + with drop tables, create tables, create indexes, reset sequences + + set work_mem to '16MB', maintenance_work_mem to '512 MB'; +|# +(defrule sqlite-option (or option-truncate + option-schema-only + option-drop-tables + option-create-tables + option-create-indexes + option-reset-sequences)) + +(defrule another-sqlite-option (and #\, ignore-whitespace sqlite-option) + (:lambda (source) + (destructuring-bind (comma ws option) source + (declare (ignore comma ws)) + option))) + +(defrule sqlite-option-list (and sqlite-option (* another-sqlite-option)) + (:lambda (source) + (destructuring-bind (opt1 opts) source + (alexandria:alist-plist (list* opt1 opts))))) + +(defrule sqlite-options (and kw-with sqlite-option-list) + (:lambda (source) + (destructuring-bind (w opts) source + (declare (ignore w)) + opts))) + +(defrule sqlite-db-uri (and "sqlite://" filename) + (:lambda (source) + (destructuring-bind (prefix filename) source + (declare (ignore prefix)) + (destructuring-bind (type path) filename + (declare (ignore type)) + (list :sqlite path))))) + +(defrule sqlite-uri (or sqlite-db-uri http-uri)) +(defrule sqlite-source (and kw-load kw-database kw-from sqlite-uri) + (:destructure (l d f u) + (declare (ignore l d f)) + u)) + +(defrule load-sqlite-database (and sqlite-source target + (? sqlite-options) + (? gucs)) + (:lambda (source) + (destructuring-bind (sqlite-uri pg-db-uri options gucs) source + (destructuring-bind (&key host port user password dbname table-name + &allow-other-keys) + pg-db-uri + `(lambda () + (let* ((state-before (pgloader.utils:make-pgstate)) + (*state* (pgloader.utils:make-pgstate)) + (db + ,(destructuring-bind (kind url) sqlite-uri + (ecase kind + (:http `(with-stats-collection + (,dbname "download" :state state-before) + (pgloader.archive:http-fetch-file ,url))) + (:sqlite url)))) + (db + (if (string= "zip" (pathname-type db)) + (progn + (with-stats-collection (,dbname "extract" + :state state-before) + (let ((d (pgloader.archive:expand-archive db))) + (merge-pathnames + (make-pathname :name (pathname-name db) + :type "db") + d)))) + db)) + (*pgconn-host* ,host) + (*pgconn-port* ,port) + (*pgconn-user* ,user) + (*pgconn-pass* ,password) + (*pg-settings* ',gucs) + (pgloader.pgsql::*pgsql-reserved-keywords* + (pgloader.pgsql:list-reserved-keywords ,dbname)) + (source + (make-instance 'pgloader.sqlite::copy-sqlite + :target-db ,dbname + :source-db db))) + (pgloader.sqlite:copy-database source + :state-before state-before + ,@(when table-name + `(:only-tables ',(list table-name))) + ,@options))))))) + + ;;; ;;; LOAD MESSAGES FROM syslog @@ -1366,7 +1463,8 @@ (defrule command (and (or load-archive load-csv-file load-dbf-file - load-database + load-mysql-database + load-sqlite-database load-syslog-messages) end-of-command) (:lambda (cmd) diff --git a/src/sources/sqlite.lisp b/src/sources/sqlite.lisp index 04031ec..141d50d 100644 --- a/src/sources/sqlite.lisp +++ b/src/sources/sqlite.lisp @@ -127,7 +127,7 @@ (defmethod copy-from ((sqlite copy-sqlite) &key (kernel nil k-s-p) truncate) "Stream the contents from a SQLite database table down to PostgreSQL." - (let* ((summary (null *state*)) + (let* ((summary (null *state*)) (*state* (or *state* (pgloader.utils:make-pgstate))) (lp:*kernel* (or kernel (make-kernel 2))) (channel (lp:make-channel)) @@ -187,6 +187,7 @@ (defmethod copy-database ((sqlite copy-sqlite) &key + state-before truncate schema-only create-tables @@ -195,7 +196,9 @@ reset-sequences only-tables) "Stream the given SQLite database down to PostgreSQL." - (let* ((*state* (make-pgstate)) + (let* ((summary (null *state*)) + (*state* (or *state* (make-pgstate))) + (state-before (or state-before (make-pgstate))) (idx-state (make-pgstate)) (seq-state (make-pgstate)) (copy-kernel (make-kernel 2)) @@ -213,8 +216,11 @@ ;; if asked, first drop/create the tables on the PostgreSQL side (when create-tables (log-message :notice "~:[~;DROP then ~]CREATE TABLES" include-drop) - (with-pgsql-transaction (pg-dbname) - (create-tables all-columns :include-drop include-drop))) + (with-stats-collection (pg-dbname "create, truncate" + :state state-before + :summary summary) + (with-pgsql-transaction (pg-dbname) + (create-tables all-columns :include-drop include-drop)))) (loop for (table-name . columns) in all-columns @@ -270,7 +276,8 @@ (lp:end-kernel)) ;; and report the total time spent on the operation - (report-summary) + (report-summary :state state-before) + (report-summary :header nil :footer nil) (format t pgloader.utils::*header-line*) (report-summary :state idx-state :header nil :footer nil) (report-summary :state seq-state :header nil :footer nil) diff --git a/test/sqlite.load b/test/sqlite.load new file mode 100644 index 0000000..0d2657b --- /dev/null +++ b/test/sqlite.load @@ -0,0 +1,7 @@ +load database + from sqlite:///Users/dim/Downloads/lastfm_tags.db + into postgresql:///tags + + with drop tables, create tables, create indexes, reset sequences + + set work_mem to '16MB', maintenance_work_mem to '512 MB';