mirrors/pgloader
1
0
Fork 0
mirror of https://github.com/dimitri/pgloader.git synced 2026-05-04 10:31:02 +02:00
Code Issues Projects Releases Wiki Activity

Implement --dry-run option, fix #264.

Browse Source 
The dry run option will currently only check database connections, but
as that happens after having correctly parsed the load file, it allows
to also check that the command file is correct for the parser.

Note that the list load-data API isn't subject to the dry-run method.

In passing, we add some more API entry points to the connection objects
and we should actually clean the code base to use the new QUERY generic
all over the place. It's for another patch tho.
This commit is contained in:
Dimitri Fontaine 2015-08-22 16:23:47 +02:00
parent 04ddf940d9
commit ea35eb575d
16 changed files with 184 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
src/connection.lisp
View File

@ -18,6 +18,9 @@
(defgeneric close-connection (connection)
(:documentation "Close a connection to the data source."))
(defgeneric check-connection (connection)
(:documentation "Check that we can actually connect."))
(defclass fd-connection (connection)
((uri :initarg :uri :accessor fd-uri)
(arch :initarg :arch :accessor fd-arch)
@ -94,6 +97,9 @@
(connection-error-user err)
(connection-error-mesg err)))))
(defgeneric query (db-connection sql &key)
(:documentation "Query DB-CONNECTION with SQL query"))
(defmacro with-connection ((var connection) &body forms)
"Connect to DB-CONNECTION and handle any condition when doing so, and when
connected execute FORMS in a protected way so that we always disconnect
@ -120,3 +126,26 @@
(progn ,@forms)
(close-connection ,var)))))
(defmethod check-connection ((fd fd-connection))
"Check that it is possible to connect to db-connection C."
(log-message :log "Attempting to open ~a" fd)
(handler-case
(with-connection (cnx fd)
(log-message :log "Success, opened ~a." fd))
(condition (e)
(log-message :fatal "Failed to connect to ~a: ~a" fd e))))
(defmethod check-connection ((c db-connection))
"Check that it is possible to connect to db-connection C."
(log-message :log "Attempting to connect to ~a" c)
(handler-case
(with-connection (cnx c)
(log-message :log "Success, opened ~a." c)
(let ((sql "SELECT 1;"))
(log-message :log "Running a simple query: ~a" sql)
(handler-case
(query cnx sql)
(condition (e)
(log-message :fatal "SQL failed on ~a: ~a" c e)))))
(condition (e)
(log-message :fatal "Failed to connect to ~a: ~a" c e))))

8
src/main.lisp
View File

@ -53,6 +53,9 @@
(("load-lisp-file" #\l) :type string :list t :optional t
:documentation "Read user code from files")
("dry-run" :type boolean
:documentation "Only check database connections, don't load anything.")
(("with") :type string :list t :optional t
:documentation "Load options")
@ -188,7 +191,7 @@
(usage argv :quit t)))
(destructuring-bind (&key help version quiet verbose debug logfile
list-encodings upgrade-config
list-encodings upgrade-config dry-run
((:load-lisp-file load))
client-min-messages log-min-messages summary
root-dir self-upgrade
@ -257,6 +260,9 @@
(uiop:quit +os-code-error+))))
(uiop:quit +os-code-success+))
;; Should we run in dry-run mode?
(setf *dry-run* dry-run)
;; Now process the arguments
(when arguments
;; Start the logs system

4
src/package.lisp
View File

@ -110,6 +110,8 @@
(defpackage #:pgloader.connection
(:use #:cl #:pgloader.archive)
(:import-from #:pgloader.monitor
#:log-message)
(:export #:connection
#:open-connection
#:close-connection
@ -119,6 +121,8 @@
#:fd-connection-error
#:db-connection-error
#:with-connection
#:query
#:check-connection
;; file based connections API for HTTP and Archives support
#:fetch-file

4
src/params.lisp
View File

@ -6,6 +6,7 @@
(defpackage #:pgloader.params
(:use #:cl)
(:export #:*version-string*
#:*dry-run*
#:*self-upgrade-immutable-systems*
#:*csv-path-root*
#:*root-dir*
@ -64,6 +65,9 @@
DEFAULT if that variable isn't set"
(or (uiop:getenv name) default)))
(defparameter *dry-run* nil
"Set to non-nil to only run checks about the load setup.")
;; we can't use pgloader.utils:make-pgstate yet because params is compiled
;; first in the asd definition, we just make the symbol a special variable.
(defparameter *state* nil

17
src/parsers/command-copy.lisp
View File

@ -166,10 +166,13 @@
(:lambda (command)
(bind (((source encoding fields pg-db-uri columns
&key ((:copy-options options)) gucs before after) command))
(lisp-code-for-loading-from-copy source fields pg-db-uri
:encoding encoding
:columns columns
:gucs gucs
:before before
:after after
:copy-options options))))
(cond (*dry-run*
(lisp-code-for-csv-dry-run pg-db-uri))
(t
(lisp-code-for-loading-from-copy source fields pg-db-uri
:encoding encoding
:columns columns
:gucs gucs
:before before
:after after
:copy-options options))))))

25
src/parsers/command-csv.lisp
View File

@ -434,6 +434,14 @@
(destructuring-bind (source encoding fields target columns clauses) command
`(,source ,encoding ,fields ,target ,columns ,@clauses))))
(defun lisp-code-for-csv-dry-run (pg-db-conn)
`(lambda ()
;; CSV connection objects are not actually implementing the generic API
;; because they support many complex options... (the file can be a
;; pattern or standard input or inline or compressed etc).
(log-message :log "DRY RUN, only checking PostgreSQL connection.")
(check-connection ,pg-db-conn)))
(defun lisp-code-for-loading-from-csv (csv-conn fields pg-db-conn
&key
(encoding :utf-8)
@ -492,10 +500,13 @@
(:lambda (command)
(bind (((source encoding fields pg-db-uri columns
&key ((:csv-options options)) gucs before after) command))
(lisp-code-for-loading-from-csv source fields pg-db-uri
:encoding encoding
:columns columns
:gucs gucs
:before before
:after after
:csv-options options))))
(cond (*dry-run*
(lisp-code-for-csv-dry-run pg-db-uri))
(t
(lisp-code-for-loading-from-csv source fields pg-db-uri
:encoding encoding
:columns columns
:gucs gucs
:before before
:after after
:csv-options options))))))

21
src/parsers/command-dbf.lisp
View File

@ -81,6 +81,12 @@
(destructuring-bind (source encoding target clauses) command
`(,source ,encoding ,target ,@clauses))))
(defun lisp-code-for-dbf-dry-run (dbf-db-conn pg-db-conn)
`(lambda ()
(let ((source-db (expand (fetch-file ,dbf-db-conn))))
(check-connection source-db)
(check-connection ,pg-db-conn))))
(defun lisp-code-for-loading-from-dbf (dbf-db-conn pg-db-conn
&key
(encoding :ascii)
@ -123,9 +129,12 @@
(:lambda (command)
(bind (((source encoding pg-db-uri
&key ((:dbf-options options)) gucs before after) command))
(lisp-code-for-loading-from-dbf source pg-db-uri
:encoding encoding
:gucs gucs
:before before
:after after
:dbf-options options))))
(cond (*dry-run*
(lisp-code-for-dbf-dry-run source pg-db-uri))
(t
(lisp-code-for-loading-from-dbf source pg-db-uri
:encoding encoding
:gucs gucs
:before before
:after after
:dbf-options options))))))

17
src/parsers/command-fixed.lisp
View File

@ -172,10 +172,13 @@
(:lambda (command)
(bind (((source encoding fields pg-db-uri columns
&key ((:fixed-options options)) gucs before after) command))
(lisp-code-for-loading-from-fixed source fields pg-db-uri
:encoding encoding
:columns columns
:gucs gucs
:before before
:after after
:fixed-options options))))
(cond (*dry-run*
(lisp-code-for-csv-dry-run pg-db-uri))
(t
(lisp-code-for-loading-from-fixed source fields pg-db-uri
:encoding encoding
:columns columns
:gucs gucs
:before before
:after after
:fixed-options options))))))

13
src/parsers/command-ixf.lisp
View File

@ -82,8 +82,11 @@
(:lambda (command)
(bind (((source pg-db-uri
&key ((:ixf-options options)) gucs before after) command))
(lisp-code-for-loading-from-ixf source pg-db-uri
:gucs gucs
:before before
:after after
:ixf-options options))))
(cond (*dry-run*
(lisp-code-for-csv-dry-run pg-db-uri))
(t
(lisp-code-for-loading-from-ixf source pg-db-uri
:gucs gucs
:before before
:after after
:ixf-options options))))))

25
src/parsers/command-mssql.lisp
View File

@ -139,6 +139,12 @@
;;; LOAD DATABASE FROM mssql://
(defun lisp-code-for-mssql-dry-run (ms-db-conn pg-db-conn)
`(lambda ()
(log-message :log "DRY RUN, only checking connections.")
(check-connection ,ms-db-conn)
(check-connection ,pg-db-conn)))
(defun lisp-code-for-loading-from-mssql (ms-db-conn pg-db-conn
&key
gucs casts before after
@ -188,12 +194,15 @@
gucs casts before after including excluding
((:mssql-options options)))
source))
(lisp-code-for-loading-from-mssql ms-db-uri pg-db-uri
:gucs gucs
:casts casts
:before before
:after after
:mssql-options options
:including including
:excluding excluding))))
(cond (*dry-run*
(lisp-code-for-mssql-dry-run ms-db-uri pg-db-uri))
(t
(lisp-code-for-loading-from-mssql ms-db-uri pg-db-uri
:gucs gucs
:casts casts
:before before
:after after
:mssql-options options
:including including
:excluding excluding))))))

29
src/parsers/command-mysql.lisp
View File

@ -152,6 +152,12 @@
;;; LOAD DATABASE FROM mysql://
(defun lisp-code-for-mysql-dry-run (my-db-conn pg-db-conn)
`(lambda ()
(log-message :log "DRY RUN, only checking connections.")
(check-connection ,my-db-conn)
(check-connection ,pg-db-conn)))
(defun lisp-code-for-loading-from-mysql (my-db-conn pg-db-conn
&key
gucs casts views before after
@ -201,14 +207,17 @@
gucs casts views before after
mysql-options including excluding decoding)
source
(lisp-code-for-loading-from-mysql my-db-uri pg-db-uri
:gucs gucs
:casts casts
:views views
:before before
:after after
:mysql-options mysql-options
:including including
:excluding excluding
:decoding decoding))))
(cond (*dry-run*
(lisp-code-for-mysql-dry-run my-db-uri pg-db-uri))
(t
(lisp-code-for-loading-from-mysql my-db-uri pg-db-uri
:gucs gucs
:casts casts
:views views
:before before
:after after
:mysql-options mysql-options
:including including
:excluding excluding
:decoding decoding))))))

21
src/parsers/command-sqlite.lisp
View File

@ -88,6 +88,12 @@ load database
(destructuring-bind (source target clauses) command
`(,source ,target ,@clauses))))
(defun lisp-code-for-sqlite-dry-run (sqlite-db-conn pg-db-conn)
`(lambda ()
(log-message :log "DRY RUN, only checking connections.")
(check-connection ,sqlite-db-conn)
(check-connection ,pg-db-conn)))
(defun lisp-code-for-loading-from-sqlite (sqlite-db-conn pg-db-conn
&key
gucs casts
@ -119,10 +125,13 @@ load database
pg-db-uri
&key gucs casts sqlite-options including excluding)
source
(lisp-code-for-loading-from-sqlite sqlite-uri pg-db-uri
:gucs gucs
:casts casts
:sqlite-options sqlite-options
:including including
:excluding excluding))))
(cond (*dry-run*
(lisp-code-for-sqlite-dry-run sqlite-uri pg-db-uri))
(t
(lisp-code-for-loading-from-sqlite sqlite-uri pg-db-uri
:gucs gucs
:casts casts
:sqlite-options sqlite-options
:including including
:excluding excluding))))))

4
src/pgsql/queries.lisp
View File

@ -45,6 +45,10 @@
(setf (conn-handle pgconn) nil)
pgconn)
(defmethod query ((pgconn pgsql-connection) sql &key)
(let ((pomo:*database* (conn-handle pgconn)))
(pomo:query sql)))
(defmacro handling-pgsql-notices (&body forms)
"The BODY is run within a PostgreSQL transaction where *pg-settings* have
been applied. PostgreSQL warnings and errors are logged at the

4
src/sources/mssql/mssql-schema.lisp
View File

@ -29,6 +29,10 @@
(setf (conn-handle msconn) nil)
msconn)
(defmethod query ((msconn mssql-connection) sql &key)
"Send SQL query to MSCONN connection."
(mssql:query sql :connection (conn-handle msconn)))
(defun mssql-query (query)
"Execute given QUERY within the current *connection*, and set proper
defaults for pgloader."

17
src/sources/mysql/mysql-schema.lisp
View File

@ -73,6 +73,23 @@
(setf (conn-handle myconn) nil)
myconn)
(defmethod query ((myconn mysql-connection)
sql
&key
row-fn
(as-text t)
(result-type 'list))
"Run SQL query against MySQL connection MYCONN."
(log-message :debug "MySQL: sending query: ~a" sql)
(qmynd:mysql-query (conn-handle myconn)
sql
:row-fn row-fn
:as-text as-text
:result-type result-type))
;;;
;;; The generic API query is recent, used to look like this:
;;;
(defun mysql-query (query &key row-fn (as-text t) (result-type 'list))
"Execute given QUERY within the current *connection*, and set proper
defaults for pgloader."

3
src/sources/sqlite/sqlite.lisp
View File

@ -23,6 +23,9 @@
(setf (conn-handle slconn) nil)
slconn)
(defmethod query ((slconn sqlite-connection) sql &key)
(sqlite:execute-to-list (conn-handle slconn) sql))
(defclass copy-sqlite (copy)
((db :accessor db :initarg :db))
(:documentation "pgloader SQLite Data Source"))