mirrors/pgloader
1
0
Fork 0
mirror of https://github.com/dimitri/pgloader.git synced 2026-05-05 02:46:10 +02:00
Code Issues Projects Releases Wiki Activity

Implement --before and --after options on the command line.

Browse Source 
That allows using SQL scripts to run before and after the main data
processing and loading done by pgloader when used only from the command
line.
This commit is contained in:
Dimitri Fontaine 2014-12-23 12:21:44 +01:00
parent 65c2043694
commit 6eac0d9dd8
5 changed files with 62 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
pgloader.1
View File

@ -149,6 +149,18 @@ Allows forcing the source type, in case when the \fISOURCE\fR parsing isn\'t sat
.IP
Set the encoding of the source file to load data from\.
.
.IP "\(bu" 4
\fB\-\-before <filename>\fR
.
.IP
Parse given filename for SQL queries and run them against the target database before loading the data from the source\. The queries are parsed by pgloader itself: they need to be terminated by a semi\-colon (;) and the file may include \fB\ei\fR or \fB\eir\fR commands to \fIinclude\fR another file\.
.
.IP "\(bu" 4
\fB\-\-after <filename>\fR
.
.IP
Parse given filename for SQL queries and run them against the target database after having loaded the data from the source\. The queries are parsed in the same way as with the \fB\-\-before\fR option, see above\.
.
.IP "" 0
.
.SS "MORE DEBUG INFORMATION"

13
pgloader.1.md
View File

@ -126,6 +126,19 @@ options:
* `--encoding <encoding>`
Set the encoding of the source file to load data from.
* `--before <filename>`
Parse given filename for SQL queries and run them against the target
database before loading the data from the source. The queries are parsed
by pgloader itself: they need to be terminated by a semi-colon (;) and
the file may include `\i` or `\ir` commands to *include* another file.
* `--after <filename>`
Parse given filename for SQL queries and run them against the target
database after having loaded the data from the source. The queries are
parsed in the same way as with the `--before` option, see above.
### MORE DEBUG INFORMATION

39
src/main.lisp
View File

@ -60,6 +60,12 @@
(("encoding") :type string :optional t
:documentation "Source expected encoding")
(("before") :type string :optional t
:documentation "SQL script to run before loading the data")
(("after") :type string :optional t
:documentation "SQL script to run after loading the data")
("self-upgrade" :type string :optional t
:documentation "Path to pgloader newer sources")))
@ -170,7 +176,7 @@
((:load-lisp-file load))
client-min-messages log-min-messages summary
root-dir self-upgrade
with set field cast type encoding)
with set field cast type encoding before after)
options
;; First thing: Self Upgrade?
@ -303,7 +309,9 @@
:gucs (parse-cli-gucs set)
:type type
:fields (parse-cli-fields type field)
:casts (parse-cli-casts cast)))))
:casts (parse-cli-casts cast)
:before (parse-sql-file before)
:after (parse-sql-file after)))))
;; process the files
(mapcar #'process-command-file arguments)))
@ -329,7 +337,8 @@
;;;
(defun load-data (&key ((:from source)) ((:into target))
(type (getf source :type))
encoding fields options gucs casts start-logger)
encoding fields options gucs casts before after
start-logger)
"Load data from SOURCE into TARGET."
(with-monitor (:start-logger start-logger)
;; some preliminary checks
@ -353,20 +362,28 @@
(:csv (lisp-code-for-loading-from-csv source fields target
:encoding encoding
:gucs gucs
:csv-options options))
:csv-options options
:before before
:after after))
(:fixed (lisp-code-for-loading-from-fixed source fields target
:encoding encoding
:gucs gucs
:fixed-options options))
:fixed-options options
:before before
:after after))
(:db3 (lisp-code-for-loading-from-dbf source target
:gucs gucs
:dbf-options options))
:dbf-options options
:before before
:after after))
(:ixf (lisp-code-for-loading-from-ixf source target
:gucs gucs
:ixf-options options))
:ixf-options options
:before before
:after after))
(:sqlite (lisp-code-for-loading-from-sqlite source target
:gucs gucs
@ -376,10 +393,14 @@
(:mysql (lisp-code-for-loading-from-mysql source target
:gucs gucs
:casts casts
:mysql-options options))
:mysql-options options
:before before
:after after))
(:mssql (lisp-code-for-loading-from-mssql source target
:gucs gucs
:casts casts
:mssql-options options))))
:mssql-options options
:before before
:after after))))
:start-logger start-logger)))

1
src/package.lisp
View File

@ -393,6 +393,7 @@
#:parse-cli-encoding
#:parse-cli-fields
#:parse-cli-casts
#:parse-sql-file
;; functions to generate lisp code from parameters
#:lisp-code-for-loading-from-mysql

6
src/parsers/command-parser.lisp
View File

@ -283,3 +283,9 @@
"Parse additional CAST rules when we get them from the CLI."
(loop :for cast :in casts
:collect (parse 'cast-rule cast)))
(defun parse-sql-file (filename)
"Parse FILENAME for SQL statements"
(when filename
(log-message :notice "reading SQL queries from ~s" filename)
(pgloader.sql:read-queries (probe-file filename))))