mirror of
https://github.com/dimitri/pgloader.git
synced 2026-02-03 21:41:47 +01:00
Rework the summary handling in LOAD FROM ARCHIVE commands.
This commit is contained in:
Dimitri Fontaine
parent
9c1592c4eb
commit
32d183efa5
56
csv.lisp
56
csv.lisp
@ -205,47 +205,37 @@ Finally returns how many rows where read and processed."
|
||||
(escape cl-csv:*quote-escape*)
|
||||
(null-as "\\N"))
|
||||
"Copy data from CSV file FILENAME into PostgreSQL DBNAME.TABLE-NAME"
|
||||
(let* ((report-header (null *state*))
|
||||
(let* ((summary (null *state*))
|
||||
(*state* (or *state* (pgloader.utils:make-pgstate)))
|
||||
(lp:*kernel* (make-kernel 2))
|
||||
(channel (lp:make-channel))
|
||||
(dataq (lq:make-queue :fixed-capacity 4096))
|
||||
(filename (get-absolute-pathname filename-or-regex)))
|
||||
|
||||
;; statistics
|
||||
(when report-header (report-header))
|
||||
(pgstate-add-table *state* dbname table-name)
|
||||
(report-table-name table-name)
|
||||
(with-stats-collection (dbname table-name :state *state* :summary summary)
|
||||
(log-message :notice "COPY ~a.~a" dbname table-name)
|
||||
(lp:submit-task channel
|
||||
;; this function update :read stats
|
||||
#'pgloader.csv:copy-to-queue table-name filename dataq
|
||||
:fields fields
|
||||
:columns columns
|
||||
:encoding encoding
|
||||
:skip-lines skip-lines
|
||||
:separator separator
|
||||
:quote quote
|
||||
:escape escape
|
||||
:null-as null-as)
|
||||
|
||||
(multiple-value-bind (res secs)
|
||||
(timing
|
||||
(lp:submit-task channel
|
||||
;; this function update :read stats
|
||||
#'pgloader.csv:copy-to-queue table-name filename dataq
|
||||
:fields fields
|
||||
:columns columns
|
||||
:encoding encoding
|
||||
:skip-lines skip-lines
|
||||
:separator separator
|
||||
:quote quote
|
||||
:escape escape
|
||||
:null-as null-as)
|
||||
;; and start another task to push that data from the queue to PostgreSQL
|
||||
(lp:submit-task channel
|
||||
;; this function update :rows stats
|
||||
#'pgloader.pgsql:copy-from-queue dbname table-name dataq
|
||||
:truncate truncate
|
||||
:transforms transforms)
|
||||
|
||||
;; and start another task to push that data from the queue to PostgreSQL
|
||||
(lp:submit-task channel
|
||||
;; this function update :rows stats
|
||||
#'pgloader.pgsql:copy-from-queue dbname table-name dataq
|
||||
:truncate truncate
|
||||
:transforms transforms)
|
||||
|
||||
;; now wait until both the tasks are over
|
||||
(loop for tasks below 2 do (lp:receive-result channel))
|
||||
(lp:end-kernel))
|
||||
|
||||
;; report stats!
|
||||
(declare (ignore res))
|
||||
(pgstate-incf *state* table-name :secs secs)
|
||||
(report-pgtable-stats *state* table-name))))
|
||||
;; now wait until both the tasks are over
|
||||
(loop for tasks below 2 do (lp:receive-result channel)
|
||||
finally (lp:end-kernel)))))
|
||||
|
||||
(defun import-database (dbname
|
||||
&key
|
||||
|
||||
57
parser.lisp
57
parser.lisp
@ -1230,39 +1230,56 @@ Here's a quick description of the format we're parsing here:
|
||||
(destructuring-bind (&key host port user password dbname &allow-other-keys)
|
||||
pg-db-uri
|
||||
`(lambda ()
|
||||
(let* ((archive-file
|
||||
(let* ((state-before (pgloader.utils:make-pgstate))
|
||||
(*state* (pgloader.utils:make-pgstate))
|
||||
(state-finally (pgloader.utils:make-pgstate))
|
||||
(archive-file
|
||||
,(destructuring-bind (kind url) source
|
||||
(ecase kind
|
||||
(:http `(pgloader.archive:http-fetch-file ,url))
|
||||
(:http `(with-stats-collection
|
||||
(,dbname "download" :state state-before)
|
||||
(pgloader.archive:http-fetch-file ,url)))
|
||||
(:filename url))))
|
||||
(*csv-path-root* (pgloader.archive:expand-archive archive-file))
|
||||
(*csv-path-root*
|
||||
(with-stats-collection (,dbname "extract" :state state-before)
|
||||
(pgloader.archive:expand-archive archive-file)))
|
||||
(*pgconn-host* ,host)
|
||||
(*pgconn-port* ,port)
|
||||
(*pgconn-user* ,user)
|
||||
(*pgconn-pass* ,password)
|
||||
(*state* (pgloader.utils:make-pgstate)))
|
||||
(*pgconn-pass* ,password))
|
||||
(progn
|
||||
;; before block
|
||||
(with-pgsql-transaction (,dbname)
|
||||
(loop for command in ',before
|
||||
do
|
||||
(log-message :notice command)
|
||||
(pgsql-execute command :client-min-messages :error)))
|
||||
(with-stats-collection (,dbname "before load" :state state-before)
|
||||
(with-pgsql-transaction (,dbname)
|
||||
(loop for command in ',before
|
||||
do
|
||||
(log-message :notice command)
|
||||
(pgsql-execute command :client-min-messages :error))))
|
||||
|
||||
;; import from files block
|
||||
(report-header)
|
||||
,@(loop for command in commands
|
||||
collect `(funcall ,command))
|
||||
(report-pgstate-stats *state* "Total import time")
|
||||
|
||||
;; finally block
|
||||
(with-pgsql-transaction (,dbname)
|
||||
(loop for command in ',finally
|
||||
for first = t then nil
|
||||
when first do (format t "~&")
|
||||
do
|
||||
(log-message :notice command)
|
||||
(pgsql-execute command :client-min-messages :error))))))))))
|
||||
(with-stats-collection (,dbname "finally" :state state-finally)
|
||||
(with-pgsql-transaction (,dbname)
|
||||
(loop for command in ',finally
|
||||
do
|
||||
(log-message :notice command)
|
||||
(pgsql-execute command :client-min-messages :error))))
|
||||
|
||||
;; reporting
|
||||
(report-summary :state state-before :footer nil)
|
||||
(format t pgloader.utils::*header-line*)
|
||||
(report-summary :state *state* :header nil :footer nil)
|
||||
(format t pgloader.utils::*header-line*)
|
||||
(report-summary :state state-finally :header nil :footer nil)
|
||||
;; add to the grand total the other sections
|
||||
(incf (pgloader.utils::pgstate-secs *state*)
|
||||
(+ (pgloader.utils::pgstate-secs state-before)
|
||||
(pgloader.utils::pgstate-secs state-finally)))
|
||||
;; and report the Grand Total
|
||||
(report-pgstate-stats *state* "Total import time"))))))))
|
||||
|
||||
|
||||
;;;
|
||||
@ -1381,6 +1398,8 @@ LOAD FROM http:///tapoueh.org/db.t
|
||||
"))
|
||||
|
||||
(defun test-parsing-load-from-archive ()
|
||||
"Use either http://pgsql.tapoueh.org/temp/foo.zip
|
||||
or /Users/dim/Downloads/GeoLiteCity-latest.zip"
|
||||
(parse-command "
|
||||
LOAD FROM ARCHIVE /Users/dim/Downloads/GeoLiteCity-latest.zip
|
||||
INTO postgresql://dim@localhost:54393/dim
|
||||
|
||||
15
utils.lisp
15
utils.lisp
@ -253,13 +253,14 @@
|
||||
given DBNAME and TABLE-NAME"
|
||||
(let ((result (gensym "result"))
|
||||
(secs (gensym "secs")))
|
||||
`(progn
|
||||
(pgstate-add-table ,pgstate ,dbname ,table-name)
|
||||
(multiple-value-bind (,result ,secs)
|
||||
(timing ,@forms)
|
||||
(if ,use-result-as-rows
|
||||
(pgstate-incf ,pgstate ,table-name :rows ,result :secs ,secs)
|
||||
(pgstate-incf ,pgstate ,table-name :secs ,secs)))
|
||||
`(prog2
|
||||
(pgstate-add-table ,pgstate ,dbname ,table-name)
|
||||
(multiple-value-bind (,result ,secs)
|
||||
(timing ,@forms)
|
||||
(if ,use-result-as-rows
|
||||
(pgstate-incf ,pgstate ,table-name :rows ,result :secs ,secs)
|
||||
(pgstate-incf ,pgstate ,table-name :secs ,secs))
|
||||
,result)
|
||||
(when ,summary (report-summary)))))
|
||||
|
||||
;;;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user