mirror of
https://github.com/dimitri/pgloader.git
synced 2026-05-04 10:31:02 +02:00
Review api around worker-count.
It was worker-count and it's now exposed as the worker in the WITH clause, but we can actually keep it as worker-count in the internal API, and it feels better that way.
This commit is contained in:
parent
eb45bf0338
commit
dcc8eb6d61
@ -28,7 +28,7 @@
|
||||
(defrule option-workers (and kw-workers equal-sign (+ (digit-char-p character)))
|
||||
(:lambda (workers)
|
||||
(bind (((_ _ nb) workers))
|
||||
(cons :workers (parse-integer (text nb))))))
|
||||
(cons :worker-count (parse-integer (text nb))))))
|
||||
|
||||
(defrule option-concurrency (and kw-concurrency
|
||||
equal-sign
|
||||
|
||||
@ -77,7 +77,7 @@
|
||||
|
||||
(defgeneric copy-database (source
|
||||
&key
|
||||
workers
|
||||
worker-count
|
||||
concurrency
|
||||
truncate
|
||||
data-only
|
||||
|
||||
@ -120,7 +120,7 @@
|
||||
;;;
|
||||
(defmethod copy-database ((copy db-copy)
|
||||
&key
|
||||
(workers 4)
|
||||
(worker-count 4)
|
||||
(concurrency 1)
|
||||
(truncate nil)
|
||||
(disable-triggers nil)
|
||||
@ -139,7 +139,7 @@
|
||||
set-table-oids
|
||||
materialize-views)
|
||||
"Export database source data and Import it into PostgreSQL"
|
||||
(let* ((copy-kernel (make-kernel workers))
|
||||
(let* ((copy-kernel (make-kernel worker-count))
|
||||
(copy-channel (let ((lp:*kernel* copy-kernel)) (lp:make-channel)))
|
||||
(catalog (fetch-metadata
|
||||
copy
|
||||
@ -264,8 +264,8 @@
|
||||
(with-stats-collection ("COPY Threads Completion" :section :post
|
||||
:use-result-as-read t
|
||||
:use-result-as-rows t)
|
||||
(let ((workers-count (* table-count (task-count concurrency))))
|
||||
(loop :for tasks :below workers-count
|
||||
(let ((worker-count (* table-count (task-count concurrency))))
|
||||
(loop :for tasks :below worker-count
|
||||
:do (destructuring-bind (task table-name seconds)
|
||||
(lp:receive-result copy-channel)
|
||||
(log-message :debug "Finished processing ~a for ~s ~50T~6$s"
|
||||
@ -273,7 +273,7 @@
|
||||
(when (eq :writer task)
|
||||
(update-stats :data table-name :secs seconds))))
|
||||
(prog1
|
||||
workers-count
|
||||
worker-count
|
||||
(lp:end-kernel :wait nil))))))
|
||||
|
||||
(when idx-kernel
|
||||
|
||||
@ -61,8 +61,9 @@
|
||||
drop-indexes
|
||||
|
||||
;; generic API, but ignored here
|
||||
workers
|
||||
concurrency
|
||||
(worker-count 8)
|
||||
(concurrency 2)
|
||||
|
||||
data-only
|
||||
schema-only
|
||||
create-tables
|
||||
@ -76,15 +77,20 @@
|
||||
excluding)
|
||||
"Copy the contents of the COPY formated file to PostgreSQL."
|
||||
(declare (ignore data-only schema-only
|
||||
create-tables include-drop
|
||||
create-indexes reset-sequences))
|
||||
create-tables include-drop foreign-keys
|
||||
create-indexes reset-sequences materialize-views
|
||||
set-table-oids including excluding))
|
||||
|
||||
;; this sets (table-index-list (target copy))
|
||||
(maybe-drop-indexes (target-db copy)
|
||||
(target copy)
|
||||
:drop-indexes drop-indexes)
|
||||
|
||||
(copy-from copy :truncate truncate :disable-triggers disable-triggers)
|
||||
(copy-from copy
|
||||
:worker-count worker-count
|
||||
:concurrency concurrency
|
||||
:truncate truncate
|
||||
:disable-triggers disable-triggers)
|
||||
|
||||
;; re-create the indexes from the target table entry
|
||||
(create-indexes-again (target-db copy)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user