mirrors/pgloader
1
0
Fork 0
mirror of https://github.com/dimitri/pgloader.git synced 2026-02-13 18:31:15 +01:00
Code Issues Projects Releases Wiki Activity

Fix with drop index option, fix #323.

Browse Source 
Have PostgreSQL always fully qualify the index related objects and SQL
definition statements when fetching the list of indexes of a table, by
playing with an empty search_path.

Also improve the whole index creation by passing the table object as the
context where to derive the table-name from, so that schema qualified
tables are taken into account properly.
This commit is contained in:
Dimitri Fontaine 2016-01-15 15:04:07 +01:00
parent 1ff204c172
commit bfdbb2145b
5 changed files with 79 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
src/pgsql/queries.lisp
View File

@ -112,6 +112,7 @@
(defmethod query ((pgconn pgsql-connection) sql &key)
(let ((pomo:*database* (conn-handle pgconn)))
(log-message :debug "~a" sql)
(pomo:query sql)))
(defmacro handling-pgsql-notices (&body forms)
@ -271,9 +272,20 @@
(defun list-indexes (table)
"List all indexes for TABLE-NAME in SCHEMA. A PostgreSQL connection must
be already established when calling that function."
(with-schema (unqualified-table-name table)
(loop :for (index-name table-name table-oid primary unique sql conname condef)
:in (pomo:query (format nil "
(let ((schema (or (table-schema table)
(cdr (assoc "search_path" *pg-settings* :test #'string-equal))))
(sql-set-search-path "SET search_path TO '';"))
;; we force the search_path to emty value so that ::regclass special
;; cast and the pg_get_indexdef() and the pg_get_contraintdef() returns
;; the fully qualified table name, even when the table is found in the
;; 'public' schema.
(log-message :debug "~a" sql-set-search-path)
(pgloader.pgsql:pgsql-execute sql-set-search-path)
(prog1
(loop
:with sql-index-list
:= (let ((sql (format nil "
select i.relname,
indrelid::regclass,
indrelid,
@ -285,18 +297,27 @@ select i.relname,
from pg_index x
join pg_class i ON i.oid = x.indexrelid
left join pg_constraint c ON c.conindid = i.oid
where indrelid = '~@[~a.~]~a'::regclass"
(table-schema table)
unqualified-table-name))
:collect (make-pgsql-index :name index-name
:table-name table-name
:table-oid table-oid
:primary primary
:unique unique
:columns nil
:sql sql
:conname (unless (eq :null conname) conname)
:condef (unless (eq :null condef) condef)))))
where indrelid = '~:[~*public.~;~a.~]~a'::regclass"
schema schema
(table-name table))))
(log-message :debug "~a" sql)
sql)
:for (index-name table-name table-oid primary unique sql conname condef)
:in (pomo:query sql-index-list)
:collect (make-pgsql-index :name index-name
:table-name table-name
:table-oid table-oid
:primary primary
:unique unique
:columns nil
:sql sql
:conname (unless (eq :null conname) conname)
:condef (unless (eq :null condef) condef)))
(let ((sql-reset-search-path "reset search_path;"))
(log-message :debug "~a" sql-reset-search-path)
(pgloader.pgsql:pgsql-execute sql-reset-search-path)))))
(defun sanitize-user-gucs (gucs)
"Forbid certain actions such as setting a client_encoding different from utf8."

57
src/pgsql/schema.lisp
View File

@ -241,18 +241,15 @@
;; indexes' option (in CSV mode and the like)
name table-name table-oid primary unique columns sql conname condef)
(defgeneric index-table-name (index)
(:documentation
"Return the name of the table to attach this index to."))
(defgeneric format-pgsql-create-index (index)
(defgeneric format-pgsql-create-index (table index)
(:documentation
"Return the PostgreSQL command to define an Index."))
(defmethod index-table-name ((index pgsql-index))
(pgsql-index-table-name index))
(defgeneric format-pgsql-drop-index (table index)
(:documentation
"Return the PostgreSQL command to drop an Index."))
(defmethod format-pgsql-create-index ((index pgsql-index))
(defmethod format-pgsql-create-index ((table table) (index pgsql-index))
"Generate the PostgreSQL statement list to rebuild a Foreign Key"
(let* ((index-name (if (and *preserve-index-names*
(not (string-equal "primary" (pgsql-index-name index)))
@ -273,18 +270,18 @@
(or (pgsql-index-sql index)
(format nil "CREATE UNIQUE INDEX ~a ON ~a (~{~a~^, ~});"
index-name
(pgsql-index-table-name index)
(format-table-name table)
(pgsql-index-columns index)))
(format nil
"ALTER TABLE ~a ADD ~a USING INDEX ~a;"
(pgsql-index-table-name index)
(format-table-name table)
(cond ((pgsql-index-primary index) "PRIMARY KEY")
((pgsql-index-unique index) "UNIQUE"))
index-name)))
((pgsql-index-condef index)
(format nil "ALTER TABLE ~a ADD ~a;"
(pgsql-index-table-name index)
(format-table-name table)
(pgsql-index-condef index)))
(t
@ -292,10 +289,10 @@
(format nil "CREATE~:[~; UNIQUE~] INDEX ~a ON ~a (~{~a~^, ~});"
(pgsql-index-unique index)
index-name
(pgsql-index-table-name index)
(format-table-name table)
(pgsql-index-columns index)))))))
(defmethod format-pgsql-drop-index ((index pgsql-index))
(defmethod format-pgsql-drop-index ((table table) (index pgsql-index))
"Generate the PostgreSQL statement to DROP the index."
(let* ((index-name (apply-identifier-case (pgsql-index-name index))))
(cond ((pgsql-index-conname index)
@ -303,7 +300,7 @@
;; comes from one source only, the PostgreSQL database catalogs,
;; so don't question it, quote it.
(format nil "ALTER TABLE ~a DROP CONSTRAINT ~s;"
(pgsql-index-table-name index)
(format-table-name table)
(pgsql-index-conname index)))
(t
@ -312,17 +309,17 @@
;;;
;;; Parallel index building.
;;;
(defun create-indexes-in-kernel (pgconn indexes kernel channel
(defun create-indexes-in-kernel (pgconn table kernel channel
&key (label "Create Indexes"))
"Create indexes for given table in dbname, using given lparallel KERNEL
and CHANNEL so that the index build happen in concurrently with the data
copying."
(let* ((lp:*kernel* kernel))
(loop
:for index :in indexes
:for index :in (table-index-list table)
:collect (multiple-value-bind (sql pkey)
;; we postpone the pkey upgrade of the index for later.
(format-pgsql-create-index index)
(format-pgsql-create-index table index)
(log-message :notice "~a" sql)
(lp:submit-task channel
@ -357,11 +354,11 @@
;;;
;;; Drop indexes before loading
;;;
(defun drop-indexes (section pgsql-index-list)
(defun drop-indexes (section table)
"Drop indexes in PGSQL-INDEX-LIST. A PostgreSQL connection must already be
active when calling that function."
(loop :for index :in pgsql-index-list
:do (let ((sql (format-pgsql-drop-index index)))
(loop :for index :in (table-index-list table)
:do (let ((sql (format-pgsql-drop-index table index)))
(log-message :notice "~a" sql)
(pgsql-execute-with-timing section "drop indexes" sql))))
@ -376,6 +373,10 @@
;; we get the list of indexes from PostgreSQL catalogs, so don't
;; question their spelling, just quote them.
(*identifier-case* :quote))
;; set the indexes list in the table structure
(setf (table-index-list table) indexes)
(cond ((and indexes (not drop-indexes))
(log-message :warning
"Target table ~s has ~d indexes defined against it."
@ -388,26 +389,24 @@
(indexes
;; drop the indexes now
(with-stats-collection ("drop indexes" :section section)
(drop-indexes section indexes))))
(drop-indexes section table)))))))
;; and return the indexes list
indexes)))
(defun create-indexes-again (target indexes &key (section :post) drop-indexes)
(defun create-indexes-again (target table &key (section :post) drop-indexes)
"Create the indexes that we dropped previously."
(when (and indexes drop-indexes)
(when (and (table-index-list table) drop-indexes)
(let* ((*preserve-index-names* t)
;; we get the list of indexes from PostgreSQL catalogs, so don't
;; question their spelling, just quote them.
(*identifier-case* :quote)
(idx-kernel (make-kernel (length indexes)))
(idx-kernel (make-kernel (count-indexes table)))
(idx-channel (let ((lp:*kernel* idx-kernel))
(lp:make-channel))))
(let ((pkeys
(create-indexes-in-kernel target indexes idx-kernel idx-channel)))
(create-indexes-in-kernel target table idx-kernel idx-channel)))
(with-stats-collection ("Index Build Completion" :section section)
(loop :for idx :in indexes :do (lp:receive-result idx-channel)))
(loop :repeat (count-indexes table)
:do (lp:receive-result idx-channel)))
;; turn unique indexes into pkeys now
(with-pgsql-connection (target)

6
src/sources/common/db-methods.lisp
View File

@ -224,12 +224,12 @@
;; index build might get unsync: indexes for different tables
;; will get built in parallel --- not a big problem.
(when (and create-indexes (not data-only))
(let* ((indexes (table-index-list table))
(*preserve-index-names* (eq :preserve index-names)))
(let* ((*preserve-index-names* (eq :preserve index-names)))
(alexandria:appendf
pkeys
(create-indexes-in-kernel (target-db copy)
indexes idx-kernel idx-channel))))))
table
idx-kernel idx-channel))))))
;; now end the kernels
(end-kernels copy-kernel copy-channel idx-kernel idx-channel

19
src/sources/common/md-methods.lisp
View File

@ -71,12 +71,15 @@
(declare (ignore data-only schema-only
create-tables include-drop
create-indexes reset-sequences))
(let ((indexes (maybe-drop-indexes (target-db copy)
(target copy)
:drop-indexes drop-indexes)))
(copy-from copy
:truncate truncate
:disable-triggers disable-triggers)
;; re-create the indexes
(create-indexes-again (target-db copy) indexes :drop-indexes drop-indexes)))
;; this sets (table-index-list (target copy))
(maybe-drop-indexes (target-db copy)
(target copy)
:drop-indexes drop-indexes)
(copy-from copy :truncate truncate :disable-triggers disable-triggers)
;; re-create the indexes from the target table entry
(create-indexes-again (target-db copy)
(target copy)
:drop-indexes drop-indexes))

2
src/sources/sqlite/sqlite-schema.lisp
View File

@ -98,7 +98,7 @@
(defmethod index-table-name ((index sqlite-idx))
(sqlite-idx-table-name index))
(defmethod format-pgsql-create-index ((index sqlite-idx))
(defmethod format-pgsql-create-index ((table table) (index sqlite-idx))
"Generate the PostgresQL statement to build the given SQLite index definition."
(sqlite-idx-sql index))