From 48af01dbbc5c2145ff508c0585c7bfd50a3589ef Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Mon, 19 Feb 2018 22:07:43 +0100 Subject: [PATCH] Fix implementation of foreign keys in data only mode. In data-only mode, the foreign keys parameter (which defaults to True) means something special: we remove the fkey definitions prior to the data only load then re-install the fkeys. This got broken in a previous commit, the WITH clause option being processed like the other DDL ones that only make sense when creating the schema. While fixing the setting in copy-database, we have to also fix a nesting bug in complete-pgsql-database that would prevent fkey to be installed again at the end of the load. This patch not only fix that choice, but also review the implementation of the drop-pgsql-fkeys support function to use more modern internal API, preparing a list of SQL statements to be sent to the psql-execute level. Fixes #745. --- src/pgsql/pgsql-create-schema.lisp | 33 ++++++++-------- src/sources/common/db-methods.lisp | 61 +++++++++++++++--------------- 2 files changed, 48 insertions(+), 46 deletions(-) diff --git a/src/pgsql/pgsql-create-schema.lisp b/src/pgsql/pgsql-create-schema.lisp index caaa5e7..7dd3614 100644 --- a/src/pgsql/pgsql-create-schema.lisp +++ b/src/pgsql/pgsql-create-schema.lisp @@ -203,21 +203,22 @@ ;;; ;;; API for Foreign Keys ;;; -(defun drop-pgsql-fkeys (catalog &key (cascade t)) +(defun drop-pgsql-fkeys (catalog &key (cascade t) (log-level :notice)) "Drop all Foreign Key Definitions given, to prepare for a clean run." - (loop :for table :in (table-list catalog) - :sum (loop :for fkey :in (table-fkey-list table) - :for sql := (format-drop-sql fkey :cascade cascade :if-exists t) - :do (pgsql-execute sql) - :count t) - ;; also DROP the foreign keys that depend on the indexes we want to DROP - :sum (loop :for index :in (table-index-list table) - :sum (loop :for fkey :in (index-fk-deps index) - :for sql := (format-drop-sql fkey :cascade t :if-exists t) - :do (progn - (log-message :debug "EXTRA FK DEPS!") - (pgsql-execute sql)) - :count t)))) + (let ((fk-sql-list + (loop :for table :in (table-list catalog) + :append (loop :for fkey :in (table-fkey-list table) + :collect (format-drop-sql fkey + :cascade cascade + :if-exists t)) + ;; also DROP the foreign keys that depend on the indexes we + ;; want to DROP + :append (loop :for index :in (table-index-list table) + :append (loop :for fkey :in (index-fk-deps index) + :collect (format-drop-sql fkey + :cascade t + :if-exists t)))))) + (pgsql-execute fk-sql-list :log-level log-level))) (defun create-pgsql-fkeys (catalog &key (section :post) label log-level) "Actually create the Foreign Key References that where declared in the @@ -287,7 +288,7 @@ ;;; ;;; Drop indexes before loading ;;; -(defun drop-indexes (table-or-catalog &key cascade) +(defun drop-indexes (table-or-catalog &key cascade (log-level :notice)) "Drop indexes in PGSQL-INDEX-LIST. A PostgreSQL connection must already be active when calling that function." (let ((sql-index-list @@ -297,7 +298,7 @@ (catalog (loop :for table :in (table-list table-or-catalog) :append (table-index-list table)))) :collect (format-drop-sql index :cascade cascade :if-exists t)))) - (pgsql-execute sql-index-list) + (pgsql-execute sql-index-list :log-level log-level) ;; return how many indexes we just DROPed (length sql-index-list))) diff --git a/src/sources/common/db-methods.lisp b/src/sources/common/db-methods.lisp index 0dae9ec..f706ed7 100644 --- a/src/sources/common/db-methods.lisp +++ b/src/sources/common/db-methods.lisp @@ -76,7 +76,7 @@ (with-stats-collection ("Drop Foreign Keys" :section :pre :use-result-as-read t :use-result-as-rows t) - (drop-pgsql-fkeys catalog))) + (drop-pgsql-fkeys catalog :log-level :notice))) (when drop-indexes (with-stats-collection ("Drop Indexes" :section :pre @@ -84,7 +84,7 @@ :use-result-as-rows t) ;; we want to error out early in case we can't DROP the ;; index, don't CASCADE - (drop-indexes catalog :cascade nil))) + (drop-indexes catalog :cascade nil :log-level :notice))) (when truncate (with-stats-collection ("Truncate" :section :pre @@ -147,28 +147,28 @@ ;; (when create-indexes (pgsql-execute-with-timing :post "Primary Keys" pkeys - :log-level :notice) + :log-level :notice)) - ;; - ;; Foreign Key Constraints - ;; - ;; We need to have finished loading both the reference and the - ;; refering tables to be able to build the foreign keys, so wait - ;; until all tables and indexes are imported before doing that. - ;; - (when foreign-keys - (create-pgsql-fkeys catalog - :section :post - :label "Create Foreign Keys" - :log-level :notice)) + ;; + ;; Foreign Key Constraints + ;; + ;; We need to have finished loading both the reference and the + ;; refering tables to be able to build the foreign keys, so wait + ;; until all tables and indexes are imported before doing that. + ;; + (when foreign-keys + (create-pgsql-fkeys catalog + :section :post + :label "Create Foreign Keys" + :log-level :notice)) - ;; - ;; Triggers and stored procedures -- includes special default values - ;; - (when create-triggers - (create-triggers catalog - :section :post - :label "Create Triggers"))) + ;; + ;; Triggers and stored procedures -- includes special default values + ;; + (when create-triggers + (create-triggers catalog + :section :post + :label "Create Triggers")) ;; ;; Add schemas that needs to be in the search_path to the database @@ -191,16 +191,16 @@ (log-message :error "Complete PostgreSQL database reconnecting to PostgreSQL.") - ;; in order to avoid Socket error in "connect": ECONNREFUSED if we - ;; try just too soon, wait a little + ;; in order to avoid Socket error in "connect": ECONNREFUSED if we + ;; try just too soon, wait a little (sleep 2) - ;; - ;; Reset Sequence can be done several times safely, and the rest of the - ;; operations run in a single transaction, so if the connection was lost, - ;; nothing has been done. Retry. - ;; + ;; + ;; Reset Sequence can be done several times safely, and the rest of the + ;; operations run in a single transaction, so if the connection was lost, + ;; nothing has been done. Retry. + ;; (complete-pgsql-database copy catalog pkeys @@ -280,7 +280,8 @@ (create-ddl (or schema-only (not data-only))) (create-tables (and create-tables create-ddl)) (create-schemas (and create-schemas create-ddl)) - (foreign-keys (and foreign-keys create-ddl)) + ;; foreign keys has a special meaning in data-only mode + (foreign-keys foreign-keys) (drop-indexes (or reindex (and include-drop create-ddl))) (create-indexes (or reindex