From 76668c2626d94c2ae3131c417f3e91ce6d8097c3 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Sun, 31 Jan 2016 18:42:01 +0100 Subject: [PATCH] Review package dependencies. The decision to use lots of different packages in pgloader has quite strong downsides at times, and the manual managment of dependencies is one of the, in particular how to avoid circular ones. --- src/package.lisp | 19 ++++++++++--------- src/parsers/command-sql-block.lisp | 1 - src/pgsql/queries.lisp | 11 +++++++---- src/pgsql/schema.lisp | 23 ++++------------------- src/sources/common/db-methods.lisp | 1 - src/utils/quoting.lisp | 2 +- src/utils/schema-structs.lisp | 2 -- 7 files changed, 22 insertions(+), 37 deletions(-) diff --git a/src/package.lisp b/src/package.lisp index 664c25c..4642838 100644 --- a/src/package.lisp +++ b/src/package.lisp @@ -23,8 +23,12 @@ #:start-logger #:stop-logger)) -(defpackage #:pgloader.schema +(defpackage #:pgloader.quoting (:use #:cl #:pgloader.params) + (:export #:apply-identifier-case)) + +(defpackage #:pgloader.schema + (:use #:cl #:pgloader.params #:pgloader.quoting) (:export #:catalog #:schema #:table @@ -35,6 +39,8 @@ #:cast ; generic function for sources + #:apply-identifier-case + #:make-catalog #:make-schema #:make-table @@ -93,13 +99,13 @@ #:push-to-end #:with-schema + #:format-table-name #:format-default-value #:format-column)) (defpackage #:pgloader.state (:use #:cl #:pgloader.params #:pgloader.schema) - (:export #:format-table-name - #:make-pgstate + (:export #:make-pgstate #:pgstate-tabnames #:pgstate-tables #:pgstate-read @@ -162,7 +168,6 @@ ;; state #:make-pgstate - #:format-table-name #:pgstate-tabnames ;; events @@ -257,6 +262,7 @@ #:push-to-end #:with-schema + #:format-table-name #:format-default-value #:format-column)) @@ -329,11 +335,6 @@ #:db-user #:db-pass)) -(defpackage #:pgloader.schema - (:use #:cl #:pgloader.params #:pgloader.utils #:pgloader.connection) - (:export #:push-to-end - #:with-schema)) - (defpackage #:pgloader.pgsql (:use #:cl #:pgloader.params #:pgloader.utils #:pgloader.connection diff --git a/src/parsers/command-sql-block.lisp b/src/parsers/command-sql-block.lisp index 5ae2f1b..6aaee6a 100644 --- a/src/parsers/command-sql-block.lisp +++ b/src/parsers/command-sql-block.lisp @@ -69,6 +69,5 @@ (with-pgsql-transaction (:pgconn ,pgconn) (loop for command in ',commands do - (log-message :notice command) (pgsql-execute command :client-min-messages :error) counting command))))) diff --git a/src/pgsql/queries.lisp b/src/pgsql/queries.lisp index 74ea9f0..97e7e43 100644 --- a/src/pgsql/queries.lisp +++ b/src/pgsql/queries.lisp @@ -194,15 +194,18 @@ (declare (ignore res)) (update-stats section label :read count :rows count :secs secs))) -(defun pgsql-execute (sql &key ((:client-min-messages level))) +(defun pgsql-execute (sql &key client-min-messages) "Execute given SQL in current transaction" - (when level + (when client-min-messages (pomo:execute - (format nil "SET LOCAL client_min_messages TO ~a;" (symbol-name level)))) + (format nil "SET LOCAL client_min_messages TO ~a;" + (symbol-name client-min-messages)))) + (log-message :notice "~a" sql) (pomo:execute sql) - (when level (pomo:execute (format nil "RESET client_min_messages;")))) + (when client-min-messages + (pomo:execute (format nil "RESET client_min_messages;")))) ;;; ;;; PostgreSQL Utility Queries diff --git a/src/pgsql/schema.lisp b/src/pgsql/schema.lisp index 9ef0fee..578510f 100644 --- a/src/pgsql/schema.lisp +++ b/src/pgsql/schema.lisp @@ -88,9 +88,7 @@ :for sql := (format-pgsql-drop-fkey fkey :all-pgsql-fkeys all-pgsql-fkeys) :when sql - :do - (log-message :notice "~a;" sql) - (pgsql-execute sql))))) + :do (pgsql-execute sql))))) (defun create-pgsql-fkeys (catalog &key @@ -102,9 +100,7 @@ (loop :for table :in (table-list catalog) :sum (loop :for fkey :in (table-fkey-list table) :for sql := (format-pgsql-create-fkey fkey) - :do (progn ; for indentation purposes - (log-message :notice "~a;" sql) - (pgsql-execute-with-timing section label sql)) + :do (pgsql-execute-with-timing section label sql) :count t)))) @@ -162,9 +158,7 @@ :include-drop include-drop) :count (not (null sql)) :into nb-tables :when sql - :do (progn - (log-message :info "~a" sql) - (pgsql-execute sql :client-min-messages client-min-messages)) + :do (pgsql-execute sql :client-min-messages client-min-messages) :finally (return nb-tables))) (defun create-tables (catalog @@ -199,7 +193,6 @@ (catalog (table-list catalog-or-table)) (schema (table-list catalog-or-table)) (table (list catalog-or-table))))))) - (log-message :notice "~a" sql) (pomo:execute sql)))) (defun disable-triggers (table-name) @@ -207,7 +200,6 @@ connection already opened." (let ((sql (format nil "ALTER TABLE ~a DISABLE TRIGGER ALL;" (apply-identifier-case table-name)))) - (log-message :info "~a" sql) (pomo:execute sql))) (defun enable-triggers (table-name) @@ -215,7 +207,6 @@ connection already opened." (let ((sql (format nil "ALTER TABLE ~a ENABLE TRIGGER ALL;" (apply-identifier-case table-name)))) - (log-message :info "~a" sql) (pomo:execute sql))) (defmacro with-disabled-triggers ((table-name &key disable-triggers) @@ -327,7 +318,6 @@ ;; we postpone the pkey upgrade of the index for later. (format-pgsql-create-index table index) - (log-message :notice "~a" sql) (lp:submit-task channel #'pgsql-connect-and-execute-with-timing ;; each thread must have its own connection @@ -365,7 +355,6 @@ active when calling that function." (loop :for index :in (table-index-list table) :do (let ((sql (format-pgsql-drop-index table index))) - (log-message :notice "~a" sql) (pgsql-execute-with-timing section "drop indexes" sql)))) ;;; @@ -420,9 +409,7 @@ (with-stats-collection ("Constraints" :section section) (loop :for sql :in pkeys :when sql - :do (progn - (log-message :notice "~a" sql) - (pgsql-execute-with-timing section "Constraints" sql))))))))) + :do (pgsql-execute-with-timing section "Constraints" sql)))))))) ;;; ;;; Sequences @@ -465,7 +452,6 @@ (table-name table) quote (table-comment table) quote)) :count (when sql - (log-message :notice "~a" sql) (pgsql-execute-with-timing :post "Comments" sql)) :sum (loop :for column :in (table-column-list table) @@ -475,5 +461,4 @@ (column-name column) quote (column-comment column) quote)) :count (when sql - (log-message :notice "~a;" sql) (pgsql-execute-with-timing :post "Comments" sql))))))) diff --git a/src/sources/common/db-methods.lisp b/src/sources/common/db-methods.lisp index f72d57b..3935623 100644 --- a/src/sources/common/db-methods.lisp +++ b/src/sources/common/db-methods.lisp @@ -34,7 +34,6 @@ (loop :for schema :in (catalog-schema-list catalog) :do (when create-schemas (let ((sql (format nil "CREATE SCHEMA ~a;" schema))) - (log-message :notice "~a" sql) (pgsql-execute sql))))) (create-tables catalog :include-drop include-drop) diff --git a/src/utils/quoting.lisp b/src/utils/quoting.lisp index e214862..f4f6364 100644 --- a/src/utils/quoting.lisp +++ b/src/utils/quoting.lisp @@ -5,7 +5,7 @@ ;;; need those bits of code in utils/ rather than pgsql/. ;;; -(in-package :pgloader.utils) +(in-package :pgloader.quoting) (defun quoted-p (s) "Return true if s is a double-quoted string" diff --git a/src/utils/schema-structs.lisp b/src/utils/schema-structs.lisp index a960f4e..6eac209 100644 --- a/src/utils/schema-structs.lisp +++ b/src/utils/schema-structs.lisp @@ -338,13 +338,11 @@ (table (if (table-schema ,table-name) (let ((sql (format nil "SET search_path TO ~a;" (table-schema ,table-name)))) - (log-message :notice "~a" sql) (pgloader.pgsql:pgsql-execute sql) (table-name ,table-name)) (table-name ,table-name))) (cons (let ((sql (format nil "SET search_path TO ~a;" (car ,table-name)))) - (log-message :notice "~a" sql) (pgloader.pgsql:pgsql-execute sql) (cdr ,table-name))) (string ,table-name))))