From a2370938b6712f12004afda7e613303a9ec4da54 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Mon, 26 May 2014 18:02:24 +0200 Subject: [PATCH] MATERIALIZE ALL VIEWS. Complete the MySQL migration feature. --- pgloader.1 | 6 ++++++ pgloader.1.md | 5 +++++ src/parser.lisp | 7 ++++++- src/sources/mysql-schema.lisp | 28 +++++++++++++++------------- src/sources/mysql.lisp | 14 +++++++++----- test/sakila.load | 3 ++- 6 files changed, 43 insertions(+), 20 deletions(-) diff --git a/pgloader.1 b/pgloader.1 index 46afd02..f7046b0 100644 --- a/pgloader.1 +++ b/pgloader.1 @@ -1219,6 +1219,12 @@ This clause expect a comma separated list of view definitions, each one being ei The \fIname\fR and the \fIsql query\fR will be used in a \fBCREATE VIEW\fR statement at the beginning of the data loading, and the resulting view will then be dropped at the end of the data loading\. . .IP "\(bu" 4 +\fIMATERIALIZE ALL VIEWS\fR +. +.IP +Same behaviour as \fIMATERIALIZE VIEWS\fR using the dynamic list of views as returned by MySQL rather than asking the user to specify the list\. +. +.IP "\(bu" 4 \fIINCLUDING ONLY TABLE NAMES MATCHING\fR . .IP diff --git a/pgloader.1.md b/pgloader.1.md index b875e63..7b9a659 100644 --- a/pgloader.1.md +++ b/pgloader.1.md @@ -1044,6 +1044,11 @@ The `database` command accepts the following clauses and options: at the beginning of the data loading, and the resulting view will then be dropped at the end of the data loading. + - *MATERIALIZE ALL VIEWS* + + Same behaviour as *MATERIALIZE VIEWS* using the dynamic list of views as + returned by MySQL rather than asking the user to specify the list. + - *INCLUDING ONLY TABLE NAMES MATCHING* Introduce a comma separated list of table names or *regular expression* diff --git a/src/parser.lisp b/src/parser.lisp index 0d28a14..3084276 100644 --- a/src/parser.lisp +++ b/src/parser.lisp @@ -910,9 +910,14 @@ (destructuring-bind (view1 views) vlist (list* view1 views)))) -(defrule materialize-views (and kw-materialize kw-views views-list) +(defrule materialize-all-views (and kw-materialize kw-all kw-views) + (:constant :all)) + +(defrule materialize-view-list (and kw-materialize kw-views views-list) (:destructure (mat views list) (declare (ignore mat views)) list)) +(defrule materialize-views (or materialize-view-list materialize-all-views)) + ;;; ;;; Including only some tables or excluding some others diff --git a/src/sources/mysql-schema.lisp b/src/sources/mysql-schema.lisp index 039158c..0636813 100644 --- a/src/sources/mysql-schema.lisp +++ b/src/sources/mysql-schema.lisp @@ -122,23 +122,25 @@ order by table_name" dbname only-tables)))) "VIEWS-ALIST associates view names with their SQL definition, which might be empty for already existing views. Create only the views for which we have an SQL definition." - (let ((views (remove-if #'null views-alist :key #'cdr))) - (when views - (loop for (name . def) in views - for sql = (format nil "CREATE VIEW ~a AS ~a" name def) - do - (log-message :info "MySQL: ~a" sql) - (mysql-query sql))))) + (unless (eq :all views-alist) + (let ((views (remove-if #'null views-alist :key #'cdr))) + (when views + (loop for (name . def) in views + for sql = (format nil "CREATE VIEW ~a AS ~a" name def) + do + (log-message :info "MySQL: ~a" sql) + (mysql-query sql)))))) (defun drop-my-views (views-alist) "See `create-my-views' for VIEWS-ALIST description. This time we DROP the views to clean out after our work." - (let ((views (remove-if #'null views-alist :key #'cdr))) - (when views - (let ((sql - (format nil "DROP VIEW ~{~a~^, ~};" (mapcar #'car views)))) - (log-message :info "MySQL: ~a" sql) - (mysql-query sql))))) + (unless (eq :all views-alist) + (let ((views (remove-if #'null views-alist :key #'cdr))) + (when views + (let ((sql + (format nil "DROP VIEW ~{~a~^, ~};" (mapcar #'car views)))) + (log-message :info "MySQL: ~a" sql) + (mysql-query sql)))))) ;;; diff --git a/src/sources/mysql.lisp b/src/sources/mysql.lisp index 2990f96..b946a6a 100644 --- a/src/sources/mysql.lisp +++ b/src/sources/mysql.lisp @@ -243,7 +243,8 @@ including excluding) "MySQL introspection to prepare the migration." - (let ((view-names (mapcar #'car materialize-views)) + (let ((view-names (unless (eq :all materialize-views) + (mapcar #'car materialize-views))) view-columns all-columns all-fkeys all-indexes) (with-stats-collection ("fetch meta data" :use-result-as-rows t @@ -252,7 +253,7 @@ (with-mysql-connection () ;; If asked to MATERIALIZE VIEWS, now is the time to create them in ;; MySQL, when given definitions rather than existing view names. - (when materialize-views + (when (and materialize-views (not (eq :all materialize-views))) (create-my-views materialize-views)) (setf all-columns (filter-column-list (list-all-columns) @@ -270,9 +271,12 @@ :including including :excluding excluding) - view-columns (when view-names - (list-all-columns :only-tables view-names - :table-type :view))) + view-columns (cond (view-names + (list-all-columns :only-tables view-names + :table-type :view)) + + ((eq :all materialize-views) + (list-all-columns :table-type :view)))) ;; return how many objects we're going to deal with in total ;; for stats collection diff --git a/test/sakila.load b/test/sakila.load index 14b2402..3b9c8aa 100644 --- a/test/sakila.load +++ b/test/sakila.load @@ -13,7 +13,8 @@ load database -- type tinyint to boolean using tinyint-to-boolean, -- type year to integer drop typemod -- now a default - MATERIALIZE VIEWS film_list, staff_list + -- MATERIALIZE VIEWS film_list, staff_list + MATERIALIZE ALL VIEWS -- INCLUDING ONLY TABLE NAMES MATCHING ~/film/, 'actor' -- EXCLUDING TABLE NAMES MATCHING ~