From 3af99051d26ff67e1d9be64e14e7449804adcbe8 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Sat, 18 Jul 2015 23:39:32 +0200 Subject: [PATCH] Fix the preserve index names option. MySQL names its primary keys "PRIMARY" and we need to always uniquify this name even when the used asked pgloader to preserve index names. Also, the create-indexes-again function now needs to ask for index names to be preserved specifically. --- pgloader.1 | 3 +++ pgloader.1.md | 5 +++++ src/params.lisp | 4 ++++ src/pgsql/schema.lisp | 16 ++++++++++------ src/sources/mysql/mysql.lisp | 13 +++++-------- src/utils/threads.lisp | 3 +++ 6 files changed, 30 insertions(+), 14 deletions(-) diff --git a/pgloader.1 b/pgloader.1 index 3c6d074..d9bca88 100644 --- a/pgloader.1 +++ b/pgloader.1 @@ -1776,6 +1776,9 @@ In somes cases like when the DDL are entirely left to a framework it might be se .IP The default is to \fIuniquify index names\fR\. . +.IP +Even when using the option \fIpreserve index names\fR, MySQL primary key indexes named "PRIMARY" will get their names uniquified\. Failing to do so would prevent the primary keys to be created again in PostgreSQL where the index names must be unique per schema\. +. .IP "\(bu" 4 \fIforeign keys\fR . diff --git a/pgloader.1.md b/pgloader.1.md index e676967..0e9febb 100644 --- a/pgloader.1.md +++ b/pgloader.1.md @@ -1529,6 +1529,11 @@ The `database` command accepts the following clauses and options: The default is to *uniquify index names*. + Even when using the option *preserve index names*, MySQL primary key + indexes named "PRIMARY" will get their names uniquified. Failing to + do so would prevent the primary keys to be created again in + PostgreSQL where the index names must be unique per schema. + - *foreign keys* When this option is listed, pgloader gets the definitions of all the diff --git a/src/params.lisp b/src/params.lisp index d283654..2767e05 100644 --- a/src/params.lisp +++ b/src/params.lisp @@ -15,6 +15,7 @@ #:*log-min-messages* #:*report-stream* #:*identifier-case* + #:*preserve-index-names* #:*copy-batch-rows* #:*copy-batch-size* #:*concurrent-batches* @@ -98,6 +99,9 @@ (defparameter *identifier-case* :downcase "Dealing with source databases casing rules.") +(defparameter *preserve-index-names* nil + "Dealing with source databases index naming.") + ;;; ;;; How to split batches in case of data loading errors. ;;; diff --git a/src/pgsql/schema.lisp b/src/pgsql/schema.lisp index 137e4fe..3668c3c 100644 --- a/src/pgsql/schema.lisp +++ b/src/pgsql/schema.lisp @@ -282,12 +282,15 @@ (defmethod format-pgsql-create-index ((index pgsql-index)) "Generate the PostgreSQL statement list to rebuild a Foreign Key" - (let* ((index-name (if (pgsql-index-table-oid index) + (let* ((index-name (if (and *preserve-index-names* + (not (string-equal "primary" (pgsql-index-name index))) + (pgsql-index-table-oid index)) + (pgsql-index-name index) + + ;; in the general case, we build our own index name. (format nil "idx_~a_~a" (pgsql-index-table-oid index) - (pgsql-index-name index)) - ;; lacking the oid means we preserve the index name - (pgsql-index-name index))) + (pgsql-index-name index)))) (table-name (apply-identifier-case (pgsql-index-table-name index))) (index-name (apply-identifier-case index-name)) @@ -306,7 +309,7 @@ table-name (cond ((pgsql-index-primary index) "PRIMARY KEY") ((pgsql-index-unique index) "UNIQUE")) - (pgsql-index-name index)))) + index-name))) ((pgsql-index-condef index) (format nil "ALTER TABLE ~a ADD ~a;" @@ -421,7 +424,8 @@ &key drop-indexes) "Create the indexes that we dropped previously." (when (and indexes drop-indexes) - (let* ((idx-kernel (make-kernel (length indexes))) + (let* ((*preserve-index-names* t) + (idx-kernel (make-kernel (length indexes))) (idx-channel (let ((lp:*kernel* idx-kernel)) (lp:make-channel)))) (let ((pkeys diff --git a/src/sources/mysql/mysql.lisp b/src/sources/mysql/mysql.lisp index 8a45e28..e91e408 100644 --- a/src/sources/mysql/mysql.lisp +++ b/src/sources/mysql/mysql.lisp @@ -152,8 +152,7 @@ &key state foreign-keys - include-drop - preserve-index-names) + include-drop) "Prepare the target PostgreSQL database: create tables casting datatypes from the MySQL definitions, prepare index definitions and create target tables for materialized views. @@ -180,8 +179,7 @@ ;; MySQL allows the same index name being used against several ;; tables, so we add the PostgreSQL table OID in the index name, ;; to differenciate. Set the table oids now. - (unless preserve-index-names - (set-table-oids all-indexes)) + (set-table-oids all-indexes) ;; We might have to MATERIALIZE VIEWS (when materialize-views @@ -409,9 +407,7 @@ view-columns :state state-before :foreign-keys foreign-keys - :include-drop include-drop - :preserve-index-names (eq :preserve - index-names))) + :include-drop include-drop)) (t (when truncate (truncate-tables (target-db mysql) (mapcar #'car all-columns))))) @@ -474,7 +470,8 @@ ;; will get built in parallel --- not a big problem. (when (and create-indexes (not data-only)) (let* ((indexes - (cdr (assoc table-name all-indexes :test #'string=)))) + (cdr (assoc table-name all-indexes :test #'string=))) + (*preserve-index-names* (eq :preserve index-names))) (alexandria:appendf pkeys (create-indexes-in-kernel (target-db mysql) diff --git a/src/utils/threads.lisp b/src/utils/threads.lisp index 0b625a1..88fc9e3 100644 --- a/src/utils/threads.lisp +++ b/src/utils/threads.lisp @@ -16,6 +16,9 @@ (*client-min-messages* . ,*client-min-messages*) (*log-min-messages* . ,*log-min-messages*) + ;; needed in create index specific kernels + (*preserve-index-names* . ,*preserve-index-names*) + ;; bindings updates for libs ;; CFFI is used by the SQLite lib (cffi:*default-foreign-encoding*