From 4155d06ae5a3954b79de5192146a3eb8a64d3198 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Fri, 18 Mar 2016 11:01:06 +0100 Subject: [PATCH] Improve support for MS SQL multicolumn indexes. Once more we can't use an aggregate over a text column in MS SQL to build the index definition from its catalog structure, so we have to do that in the lisp part of the code. Multi-column indexes are now supported, but filtered indexes still are a problem: the WHERE clause in MS SQL is not compatible with the PostgreSQL syntax (because of [names] and type casting. For example we cast MS SQL bit to PostgreSQL boolean, so WHERE ([deleted]=(0)) should be translated to WHERE not deleted And the code to do that is not included yet. The following documentation page offers more examples of WHERE expression we might want to support: https://technet.microsoft.com/en-us/library/cc280372.aspx WHERE EndDate IS NOT NULL AND ComponentID = 5 AND StartDate > '01/01/2008' EndDate IN ('20000825', '20000908', '20000918') It might be worth automating the translation to PostgreSQL syntax and operators, but it's not done in this patch. See #365, where the created index will now be as follows, which is a problem because of being UNIQUE: some existing data won't reload fine. CREATE UNIQUE INDEX idx__foo_name_unique ON dbo.foo (name, type, deleted); --- src/package.lisp | 4 ++++ src/sources/mssql/mssql-schema.lisp | 8 +++++--- src/utils/schema-structs.lisp | 29 +++++++++++++++++++++++------ 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/src/package.lisp b/src/package.lisp index 68af7bc..8d15316 100644 --- a/src/package.lisp +++ b/src/package.lisp @@ -87,6 +87,8 @@ #:add-field #:add-column #:add-index + #:find-index + #:maybe-add-index #:add-fkey #:find-fkey #:maybe-add-fkey @@ -249,6 +251,8 @@ #:add-field #:add-column #:add-index + #:find-index + #:maybe-add-index #:add-fkey #:find-fkey #:maybe-add-fkey diff --git a/src/sources/mssql/mssql-schema.lisp b/src/sources/mssql/mssql-schema.lisp index 9871b14..da49202 100644 --- a/src/sources/mssql/mssql-schema.lisp +++ b/src/sources/mssql/mssql-schema.lisp @@ -196,11 +196,13 @@ order by SchemaName, :do (let* ((schema (find-schema catalog schema-name)) (table (find-table schema table-name)) - (index (make-pgsql-index :name index-name + (pg-index (make-pgsql-index :name index-name :primary (= pkey 1) :unique (= unique 1) - :columns (list col)))) - (add-index table index)) + :columns nil)) + (index (maybe-add-index table index-name pg-index + :key #'pgloader.pgsql::pgsql-index-name))) + (push-to-end col (pgloader.pgsql::pgsql-index-columns index))) :finally (return catalog))) (defun list-all-fkeys (catalog &key including excluding) diff --git a/src/utils/schema-structs.lisp b/src/utils/schema-structs.lisp index 32a605a..0c4f7dc 100644 --- a/src/utils/schema-structs.lisp +++ b/src/utils/schema-structs.lisp @@ -77,9 +77,13 @@ (:documentation "Find a table by TABLE-NAME in a schema OBJECT and return the table")) +(defgeneric find-index (object index-name &key key test) + (:documentation + "Find an index by INDEX-NAME in a table OBJECT and return the index")) + (defgeneric find-fkey (object fkey-name &key key test) (:documentation - "Find a table by FKEY-NAME in a table OBJECT and return the fkey")) + "Find a foreign key by FKEY-NAME in a table OBJECT and return the fkey")) (defgeneric maybe-add-schema (object schema-name &key) (:documentation "Add a new schema or return existing one.")) @@ -90,6 +94,9 @@ (defgeneric maybe-add-view (object view-name &key) (:documentation "Add a new view or return existing one.")) +(defgeneric maybe-add-index (object index-name index &key key test) + (:documentation "Add a new index or return existing one.")) + (defgeneric maybe-add-fkey (object fkey-name fkey &key key test) (:documentation "Add a new fkey or return existing one.")) @@ -231,19 +238,29 @@ (loop :for schema :in (catalog-schema-list catalog) :do (cast schema))) +;;; +;;; There's no simple equivalent to array_agg() in MS SQL, so the index and +;;; fkey queries return a row per index|fkey column rather than per +;;; index|fkey. Hence this extra API: +;;; (defmethod add-index ((table table) index &key) "Add INDEX to TABLE and return the TABLE." (push-to-end index (table-index-list table))) +(defmethod find-index ((table table) index-name &key key (test #'string=)) + "Find INDEX-NAME in TABLE and return the INDEX object of this name." + (find index-name (table-index-list table) :key key :test test)) + +(defmethod maybe-add-index ((table table) index-name index &key key (test #'string=)) + "Add the index INDEX to the table-index-list of TABLE unless it already + exists, and return the INDEX object." + (let ((current-index (find-index table index-name :key key :test test))) + (or current-index (add-index table index)))) + (defmethod add-fkey ((table table) fkey &key) "Add FKEY to TABLE and return the TABLE." (push-to-end fkey (table-fkey-list table))) -;;; -;;; There's no simple equivalent to array_agg() in MS SQL, so the fkey query -;;; returns a row per fkey column rather than per fkey. Hence this extra -;;; API: -;;; (defmethod find-fkey ((table table) fkey-name &key key (test #'string=)) "Find FKEY-NAME in TABLE and return the FKEY object of this name." (find fkey-name (table-fkey-list table) :key key :test test))