From 3eab88b1440a8166786e90b95f563d153e2ba4dc Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Tue, 4 Jul 2017 00:15:58 +0200 Subject: [PATCH] Add a new "drop indexes" option for databases. This allows to use a combination of "data only, drop indexes" so that when the target database already exists, pgloader will use the existing schema and still DROP INDEX before loading the data and do the CREATE INDEX dance in parallel and all at the end of it. Also, as I couldn't reproduce neither #539 (which is good, it's supposed to be fixed now) nor #550 (that was open due to a regression): fixes #550. --- pgloader.1 | 16 ++++++++++++++-- pgloader.1.md | 19 ++++++++++++++++--- src/parsers/command-options.lisp | 3 +++ src/parsers/command-sqlite.lisp | 1 + src/sources/common/db-methods.lisp | 7 +++++-- 5 files changed, 39 insertions(+), 7 deletions(-) diff --git a/pgloader.1 b/pgloader.1 index d2a13b3..ee22bf0 100644 --- a/pgloader.1 +++ b/pgloader.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "PGLOADER" "1" "June 2017" "ff" "" +.TH "PGLOADER" "1" "July 2017" "ff" "" . .SH "NAME" \fBpgloader\fR \- PostgreSQL data loader @@ -1846,7 +1846,7 @@ See the \fBSOURCE CONNECTION STRING\fR section above for details on how to write \fIWITH\fR . .IP -When loading from a \fBMySQL\fR database, the following options are supported, and the efault \fIWITH\fR clause is: \fIno truncate\fR, \fIcreate tables\fR, \fIinclude drop\fR, \fIcreate indexes\fR, \fIreset sequences\fR, \fIforeign keys\fR, \fIdowncase identifiers\fR\. +When loading from a \fBMySQL\fR database, the following options are supported, and the default \fIWITH\fR clause is: \fIno truncate\fR, \fIcreate schema\fR, \fIcreate tables\fR, \fIinclude drop\fR, \fIcreate indexes\fR, \fIreset sequences\fR, \fIforeign keys\fR, \fIdowncase identifiers\fR, \fIuniquify index names\fR\. . .IP \fIWITH\fR options: @@ -1915,6 +1915,12 @@ When this option is listed, pgloader gets the definitions of all the indexes fou When this option is listed, pgloader skips the creating indexes\. . .IP "\(bu" 4 +\fIdrop indexes\fR +. +.IP +When this option is listed, pgloader drops the indexes in the target database before loading the data, and creates them again at the end of the data copy\. +. +.IP "\(bu" 4 \fIuniquify index names\fR, \fIpreserve index names\fR . .IP @@ -2556,6 +2562,12 @@ When this option is listed, pgloader gets the definitions of all the indexes fou When this option is listed, pgloader skips the creating indexes\. . .IP "\(bu" 4 +\fIdrop indexes\fR +. +.IP +When this option is listed, pgloader drops the indexes in the target database before loading the data, and creates them again at the end of the data copy\. +. +.IP "\(bu" 4 \fIreset sequences\fR . .IP diff --git a/pgloader.1.md b/pgloader.1.md index a69f48d..334acee 100644 --- a/pgloader.1.md +++ b/pgloader.1.md @@ -1578,9 +1578,10 @@ The `database` command accepts the following clauses and options: - *WITH* When loading from a `MySQL` database, the following options are - supported, and the efault *WITH* clause is: *no truncate*, *create - tables*, *include drop*, *create indexes*, *reset sequences*, *foreign - keys*, *downcase identifiers*. + supported, and the default *WITH* clause is: *no truncate*, *create + schema*, *create tables*, *include drop*, *create indexes*, *reset + sequences*, *foreign keys*, *downcase identifiers*, *uniquify index + names*. *WITH* options: @@ -1652,6 +1653,12 @@ The `database` command accepts the following clauses and options: - *create no indexes* When this option is listed, pgloader skips the creating indexes. + + - *drop indexes* + + When this option is listed, pgloader drops the indexes in the target + database before loading the data, and creates them again at the end + of the data copy. - *uniquify index names*, *preserve index names* @@ -2164,6 +2171,12 @@ The `sqlite` command accepts the following clauses and options: When this option is listed, pgloader skips the creating indexes. + - *drop indexes* + + When this option is listed, pgloader drops the indexes in the target + database before loading the data, and creates them again at the end + of the data copy. + - *reset sequences* When this option is listed, at the end of the data loading and after diff --git a/src/parsers/command-options.lisp b/src/parsers/command-options.lisp index d024594..76ca0f7 100644 --- a/src/parsers/command-options.lisp +++ b/src/parsers/command-options.lisp @@ -137,6 +137,9 @@ (make-option-rule reset-sequences (and kw-reset (? kw-no) kw-sequences)) (make-option-rule foreign-keys (and (? kw-no) kw-foreign kw-keys)) +(defrule option-reindex (and kw-drop kw-indexes) + (:constant (cons :reindex t))) + (defrule option-single-reader (and kw-single kw-reader kw-per kw-thread) (:constant (cons :multiple-readers nil))) diff --git a/src/parsers/command-sqlite.lisp b/src/parsers/command-sqlite.lisp index eef03e0..14fb500 100644 --- a/src/parsers/command-sqlite.lisp +++ b/src/parsers/command-sqlite.lisp @@ -20,6 +20,7 @@ load database option-batch-size option-prefetch-rows option-max-parallel-create-index + option-reindex option-truncate option-disable-triggers option-data-only diff --git a/src/sources/common/db-methods.lisp b/src/sources/common/db-methods.lisp index 9b888f1..55c4eb8 100644 --- a/src/sources/common/db-methods.lisp +++ b/src/sources/common/db-methods.lisp @@ -251,6 +251,7 @@ (index-names :uniquify) (reset-sequences t) (foreign-keys t) + (reindex nil) only-tables including excluding @@ -265,8 +266,10 @@ (create-tables (and create-tables create-ddl)) (create-schemas (and create-schemas create-ddl)) (foreign-keys (and foreign-keys create-ddl)) - (drop-indexes (and include-drop create-ddl)) - (create-indexes (and create-indexes drop-indexes create-ddl)) + (drop-indexes (or reindex + (and include-drop create-ddl))) + (create-indexes (or reindex + (and create-indexes drop-indexes create-ddl))) (*preserve-index-names* (or (eq :preserve index-names)