From 9e2b95d9b73cfac059d7ce9fa746e69894645bbd Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Sat, 25 Feb 2017 21:49:06 +0100 Subject: [PATCH] Implement support for PostgreSQL storage parameters. In PostgreSQL it is possible at CREATE TABLE time to set some extra storage parameters, the most useful of them in the context of pgloader being the FILLFACTOR. For the setting to be useful, it needs to be positionned at CREATE TABLE time, before we load the data. The BEFORE LOAD clause of the pgloader command allows to run SQL scripts that will be executed before the load, and even before the creation of the target schema when pgloader does that, which is nice for other use case. Here we implement a new `ALTER TABLE` rule that one can set in the pgloader command in order to change storage parameters at CREATE TABLE time: ALTER TABLE NAMES MATCHING ~/\./ SET (fillfactor='40') Fix #516. --- pgloader.1 | 19 +++++++++++++++++-- pgloader.1.md | 18 +++++++++++++++++- src/package.lisp | 1 + src/parsers/command-alter-table.lisp | 8 +++++++- src/pgsql/pgsql-ddl.lisp | 9 ++++++++- src/utils/alter-table.lisp | 4 ++++ src/utils/catalog.lisp | 2 +- test/sakila.load | 1 + 8 files changed, 56 insertions(+), 6 deletions(-) diff --git a/pgloader.1 b/pgloader.1 index f45451a..006498d 100644 --- a/pgloader.1 +++ b/pgloader.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "PGLOADER" "1" "January 2017" "ff" "" +.TH "PGLOADER" "1" "February 2017" "ff" "" . .SH "NAME" \fBpgloader\fR \- PostgreSQL data loader @@ -1819,8 +1819,18 @@ LOAD DATABASE \-\- ALTER TABLE NAMES MATCHING \'film\' RENAME TO \'films\' \-\- ALTER TABLE NAMES MATCHING ~/_list$/ SET SCHEMA \'mv\' + ALTER TABLE NAMES MATCHING ~/_list$/, \'sales_by_store\', ~/sales_by/ + SET SCHEMA \'mv\' + + ALTER TABLE NAMES MATCHING \'film\' RENAME TO \'films\' + ALTER TABLE NAMES MATCHING ~/\./ SET (fillfactor=\'40\') + + ALTER SCHEMA \'sakila\' RENAME TO \'pagila\' + BEFORE LOAD DO - $$ create schema if not exists sakila; $$; + $$ create schema if not exists pagila; $$, + $$ create schema if not exists mv; $$, + $$ alter database sakila set search_path to pagila, mv, public; $$; . .fi . @@ -2220,6 +2230,8 @@ ALTER TABLE NAMES MATCHING ~/_list$/, \'sales_by_store\', ~/sales_by/ SET SCHEMA \'mv\' ALTER TABLE NAMES MATCHING \'film\' RENAME TO \'films\' + +ALTER TABLE NAMES MATCHING ~/\./ SET (fillfactor=\'40\') . .fi . @@ -2231,6 +2243,9 @@ You can use as many such rules as you need\. The list of tables to be migrated i .IP No \fIALTER TABLE\fR command is sent to PostgreSQL, the modification happens at the level of the pgloader in\-memory representation of your source database schema\. In case of a name change, the mapping is kept and reused in the \fIforeign key\fR and \fIindex\fR support\. . +.IP +The \fISET ()\fR action takes effect as a \fIWITH\fR clause for the \fBCREATE TABLE\fR command that pgloader will run when it has to create a table\. +. .IP "" 0 . .SS "LIMITATIONS" diff --git a/pgloader.1.md b/pgloader.1.md index 80d7bc3..22806ba 100644 --- a/pgloader.1.md +++ b/pgloader.1.md @@ -1556,9 +1556,20 @@ Here's an example: -- DECODING TABLE NAMES MATCHING ~/messed/, ~/encoding/ AS utf8 -- ALTER TABLE NAMES MATCHING 'film' RENAME TO 'films' -- ALTER TABLE NAMES MATCHING ~/_list$/ SET SCHEMA 'mv' + + ALTER TABLE NAMES MATCHING ~/_list$/, 'sales_by_store', ~/sales_by/ + SET SCHEMA 'mv' + + ALTER TABLE NAMES MATCHING 'film' RENAME TO 'films' + ALTER TABLE NAMES MATCHING ~/./ SET (fillfactor='40') + + ALTER SCHEMA 'sakila' RENAME TO 'pagila' BEFORE LOAD DO - $$ create schema if not exists sakila; $$; + $$ create schema if not exists pagila; $$, + $$ create schema if not exists mv; $$, + $$ alter database sakila set search_path to pagila, mv, public; $$; + The `database` command accepts the following clauses and options: @@ -1904,6 +1915,8 @@ The `database` command accepts the following clauses and options: SET SCHEMA 'mv' ALTER TABLE NAMES MATCHING 'film' RENAME TO 'films' + + ALTER TABLE NAMES MATCHING ~/./ SET (fillfactor='40') You can use as many such rules as you need. The list of tables to be migrated is searched in pgloader memory against the *ALTER TABLE* @@ -1914,6 +1927,9 @@ The `database` command accepts the following clauses and options: at the level of the pgloader in-memory representation of your source database schema. In case of a name change, the mapping is kept and reused in the *foreign key* and *index* support. + + The *SET ()* action takes effect as a *WITH* clause for the `CREATE + TABLE` command that pgloader will run when it has to create a table. ### LIMITATIONS diff --git a/src/package.lisp b/src/package.lisp index cc34e90..808557d 100644 --- a/src/package.lisp +++ b/src/package.lisp @@ -80,6 +80,7 @@ #:table-schema #:table-oid #:table-comment + #:table-storage-parameter-list #:table-field-list #:table-column-list #:table-index-list diff --git a/src/parsers/command-alter-table.lisp b/src/parsers/command-alter-table.lisp index 6493602..3e11c7a 100644 --- a/src/parsers/command-alter-table.lisp +++ b/src/parsers/command-alter-table.lisp @@ -42,8 +42,14 @@ (bind (((_ _ schema) stmt)) (list #'pgloader.catalog::alter-table-set-schema schema)))) +(defrule set-storage-parameters (and kw-set #\( generic-option-list #\)) + (:lambda (stmt) + (bind (((_ _ parameters _) stmt)) + (list #'pgloader.catalog::alter-table-set-storage-parameters parameters)))) + (defrule alter-table-action (or rename-to - set-schema)) + set-schema + set-storage-parameters)) (defrule alter-table-command (and alter-table-names-matching (? in-schema) diff --git a/src/pgsql/pgsql-ddl.lisp b/src/pgsql/pgsql-ddl.lisp index dc29b1a..9faeb64 100644 --- a/src/pgsql/pgsql-ddl.lisp +++ b/src/pgsql/pgsql-ddl.lisp @@ -59,7 +59,14 @@ :pretty-print t :max-column-name-length max) (format s "~:[~;,~]~%" last?)))) - (format s ");~%")))) + (format s ")") + + (when (table-storage-parameter-list table) + (format s "~%WITH (~{~a = '~a'~^,~% ~})" + (alexandria:alist-plist + (table-storage-parameter-list table)))) + + (format s ";~%")))) (defmethod format-drop-sql ((table table) &key (stream nil) cascade (if-exists t)) "Return the PostgreSQL DROP TABLE IF EXISTS statement for TABLE-NAME." diff --git a/src/utils/alter-table.lisp b/src/utils/alter-table.lisp index dceafac..56536e4 100644 --- a/src/utils/alter-table.lisp +++ b/src/utils/alter-table.lisp @@ -71,6 +71,10 @@ "Alter the name of TABLE to NEW-NAME." (setf (table-name table) new-name)) +(defun alter-table-set-storage-parameters (table parameters) + "Alter the storage parameters of TABLE." + (setf (table-storage-parameter-list table) parameters)) + ;;; ;;; Apply the match rules as given by the parser to a table name. diff --git a/src/utils/catalog.lisp b/src/utils/catalog.lisp index 5fc7c1d..0e87b49 100644 --- a/src/utils/catalog.lisp +++ b/src/utils/catalog.lisp @@ -44,7 +44,7 @@ ;;; (defstruct catalog name schema-list) (defstruct schema source-name name catalog table-list view-list) -(defstruct table source-name name schema oid comment +(defstruct table source-name name schema oid comment storage-parameter-list ;; field is for SOURCE ;; column is for TARGET field-list column-list index-list fkey-list trigger-list) diff --git a/test/sakila.load b/test/sakila.load index 441b169..45d1f94 100644 --- a/test/sakila.load +++ b/test/sakila.load @@ -29,6 +29,7 @@ load database ALTER TABLE NAMES MATCHING 'sales_by_store' RENAME TO 'sales_by_store_list' ALTER TABLE NAMES MATCHING 'film' RENAME TO 'films' + ALTER TABLE NAMES MATCHING ~/./ SET (fillfactor='40') ALTER SCHEMA 'sakila' RENAME TO 'pagila'