From dbadab9e9e66cf9803df7f64090cf0245a05e9d8 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Wed, 13 Sep 2017 22:55:10 +0200 Subject: [PATCH] =?UTF-8?q?Implement=20a=20new=20=E2=80=9Csnake=5Fcase?= =?UTF-8?q?=E2=80=9D=20quoting=20rule.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In passing, add the identifiers case option to SQLite support, which makes it easier to test here, and add a table named "TableName" to our local test database. Fix #631. --- pgloader.asd | 2 +- src/package.lisp | 3 ++- src/parsers/command-keywords.lisp | 1 + src/parsers/command-options.lisp | 5 ++++- src/parsers/command-sqlite.lisp | 4 +++- src/utils/quoting.lisp | 30 +++++++++++++++++++++++++----- src/utils/threads.lisp | 1 + src/utils/utils.lisp | 19 ------------------- test/sqlite.load | 1 + test/sqlite/sqlite.db | Bin 102400 -> 106496 bytes 10 files changed, 38 insertions(+), 28 deletions(-) diff --git a/pgloader.asd b/pgloader.asd index 7f03405..6c4b10a 100644 --- a/pgloader.asd +++ b/pgloader.asd @@ -66,7 +66,7 @@ ;; PostgreSQL related utils (:file "read-sql-files") (:file "queries") - (:file "quoting") + (:file "quoting" :depends-on ("utils")) (:file "catalog" :depends-on ("quoting")) (:file "alter-table" :depends-on ("catalog")) diff --git a/src/package.lisp b/src/package.lisp index d28a89c..57befea 100644 --- a/src/package.lisp +++ b/src/package.lisp @@ -35,7 +35,8 @@ (:export #:apply-identifier-case #:build-identifier #:quoted-p - #:ensure-unquoted)) + #:ensure-unquoted + #:camelCase-to-colname)) (defpackage #:pgloader.catalog (:use #:cl #:pgloader.params #:pgloader.quoting) diff --git a/src/parsers/command-keywords.lisp b/src/parsers/command-keywords.lisp index 71c4f19..8d844c5 100644 --- a/src/parsers/command-keywords.lisp +++ b/src/parsers/command-keywords.lisp @@ -125,6 +125,7 @@ (def-keyword-rule "keys") (def-keyword-rule "downcase") (def-keyword-rule "quote") + (def-keyword-rule "snake_case") (def-keyword-rule "identifiers") (def-keyword-rule "including") (def-keyword-rule "excluding") diff --git a/src/parsers/command-options.lisp b/src/parsers/command-options.lisp index 4842144..b136088 100644 --- a/src/parsers/command-options.lisp +++ b/src/parsers/command-options.lisp @@ -160,7 +160,10 @@ (defrule option-on-error-stop (and kw-on kw-error kw-stop) (:constant (cons :on-error-stop t))) -(defrule option-identifiers-case (and (or kw-downcase kw-quote) kw-identifiers) +(defrule option-identifiers-case (and (or kw-snake_case + kw-downcase + kw-quote) + kw-identifiers) (:lambda (id-case) (bind (((action _) id-case)) (cons :identifier-case action)))) diff --git a/src/parsers/command-sqlite.lisp b/src/parsers/command-sqlite.lisp index f484e57..0c9f80b 100644 --- a/src/parsers/command-sqlite.lisp +++ b/src/parsers/command-sqlite.lisp @@ -31,7 +31,8 @@ load database option-index-names option-reset-sequences option-foreign-keys - option-encoding)) + option-encoding + option-identifiers-case)) (defrule sqlite-options (and kw-with (and sqlite-option (* (and comma sqlite-option)))) @@ -100,6 +101,7 @@ load database (*cast-rules* ',casts) ,@(pgsql-connection-bindings pg-db-conn gucs) ,@(batch-control-bindings options) + ,@(identifier-case-binding options) (source-db (with-stats-collection ("fetch" :section :pre) (expand (fetch-file ,sqlite-db-conn)))) (source diff --git a/src/utils/quoting.lisp b/src/utils/quoting.lisp index 2be0a7a..f6cc05d 100644 --- a/src/utils/quoting.lisp +++ b/src/utils/quoting.lisp @@ -40,12 +40,12 @@ ;; in other cases follow user directive (t *identifier-case*)))) - (ecase *identifier-case* - (:downcase lowercase-identifier) - (:quote (format nil "~s" - (cl-ppcre:regex-replace-all "\"" identifier "\"\""))) - (:none identifier)))) + (:snake_case (camelCase-to-colname identifier)) + (:downcase lowercase-identifier) + (:quote (format nil "~s" + (cl-ppcre:regex-replace-all "\"" identifier "\"\""))) + (:none identifier)))) (defun ensure-unquoted (identifier) (cond ((quoted-p identifier) @@ -68,3 +68,23 @@ (string part) (t (princ-to-string part)))) :when more? :collect sep)))) + +;;; +;;; Camel Case converter +;;; +(defun camelCase-to-colname (string) + "Transform input STRING into a suitable column name. + lahmanID lahman_id + playerID player_id + birthYear birth_year" + (coerce + (loop + :for first := t :then nil + :for char :across string + :for previous-upper-p := nil :then char-upper-p + :for char-upper-p := (and (alpha-char-p char) + (eq char (char-upcase char))) + :for new-word := (and (not first) char-upper-p (not previous-upper-p)) + :when (and new-word (not (char= char #\_))) :collect #\_ + :collect (char-downcase char)) + 'string)) diff --git a/src/utils/threads.lisp b/src/utils/threads.lisp index f09d362..a2776ce 100644 --- a/src/utils/threads.lisp +++ b/src/utils/threads.lisp @@ -23,6 +23,7 @@ ;; needed in create index specific kernels (*pgsql-reserved-keywords* . ',*pgsql-reserved-keywords*) (*preserve-index-names* . ,*preserve-index-names*) + (*identifier-case* . ,*identifier-case*) ;; bindings updates for libs ;; CFFI is used by the SQLite lib diff --git a/src/utils/utils.lisp b/src/utils/utils.lisp index 76844ac..157f811 100644 --- a/src/utils/utils.lisp +++ b/src/utils/utils.lisp @@ -3,25 +3,6 @@ ;;; (in-package :pgloader.utils) -;;; -;;; Camel Case converter -;;; -(defun camelCase-to-colname (string) - "Transform input STRING into a suitable column name. - lahmanID lahman_id - playerID player_id - birthYear birth_year" - (coerce - (loop - for first = t then nil - for char across string - for previous-upper-p = nil then char-upper-p - for char-upper-p = (eq char (char-upcase char)) - for new-word = (and (not first) char-upper-p (not previous-upper-p)) - when (and new-word (not (char= char #\_))) collect #\_ - collect (char-downcase char)) - 'string)) - ;;; ;;; Unquote SQLite default values, might be useful elsewhere ;;; diff --git a/test/sqlite.load b/test/sqlite.load index 2de3075..b7f86e2 100644 --- a/test/sqlite.load +++ b/test/sqlite.load @@ -3,6 +3,7 @@ load database into postgresql:///pgloader -- with include drop, create tables, create indexes, reset sequences + with snake_case identifiers before load do $$ drop schema if exists sqlite cascade; $$, diff --git a/test/sqlite/sqlite.db b/test/sqlite/sqlite.db index ac00c762eff725e65551bbac92cbdfc5f576d82f..e5522101333b5ab9a3bd3a2c8370109f37d6da18 100644 GIT binary patch delta 141 zcmZozz}9epZGyC*Ap--06cEFJ^h6zF9zzB_RtH}0e+)vL-V?KxnK->SPTbAm?J2}2 zE-%lRT#}fSlNtgd{1S5!3@PUzSH}=ng%C$4A0P>n*T_s!$jmEAO;0V-QAjgXC`qj- T(QKZ^xqTWZV^-U;07d}-RqQ6G delta 59 zcmZoTz}B#UZGyC*9s>h|BoM=Z#6%rq9z6y