From c108b8529073b27af4bcbb6085b40eb80a96fa1e Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 4 Feb 2016 16:05:53 +0100 Subject: [PATCH] Allow package prefix in CAST ... USING clause. Also, in passing, ass a new transformation function for MySQL allowing to transform from varbinary to text. --- pgloader.1 | 10 ++++++++-- pgloader.1.md | 6 ++++++ src/parsers/command-cast-rules.lisp | 27 ++++++++++++++++++++------- src/utils/transforms.lisp | 10 +++++++++- test/parse/hans.goeuro.load | 5 ++++- 5 files changed, 47 insertions(+), 11 deletions(-) diff --git a/pgloader.1 b/pgloader.1 index bb65eb0..24da748 100644 --- a/pgloader.1 +++ b/pgloader.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "PGLOADER" "1" "January 2016" "ff" "" +.TH "PGLOADER" "1" "February 2016" "ff" "" . .SH "NAME" \fBpgloader\fR \- PostgreSQL data loader @@ -1758,7 +1758,7 @@ When loading from a \fBMySQL\fR database, the following options are supported, a \fIinclude drop\fR . .IP -When this option is listed, pgloader drops all the tables in the target PostgreSQL database whose names appear in the SQLite database\. This option allows for using the same command several times in a row until you figure out all the options, starting automatically from a clean environment\. Please note that \fBCASCADE\fR is used to ensure that tables are dropped even if there are foreign keys pointing to them\. This is precisely what \fBinclude drop\fR is intended to do: drop all target tables and recreate them\. +When this option is listed, pgloader drops all the tables in the target PostgreSQL database whose names appear in the MySQL database\. This option allows for using the same command several times in a row until you figure out all the options, starting automatically from a clean environment\. Please note that \fBCASCADE\fR is used to ensure that tables are dropped even if there are foreign keys pointing to them\. This is precisely what \fBinclude drop\fR is intended to do: drop all target tables and recreate them\. . .IP Great care needs to be taken when using \fBinclude drop\fR, as it will cascade to \fIall\fR objects referencing the target tables, possibly including other tables that are not being loaded from the source DB\. @@ -2945,6 +2945,12 @@ The SQL Server driver receives data fo type uniqueidentifier as byte vector that .IP Converts a unix timestamp (number of seconds elapsed since beginning of 1970) into a proper PostgreSQL timestamp format\. . +.IP "\(bu" 4 +\fIvarbinary\-to\-string\fR +. +.IP +Converts binary encoded string (such as a MySQL \fBvarbinary\fR entry) to a decoded text, using the table\'s encoding that may be overloaded with the \fIDECODING TABLE NAMES MATCHING\fR clause\. +. .IP "" 0 . .SH "LOAD MESSAGES" diff --git a/pgloader.1.md b/pgloader.1.md index 097e2d9..33af8f0 100644 --- a/pgloader.1.md +++ b/pgloader.1.md @@ -2318,6 +2318,12 @@ The provided transformation functions are: Converts a unix timestamp (number of seconds elapsed since beginning of 1970) into a proper PostgreSQL timestamp format. + - *varbinary-to-string* + + Converts binary encoded string (such as a MySQL `varbinary` entry) to a + decoded text, using the table's encoding that may be overloaded with the + *DECODING TABLE NAMES MATCHING* clause. + ## LOAD MESSAGES This command is still experimental and allows receiving messages via diff --git a/src/parsers/command-cast-rules.lisp b/src/parsers/command-cast-rules.lisp index 1dd5c61..5417004 100644 --- a/src/parsers/command-cast-rules.lisp +++ b/src/parsers/command-cast-rules.lisp @@ -92,16 +92,29 @@ :drop-not-null drop-not-null)))) (defun function-name-character-p (char) - (or (member char #.(quote (coerce "/:.-%" 'list))) + (or (member char #.(quote (coerce "/.-%" 'list))) (alphanumericp char))) -(defrule function-name (* (function-name-character-p character)) - (:text t)) +(defrule function-name (+ (function-name-character-p character)) + (:lambda (fname) + (text fname))) -(defrule cast-function (and kw-using function-name) - (:lambda (function) - (bind (((_ fname) function)) - (intern (string-upcase fname) :pgloader.transforms)))) +(defrule package-and-function-names (and function-name + (or ":" "::") + function-name) + (:lambda (pfn) + (bind (((pname _ fname) pfn)) + (intern (string-upcase fname) (find-package (string-upcase pname)))))) + +(defrule maybe-qualified-function-name (or package-and-function-names + function-name) + (:lambda (fname) + (typecase fname + (string (intern (string-upcase fname) :pgloader.transforms)) + (symbol fname)))) + +(defrule cast-function (and kw-using maybe-qualified-function-name) + (:destructure (using symbol) (declare (ignore using)) symbol)) (defun fix-target-type (source target) "When target has :type nil, steal the source :type definition." diff --git a/src/utils/transforms.lisp b/src/utils/transforms.lisp index 757cded..3552fbb 100644 --- a/src/utils/transforms.lisp +++ b/src/utils/transforms.lisp @@ -69,7 +69,8 @@ byte-vector-to-bytea sqlite-timestamp-to-timestamp sql-server-uniqueidentifier-to-uuid - sql-server-bit-to-boolean)) + sql-server-bit-to-boolean + varbinary-to-string)) ;;; @@ -314,3 +315,10 @@ ((string= "((1))" bit-string-or-integer) "t") (t nil))))) +(defun varbinary-to-string (string) + (let ((babel::*default-character-encoding* + (or qmynd::*mysql-encoding* + babel::*default-character-encoding*))) + (etypecase string + (string string) + (vector (babel:octets-to-string string))))) diff --git a/test/parse/hans.goeuro.load b/test/parse/hans.goeuro.load index d570df8..46c79fa 100644 --- a/test/parse/hans.goeuro.load +++ b/test/parse/hans.goeuro.load @@ -28,7 +28,10 @@ LOAD DATABASE column ip.ip_address to inet keep not null drop typemod, - type decimal when (= precision 1) to boolean drop typemod + type decimal when (= precision 1) to boolean drop typemod, + + -- type varbinary to text drop typemod using babel:octets-to-string + type varbinary to text drop typemod using varbinary-to-string MATERIALIZE VIEWS nonexisting, d as $$