From d8b0bd51455a18760f1aabd7244aa5e9078c644a Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Tue, 4 Jun 2019 15:11:42 +0200 Subject: [PATCH] Allow casting rules to guard on signed data types. It used to be that our casting rules mechanism would allow for matching unsigned data types only, and we sometimes have a need to do special behavior on signed data types. In particular, a signed bigint(20) in MySQL has the same values range as a PostgreSQL bigint, so we don't need to target a numeric in that case. It's only when the bigint is unsigned that we need to target a numeric. In passing update some of the default casting rules documentation to match the code. Fix #982. --- docs/ref/mysql.rst | 22 +++++++++++++++------- src/parsers/command-cast-rules.lisp | 6 ++++++ src/parsers/command-keywords.lisp | 1 + src/sources/common/casting-rules.lisp | 2 ++ src/sources/mysql/mysql-cast-rules.lisp | 10 +++++++--- test/mysql/my.load | 3 +++ 6 files changed, 34 insertions(+), 10 deletions(-) diff --git a/docs/ref/mysql.rst b/docs/ref/mysql.rst index e20bfbe..e830c9d 100644 --- a/docs/ref/mysql.rst +++ b/docs/ref/mysql.rst @@ -567,6 +567,12 @@ Numbers:: type tinyint to boolean when (= 1 precision) using tinyint-to-boolean + type bit when (= 1 precision) to boolean drop typemod using bits-to-boolean + type bit to bit drop typemod using bits-to-hex-bitstring + + type bigint when signed to bigint drop typemod + type bigint when (< 19 precision) to numeric drop typemod + type tinyint when unsigned to smallint drop typemod type smallint when unsigned to integer drop typemod type mediumint when unsigned to integer drop typemod @@ -595,12 +601,12 @@ Texts:: Binary:: - type binary to bytea - type varbinary to bytea - type tinyblob to bytea - type blob to bytea - type mediumblob to bytea - type longblob to bytea + type binary to bytea using byte-vecotr-to-bytea + type varbinary to bytea using byte-vecotr-to-bytea + type tinyblob to bytea using byte-vecotr-to-bytea + type blob to bytea using byte-vecotr-to-bytea + type mediumblob to bytea using byte-vecotr-to-bytea + type longblob to bytea using byte-vecotr-to-bytea Date:: @@ -638,7 +644,9 @@ Date:: Geometric:: - type point to point using pgloader.transforms::convert-mysql-point + type geometry to point using convert-mysql-point + type point to point using convert-mysql-point + type linestring to path using convert-mysql-linestring Enum types are declared inline in MySQL and separately with a `CREATE TYPE` command in PostgreSQL, so each column of Enum Type is converted to a type diff --git a/src/parsers/command-cast-rules.lisp b/src/parsers/command-cast-rules.lisp index eb200b3..509c142 100644 --- a/src/parsers/command-cast-rules.lisp +++ b/src/parsers/command-cast-rules.lisp @@ -13,6 +13,9 @@ (defrule cast-unsigned-guard (and kw-when kw-unsigned) (:constant (cons :unsigned t))) +(defrule cast-signed-guard (and kw-when kw-signed) + (:constant (cons :signed t))) + ;; at the moment we only know about extra auto_increment (defrule cast-source-extra (and kw-with kw-extra (or kw-auto-increment @@ -35,6 +38,7 @@ (:destructure (kw name) (declare (ignore kw)) name)) (defrule cast-source-extra-or-guard (* (or cast-unsigned-guard + cast-signed-guard cast-default-guard cast-typemod-guard cast-source-extra)) @@ -46,6 +50,7 @@ (bind (((name-and-type extra-and-guards) source) ((&key (default nil d-s-p) (typemod nil t-s-p) + (signed nil s-s-p) (unsigned nil u-s-p) (auto-increment nil ai-s-p) (on-update-current-timestamp nil ouct-s-p) @@ -54,6 +59,7 @@ `(,@name-and-type ,@(when t-s-p (list :typemod typemod)) ,@(when d-s-p (list :default default)) + ,@(when s-s-p (list :signed signed)) ,@(when u-s-p (list :unsigned unsigned)) ,@(when ai-s-p (list :auto-increment auto-increment)) ,@(when ouct-s-p (list :on-update-current-timestamp diff --git a/src/parsers/command-keywords.lisp b/src/parsers/command-keywords.lisp index 84d8bb3..d267288 100644 --- a/src/parsers/command-keywords.lisp +++ b/src/parsers/command-keywords.lisp @@ -143,6 +143,7 @@ (def-keyword-rule "per") (def-keyword-rule "thread") (def-keyword-rule "range") + (def-keyword-rule "signed") (def-keyword-rule "unsigned") ;; option for loading from an archive (def-keyword-rule "archive") diff --git a/src/sources/common/casting-rules.lisp b/src/sources/common/casting-rules.lisp index 1c4c5a9..f9bfead 100644 --- a/src/sources/common/casting-rules.lisp +++ b/src/sources/common/casting-rules.lisp @@ -28,6 +28,7 @@ ((:column rule-source-column) nil c-s-p) ((:typemod typemod-expr) nil tm-s-p) ((:default rule-source-default) nil d-s-p) + ((:signed rule-signed) nil s-s-p) ((:unsigned rule-unsigned) nil u-s-p) ((:not-null rule-source-not-null) nil n-s-p) ((:auto-increment rule-source-auto-increment)) @@ -61,6 +62,7 @@ (or (null tm-s-p) (when typemod (typemod-expr-matches-p typemod-expr typemod))) (or (null d-s-p) (string-equal default rule-source-default)) + (or (null s-s-p) (eq unsigned (not rule-signed))) (or (null u-s-p) (eq unsigned rule-unsigned)) (or (null n-s-p) (eq not-null rule-source-not-null)) diff --git a/src/sources/mysql/mysql-cast-rules.lisp b/src/sources/mysql/mysql-cast-rules.lisp index a219dff..6cb27f9 100644 --- a/src/sources/mysql/mysql-cast-rules.lisp +++ b/src/sources/mysql/mysql-cast-rules.lisp @@ -46,10 +46,14 @@ :target (:type "bit" :drop-typemod nil) :using pgloader.transforms::bits-to-hex-bitstring) - ;; bigint(20) unsigned (or not, actually) does not fit into PostgreSQL - ;; bigint (-9223372036854775808 to +9223372036854775807): + ;; bigint(20) signed do fit into PostgreSQL bigint + ;; (-9223372036854775808 to +9223372036854775807): + (:source (:type "bigint" :signed t) + :target (:type "bigint" :drop-typemod t)) + + ;; bigint(20) unsigned does not fit into PostgreSQL bigint (:source (:type "bigint" :typemod (< 19 precision)) - :target (:type "numeric" :drop-typemod t)) + :target (:type "numeric" :drop-typemod t)) ;; now unsigned types (:source (:type "tinyint" :unsigned t) diff --git a/test/mysql/my.load b/test/mysql/my.load index e67d603..e4aa4c9 100644 --- a/test/mysql/my.load +++ b/test/mysql/my.load @@ -19,6 +19,9 @@ load database column base64.id to uuid drop typemod, column base64.data to jsonb using base64-decode, + -- This is now a default casting rule for MySQL + -- type bigint when signed to bigint drop typemod, + type decimal when (and (= 18 precision) (= 6 scale)) to "double precision" drop typemod,