From 63005168fd16fffbfc476e8b58306bc416b2b959 Mon Sep 17 00:00:00 2001
From: Dimitri Fontaine <dim@tapoueh.org>
Date: Fri, 1 Nov 2013 11:34:43 +0000
Subject: [PATCH] MySQL SET datatype support: convert to ENUM Arrays.

---
 src/mysql-cast-rules.lisp | 32 +++++++++++++++++++++++---------
 src/mysql.lisp            |  5 +++--
 src/transforms.lisp       |  7 ++++++-
 test/archive.load         |  8 ++++----
 4 files changed, 36 insertions(+), 16 deletions(-)

diff --git a/src/mysql-cast-rules.lisp b/src/mysql-cast-rules.lisp
index fa4cbe9..ca0a7b0 100644
--- a/src/mysql-cast-rules.lisp
+++ b/src/mysql-cast-rules.lisp
@@ -34,6 +34,13 @@
   (declare (ignore type ctype typemod))
   (format nil "~a_~a" table-name column-name))
 
+(defun cast-set (table-name column-name type ctype typemod)
+  "Cast MySQL inline SET type to using a PostgreSQL ENUM Array.
+
+   The ENUM data type name is hardcoded to be table-name_column-name"
+  (declare (ignore type ctype typemod))
+  (format nil "~a_~a[]" table-name column-name))
+
 ;;;
 ;;; The default MySQL Type Casting Rules
 ;;;
@@ -106,9 +113,14 @@
     (:source (:type "timestamp") :target (:type "timestamptz"))
     (:source (:type "year")      :target (:type "integer"))
 
+    ;; Inline MySQL "interesting" datatype
     (:source (:type "enum")
      :target (:type ,#'cast-enum))
 
+    (:source (:type "set")
+     :target (:type ,#'cast-set)
+     :using pgloader.transforms::set-to-enum-array)
+
     ;; geometric data types, just POINT for now
     (:source (:type "point")
      :target (:type "point")
@@ -125,7 +137,8 @@
 
    Beware that some data-type are using a typmod looking definition for
    things that are not typmods at all: enum."
-  (unless (string= "enum" data-type)
+  (unless (or (string= "enum" data-type)
+	      (string= "set" data-type))
     (let ((start-1 (position #\( column-type))	; just before start position
 	  (end     (position #\) column-type)))	; just before end position
       (when start-1
@@ -313,14 +326,15 @@ that would be int and int(7) or varchar and varchar(25)."
 	   ("e"  "datetime"  "datetime"    "0000-00-00 00:00:00" nil nil)
 	   ("f"  "date"      "date"        "0000-00-00" "NO" nil)
 	   ("g"  "enum"      "ENUM('a', 'b')"  nil nil nil)
-	   ("h"  "int"       "int(11)"         nil nil nil)
-	   ("i"  "float"     "float(12,2)"     nil nil nil)
-	   ("j"  "double"    "double unsigned" nil nil nil)
-	   ("k"  "bigint"    "bigint(20)"      nil nil nil)
-	   ("l"  "numeric"   "numeric(18,3)"   nil nil nil)
-	   ("m"  "decimal"   "decimal(15,5)"   nil nil nil)
-	   ("n"  "timestamp" "timestamp" "CURRENT_TIMESTAMP" "NO" "on update CURRENT_TIMESTAMP")
-	   ("o"  "point"     "point"     nil "YES" nil))))
+	   ("h"  "set"       "SET('a', 'b')"   nil nil nil)
+	   ("i"  "int"       "int(11)"         nil nil nil)
+	   ("j"  "float"     "float(12,2)"     nil nil nil)
+	   ("k"  "double"    "double unsigned" nil nil nil)
+	   ("l"  "bigint"    "bigint(20)"      nil nil nil)
+	   ("m"  "numeric"   "numeric(18,3)"   nil nil nil)
+	   ("n"  "decimal"   "decimal(15,5)"   nil nil nil)
+	   ("o"  "timestamp" "timestamp" "CURRENT_TIMESTAMP" "NO" "on update CURRENT_TIMESTAMP")
+	   ("p"  "point"     "point"     nil "YES" nil))))
 
     (loop
        for (name dtype ctype nullable default extra) in columns
diff --git a/src/mysql.lisp b/src/mysql.lisp
index ecbfc55..e901ee5 100644
--- a/src/mysql.lisp
+++ b/src/mysql.lisp
@@ -101,7 +101,8 @@ order by table_name, ordinal_position" dbname only-tables)))
   "MySQL declares ENUM types inline, PostgreSQL wants explicit CREATE TYPE."
   (loop
      for (name dtype ctype default nullable extra) in cols
-     when (string-equal "enum" dtype)
+     when (or (string-equal "enum" dtype)
+	      (string-equal "set" dtype)) ; a SET is an Array of ENUMs
      collect (when include-drop
 	       (let* ((type-name
 		       (get-enum-type-name table-name name identifier-case)))
@@ -520,7 +521,7 @@ order by ordinal_position" dbname table-name)))
 					    :identifier-case identifier-case)
        do
 	 (log-message :notice "~a" sql)
-	 (lp:submit-task channel #'psql-execute-with-timing
+	 (lp:submit-task channel #'pgsql-execute-with-timing
 			 dbname label sql state))))
 
 (defun stream-database (dbname
diff --git a/src/transforms.lisp b/src/transforms.lisp
index 8d06bec..2595ae4 100644
--- a/src/transforms.lisp
+++ b/src/transforms.lisp
@@ -18,7 +18,8 @@
 		 int-to-ip
 		 ip-range
 		 convert-mysql-point
-		 float-to-string))
+		 float-to-string
+		 set-to-enum-array))
 
 (defun intern-symbol (symbol-name)
   (intern (string-upcase symbol-name)
@@ -104,3 +105,7 @@
   (declare (type float float))
   (let ((*read-default-float-format* 'double-float))
     (princ-to-string float)))
+
+(defun set-to-enum-array (set-string)
+  "Transform a MySQL SET value into a PostgreSQL ENUM Array"
+  (format nil "{~a}" set-string))
diff --git a/test/archive.load b/test/archive.load
index 05f6846..b2c0f9c 100644
--- a/test/archive.load
+++ b/test/archive.load
@@ -8,8 +8,8 @@
  */
 
 LOAD ARCHIVE
-   FROM http://pgsql.tapoueh.org/temp/foo.zip
-   INTO postgresql:///pgloader
+   FROM /Users/dim/Downloads/GeoLiteCity-latest.zip
+   INTO postgresql:///ip4r
 
    BEFORE LOAD DO
      $$ create extension if not exists ip4r; $$,
@@ -49,7 +49,7 @@ LOAD ARCHIVE
                 metroCode  null if blanks,
                 areaCode   null if blanks
              )
-        INTO postgresql:///pgloader?geolite.location
+        INTO postgresql:///ip4r?geolite.location
              (
                 locid,country,region,city,postalCode,
                 location point using (format nil "(~a,~a)" longitude latitude),
@@ -66,7 +66,7 @@ LOAD ARCHIVE
              (
                 startIpNum, endIpNum, locId
              )
-        INTO postgresql:///pgloader?geolite.blocks
+        INTO postgresql:///ip4r?geolite.blocks
              (
                 iprange ip4r using (ip-range startIpNum endIpNum),
                 locId