From 957caa877e1850791437d722ddfe687e64f35e47 Mon Sep 17 00:00:00 2001
From: Dimitri Fontaine <dim@tapoueh.org>
Date: Mon, 18 Mar 2019 14:21:33 +0100
Subject: [PATCH] Add a new transform function for byte-vector to hexadecimal
 string.

In some cases when migrating from MySQL we want to transform data from
binary representation to an hexadecimal number. One such case is going from
MySQL binary(16) to PostgreSQL UUID data type.

Fixes #904.
---
 src/utils/transforms.lisp | 29 ++++++++++++++++++++++++++++-
 test/mysql/hex.sql        | 25 +++++++++++++++++++++++++
 test/mysql/my.load        | 10 ++++++++--
 test/mysql/my.sql         | 30 ++++++++++++++++++++++++++++++
 4 files changed, 91 insertions(+), 3 deletions(-)
 create mode 100644 test/mysql/hex.sql

diff --git a/src/utils/transforms.lisp b/src/utils/transforms.lisp
index 0c6961d..d1b6f8d 100644
--- a/src/utils/transforms.lisp
+++ b/src/utils/transforms.lisp
@@ -96,7 +96,8 @@
                  sql-server-bit-to-boolean
                  varbinary-to-string
                  base64-decode
-		 hex-to-dec))
+		 hex-to-dec
+                 byte-vector-to-hexstring))
 
 
 ;;;
@@ -413,6 +414,32 @@
            ((string= "((1))" bit-string-or-integer) "t")
            (t nil)))))
 
+(defun byte-vector-to-hexstring (vector)
+  "Transform binary input received as a vector of bytes into a string of
+  hexadecimal digits, as per the following example:
+
+  Input:   #(136 194 152 47 66 138 70 183 183 27 33 6 24 174 22 88)
+  Output:  88C2982F428A46B7B71B210618AE1658"
+  (declare (type (or null string (simple-array (unsigned-byte 8) (*))) vector))
+  (etypecase vector
+    (null nil)
+    (string (if (string= "" vector)
+                nil
+                (error "byte-vector-to-bytea called on a string: ~s" vector)))
+    (simple-array
+     (let ((hex-digits "0123456789abcdef")
+           (bytea (make-array (* 2 (length vector))
+                              :initial-element #\0
+                              :element-type 'standard-char)))
+
+       (loop for pos from 0 by 2
+          for byte across vector
+          do (let ((high (ldb (byte 4 4) byte))
+                   (low  (ldb (byte 4 0) byte)))
+               (setf (aref bytea pos)       (aref hex-digits high))
+               (setf (aref bytea (+ pos 1)) (aref hex-digits low)))
+          finally (return bytea))))))
+
 (defun varbinary-to-string (string)
   (let ((babel::*default-character-encoding*
          (or qmynd::*mysql-encoding*
diff --git a/test/mysql/hex.sql b/test/mysql/hex.sql
new file mode 100644
index 0000000..cbd430c
--- /dev/null
+++ b/test/mysql/hex.sql
@@ -0,0 +1,25 @@
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `encryption_key_canary` (
+  `encrypted_value` blob,
+  `nonce` tinyblob,
+  `uuid` binary(16) NOT NULL,
+  `salt` tinyblob,
+  PRIMARY KEY (`uuid`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Dumping data for table `encryption_key_canary`
+--
+
+LOCK TABLES `encryption_key_canary` WRITE;
+/*!40000 ALTER TABLE `encryption_key_canary` DISABLE KEYS */;
+INSERT INTO `encryption_key_canary` VALUES (
+  0x1F36F183D7EE47C71453850B756945C16D9D711B2F0594E5D5E54D1EC94E081716AB8642AA60F84B50F69454D098122B7136A0DEB3AF200C2C5C7500BDFA0BD9689CCBF10A76972374882B304F7F15A227E815989FC87EEB72612396F569C662E72A2A7555E654605A3B83C1C753297832E52C5961E81EBC60DC43D929ABAB8CB14601DEFED121604CEB26210AB6D724,
+  0x044AA707DF17021E55E9A1E4,
+  0x88C2982F428A46B7B71B210618AE1658,
+  0xAE7F18028E7984FB5630F7D23FB77999C6CA7CF5355EF0194F3F16521EA7EC503F566229ED8DC5EFBBE9C12BA491BDDC939FE60FA31FB9AF123B2B4D5B7A61FE
+);
+/*!40000 ALTER TABLE `encryption_key_canary` ENABLE KEYS */;
+UNLOCK TABLES;
diff --git a/test/mysql/my.load b/test/mysql/my.load
index 158ed95..f0f308f 100644
--- a/test/mysql/my.load
+++ b/test/mysql/my.load
@@ -11,6 +11,8 @@ load database
  ALTER SCHEMA 'pgloader' RENAME TO 'mysql'
  ALTER TABLE NAMES MATCHING ~/./ SET TABLESPACE 'pg_default'
 
+ INCLUDING ONLY TABLE NAMES MATCHING 'encryption_key_canary'
+
  CAST column utilisateurs__Yvelines2013-06-28.sexe
           to text drop not null using empty-string-to-null,
 
@@ -31,6 +33,10 @@ load database
              using zero-dates-to-null,
 
       type timestamp with extra on update current timestamp
-        to "timestamp with time zone" drop extra
+        to "timestamp with time zone" drop extra,
 
- BEFORE LOAD DO $$ create schema if not exists mysql; $$;
+      column encryption_key_canary.uuid
+          to uuid drop typemod using byte-vector-to-hexstring
+
+ BEFORE LOAD DO
+   $$ create schema if not exists mysql; $$;
diff --git a/test/mysql/my.sql b/test/mysql/my.sql
index a9daefd..439ea83 100644
--- a/test/mysql/my.sql
+++ b/test/mysql/my.sql
@@ -118,6 +118,36 @@ CREATE TABLE `domain_filter` (
   KEY `domain_filter` (`type`)
 ) ENGINE=InnoDB DEFAULT CHARSET=ascii;
 
+/*
+ * https://github.com/dimitri/pgloader/issues/904
+ */
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `encryption_key_canary` (
+  `encrypted_value` blob,
+  `nonce` tinyblob,
+  `uuid` binary(16) NOT NULL,
+  `salt` tinyblob,
+  PRIMARY KEY (`uuid`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Dumping data for table `encryption_key_canary`
+--
+
+LOCK TABLES `encryption_key_canary` WRITE;
+/*!40000 ALTER TABLE `encryption_key_canary` DISABLE KEYS */;
+INSERT INTO `encryption_key_canary` VALUES (
+  0x1F36F183D7EE47C71453850B756945C16D9D711B2F0594E5D5E54D1EC94E081716AB8642AA60F84B50F69454D098122B7136A0DEB3AF200C2C5C7500BDFA0BD9689CCBF10A76972374882B304F7F15A227E815989FC87EEB72612396F569C662E72A2A7555E654605A3B83C1C753297832E52C5961E81EBC60DC43D929ABAB8CB14601DEFED121604CEB26210AB6D724,
+  0x044AA707DF17021E55E9A1E4,
+  0x88C2982F428A46B7B71B210618AE1658,
+  0xAE7F18028E7984FB5630F7D23FB77999C6CA7CF5355EF0194F3F16521EA7EC503F566229ED8DC5EFBBE9C12BA491BDDC939FE60FA31FB9AF123B2B4D5B7A61FE
+);
+/*!40000 ALTER TABLE `encryption_key_canary` ENABLE KEYS */;
+UNLOCK TABLES;
+
+
 /*
  * https://github.com/dimitri/pgloader/issues/703
  */