Fix more table name quoting, fix #163 again.

Now that we can have several threads doing COPY, each of them need to
know about the *pgsql-reserved-keywords* list. Make sure that's the case
and in passing fix some call sites to apply-identifier-case.

Also, more disturbingly, fix the code so that TRUNCATE is called from
the main thread before giving control to the COPY threads, rather than
having two concurrent threads doing the TRUNCATE twice. It's rather
strange that we got no complaint from the field on that part...

src/main.lisp

@@ -407,23 +407,21 @@
 
     ;; once we are done running the load-file, compare the loaded data with
     ;; our expected data file
-    (let* ((expected-subdir    (directory-namestring
+    (bind ((expected-subdir        (directory-namestring
-                                (asdf:system-relative-pathname
+                                    (asdf:system-relative-pathname
-                                 :pgloader "test/regress/expected/")))
+                                     :pgloader "test/regress/expected/")))
-           (expected-data-file (make-pathname :defaults load-file
+           (expected-data-file     (make-pathname :defaults load-file
-                                              :type "out"
+                                                  :type "out"
-                                              :directory expected-subdir)))
+                                                  :directory expected-subdir))
-      (destructuring-bind (target *pg-settings*)
+           ((target *pg-settings*) (parse-target-pg-db-uri load-file))
-          (parse-target-pg-db-uri load-file)
+           (*pgsql-reserved-keywords* (list-reserved-keywords target))
-       (log-message :log "Comparing loaded data against ~s"
-                    expected-data-file)
 
-       (let ((expected-data-source
+           (expected-data-source
               (parse-source-string-for-type
                :copy (uiop:native-namestring expected-data-file)))
 
-             ;; change target table-name schema
+           ;; change target table-name schema
-             (expected-data-target
+           (expected-data-target
               (let ((e-d-t (clone-connection target)))
                 (setf (pgconn-table-name e-d-t)
                       (cons "expected"
@@ -432,29 +430,33 @@
                               (cons   (cdr (pgconn-table-name e-d-t))))))
                 e-d-t)))
 
-         ;; prepare expected table in "expected" schema
+      (log-message :log "Comparing loaded data against ~s" expected-data-file)
-         (with-pgsql-connection (target)
-           (with-schema (unqualified-table-name (pgconn-table-name target))
-             (let ((drop   (format nil "drop table if exists expected.~a;"
-                                   unqualified-table-name))
-                   (create (format nil "create table expected.~a(like ~a);"
-                                   unqualified-table-name unqualified-table-name)))
-               (log-message :notice "~a" drop)
-               (pomo:query drop)
-               (log-message :notice "~a" create)
-               (pomo:query create))))
 
-         ;; load expected data
+      ;; prepare expected table in "expected" schema
-         (load-data :from expected-data-source
+      (with-pgsql-connection (target)
-                    :into expected-data-target
+        (with-schema (unqualified-table-name (pgconn-table-name target))
-                    :options '(:truncate t)
+          (let* ((tname  (apply-identifier-case unqualified-table-name))
-                    :start-logger nil))
+                 (drop   (format nil "drop table if exists expected.~a;"
+                                 tname))
+                 (create (format nil "create table expected.~a(like ~a);"
+                                 tname tname)))
+            (log-message :notice "~a" drop)
+            (pomo:query drop)
+            (log-message :notice "~a" create)
+            (pomo:query create))))
+
+      ;; load expected data
+      (load-data :from expected-data-source
+                 :into expected-data-target
+                 :options '(:truncate t)
+                 :start-logger nil)
 
        ;; now compare both
        (with-pgsql-connection (target)
          (with-schema (unqualified-table-name (pgconn-table-name target))
-           (let* ((cols (loop :for (name type)
+           (let* ((tname  (apply-identifier-case unqualified-table-name))
-                           :in (list-columns-query unqualified-table-name)
+                  (cols (loop :for (name type)
+                           :in (list-columns-query tname)
                            ;;
                            ;; We can't just use table names here, because
                            ;; PostgreSQL support for the POINT datatype fails
@@ -469,9 +471,9 @@
                   (sql  (format nil
                                 "select count(*) from (select ~{~a~^, ~} from ~a except select ~{~a~^, ~} from expected.~a) ss"
                                 cols
-                                unqualified-table-name
+                                tname
                                 cols
-                                unqualified-table-name))
+                                tname))
                   (diff-count (pomo:query sql :single)))
              (log-message :notice "~a" sql)
              (log-message :notice "Got a diff of ~a rows" diff-count)
@@ -483,7 +485,7 @@
                  (progn
                    (log-message :log "Regress fail.")
                    #-pgloader-image (values diff-count +os-code-error-regress+)
-                   #+pgloader-image (uiop:quit +os-code-error-regress+))))))))))
+                   #+pgloader-image (uiop:quit +os-code-error-regress+)))))))))
 
 
 ;;;

src/package.lisp

@@ -601,7 +601,7 @@
 (defpackage #:pgloader
   (:use #:cl
         #:pgloader.params #:pgloader.utils #:pgloader.parser
-        #:pgloader.connection)
+        #:pgloader.connection #:metabang.bind)
   (:import-from #:pgloader.pgsql
                 #:pgconn-table-name
                 #:pgsql-connection
@@ -611,7 +611,9 @@
 		#:list-columns-query)
   (:import-from #:pgloader.pgsql
                 #:with-pgsql-connection
-                #:with-schema)
+                #:with-schema
+                #:list-reserved-keywords
+                #:apply-identifier-case)
   (:export #:*version-string*
 	   #:*state*
 	   #:*fd-path-root*

src/params.lisp

@@ -15,6 +15,7 @@
 	   #:*client-min-messages*
 	   #:*log-min-messages*
            #:*report-stream*
+           #:*pgsql-reserved-keywords*
            #:*identifier-case*
            #:*preserve-index-names*
 	   #:*copy-batch-rows*
@@ -94,6 +95,9 @@
 ;;;
 ;;; When converting from other databases, how to deal with case sensitivity?
 ;;;
+(defvar *pgsql-reserved-keywords* nil
+  "We need to always quote PostgreSQL reserved keywords")
+
 (defparameter *identifier-case* :downcase
   "Dealing with source databases casing rules.")
 

src/pgsql/pgsql.lisp

@@ -48,9 +48,6 @@
    *writer-batch* for us to send down to PostgreSQL, and when that's done
    update stats."
   (let ((seconds 0))
-    (when truncate
-      (truncate-tables pgconn (list table-name)))
-
     (with-pgsql-connection (pgconn)
       (with-schema (unqualified-table-name table-name)
         (with-disabled-triggers (unqualified-table-name
@@ -63,7 +60,8 @@
              :until (eq :end-of-data mesg)
              :for (rows batch-seconds) :=
              (let ((start-time (get-internal-real-time)))
-               (list (copy-batch unqualified-table-name columns batch read)
+               (list (copy-batch (apply-identifier-case unqualified-table-name)
+                                 columns batch read)
                      (elapsed-time-since start-time)))
              :do (progn
                    ;; The SBCL implementation needs some Garbage Collection

src/pgsql/schema.lisp

@@ -3,9 +3,6 @@
 ;;;
 (in-package pgloader.pgsql)
 
-(defvar *pgsql-reserved-keywords* nil
-  "We need to always quote PostgreSQL reserved keywords")
-
 (defun quoted-p (s)
   "Return true if s is a double-quoted string"
   (and (eq (char s 0) #\")

src/sources/common/methods.lisp

@@ -120,6 +120,13 @@
           ;; http://www.anarazel.de/talks/pgconf-eu-2015-10-30/concurrency.pdf
           ;;
           ;; Let's just hardcode 2 threads for that then.
+          ;;
+          ;; Also, we need to do the TRUNCATE here before starting the
+          ;; threads, so that it's done just once.
+          (when truncate
+            (truncate-tables (clone-connection (target-db copy))
+                             (list (target copy))))
+
           (loop :for w :below 2
              :do (lp:submit-task channel
                                  #'pgloader.pgsql:copy-from-queue
@@ -127,7 +134,6 @@
                                  (target copy)
                                  fmtq
                                  :columns (copy-column-list copy)
-                                 :truncate truncate
                                  :disable-triggers disable-triggers))
 
           ;; now wait until both the tasks are over, and kill the kernel

src/utils/threads.lisp

@@ -17,6 +17,7 @@
                             (*log-min-messages*    . ,*log-min-messages*)
 
                             ;; needed in create index specific kernels
+                            (*pgsql-reserved-keywords* . ',*pgsql-reserved-keywords*)
                             (*preserve-index-names* . ,*preserve-index-names*)
 
                             ;; bindings updates for libs

test/csv-header.load

@@ -1,14 +1,14 @@
 LOAD CSV
      FROM INLINE
-     INTO postgresql:///pgloader?public.header
+     INTO postgresql:///pgloader?"group"
 
      WITH truncate,
           fields terminated by ',',
           csv header
 
    BEFORE LOAD DO
-    $$ drop table if exists header; $$,
+    $$ drop table if exists "group"; $$,
-    $$ CREATE TABLE header
+    $$ CREATE TABLE "group"
        (
           somefields     text,
           rekplcode      text,