From 4f2385fa4c75840ff846ebc9dcfe91a9ea6c5370 Mon Sep 17 00:00:00 2001
From: Dimitri Fontaine <dim@tapoueh.org>
Date: Thu, 16 Jul 2015 19:35:34 +0200
Subject: [PATCH] Refactor code from previous commit.

The goal is to make it easy to add support for the 'drop indexes' option
in other source types (fixed, ixf, db3, file-based sources).
---
 src/package.lisp         |  9 ++++----
 src/pgsql/schema.lisp    | 46 ++++++++++++++++++++++++++++++++++++++++
 src/sources/csv/csv.lisp | 46 ++++++----------------------------------
 3 files changed, 56 insertions(+), 45 deletions(-)

diff --git a/src/package.lisp b/src/package.lisp
index dad8b6a..6f00fa1 100644
--- a/src/package.lisp
+++ b/src/package.lisp
@@ -201,6 +201,8 @@
 	   #:create-indexes-in-kernel
            #:set-table-oids
            #:drop-indexes
+           #:maybe-drop-indexes
+           #:create-indexes-again
            #:reset-sequences))
 
 (defpackage #:pgloader.queue
@@ -247,11 +249,8 @@
         #:pgloader.params #:pgloader.utils #:pgloader.connection
         #:pgloader.sources #:pgloader.queue)
   (:import-from #:pgloader.pgsql
-                #:list-indexes
-                #:drop-indexes
-                #:with-pgsql-connection
-                #:pgsql-execute-with-timing
-                #:create-indexes-in-kernel)
+                #:maybe-drop-indexes
+                #:create-indexes-again)
   (:export #:*csv-path-root*
            #:csv-connection
            #:specs
diff --git a/src/pgsql/schema.lisp b/src/pgsql/schema.lisp
index 4cf22dc..26d4543 100644
--- a/src/pgsql/schema.lisp
+++ b/src/pgsql/schema.lisp
@@ -380,6 +380,52 @@
            (log-message :notice "~a" sql)
            (pgsql-execute-with-timing "drop indexes" sql state))))
 
+;;;
+;;; Higher level API to care about indexes
+;;;
+(defun maybe-drop-indexes (target table-name state &key drop-indexes)
+  "Drop the indexes for TABLE-NAME on TARGET PostgreSQL connection, and
+   returns a list of indexes to create again."
+  (with-pgsql-connection (target)
+    (let ((indexes (list-indexes table-name)))
+      (cond ((and indexes (not drop-indexes))
+             (log-message :warning
+                          "Target table ~s has ~d indexes defined against it."
+                          table-name (length indexes))
+             (log-message :warning
+                          "That could impact loading performance badly.")
+             (log-message :warning
+                          "Consider the option 'drop indexes'."))
+
+            (indexes
+             ;; drop the indexes now
+             (with-stats-collection ("drop indexes" :state state)
+                 (drop-indexes state indexes))))
+
+      ;; and return the indexes list
+      indexes)))
+
+(defun create-indexes-again (target indexes state &key drop-indexes)
+  "Create the indexes that we dropped previously."
+  (when (and indexes drop-indexes)
+    (let* ((idx-kernel  (make-kernel (length indexes)))
+           (idx-channel (let ((lp:*kernel* idx-kernel))
+                          (lp:make-channel))))
+      (let ((pkeys
+             (create-indexes-in-kernel target indexes idx-kernel idx-channel
+                                       :state state)))
+
+        (with-stats-collection ("Index Build Completion" :state *state*)
+            (loop :for idx :in indexes :do (lp:receive-result idx-channel)))
+
+        ;; turn unique indexes into pkeys now
+        (with-pgsql-connection (target)
+          (with-stats-collection ("Primary Keys" :state state)
+              (loop :for sql :in pkeys
+                 :when sql
+                 :do (progn
+                       (log-message :notice "~a" sql)
+                       (pgsql-execute-with-timing "Primary Keys" sql state)))))))))
 
 ;;;
 ;;; Sequences
diff --git a/src/sources/csv/csv.lisp b/src/sources/csv/csv.lisp
index 9779528..f607b0c 100644
--- a/src/sources/csv/csv.lisp
+++ b/src/sources/csv/csv.lisp
@@ -186,23 +186,10 @@
 	 (lp:*kernel*    (make-kernel 2))
 	 (channel        (lp:make-channel))
 	 (queue          (lq:make-queue :fixed-capacity *concurrent-batches*))
-         indexes)
-
-    ;; issue a performance warning against pre-existing indexes
-    (with-pgsql-connection ((target-db csv))
-      (setf indexes (list-indexes (target csv)))
-      (cond ((and indexes (not drop-indexes))
-             (log-message :warning
-                          "Target table ~s has ~d indexes defined against it."
-                          (target csv) (length indexes))
-             (log-message :warning "That could impact loading performance badly"))
-
-            (indexes
-
-             ;; drop the indexes now
-             (with-stats-collection ("drop indexes" :state state-before
-                                                    :summary summary)
-                 (drop-indexes state-before indexes)))))
+         (indexes        (maybe-drop-indexes (target-db csv)
+                                             (target csv)
+                                             state-before
+                                             :drop-indexes drop-indexes)))
 
     (with-stats-collection ((target csv)
                             :dbname (db-name (target-db csv))
@@ -229,26 +216,5 @@
            finally (lp:end-kernel))))
 
     ;; re-create the indexes
-    (when (and indexes drop-indexes)
-      (let* ((idx-kernel  (make-kernel (length indexes)))
-             (idx-channel (let ((lp:*kernel* idx-kernel))
-                            (lp:make-channel))))
-        (let ((pkeys
-               (create-indexes-in-kernel (target-db csv)
-                                         indexes
-                                         idx-kernel
-                                         idx-channel
-                                         :state state-after)))
-          (with-stats-collection ("Index Build Completion" :state *state*)
-              (loop :for idx :in indexes :do (lp:receive-result idx-channel)))
-
-          ;; turn unique indexes into pkeys now
-          (with-pgsql-connection ((target-db csv))
-            (with-stats-collection ("Primary Keys" :state state-after)
-                (loop :for sql :in pkeys
-                   :when sql
-                   :do (progn
-                         (log-message :notice "~a" sql)
-                         (pgsql-execute-with-timing "Primary Keys"
-                                                    sql
-                                                    state-after))))))))))
+    (create-indexes-again (target-db csv) indexes state-after
+                          :drop-indexes drop-indexes)))