Fix the logging system, we now have a proper logfile.

src/main.lisp

@@ -17,7 +17,7 @@
     (("version" #\V) :type boolean
      :documentation "Displays pgloader version and exit.")
 
-    (("quiet" #\q)   :type boolean :documentation "Be quiet")
+    (("quiet"   #\q) :type boolean :documentation "Be quiet")
     (("verbose" #\v) :type boolean :documentation "Be verbose")
     (("debug"   #\d) :type boolean :documentation "Diplay debug level information.")
 
@@ -27,6 +27,9 @@
     (("list-encodings" #\E) :type boolean
      :documentation "List pgloader known encodings and exit.")
 
+    (("logfile" #\L) :type string :initial-value "/tmp/pgloader/pgloader.log"
+     :documentation "Filename where to send the logs.")
+
     (("load" #\l) :type string :list t :optional t
      :documentation "Read user code from file")))
 
@@ -36,7 +39,7 @@
     (multiple-value-bind (options arguments)
 	(command-line-arguments:process-command-line-options *opt-spec* args)
 
-      (destructuring-bind (&key help version quiet verbose debug
+      (destructuring-bind (&key help version quiet verbose debug logfile
 				list-encodings upgrade-config load)
 	  options
 
@@ -59,26 +62,24 @@
 	       (format t "~%~%"))
 	  (uiop:quit))
 
-	(setf *client-min-messages* (cond (debug   :debug)
-					  (verbose :info)
-					  (quiet   :warning)
-					  (t       :notice)))
-
 	(when load
 	  (loop for filename in load
 	     do (load (compile-file filename :verbose nil :print nil))))
 
 	(when arguments
-	  ;; Start the logger
-	  (start-logger)
-
 	  ;; process the files
 	  (handler-case
-	      (loop for filename in arguments
+	      (let ((min-messages (cond (debug   :debug)
-		 do
+					(verbose :info)
-		   (log-message :notice "Processing ~s~%" filename)
+					(quiet   :warning)
-		   (run-commands filename)
+					(t       :notice))))
-		   (format t "~&"))
+		(loop for filename in arguments
+		   do
+		     (run-commands filename
+				   :log-filename logfile
+				   :log-min-messages min-messages
+				   :client-min-messages min-messages)
+		     (format t "~&")))
 	    (condition (e)
 	      (if debug
 		  (trivial-backtrace:print-backtrace e :verbose t)

src/package.lisp

@@ -15,6 +15,7 @@
 		#:text-stream-messenger
 		#:formatted-message)
   (:export #:start-logger
+	   #:stop-logger
 	   #:log-message
 	   #:report-header
 	   #:report-table-name

src/parser.lisp

@@ -1718,6 +1718,8 @@ load database
   "The command could be using from :inline, in which case we want to parse
    as much as possible then use the command against an already opened stream
    where we moved at the beginning of the data."
+  (log-message :notice "Parsing commands from file '~s'~%" filename)
+
   (let ((*data-expected-inline* nil)
 	(content (slurp-file-into-string filename)))
     (multiple-value-bind (commands end-commands-position)
@@ -1753,10 +1755,17 @@ load database
 
 (defun run-commands (source
 		     &key
+		       ((:log-filename *log-filename*) *log-filename*)
+		       ((:log-min-messages *log-min-messages*) *log-min-messages*)
 		       ((:client-min-messages *client-min-messages*) *client-min-messages*))
   "SOURCE can be a function, which is run, a list, which is compiled as CL
    code then run, a pathname containing one or more commands that are parsed
    then run, or a commands string that is then parsed and each command run."
+
+  (start-logger :log-filename *log-filename*
+		:log-min-messages *log-min-messages*
+		:client-min-messages *client-min-messages*)
+
   (let* ((funcs
 	  (typecase source
 	    (function (list source))
@@ -1772,7 +1781,10 @@ load database
 				  (parse-commands source)))))))
 
     ;; run the commands
-    (loop for func in funcs do (funcall func))))
+    (loop for func in funcs do (funcall func))
+
+    ;; close the logs
+    (stop-logger)))
 
 
 ;;;

src/sources/mysql.lisp

@@ -238,6 +238,10 @@
     ;; if asked, first drop/create the tables on the PostgreSQL side
     (when (and (or create-tables schema-only) (not data-only))
       (log-message :notice "~:[~;DROP then ~]CREATE TABLES" include-drop)
+      (log-message :debug  (if include-drop
+			       "drop then create ~d tables with ~d indexes."
+			       "create ~d tables with ~d indexes.")
+		   (length all-columns) (length all-indexes))
       (with-stats-collection (pg-dbname "create, drop"
 					:use-result-as-rows t
 					:state state-before)

src/utils.lisp

@@ -8,31 +8,50 @@
 ;;;
 ;;; First define the log categories
 (defcategory :critical)
-(defcategory :error   (or :error :critical))
+(defcategory :log     (or :log :critical))
+(defcategory :error   (or :error :log))
 (defcategory :warning (or :warning :error))
 (defcategory :notice  (or :notice :warning))
 (defcategory :info    (or :info :notice))
 (defcategory :debug   (or :debug :info))
 
-;; Now define the Logger
+(defvar *log-messengers* nil
-(setf (log-manager)
+  "Currently active log-messengers")
-      (make-instance 'log-manager :message-class 'formatted-message))
 
 ;; Start a messenger to store our message into
-(defun start-logger (&key (log-filename *log-filename*))
+(defun start-logger (&key
-  "Start the parch log manager and messenger."
+		       (log-filename *log-filename*)
-  (let ((log-pathname (typecase log-filename
+		       ((:log-min-messages *log-min-messages*) *log-min-messages*)
-			(string (pathname log-filename))
+		       ((:client-min-messages *client-min-messages*) *client-min-messages*))
-			(t      log-filename))))
+  "Start the pgloader log manager and messenger."
-    (ensure-directories-exist (directory-namestring log-filename))
+
-    (cl-log:start-messenger 'text-file-messenger
+  (setf (log-manager)
-			    :filter *log-min-messages*
+	(make-instance 'log-manager
-			    :filename log-filename)
+		       :message-class 'formatted-message))
-    (cl-log:start-messenger 'text-stream-messenger
+
-			    :filter *client-min-messages*
+  ;; we need an existing place where to log our messages
-			    :stream *standard-output*)
+  (ensure-directories-exist (directory-namestring log-filename))
-    (format t "~&Now logging in '~a'.~%" log-pathname)
+
-    (log-message :notice "Starting pgloader, log system is ready.")))
+  (push (cl-log:start-messenger 'text-file-messenger
+				:name "logfile"
+				:filter *log-min-messages*
+				:filename log-filename)
+	*log-messengers*)
+
+  (push (cl-log:start-messenger 'text-stream-messenger
+				:name "client"
+				:filter *client-min-messages*
+				:stream *standard-output*)
+	*log-messengers*)
+
+  (log-message :log "Starting pgloader, log system is ready."))
+
+(defun stop-logger ()
+  "Stop the pgloader manager and messengers."
+
+  (loop for messenger = (pop *log-messengers*)
+     while messenger
+     do (cl-log:stop-messenger messenger)))
 
 ;; monkey patch the print-object method for cl-log timestamp
 (defconstant +nsec+ (* 1000 1000 1000)