diff --git a/pgloader.1.md b/pgloader.1.md index eeb65b8..e391cfa 100644 --- a/pgloader.1.md +++ b/pgloader.1.md @@ -1235,7 +1235,7 @@ an example: LOAD IXF FROM data/nsitra.test1.ixf INTO postgresql:///pgloader?nsitra.test1 - WITH truncate, create table + WITH truncate, create table, timezone UTC BEFORE LOAD DO $$ create schema if not exists nsitra; $$, @@ -1282,6 +1282,13 @@ The `ixf` format command accepts the following clauses and options: This options expects as its value the possibly qualified name of the table to create. + - *timezone* + + This options allows to specify which timezone is used when parsing + timestamps from an IXF file, and defaults to *UTC*. Expected values + are either `UTC`, `GMT` or a single quoted location name such as + `'Universal'` or `'Europe/Paris'`. + ## LOAD ARCHIVE This command instructs pgloader to load data from one or more files contained diff --git a/src/parsers/command-ixf.lisp b/src/parsers/command-ixf.lisp index fbad04c..ae508a3 100644 --- a/src/parsers/command-ixf.lisp +++ b/src/parsers/command-ixf.lisp @@ -6,11 +6,42 @@ (in-package #:pgloader.parser) -(defrule option-create-table (and kw-create kw-table) - (:constant (cons :create-tables t))) +(defrule tz-utc (~ "UTC") (:constant local-time:+utc-zone+)) +(defrule tz-gmt (~ "GMT") (:constant local-time:+gmt-zone+)) +(defrule tz-name (and #\' (+ (not #\')) #\') + (:lambda (tzn) + (bind (((_ chars _) tzn)) + (local-time:reread-timezone-repository) + (local-time:find-timezone-by-location-name (text chars))))) + +(defrule option-timezone (and kw-timezone (or tz-utc tz-gmt tz-name)) + (:lambda (tzopt) + (bind (((_ tz) tzopt)) (cons :timezone tz)))) + +(defrule ixf-option (or option-batch-rows + option-batch-size + option-batch-concurrency + option-truncate + option-disable-triggers + option-data-only + option-schema-only + option-include-drop + option-create-table + option-create-tables + option-table-name + option-timezone)) + +(defrule another-ixf-option (and comma ixf-option) + (:lambda (source) + (bind (((_ option) source)) option))) + +(defrule ixf-option-list (and ixf-option (* another-ixf-option)) + (:lambda (source) + (destructuring-bind (opt1 opts) source + (alexandria:alist-plist `(,opt1 ,@opts))))) ;;; piggyback on DBF parsing -(defrule ixf-options (and kw-with dbf-option-list) +(defrule ixf-options (and kw-with ixf-option-list) (:lambda (source) (bind (((_ opts) source)) (cons :ixf-options opts)))) @@ -52,19 +83,23 @@ (let* (,@(pgsql-connection-bindings pg-db-conn gucs) ,@(batch-control-bindings options) ,@(identifier-case-binding options) + (timezone (getf ',options :timezone)) (table-name ',(pgconn-table-name pg-db-conn)) - (source-db (with-stats-collection ("fetch" :section :pre) + (source-db (with-stats-collection ("fetch" :section :pre) (expand (fetch-file ,ixf-db-conn)))) (source (make-instance 'pgloader.ixf:copy-ixf :target-db ,pg-db-conn :source-db source-db - :target table-name))) + :target table-name + :timezone timezone))) ,(sql-code-block pg-db-conn :pre before "before load") (pgloader.sources:copy-database source - ,@(remove-batch-control-option options)) + ,@(remove-batch-control-option + options + :extras '(:timezone))) ,(sql-code-block pg-db-conn :post after "after load")))) diff --git a/src/parsers/command-keywords.lisp b/src/parsers/command-keywords.lisp index 2271ed4..1e2bb0b 100644 --- a/src/parsers/command-keywords.lisp +++ b/src/parsers/command-keywords.lisp @@ -59,6 +59,7 @@ (def-keyword-rule "log") (def-keyword-rule "level") (def-keyword-rule "encoding") + (def-keyword-rule "timezone") (def-keyword-rule "decoding") (def-keyword-rule "truncate") (def-keyword-rule "disable") diff --git a/src/sources/ixf/ixf.lisp b/src/sources/ixf/ixf.lisp index 695ec6b..c37290b 100644 --- a/src/sources/ixf/ixf.lisp +++ b/src/sources/ixf/ixf.lisp @@ -8,7 +8,10 @@ ;;; ;;; Integration with pgloader ;;; -(defclass copy-ixf (copy) () +(defclass copy-ixf (copy) + ((timezone :accessor timezone ; timezone + :initarg :timezone + :initform local-time:+utc-zone+)) (:documentation "pgloader IXF Data Source")) (defmethod initialize-instance :after ((source copy-ixf) &key) @@ -16,6 +19,10 @@ (setf (slot-value source 'source) (pathname-name (fd-path (source-db source)))) + ;; force default timezone when nil + (when (null (timezone source)) + (setf (timezone source) local-time:+utc-zone+)) + (with-connection (conn (source-db source)) (unless (and (slot-boundp source 'columns) (slot-value source 'columns)) (setf (slot-value source 'columns) @@ -58,13 +65,16 @@ (defmethod map-rows ((copy-ixf copy-ixf) &key process-row-fn) "Extract IXF data and call PROCESS-ROW-FN function with a single argument (a list of column values) for each row." - (with-connection (conn (source-db copy-ixf)) - (let ((ixf (ixf:make-ixf-file :stream (conn-handle conn))) - (row-fn (lambda (row) - (update-stats :data (target copy-ixf) :read 1) - (funcall process-row-fn row)))) - (ixf:read-headers ixf) - (ixf:map-data ixf row-fn)))) + (let ((local-time:*default-timezone* (timezone copy-ixf))) + (log-message :notice "Parsing IXF with TimeZone: ~a" + (local-time::timezone-name local-time:*default-timezone*)) + (with-connection (conn (source-db copy-ixf)) + (let ((ixf (ixf:make-ixf-file :stream (conn-handle conn))) + (row-fn (lambda (row) + (update-stats :data (target copy-ixf) :read 1) + (funcall process-row-fn row)))) + (ixf:read-headers ixf) + (ixf:map-data ixf row-fn))))) (defmethod copy-to-queue ((ixf copy-ixf) queue) "Copy data from IXF file FILENAME into queue DATAQ" diff --git a/test/ixf.load b/test/ixf.load index 6064443..54caaf4 100644 --- a/test/ixf.load +++ b/test/ixf.load @@ -1,7 +1,8 @@ LOAD IXF FROM data/nsitra.test1.ixf INTO postgresql:///pgloader?nsitra.test1 - WITH truncate, create table + + WITH truncate, create table, timezone UTC BEFORE LOAD DO $$ drop schema if exists nsitra cascade; $$,