Add support for comments (-- and /* */ style) in the command language.

parser.lisp

@@ -11,67 +11,21 @@
 (defvar *data-expected-inline* nil
   "Set to :inline when parsing an INLINE keyword in a FROM clause.")
 
-#|
-Here's a quick description of the format we're parsing here:
-
-    LOAD FROM '/path/to/filename.txt'
-               stdin
-               http://url.to/some/file.txt
-               mysql://[user[:pass]@][host[:port]]/dbname?table-name
-               postgresql://[user[:pass]@][host[:port]]/dbname?table-name
-
-        [ COMPRESSED WITH zip | bzip2 | gzip ]
-
-    WITH workers = 2,
-         batch size = 25000,
-         batch split = 5,
-         reject file = '/tmp/pgloader/<table-name>.dat'
-         log file = '/tmp/pgloader/pgloader.log',
-         log level = debug | info | notice | warning | error | critical,
-         truncate,
-         fields [ optionally ] enclosed by '"',
-         fields escaped by '\\',
-         fields terminated by '\t',
-         lines terminated by '\r\n',
-         encoding = 'latin9',
-         drop table,
-         create table,
-         create indexes,
-         reset sequences
-
-     SET guc-1 = 'value', guc-2 = 'value'
-
-     PREPARE CLIENT WITH ( <lisp> )
-     PREPARE SERVER WITH ( <sql> )
-
-     INTO table-name [ WITH <options> SET <gucs> ]
-          (
-               field-name data-type field-desc [ with column options ],
-               ...
-          )
-    USING (expression field-name other-field-name) as column-name,
-          ...
-
-    INTO table-name  [ WITH <options> SET <gucs> ]
-         (
-           *
-         )
-
-    TODO WHEN
-
-    FINALLY ON CLIENT DO ( <lisp> )
-            ON SERVER DO ( <lisp> )
-
-    < data here if loading from stdin >
-|#
-
 ;;
 ;; Some useful rules
 ;;
+(defrule single-line-comment (and "--" (+ (not #\Newline)) #\Newline)
+  (:constant :comment))
+
+(defrule multi-line-comment (and "/*" (+ (not "*/")) "*/")
+  (:constant :comment))
+
+(defrule comments (or single-line-comment multi-line-comment))
+
 (defrule keep-a-single-whitespace (+ (or #\space #\tab #\newline #\linefeed))
   (:constant " "))
 
-(defrule whitespace (+ (or #\space #\tab #\newline #\linefeed))
+(defrule whitespace (+ (or #\space #\tab #\newline #\linefeed comments))
   (:constant 'whitespace))
 
 (defrule ignore-whitespace (* whitespace)

test/allcols.load

@@ -1,3 +1,16 @@
+/*
+ * This test is ported from pgloader 2.x where it was defined as:
+ *
+ * [allcols]
+ * table     = allcols
+ * format    = csv
+ * filename  = allcols/allcols.data
+ * field_sep = :
+ * columns   = *
+ * pg_option_work_mem = 14MB
+ *
+ */
+
 LOAD CSV
      FROM inline (a, b, c)
      INTO postgresql://dim:pgpass@localhost:54393/pgloader?allcols

test/archive.load

@@ -1,3 +1,12 @@
+/*
+ * Loading from a ZIP archive containing CSV files. The full test can be
+ * done with using the archive found at
+ * http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.zip
+ *
+ * And a very light version of this data set is found at
+ * http://pgsql.tapoueh.org/temp/foo.zip for quick testing.
+ */
+
 LOAD ARCHIVE
    FROM http://pgsql.tapoueh.org/temp/foo.zip
    INTO postgresql://dim@localhost:54393/ip4r

test/csv.load

@@ -1,3 +1,20 @@
+/*
+ * This test is ported from pgloader 2.x where it was defined as:
+ *
+ * [csv]
+ * table            = csv
+ * format           = csv
+ * filename         = csv/csv.data
+ * field_size_limit = 512kB
+ * field_sep        = ,
+ * quotechar        = "
+ * columns          = x, y, a, b, d:6, c:5
+ * only_cols        = 3-6
+ * skip_head_lines  = 1
+ * truncate         = True
+ *
+ */
+
 LOAD CSV
      FROM inline 
         (

test/errors.load

@@ -1,3 +1,16 @@
+/*
+ * This test is ported from pgloader 2.x where it was defined as:
+ *
+ * [errors]
+ * table        = errors
+ * format       = text
+ * filename     = errors/errors.data
+ * field_sep    = |
+ * trailing_sep = True
+ * columns      = a:1, b:3, c:2
+ *
+ */
+
 LOAD CSV
      FROM inline (a, c, b, trailing)
      INTO postgresql://dim:pgpass@localhost:54393/pgloader?errors

test/partial.load

@@ -1,3 +1,15 @@
+/*
+ * This test is ported from pgloader 2.x where it was defined as:
+ *
+ * [partial]
+ * table        = partial
+ * format       = text
+ * filename     = partial/partial.data
+ * field_sep    = %
+ * columns      = *
+ * only_cols    = 1-3, 5
+ *
+ */
 LOAD CSV
      FROM inline (a, b, c, d, e)
      INTO postgresql://dim:pgpass@localhost:54393/pgloader?partial

test/reformat.load

@@ -1,3 +1,17 @@
+/*
+ * This test is ported from pgloader 2.x where it was defined as:
+ *
+ * [reformat]
+ * table           = reformat
+ * format          = text
+ * filename        = reformat/reformat.data
+ * field_sep       = |
+ * columns         = id, timestamp
+ * reformat        = timestamp:mysql:timestamp
+ *
+ */
+
+
 LOAD CSV
      FROM inline (id, timestamp)
      INTO postgresql://dim:pgpass@localhost:54393/pgloader?reformat

test/serial.load

@@ -1,3 +1,15 @@
+/*
+ * This test is ported from pgloader 2.x where it was defined as:
+ *
+ * [serial]
+ * table        = serial
+ * format       = text
+ * filename     = serial/serial.data
+ * field_sep    = ;
+ * columns      = b:2, c:1
+ *
+ */
+
 LOAD CSV
      FROM inline (c, b)
      INTO postgresql://dim:pgpass@localhost:54393/pgloader?serial

test/simple.load

@@ -1,3 +1,25 @@
+/*
+ * This test is ported from pgloader 2.x where it was defined as:
+ *
+ * [simple_tmpl]
+ * template     = True
+ * format       = text
+ * datestyle    = dmy
+ * field_sep    = |
+ * trailing_sep = True
+ * 
+ * [simple]
+ * use_template    = simple_tmpl
+ * table           = simple
+ * filename        = simple/simple.data
+ * columns         = a:1, b:3, c:2
+ * skip_head_lines = 2
+ * 
+ * # those reject settings are defaults one
+ * reject_log   = /tmp/simple.rej.log
+ * reject_data  = /tmp/simple.rej
+ */
+
 LOAD CSV
      FROM inline (a, c, b, trailing)
      INTO postgresql://dim:pgpass@localhost:54393/pgloader?simple

test/udc.load

@@ -1,3 +1,18 @@
+/*
+ * This test is ported from pgloader 2.x where it was defined as:
+ *
+ * [udc]
+ * table           = udc
+ * format          = text
+ * filename        = udc/udc.data
+ * input_encoding  = 'latin1'
+ * field_sep       = %
+ * columns         = b:2, d:1, x:3, y:4
+ * udc_c           = constant value
+ * copy_columns    = b, c, d
+ *
+ */
+
 LOAD CSV
      FROM inline WITH ENCODING latin1
         (d, b, x, y)