Improve database connection parsing to use sensible defaults.

parser.lisp

@@ -4,9 +4,10 @@
 
 (in-package :pgloader.parser)
 
-(defparameter *default-host* "localhost")
+(defun getenv-default (name &optional default)
-(defparameter *default-postgresql-port* 5432)
+  "Return the value of the NAME variable as found in the environment, or
-(defparameter *default-mysql-port* 3306)
+     DEFAULT if that variable isn't set"
+  (or (uiop:getenv name) default))
 
 (defvar *data-expected-inline* nil
   "Set to :inline when parsing an INLINE keyword in a FROM clause.")
@@ -229,7 +230,7 @@
   (:destructure (hostname &optional port)
 		(append (list :host hostname) port)))
 
-(defrule dsn-dbname (and "/" namestring)
+(defrule dsn-dbname (and "/" (? namestring))
   (:destructure (slash dbname)
 		(declare (ignore slash))
 		(list :dbname dbname)))
@@ -266,15 +267,34 @@
 			      dbname
 			      table-name)
 	(apply #'append uri)
-      (list :type type
+      ;;
-	    :user user
+      ;; Default to environment variables as described in
-	    :password password
+      ;;  http://www.postgresql.org/docs/9.3/static/app-psql.html
-	    :host (or host *default-host*)
+      ;;  http://dev.mysql.com/doc/refman/5.0/en/environment-variables.html
-	    :port (or port (case type
+      ;;
-			     (:postgresql *default-postgresql-port*)
+      (let ((user
-			     (:mysql      *default-mysql-port*)))
+	     (or user
-	    :dbname dbname
+		 (case type
-	    :table-name table-name))))
+		   (:postgresql (getenv-default "PGUSER" (getenv-default "USER")))
+		   (:mysql      (getenv-default "USER"))))))
+       (list :type type
+	     :user user
+	     :password password
+	     :host (or host
+		       (case type
+			 (:postgresql (getenv-default "PGHOST" "localhost"))
+			 (:mysql      (getenv-default "MYSQL_HOST" "localhost"))))
+	     :port (or port
+		       (parse-integer
+			;; avoid a NIL is not a STRING style warning by
+			;; using ecase here
+			(ecase type
+			  (:postgresql (getenv-default "PGPORT" "5432"))
+			  (:mysql      (getenv-default "MYSQL_TCP_PORT" "3306")))))
+	     :dbname (or dbname
+			 (case type
+			   (:postgresql (getenv-default "PGDATABASE" user))))
+	     :table-name table-name)))))
 
 (defrule target (and kw-into db-connection-uri)
   (:destructure (into target)

pgloader.1.md

@@ -82,6 +82,15 @@ The main clauses are the `LOAD`, `FROM`, `INTO` and `WITH` clauses that each
 command implements. Some command then implement the `SET` command, or some
 specific clauses such as the `CAST` clause.
 
+The `<source-url>` parameter is expected to be given as a *Connection URI*
+as documented in the PostgreSQL documentation at
+http://www.postgresql.org/docs/9.3/static/libpq-connect.html#LIBPQ-CONNSTRING.
+
+    postgresql://[user[:password]@][netloc][:port][/dbname][?schema.table]
+
+The connection string in pgloader only accepts a single optionnal parameter
+which should be a possibly qualified table name.
+
 ## LOAD CSV
 
 This command instructs pgloader to load data from a `CSV` file. Here's an
@@ -92,7 +101,7 @@ example:
             (
                startIpNum, endIpNum, locId
             )
-       INTO postgresql://dim@localhost:54393/dim?geolite.blocks
+       INTO postgresql://user@localhost:54393/dbname?geolite.blocks
             (
                iprange ip4r using (ip-range startIpNum endIpNum),
                locId
@@ -272,7 +281,7 @@ example:
 
     LOAD DBF
 	    FROM http://www.insee.fr/fr/methodes/nomenclatures/cog/telechargement/2013/dbf/reg2013.dbf
-        INTO postgresql://dim@localhost:54393/dim
+        INTO postgresql://user@localhost/dbname
         WITH truncate, create table; 
 
 The `dbf` format command accepts the following clauses and options:
@@ -328,7 +337,7 @@ Here's an example:
 
     LOAD ARCHIVE
        FROM /Users/dim/Downloads/GeoLiteCity-latest.zip
-       INTO postgresql://dim@localhost:54393/ip4r
+       INTO postgresql:///ip4r
     
        BEFORE LOAD DO
          $$ create extension if not exists ip4r; $$,
@@ -368,7 +377,7 @@ Here's an example:
                     metroCode  null if blanks,
                     areaCode   null if blanks
                  )
-            INTO postgresql://dim@localhost:54393/ip4r?geolite.location
+            INTO postgresql:///ip4r?geolite.location
                  (
                     locid,country,region,city,postalCode,
                     location point using (format nil "(~a,~a)" longitude latitude),
@@ -385,7 +394,7 @@ Here's an example:
                  (
                     startIpNum, endIpNum, locId
                  )
-            INTO postgresql://dim@localhost:54393/ip4r?geolite.blocks
+            INTO postgresql:///ip4r?geolite.blocks
                  (
                     iprange ip4r using (ip-range startIpNum endIpNum),
                     locId
@@ -454,7 +463,7 @@ Here's an example:
 
     load database
 	   from mysql://localhost/adv
-       into postgresql://dim@localhost/adv
+       into postgresql:///adv
     
     with drop tables, truncate, create tables, create indexes,
          reset sequences,

test/allcols.load

@@ -13,8 +13,7 @@
 
 LOAD CSV
      FROM inline (a, b, c)
-     INTO postgresql://dim:pgpass@localhost:54393/pgloader?allcols
+     INTO postgresql:///pgloader?allcols (a, b, c)
-        (a, b, c)
 
      WITH fields optionally enclosed by '"',
           fields escaped by double-quote,
@@ -25,7 +24,8 @@ LOAD CSV
           standard_conforming_strings to 'on'
 
    BEFORE LOAD DO
-     $$ create table if not exists allcols (
+     $$ drop table if exists allcols; $$,
+     $$ create table allcols (
          a integer primary key,
          b date,
          c text

test/archive.load

@@ -9,7 +9,7 @@
 
 LOAD ARCHIVE
    FROM http://pgsql.tapoueh.org/temp/foo.zip
-   INTO postgresql://dim@localhost:54393/ip4r
+   INTO postgresql:///pgloader
 
    BEFORE LOAD DO
      $$ create extension if not exists ip4r; $$,
@@ -49,7 +49,7 @@ LOAD ARCHIVE
                 metroCode  null if blanks,
                 areaCode   null if blanks
              )
-        INTO postgresql://dim@localhost:54393/ip4r?geolite.location
+        INTO postgresql:///pgloader?geolite.location
              (
                 locid,country,region,city,postalCode,
                 location point using (format nil "(~a,~a)" longitude latitude),
@@ -66,7 +66,7 @@ LOAD ARCHIVE
              (
                 startIpNum, endIpNum, locId
              )
-        INTO postgresql://dim@localhost:54393/ip4r?geolite.blocks
+        INTO postgresql:///pgloader?geolite.blocks
              (
                 iprange ip4r using (ip-range startIpNum endIpNum),
                 locId

test/csv-before-after.load

@@ -4,7 +4,7 @@ LOAD CSV
          (
             startIpNum, endIpNum, locId
          )
-    INTO postgresql://dim@localhost:54393/ip4r?csv.blocks
+    INTO postgresql:///pgloader?csv.blocks
          (
             iprange ip4r using (ip-range startIpNum endIpNum),
             locId

test/csv-empty-as-null.load

@@ -1,6 +1,6 @@
 LOAD CSV
     FROM INLINE
-    INTO postgresql://dim@localhost:54393/dim?nulls (f1, f2, f3)
+    INTO postgresql:///pgloader?nulls (f1, f2, f3)
     WITH truncate,
     	 keep unquoted blanks,
          fields optionally enclosed by '"',

test/csv.load

@@ -16,22 +16,8 @@
  */
 
 LOAD CSV
-     FROM inline 
+     FROM inline (x, y, a, b, c, d)
-        (
+     INTO postgresql:///pgloader?csv (a, b, d, c)
-          x,
-          y,
-          a,
-          b,
-          c,
-          d
-        )
-     INTO postgresql://dim:pgpass@localhost:54393/pgloader?csv
-        (
-          a,
-          b,
-          d,
-          c
-        )
 
      WITH truncate,
           skip header = 1,
@@ -44,13 +30,14 @@ LOAD CSV
           standard_conforming_strings to 'on'
 
    BEFORE LOAD DO
-   $$ CREATE TABLE csv (
+    $$ drop table if exists csv; $$,
- a bigint,
+    $$ create table csv (
- b bigint,
+        a bigint,
- c char(2),
+        b bigint,
- d text
+        c char(2),
-); $$;
+        d text
-
+       );
+    $$;
 
 
 

test/dbf-zip.load

@@ -1,5 +1,5 @@
 LOAD DBF
     FROM http://www.insee.fr/fr/methodes/nomenclatures/cog/telechargement/2013/dbf/historiq2013.zip
-    INTO postgresql://dim@localhost:54393/dim
+    INTO postgresql:///pgloader
     WITH truncate, create table
      SET client_encoding TO 'latin1';

test/dbf.load

@@ -1,5 +1,5 @@
 LOAD DBF
     FROM http://www.insee.fr/fr/methodes/nomenclatures/cog/telechargement/2013/dbf/reg2013.dbf
-    INTO postgresql://dim@localhost:54393/dim
+    INTO postgresql:///pgloader
     WITH truncate, create table
      SET client_encoding TO 'latin1';

test/errors.load

@@ -13,8 +13,7 @@
 
 LOAD CSV
      FROM inline (a, c, b, trailing)
-     INTO postgresql://dim:pgpass@localhost:54393/pgloader?errors
+     INTO postgresql:///pgloader?errors (a, b, c)
-        (a, b, c)
 
      WITH fields optionally enclosed by '"',
           fields escaped by double-quote,
@@ -25,7 +24,8 @@ LOAD CSV
           standard_conforming_strings to 'on'
 
    BEFORE LOAD DO
-     $$ create table if not exists errors (
+     $$ drop table if exists errors; $$,
+     $$ create table errors (
          a integer primary key,
          b date,
          c text

test/partial.load

@@ -12,8 +12,7 @@
  */
 LOAD CSV
      FROM inline (a, b, c, d, e)
-     INTO postgresql://dim:pgpass@localhost:54393/pgloader?partial
+     INTO postgresql:///pgloader?partial (a, b, c, e)
-        (a, b, c, e)
 
      WITH fields optionally enclosed by '"',
           fields escaped by double-quote,
@@ -24,7 +23,8 @@ LOAD CSV
           standard_conforming_strings to 'on'
 
    BEFORE LOAD DO
-   $$ create table if not exists partial (
+   $$ drop table if exists partial; $$,
+   $$ create table partial (
        a integer primary key,
        b text,
        c text,

test/reformat.load

@@ -14,7 +14,7 @@
 
 LOAD CSV
      FROM inline (id, timestamp)
-     INTO postgresql://dim:pgpass@localhost:54393/pgloader?reformat
+     INTO postgresql:///pgloader?reformat
         (
           id,
           timestamp timestamptz using (date-with-no-separator timestamp)
@@ -29,7 +29,8 @@ LOAD CSV
           standard_conforming_strings to 'on'
 
    BEFORE LOAD DO
-   $$ create table if not exists reformat (
+   $$ drop table if exists reformat; $$,
+   $$ create table reformat (
        id integer primary key,
        timestamp timestamp with time zone
       );

test/serial.load

@@ -12,8 +12,7 @@
 
 LOAD CSV
      FROM inline (c, b)
-     INTO postgresql://dim:pgpass@localhost:54393/pgloader?serial
+     INTO postgresql:///pgloader?serial (b, c)
-        (b, c)
 
      WITH fields optionally enclosed by '"',
           fields escaped by double-quote,
@@ -24,7 +23,8 @@ LOAD CSV
           standard_conforming_strings to 'on'
 
    BEFORE LOAD DO
-   $$ create table if not exists serial (
+   $$ drop table if exists serial; $$,
+   $$ create table serial (
        a serial primary key,
        b date,
        c text

test/simple.load

@@ -22,8 +22,7 @@
 
 LOAD CSV
      FROM inline (a, c, b, trailing)
-     INTO postgresql://dim:pgpass@localhost:54393/pgloader?simple
+     INTO postgresql:///pgloader?simple (a, b, c)
-        (a, b, c)
 
      WITH truncate,
           skip header = 2,
@@ -37,7 +36,8 @@ LOAD CSV
           standard_conforming_strings to 'on'
 
    BEFORE LOAD DO
-   $$ CREATE TABLE if not exists simple (
+   $$ drop table if exists simple; $$,
+   $$ create table simple (
        a integer primary key,
        b date,
        c text

test/udc.load

@@ -16,7 +16,7 @@
 LOAD CSV
      FROM inline WITH ENCODING latin1
         (d, b, x, y)
-     INTO postgresql://dim:pgpass@localhost:54393/pgloader?udc
+     INTO postgresql:///pgloader?udc
         (
           b,
           c text using "constant value",
@@ -32,7 +32,8 @@ LOAD CSV
           standard_conforming_strings to 'on'
 
    BEFORE LOAD DO
-   $$ create table if not exists udc (
+   $$ drop table if exists udc; $$,
+   $$ create table udc (
        b integer primary key,
        c text,
        d integer