From f619054c79f3e8875d24d4213582832b38870f77 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Sat, 19 Oct 2013 22:39:09 +0200 Subject: [PATCH] Improve database connection parsing to use sensible defaults. --- parser.lisp | 46 ++++++++++++++++++++++++++---------- pgloader.1.md | 21 +++++++++++----- test/allcols.load | 6 ++--- test/archive.load | 6 ++--- test/csv-before-after.load | 2 +- test/csv-empty-as-null.load | 2 +- test/csv.load | 33 ++++++++------------------ test/dbf-zip.load | 2 +- test/dbf.load | 2 +- test/errors.load | 6 ++--- test/partial.load | 6 ++--- test/reformat.load | 5 ++-- test/serial.load | 6 ++--- test/simple.load | 6 ++--- test/udc.load | 5 ++-- test/xzero.load | Bin 1305 -> 1271 bytes 16 files changed, 86 insertions(+), 68 deletions(-) diff --git a/parser.lisp b/parser.lisp index 552d626..d943dcc 100644 --- a/parser.lisp +++ b/parser.lisp @@ -4,9 +4,10 @@ (in-package :pgloader.parser) -(defparameter *default-host* "localhost") -(defparameter *default-postgresql-port* 5432) -(defparameter *default-mysql-port* 3306) +(defun getenv-default (name &optional default) + "Return the value of the NAME variable as found in the environment, or + DEFAULT if that variable isn't set" + (or (uiop:getenv name) default)) (defvar *data-expected-inline* nil "Set to :inline when parsing an INLINE keyword in a FROM clause.") @@ -229,7 +230,7 @@ (:destructure (hostname &optional port) (append (list :host hostname) port))) -(defrule dsn-dbname (and "/" namestring) +(defrule dsn-dbname (and "/" (? namestring)) (:destructure (slash dbname) (declare (ignore slash)) (list :dbname dbname))) @@ -266,15 +267,34 @@ dbname table-name) (apply #'append uri) - (list :type type - :user user - :password password - :host (or host *default-host*) - :port (or port (case type - (:postgresql *default-postgresql-port*) - (:mysql *default-mysql-port*))) - :dbname dbname - :table-name table-name)))) + ;; + ;; Default to environment variables as described in + ;; http://www.postgresql.org/docs/9.3/static/app-psql.html + ;; http://dev.mysql.com/doc/refman/5.0/en/environment-variables.html + ;; + (let ((user + (or user + (case type + (:postgresql (getenv-default "PGUSER" (getenv-default "USER"))) + (:mysql (getenv-default "USER")))))) + (list :type type + :user user + :password password + :host (or host + (case type + (:postgresql (getenv-default "PGHOST" "localhost")) + (:mysql (getenv-default "MYSQL_HOST" "localhost")))) + :port (or port + (parse-integer + ;; avoid a NIL is not a STRING style warning by + ;; using ecase here + (ecase type + (:postgresql (getenv-default "PGPORT" "5432")) + (:mysql (getenv-default "MYSQL_TCP_PORT" "3306"))))) + :dbname (or dbname + (case type + (:postgresql (getenv-default "PGDATABASE" user)))) + :table-name table-name))))) (defrule target (and kw-into db-connection-uri) (:destructure (into target) diff --git a/pgloader.1.md b/pgloader.1.md index 26f0867..a4467ae 100644 --- a/pgloader.1.md +++ b/pgloader.1.md @@ -82,6 +82,15 @@ The main clauses are the `LOAD`, `FROM`, `INTO` and `WITH` clauses that each command implements. Some command then implement the `SET` command, or some specific clauses such as the `CAST` clause. +The `` parameter is expected to be given as a *Connection URI* +as documented in the PostgreSQL documentation at +http://www.postgresql.org/docs/9.3/static/libpq-connect.html#LIBPQ-CONNSTRING. + + postgresql://[user[:password]@][netloc][:port][/dbname][?schema.table] + +The connection string in pgloader only accepts a single optionnal parameter +which should be a possibly qualified table name. + ## LOAD CSV This command instructs pgloader to load data from a `CSV` file. Here's an @@ -92,7 +101,7 @@ example: ( startIpNum, endIpNum, locId ) - INTO postgresql://dim@localhost:54393/dim?geolite.blocks + INTO postgresql://user@localhost:54393/dbname?geolite.blocks ( iprange ip4r using (ip-range startIpNum endIpNum), locId @@ -272,7 +281,7 @@ example: LOAD DBF FROM http://www.insee.fr/fr/methodes/nomenclatures/cog/telechargement/2013/dbf/reg2013.dbf - INTO postgresql://dim@localhost:54393/dim + INTO postgresql://user@localhost/dbname WITH truncate, create table; The `dbf` format command accepts the following clauses and options: @@ -328,7 +337,7 @@ Here's an example: LOAD ARCHIVE FROM /Users/dim/Downloads/GeoLiteCity-latest.zip - INTO postgresql://dim@localhost:54393/ip4r + INTO postgresql:///ip4r BEFORE LOAD DO $$ create extension if not exists ip4r; $$, @@ -368,7 +377,7 @@ Here's an example: metroCode null if blanks, areaCode null if blanks ) - INTO postgresql://dim@localhost:54393/ip4r?geolite.location + INTO postgresql:///ip4r?geolite.location ( locid,country,region,city,postalCode, location point using (format nil "(~a,~a)" longitude latitude), @@ -385,7 +394,7 @@ Here's an example: ( startIpNum, endIpNum, locId ) - INTO postgresql://dim@localhost:54393/ip4r?geolite.blocks + INTO postgresql:///ip4r?geolite.blocks ( iprange ip4r using (ip-range startIpNum endIpNum), locId @@ -454,7 +463,7 @@ Here's an example: load database from mysql://localhost/adv - into postgresql://dim@localhost/adv + into postgresql:///adv with drop tables, truncate, create tables, create indexes, reset sequences, diff --git a/test/allcols.load b/test/allcols.load index 7d08995..e5c4e29 100644 --- a/test/allcols.load +++ b/test/allcols.load @@ -13,8 +13,7 @@ LOAD CSV FROM inline (a, b, c) - INTO postgresql://dim:pgpass@localhost:54393/pgloader?allcols - (a, b, c) + INTO postgresql:///pgloader?allcols (a, b, c) WITH fields optionally enclosed by '"', fields escaped by double-quote, @@ -25,7 +24,8 @@ LOAD CSV standard_conforming_strings to 'on' BEFORE LOAD DO - $$ create table if not exists allcols ( + $$ drop table if exists allcols; $$, + $$ create table allcols ( a integer primary key, b date, c text diff --git a/test/archive.load b/test/archive.load index 6f78a6b..05f6846 100644 --- a/test/archive.load +++ b/test/archive.load @@ -9,7 +9,7 @@ LOAD ARCHIVE FROM http://pgsql.tapoueh.org/temp/foo.zip - INTO postgresql://dim@localhost:54393/ip4r + INTO postgresql:///pgloader BEFORE LOAD DO $$ create extension if not exists ip4r; $$, @@ -49,7 +49,7 @@ LOAD ARCHIVE metroCode null if blanks, areaCode null if blanks ) - INTO postgresql://dim@localhost:54393/ip4r?geolite.location + INTO postgresql:///pgloader?geolite.location ( locid,country,region,city,postalCode, location point using (format nil "(~a,~a)" longitude latitude), @@ -66,7 +66,7 @@ LOAD ARCHIVE ( startIpNum, endIpNum, locId ) - INTO postgresql://dim@localhost:54393/ip4r?geolite.blocks + INTO postgresql:///pgloader?geolite.blocks ( iprange ip4r using (ip-range startIpNum endIpNum), locId diff --git a/test/csv-before-after.load b/test/csv-before-after.load index a1bd6c0..0a8b123 100644 --- a/test/csv-before-after.load +++ b/test/csv-before-after.load @@ -4,7 +4,7 @@ LOAD CSV ( startIpNum, endIpNum, locId ) - INTO postgresql://dim@localhost:54393/ip4r?csv.blocks + INTO postgresql:///pgloader?csv.blocks ( iprange ip4r using (ip-range startIpNum endIpNum), locId diff --git a/test/csv-empty-as-null.load b/test/csv-empty-as-null.load index 92ef9f3..2098946 100644 --- a/test/csv-empty-as-null.load +++ b/test/csv-empty-as-null.load @@ -1,6 +1,6 @@ LOAD CSV FROM INLINE - INTO postgresql://dim@localhost:54393/dim?nulls (f1, f2, f3) + INTO postgresql:///pgloader?nulls (f1, f2, f3) WITH truncate, keep unquoted blanks, fields optionally enclosed by '"', diff --git a/test/csv.load b/test/csv.load index e494fb5..9e1ab9d 100644 --- a/test/csv.load +++ b/test/csv.load @@ -16,22 +16,8 @@ */ LOAD CSV - FROM inline - ( - x, - y, - a, - b, - c, - d - ) - INTO postgresql://dim:pgpass@localhost:54393/pgloader?csv - ( - a, - b, - d, - c - ) + FROM inline (x, y, a, b, c, d) + INTO postgresql:///pgloader?csv (a, b, d, c) WITH truncate, skip header = 1, @@ -44,13 +30,14 @@ LOAD CSV standard_conforming_strings to 'on' BEFORE LOAD DO - $$ CREATE TABLE csv ( - a bigint, - b bigint, - c char(2), - d text -); $$; - + $$ drop table if exists csv; $$, + $$ create table csv ( + a bigint, + b bigint, + c char(2), + d text + ); + $$; diff --git a/test/dbf-zip.load b/test/dbf-zip.load index cfc2961..bc6efba 100644 --- a/test/dbf-zip.load +++ b/test/dbf-zip.load @@ -1,5 +1,5 @@ LOAD DBF FROM http://www.insee.fr/fr/methodes/nomenclatures/cog/telechargement/2013/dbf/historiq2013.zip - INTO postgresql://dim@localhost:54393/dim + INTO postgresql:///pgloader WITH truncate, create table SET client_encoding TO 'latin1'; diff --git a/test/dbf.load b/test/dbf.load index 4a21d7a..8994eb4 100644 --- a/test/dbf.load +++ b/test/dbf.load @@ -1,5 +1,5 @@ LOAD DBF FROM http://www.insee.fr/fr/methodes/nomenclatures/cog/telechargement/2013/dbf/reg2013.dbf - INTO postgresql://dim@localhost:54393/dim + INTO postgresql:///pgloader WITH truncate, create table SET client_encoding TO 'latin1'; diff --git a/test/errors.load b/test/errors.load index 4182cd2..89b0a46 100644 --- a/test/errors.load +++ b/test/errors.load @@ -13,8 +13,7 @@ LOAD CSV FROM inline (a, c, b, trailing) - INTO postgresql://dim:pgpass@localhost:54393/pgloader?errors - (a, b, c) + INTO postgresql:///pgloader?errors (a, b, c) WITH fields optionally enclosed by '"', fields escaped by double-quote, @@ -25,7 +24,8 @@ LOAD CSV standard_conforming_strings to 'on' BEFORE LOAD DO - $$ create table if not exists errors ( + $$ drop table if exists errors; $$, + $$ create table errors ( a integer primary key, b date, c text diff --git a/test/partial.load b/test/partial.load index 208c2e7..ef52075 100644 --- a/test/partial.load +++ b/test/partial.load @@ -12,8 +12,7 @@ */ LOAD CSV FROM inline (a, b, c, d, e) - INTO postgresql://dim:pgpass@localhost:54393/pgloader?partial - (a, b, c, e) + INTO postgresql:///pgloader?partial (a, b, c, e) WITH fields optionally enclosed by '"', fields escaped by double-quote, @@ -24,7 +23,8 @@ LOAD CSV standard_conforming_strings to 'on' BEFORE LOAD DO - $$ create table if not exists partial ( + $$ drop table if exists partial; $$, + $$ create table partial ( a integer primary key, b text, c text, diff --git a/test/reformat.load b/test/reformat.load index c98e673..0a7ecfc 100644 --- a/test/reformat.load +++ b/test/reformat.load @@ -14,7 +14,7 @@ LOAD CSV FROM inline (id, timestamp) - INTO postgresql://dim:pgpass@localhost:54393/pgloader?reformat + INTO postgresql:///pgloader?reformat ( id, timestamp timestamptz using (date-with-no-separator timestamp) @@ -29,7 +29,8 @@ LOAD CSV standard_conforming_strings to 'on' BEFORE LOAD DO - $$ create table if not exists reformat ( + $$ drop table if exists reformat; $$, + $$ create table reformat ( id integer primary key, timestamp timestamp with time zone ); diff --git a/test/serial.load b/test/serial.load index c8b2e79..e6896da 100644 --- a/test/serial.load +++ b/test/serial.load @@ -12,8 +12,7 @@ LOAD CSV FROM inline (c, b) - INTO postgresql://dim:pgpass@localhost:54393/pgloader?serial - (b, c) + INTO postgresql:///pgloader?serial (b, c) WITH fields optionally enclosed by '"', fields escaped by double-quote, @@ -24,7 +23,8 @@ LOAD CSV standard_conforming_strings to 'on' BEFORE LOAD DO - $$ create table if not exists serial ( + $$ drop table if exists serial; $$, + $$ create table serial ( a serial primary key, b date, c text diff --git a/test/simple.load b/test/simple.load index 213de8c..315a8fd 100644 --- a/test/simple.load +++ b/test/simple.load @@ -22,8 +22,7 @@ LOAD CSV FROM inline (a, c, b, trailing) - INTO postgresql://dim:pgpass@localhost:54393/pgloader?simple - (a, b, c) + INTO postgresql:///pgloader?simple (a, b, c) WITH truncate, skip header = 2, @@ -37,7 +36,8 @@ LOAD CSV standard_conforming_strings to 'on' BEFORE LOAD DO - $$ CREATE TABLE if not exists simple ( + $$ drop table if exists simple; $$, + $$ create table simple ( a integer primary key, b date, c text diff --git a/test/udc.load b/test/udc.load index 617ed2f..197cc2b 100644 --- a/test/udc.load +++ b/test/udc.load @@ -16,7 +16,7 @@ LOAD CSV FROM inline WITH ENCODING latin1 (d, b, x, y) - INTO postgresql://dim:pgpass@localhost:54393/pgloader?udc + INTO postgresql:///pgloader?udc ( b, c text using "constant value", @@ -32,7 +32,8 @@ LOAD CSV standard_conforming_strings to 'on' BEFORE LOAD DO - $$ create table if not exists udc ( + $$ drop table if exists udc; $$, + $$ create table udc ( b integer primary key, c text, d integer diff --git a/test/xzero.load b/test/xzero.load index e32f9ad0c6469c173081443bd2113038faa61bd1..9fc84da8d7fa99afb76516e3be6bf3f4a282c819 100644 GIT binary patch delta 28 jcmbQq^__FWOGbYEg7lpH#FW$``--a6qWsNLO#hhxqz?-b delta 62 zcmey)Ig@L{OGd+#%v`I2^n%3VVuzgk