Improve tests and add test cases.

bootstrap.sh

@@ -19,8 +19,6 @@ echo "host  all all 127.0.0.1/32 trust" | sudo tee -a $HBA
 
 sudo pg_ctlcluster 9.3 main reload
 createuser -U postgres -SdR `whoami`
-createdb -U postgres -O `whoami` pgloader
-psql -U postgres -d pgloader -c 'create extension ip4r'
 
 make -C /vagrant pgloader
 make -C /vagrant test

test/Makefile

@@ -1,8 +1,46 @@
 TESTS = $(wildcard *.load)
 OUT   = $(TESTS:.load=.out)
 
-all: $(OUT)
+REMOTE = archive.load bossa-all.load bossa.load census-places.load dbf-zip.load
+LOCAL  = $(filter-out $(REMOTE:.load=.out),$(OUT))
 
-%.out: %.load
+PGLOADER ?= ../build/pgloader.exe
-	$(PGLOADER) --verbose $<
+
+local: prepare $(LOCAL)
+
+remote: prepare $(REMOTE:.load=.out)
+
+all: prepare $(OUT)
+
+sakila: ;
+# 	unzip data/sakila-db.zip
+# 	echo "SOURCE data/sakila-db/sakila-schema.sql" | mysql -u root
+# 	echo "SOURCE data/sakila-db/sakila-data.sql"   | mysql -u root
+
+prepare: bossa.sql sakila
+	-dropdb -U postgres pgloader
+	-dropdb -U postgres stocks
+	-dropdb -U postgres ip4r
+	-createdb -U postgres -O `whoami` pgloader
+	-createdb -U postgres -O `whoami` stocks
+	-createdb -U postgres -O `whoami` ip4r
+	-psql -U postgres -d pgloader -c 'create extension ip4r'
+	-psql -U postgres -d ip4r -c 'create extension ip4r'
+	-psql -d stocks -f bossa.sql
+
+# Some special cases where we expect errors and want to continue testing
+sakila.out: sakila.load
+	@echo "no MySQL installation here."
+
+errors.out: errors.load
+	-$(PGLOADER) $<
+	@echo
+
+nofile.out: nofile.load
+	-$(PGLOADER) $<
+	@echo
+
+# General case where we do NOT expect any error
+%.out: %.load
+	$(PGLOADER) $<
 	@echo

test/archive.load

@@ -8,7 +8,7 @@
  */
 
 LOAD ARCHIVE
-   FROM /Users/dim/Downloads/GeoLiteCity-latest.zip
+   FROM http://pgsql.tapoueh.org/temp/foo.zip
    INTO postgresql:///ip4r
 
    BEFORE LOAD DO

test/bossa-all.load

@@ -0,0 +1,25 @@
+/*
+ * We load all the files in the archive, all of them into the same target table.
+ */
+
+load archive
+     from http://bossa.pl/pub/futures/mstock/mstfut.zip
+
+  load csv
+    from all filenames matching ~/./
+      with encoding iso-8859-2
+      ( ticker
+      , quote_date
+      , open
+      , high
+      , low
+      , close
+      , volume
+      , openint
+      )
+    into postgresql:///stocks?intf_derivatives
+    with
+      skip header=1
+    , fields optionally enclosed by '"'
+    , fields terminated by ','
+    , truncate;

test/bossa.load

@@ -0,0 +1,25 @@
+LOAD ARCHIVE
+  FROM http://bossa.pl/pub/metastock/mstock/mstall.zip
+  -- FROM /Users/dim/dev/temp/mstall.zip
+  INTO postgresql:///stocks
+
+  LOAD CSV
+    FROM FILENAME MATCHING ~/ALIOR/
+      WITH ENCODING iso-8859-2
+      (ticker, quote_date, open, high, low, close, volume)
+    INTO postgresql:///stocks?intf_stocks
+    WITH
+      SKIP HEADER=1,
+      FIELDS OPTIONALLY ENCLOSED BY '"',
+      FIELDS TERMINATED BY ','
+
+  AND LOAD CSV
+    FROM FILENAME MATCHING ~/FPGNH14/
+      WITH ENCODING iso-8859-2
+      (ticker, quote_date, open, high, low, close, volume, openint)
+    INTO postgresql:///stocks?intf_derivatives
+    WITH
+      SKIP HEADER = 1,
+      FIELDS OPTIONALLY ENCLOSED BY '"',
+      FIELDS TERMINATED BY ','
+  ;

test/bossa.sql

@@ -0,0 +1,30 @@
+---
+--- We need to run some tests where we don't provide for the schema within
+--- the pgloader command itself, so we prepare the schema "manually" here in
+--- advance.
+---
+
+drop table if exists intf_derivatives, intf_stocks;
+
+create table intf_stocks
+  (
+   ticker     text,
+   quote_date date,
+   open       numeric,
+   high       numeric,
+   low        numeric,
+   close      numeric,
+   volume     bigint
+  );
+
+create table intf_derivatives
+  (
+   ticker     text,
+   quote_date date,
+   open       numeric,
+   high       numeric,
+   low        numeric,
+   close      numeric,
+   volume     bigint,
+   openint    bigint
+  );

test/dbf.load

@@ -1,5 +1,10 @@
+/*
+ * The file comes from:
+ *
+ * http://www.insee.fr/fr/methodes/nomenclatures/cog/telechargement/2013/dbf/reg2013.dbf
+ */ 
 LOAD DBF
-    FROM http://www.insee.fr/fr/methodes/nomenclatures/cog/telechargement/2013/dbf/reg2013.dbf
+    FROM data/reg2013.dbf
     INTO postgresql:///pgloader
     WITH truncate, create table
      SET client_encoding TO 'latin1';

test/nofile.load

@@ -0,0 +1,22 @@
+LOAD CSV
+     FROM 'this/file/does/not/exists.csv' WITH ENCODING latin1
+          (a, b, c, d, e)
+     INTO postgresql:///pgloader?csv.nofile
+
+     WITH truncate,
+          skip header = 1,
+          fields optionally enclosed by '"',
+          fields escaped by double-quote,
+          fields terminated by ','
+
+  BEFORE LOAD DO
+   $$ create schema if not exists csv; $$,
+   $$ create table if not exists csv.nofile
+     (
+        a      text,
+        b      text,
+        c      text,
+        d      text,
+        e      text
+     );
+   $$;