diff --git a/bootstrap.sh b/bootstrap.sh index b1412e9..72faf86 100644 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -19,8 +19,6 @@ echo "host all all 127.0.0.1/32 trust" | sudo tee -a $HBA sudo pg_ctlcluster 9.3 main reload createuser -U postgres -SdR `whoami` -createdb -U postgres -O `whoami` pgloader -psql -U postgres -d pgloader -c 'create extension ip4r' make -C /vagrant pgloader make -C /vagrant test diff --git a/test/Makefile b/test/Makefile index 493077d..f68ee56 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,8 +1,46 @@ TESTS = $(wildcard *.load) OUT = $(TESTS:.load=.out) -all: $(OUT) +REMOTE = archive.load bossa-all.load bossa.load census-places.load dbf-zip.load +LOCAL = $(filter-out $(REMOTE:.load=.out),$(OUT)) -%.out: %.load - $(PGLOADER) --verbose $< +PGLOADER ?= ../build/pgloader.exe + +local: prepare $(LOCAL) + +remote: prepare $(REMOTE:.load=.out) + +all: prepare $(OUT) + +sakila: ; +# unzip data/sakila-db.zip +# echo "SOURCE data/sakila-db/sakila-schema.sql" | mysql -u root +# echo "SOURCE data/sakila-db/sakila-data.sql" | mysql -u root + +prepare: bossa.sql sakila + -dropdb -U postgres pgloader + -dropdb -U postgres stocks + -dropdb -U postgres ip4r + -createdb -U postgres -O `whoami` pgloader + -createdb -U postgres -O `whoami` stocks + -createdb -U postgres -O `whoami` ip4r + -psql -U postgres -d pgloader -c 'create extension ip4r' + -psql -U postgres -d ip4r -c 'create extension ip4r' + -psql -d stocks -f bossa.sql + +# Some special cases where we expect errors and want to continue testing +sakila.out: sakila.load + @echo "no MySQL installation here." + +errors.out: errors.load + -$(PGLOADER) $< + @echo + +nofile.out: nofile.load + -$(PGLOADER) $< + @echo + +# General case where we do NOT expect any error +%.out: %.load + $(PGLOADER) $< @echo diff --git a/test/archive.load b/test/archive.load index b2c0f9c..efb894d 100644 --- a/test/archive.load +++ b/test/archive.load @@ -8,7 +8,7 @@ */ LOAD ARCHIVE - FROM /Users/dim/Downloads/GeoLiteCity-latest.zip + FROM http://pgsql.tapoueh.org/temp/foo.zip INTO postgresql:///ip4r BEFORE LOAD DO diff --git a/test/bossa-all.load b/test/bossa-all.load new file mode 100644 index 0000000..5e43dcb --- /dev/null +++ b/test/bossa-all.load @@ -0,0 +1,25 @@ +/* + * We load all the files in the archive, all of them into the same target table. + */ + +load archive + from http://bossa.pl/pub/futures/mstock/mstfut.zip + + load csv + from all filenames matching ~/./ + with encoding iso-8859-2 + ( ticker + , quote_date + , open + , high + , low + , close + , volume + , openint + ) + into postgresql:///stocks?intf_derivatives + with + skip header=1 + , fields optionally enclosed by '"' + , fields terminated by ',' + , truncate; diff --git a/test/bossa.load b/test/bossa.load new file mode 100644 index 0000000..437f094 --- /dev/null +++ b/test/bossa.load @@ -0,0 +1,25 @@ +LOAD ARCHIVE + FROM http://bossa.pl/pub/metastock/mstock/mstall.zip + -- FROM /Users/dim/dev/temp/mstall.zip + INTO postgresql:///stocks + + LOAD CSV + FROM FILENAME MATCHING ~/ALIOR/ + WITH ENCODING iso-8859-2 + (ticker, quote_date, open, high, low, close, volume) + INTO postgresql:///stocks?intf_stocks + WITH + SKIP HEADER=1, + FIELDS OPTIONALLY ENCLOSED BY '"', + FIELDS TERMINATED BY ',' + + AND LOAD CSV + FROM FILENAME MATCHING ~/FPGNH14/ + WITH ENCODING iso-8859-2 + (ticker, quote_date, open, high, low, close, volume, openint) + INTO postgresql:///stocks?intf_derivatives + WITH + SKIP HEADER = 1, + FIELDS OPTIONALLY ENCLOSED BY '"', + FIELDS TERMINATED BY ',' + ; diff --git a/test/bossa.sql b/test/bossa.sql new file mode 100644 index 0000000..53fce4b --- /dev/null +++ b/test/bossa.sql @@ -0,0 +1,30 @@ +--- +--- We need to run some tests where we don't provide for the schema within +--- the pgloader command itself, so we prepare the schema "manually" here in +--- advance. +--- + +drop table if exists intf_derivatives, intf_stocks; + +create table intf_stocks + ( + ticker text, + quote_date date, + open numeric, + high numeric, + low numeric, + close numeric, + volume bigint + ); + +create table intf_derivatives + ( + ticker text, + quote_date date, + open numeric, + high numeric, + low numeric, + close numeric, + volume bigint, + openint bigint + ); diff --git a/test/data/reg2013.dbf b/test/data/reg2013.dbf new file mode 100644 index 0000000..fac4d33 Binary files /dev/null and b/test/data/reg2013.dbf differ diff --git a/test/data/sakila-db.zip b/test/data/sakila-db.zip new file mode 100644 index 0000000..9fd458c Binary files /dev/null and b/test/data/sakila-db.zip differ diff --git a/test/dbf.load b/test/dbf.load index 8994eb4..a89f9a8 100644 --- a/test/dbf.load +++ b/test/dbf.load @@ -1,5 +1,10 @@ +/* + * The file comes from: + * + * http://www.insee.fr/fr/methodes/nomenclatures/cog/telechargement/2013/dbf/reg2013.dbf + */ LOAD DBF - FROM http://www.insee.fr/fr/methodes/nomenclatures/cog/telechargement/2013/dbf/reg2013.dbf + FROM data/reg2013.dbf INTO postgresql:///pgloader WITH truncate, create table SET client_encoding TO 'latin1'; diff --git a/test/nofile.load b/test/nofile.load new file mode 100644 index 0000000..feb4b5c --- /dev/null +++ b/test/nofile.load @@ -0,0 +1,22 @@ +LOAD CSV + FROM 'this/file/does/not/exists.csv' WITH ENCODING latin1 + (a, b, c, d, e) + INTO postgresql:///pgloader?csv.nofile + + WITH truncate, + skip header = 1, + fields optionally enclosed by '"', + fields escaped by double-quote, + fields terminated by ',' + + BEFORE LOAD DO + $$ create schema if not exists csv; $$, + $$ create table if not exists csv.nofile + ( + a text, + b text, + c text, + d text, + e text + ); + $$;