diff --git a/.gitignore b/.gitignore index ebe21a6..2358c87 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ -galaxya -csv \ No newline at end of file +.vagrant +local-data diff --git a/bootstrap.sh b/bootstrap.sh index 27408f7..8e8ad06 100644 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -31,7 +31,7 @@ git clone -b empty-strings-and-nil https://github.com/dimitri/cl-csv.git git clone https://github.com/marijnh/Postmodern.git cd ~/quicklisp/local-projects/Postmodern/ -patch -p1 < /vagrant/postmodern-send-copy-done.patch +patch -p1 < /vagrant/patches/postmodern-send-copy-done.patch cd ~ @@ -55,7 +55,7 @@ echo "(:tree \"/vagrant/\")" > $REGISTRY/projects.conf # --load-system pgloader \ # --entry pgloader:main \ # --dynamic-space-size 4096 \ -# --output pgloader.exe +# --output /home/vagrant/pgloader.exe echo "TESTING" @@ -63,7 +63,8 @@ echo "TESTING" for test in /vagrant/test/*.load do - echo " TEST: $test" + echo "# TEST: $test" + echo /vagrant/pgloader.lisp $test echo done diff --git a/postmodern-send-copy-done.patch b/patches/postmodern-send-copy-done.patch similarity index 100% rename from postmodern-send-copy-done.patch rename to patches/postmodern-send-copy-done.patch diff --git a/pgloader.asd b/pgloader.asd index 9703d5f..07b0790 100644 --- a/pgloader.asd +++ b/pgloader.asd @@ -1,34 +1,37 @@ ;;;; pgloader.asd (asdf:defsystem #:pgloader - :serial t - :description "Load data into PostgreSQL" - :author "Dimitri Fontaine " - :license "The PostgreSQL Licence" - :depends-on (#:uiop ; host system integration - #:cl-log ; logging - #:postmodern ; PostgreSQL protocol implementation - #:cl-postgres ; low level bits for COPY streaming - #:simple-date ; FIXME: recheck dependency - #:cl-mysql ; CFFI binding to libmysqlclient-dev - #:split-sequence ; some parsing is made easy - #:cl-csv ; full CSV reader - #:cl-fad ; file and directories - #:lparallel ; threads, workers, queues - #:esrap ; parser generator - #:alexandria ; utils - #:drakma ; http client, download archives - #:zip ; support for zip archive files - #:flexi-streams ; streams - #:com.informatimago.clext ; portable character-sets listings - #:usocket ; UDP / syslog - #:local-time ; UDP date parsing - #:command-line-arguments ; for the main function - #:abnf ; ABNF parser generator (for syslog) - #:db3 ; DBF version 3 file reader - #:py-configparser ; Read old-style INI config files - ) - :components ((:file "params") + :serial t + :description "Load data into PostgreSQL" + :author "Dimitri Fontaine " + :license "The PostgreSQL Licence" + :depends-on (#:uiop ; host system integration + #:cl-log ; logging + #:postmodern ; PostgreSQL protocol implementation + #:cl-postgres ; low level bits for COPY streaming + #:simple-date ; FIXME: recheck dependency + #:cl-mysql ; CFFI binding to libmysqlclient-dev + #:split-sequence ; some parsing is made easy + #:cl-csv ; full CSV reader + #:cl-fad ; file and directories + #:lparallel ; threads, workers, queues + #:esrap ; parser generator + #:alexandria ; utils + #:drakma ; http client, download archives + #:zip ; support for zip archive files + #:flexi-streams ; streams + #:com.informatimago.clext ; portable character-sets listings + #:usocket ; UDP / syslog + #:local-time ; UDP date parsing + #:command-line-arguments ; for the main function + #:abnf ; ABNF parser generator (for syslog) + #:db3 ; DBF version 3 file reader + #:py-configparser ; Read old-style INI config files + ) + :components + ((:module "src" + :components + ((:file "params") (:file "package" :depends-on ("params")) (:file "utils" :depends-on ("package")) @@ -70,6 +73,6 @@ "db3" "archive" "syslog" - "mysql")))) + "mysql")))))) diff --git a/archive.lisp b/src/archive.lisp similarity index 100% rename from archive.lisp rename to src/archive.lisp diff --git a/csv.lisp b/src/csv.lisp similarity index 100% rename from csv.lisp rename to src/csv.lisp diff --git a/db3.lisp b/src/db3.lisp similarity index 100% rename from db3.lisp rename to src/db3.lisp diff --git a/main.lisp b/src/main.lisp similarity index 100% rename from main.lisp rename to src/main.lisp diff --git a/mysql-cast-rules.lisp b/src/mysql-cast-rules.lisp similarity index 100% rename from mysql-cast-rules.lisp rename to src/mysql-cast-rules.lisp diff --git a/mysql.lisp b/src/mysql.lisp similarity index 100% rename from mysql.lisp rename to src/mysql.lisp diff --git a/mytest.lisp b/src/mytest.lisp similarity index 100% rename from mytest.lisp rename to src/mytest.lisp diff --git a/package.lisp b/src/package.lisp similarity index 100% rename from package.lisp rename to src/package.lisp diff --git a/params.lisp b/src/params.lisp similarity index 100% rename from params.lisp rename to src/params.lisp diff --git a/parse-ini.lisp b/src/parse-ini.lisp similarity index 100% rename from parse-ini.lisp rename to src/parse-ini.lisp diff --git a/parser.lisp b/src/parser.lisp similarity index 100% rename from parser.lisp rename to src/parser.lisp diff --git a/pgsql.lisp b/src/pgsql.lisp similarity index 100% rename from pgsql.lisp rename to src/pgsql.lisp diff --git a/queue.lisp b/src/queue.lisp similarity index 100% rename from queue.lisp rename to src/queue.lisp diff --git a/run.lisp b/src/run.lisp similarity index 100% rename from run.lisp rename to src/run.lisp diff --git a/syslog.lisp b/src/syslog.lisp similarity index 100% rename from syslog.lisp rename to src/syslog.lisp diff --git a/transforms.lisp b/src/transforms.lisp similarity index 98% rename from transforms.lisp rename to src/transforms.lisp index ad31528..938797f 100644 --- a/transforms.lisp +++ b/src/transforms.lisp @@ -42,6 +42,7 @@ (:seconds 12 14)))) "Apply this function when input date in like '20041002152952'" ;; only process non-zero dates + (declare (type string date-string)) (if (null (zero-dates-to-null date-string)) nil (destructuring-bind (&key year month day hour minute seconds &allow-other-keys) diff --git a/utils.lisp b/src/utils.lisp similarity index 100% rename from utils.lisp rename to src/utils.lisp