diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 0000000..79af31e --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,43 @@ +# $Id: Makefile,v 1.1 2005-11-21 16:05:50 jpargudo Exp $ + +rest = $(wildcard *.rest) +html = $(addsuffix .html, $(basename $(rest))) +pdf = $(addsuffix .pdf, $(basename $(rest))) + +pdf: $(pdf) clean + +html: $(html) + +dist-clean: clean + @rm -f $(pdf) $(html) + +clean: + @rm -f *.aux *.log *~ *.tex *.out *.toc *.dvi + +%.html: %.rest + rest2html --stylesheet lib/stylesheet.sty \ + --no-section-numbering \ + --language=fr \ + $< > $@ + +%.pdf: %.dvi + dvipdf $< + +%.dvi: %.tex + latex $< >> /dev/null + latex $< >> /dev/null + +%.tex: %.rest + rest2latex --use-latex-toc \ + --stylesheet lib/stylesheet.sty \ + --use-latex-footnotes \ + --no-section-numbering \ + --language=fr \ + --input-encoding=iso-8859-15 \ + --table-style=booktabs \ + --output-encoding=iso-8859-15 \ + $< > $@ +help: + @echo " Programmes nécessaires: docbook, latex, dvipdf, kpdf" + +.PHONY: pdf diff --git a/doc/README b/doc/README new file mode 100644 index 0000000..3c8ca86 --- /dev/null +++ b/doc/README @@ -0,0 +1,34 @@ +====== +README +====== + +How to compile documentation for pgloader +----------------------------------------- + +**doc_pgloader.rest** + +Some documentation for pgloader project, in english. + +ReST format (see +http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html for more info +on this format and tools). + +Since its plain text, you don't need to compile anything to read it ;-) + +But you can still get a PDF or an HTML file: + +compile PDF version : + + $ make doc_pgloader.pdf + +compile HTML version : + + $ make doc_pgloader.html + +Contact the authors +------------------- + +Feel free to send me questions / comments / patches / whatever : + +Jean-Paul Argudo + diff --git a/doc/doc_pgloader.rest b/doc/doc_pgloader.rest new file mode 100644 index 0000000..97bfd80 --- /dev/null +++ b/doc/doc_pgloader.rest @@ -0,0 +1,179 @@ +======== +pgloader +======== + +:Author: + Jean-Paul Argudo + +:Version: + $Id: doc_pgloader.rest,v 1.1 2005-11-21 16:05:50 jpargudo Exp $ + +:Comment: + pgLoader v.1.x documentation (install, usage and example) + +:Licence: + BSD + +About +===== + +pgloader (http://pgfoundry.org/projects/pgloader/) is a new project allowing +you to import data in a PostgreSQL database. + +You have to launch pgloader as many times you have tables. pgloader handles +just one table at a time. + +All bad records are put together in a file, with a logfile explaining origins +of errors. + + +Installation +============ + +Under Debian, the current installation is a bit tricky (as per 200510xx): :: + + wget http://debian.wow-vision.com.sg/debian/pool/main/p/postgresql libpgtcl_7.4.7-6sarge1_i386.deb + dpgk -i libpgtcl_7.4.7-6sarge1_i386.deb + apt-get install tcllib + wget http://pgfoundry.org/frs/download.php/233/pgloader-1.0.tar.gz + tar zxvf pgloader-1.0.tar.gz + +Then you can eventually put the binary into /usr/local/bin to facilitate +comandlines: :: + + $ cp pgloader-1.0/pgloader /usr/local/bin + +Principle +========= + +You must fill two files per table: + + * a parameter file, let's call it .conf + * a datafile, let's call it
.data + +You need also all necessary parameters to the db connexion you want to use: + +Common ones are the following: + + * host : name of the server where your PostgreSQL db lives (localhost ?) + * user : username (you?) + * password : username's password (mybigsecret) + * dbname : name of the PostgreSQL db + +This parameters are put together in a double-quoted string: + + "host=localhost user=me password=mybigsecret dbname=mydatabase" + +This string as the same type that PQconnectdb awaits for in the libpq. Its +complete documentation can be read at: +http://www.postgresql.org/docs/current/static/libpq.html#LIBPQ-CONNECT + +You can for sure add much more parameters, depending your db configuration. + +Example +======= + +We want to insert records in "foo" table: :: + + test=> \d foo + Table «public.foo» + Colonne | Type | Modificateurs + ---------+---------+--------------- + a | integer | not null + b | date | + c | text | + Index : + «foo_pkey» PRIMARY KEY, btree (a) + +The datafile +------------ + +Our datafile "foo.data" as following records: :: + + 1;1987-12-04;"This is a test of data file" + 2;2005-03-02;"diziz'another test with som'o'lil'quotes" + 42;;"No need to date this" + 67;1999-01-02;Oops I didn't escape this string?! + +Please note that: + + * fields are separated with a semicolon + * you can handle presence of empty data: the empty field is represented with + two semicolons following + * we have a record per line + * theres is no other line separator excepted \n + * dates are in ISO format: YYYY-MM-DD (a fix is coming to handle "set datestyle + to" in the conf file) + * you can escape strings, optionnaly, double quoting them + +Configuration file +------------------ + +The corresponding file "foo.conf" for the above datafile is the following: :: + + # ---- + # Conversion parameter file for pgloader + # + # Possible file formats: + # COPY native PostgreSQL COPY format (default) + # CSV Comma separated variables + # MSCSV Comma separated variables alternate format + # + # The COPY command is constructed from the table_name, the + # table_columns and the eventual nulls string definition. + # + # The default column separator character is comma. + # ---- + + table_name = foo + table_columns = a,b,c + file_format = CSV + group_size = 1000 + file_sepchar = ; + #nulls = NULL + quote = " + +Note that separation character is set to ";" and that quoting is specifyied +with the character double-quote: " + +Inserts will be commited each 1000, per blocks of 1000 rows at a time. + + +pgloader execution +------------------ + +The execution is quite simple: :: + + $ pgloader foo.conf foo.data "host=localhost user=me password=mybigsecret \ + dbname=mydatabase" + + 4 row(s) loaded + 0 row(s) rejected + +A simple verification of what has been inserted: :: + + test=> select * from foo ; + a | b | c + ----+------------+------------------------------------------ + 1 | 1987-12-04 | This is a test of data file + 2 | 2005-03-02 | diziz'another test with som'o'lil'quotes + 42 | | No need to date this + 67 | 1999-01-02 | Oops I didn't escape this string?! + (4 lines) + +**Note**: You will find this example in the doc/example/ directory. + +when errors occurs +------------------ + +Check the following: + + * if your configuration file is not okay, pgloader will tell you whats wrong + + * if you have a problem with the data you try to import, you'll find in the + .rej file data that have bee rejected. In the .rejlog file given problems + will be explicited: a group of error messages per rejected row. + +Then you'll have to correct errors in .rej file and import *that* file like all +the others: don't reimport anything else, all the good data is already in the +box :) diff --git a/doc/example/create_table.sql b/doc/example/create_table.sql new file mode 100644 index 0000000..15cb8ad --- /dev/null +++ b/doc/example/create_table.sql @@ -0,0 +1,2 @@ +drop table foo; +create table foo (a integer primary key, b date, c text); diff --git a/doc/example/foo.conf b/doc/example/foo.conf new file mode 100644 index 0000000..33d3e25 --- /dev/null +++ b/doc/example/foo.conf @@ -0,0 +1,21 @@ +# ---- +# Conversion parameter file for pgloader +# +# Possible file formats: +# COPY native PostgreSQL COPY format (default) +# CSV Comma separated variables +# MSCSV Comma separated variables alternate format +# +# The COPY command is constructed from the table_name, the +# table_columns and the eventual nulls string definition. +# +# The default column separator character is comma. +# ---- + +table_name = foo +table_columns = a,b,c +file_format = CSV +group_size = 1000 +file_sepchar = ; +#nulls = NULL +quote = " diff --git a/doc/example/foo.data b/doc/example/foo.data new file mode 100644 index 0000000..6ee9e96 --- /dev/null +++ b/doc/example/foo.data @@ -0,0 +1,4 @@ +1;1987-12-04;"This is a test of data file" +2;2005-03-02;"diziz'another test with som'o'lil'quotes" +42;;"No need to date this" +67;1999-01-02;Oops I didn't escape this string?! diff --git a/doc/lib/stylesheet.sty b/doc/lib/stylesheet.sty new file mode 100644 index 0000000..8147887 --- /dev/null +++ b/doc/lib/stylesheet.sty @@ -0,0 +1,52 @@ +\let\oldAuthor\author +\renewcommand{\author}[1]{\newcommand{\myAuthor}{#1}\oldAuthor{#1}} +\let\oldTitle\title +\renewcommand{\title}[1]{\newcommand{\myTitle}{#1}\oldTitle{#1}} + +\usepackage{eurosym} +\usepackage[latin9]{inputenc} +\let ¤ = \euro + +\usepackage{fancyhdr} +\pagestyle{fancy} + +\lhead{} +\chead{} +\rhead{\myTitle} +\lfoot{\textsf{pgFoundry} +} +\cfoot{\small{pgloader documentation \\ +http://pgfoundry.org/projects/pgloader/}} +\rfoot{\thepage\ / \pageref*{LastPage}} + +\renewcommand{\headrulewidth}{0.4pt} +\renewcommand{\footrulewidth}{0.4pt} + +\usepackage{helvet} + +\renewcommand{\familydefault}{phv} + +%Parametrage pour une feuille A4 pleine (merci SBI) +\evensidemargin = 30mm +\oddsidemargin = 30mm +\voffset=-1in +\topmargin = 17mm +\headheight = 14.5mm +\headsep = 15mm +\hoffset=-1in +\marginparsep = 0pt +\marginparwidth = 0pt +\footskip = 20mm +\textwidth=162mm +\textheight=200mm +\paperwidth=210mm +\paperheight=297mm +\parindent=0pt +\parskip=5pt +%fin parametrage A4 plein + +\usepackage{lastpage} + +\hypersetup{colorlinks=true} + +\usepackage{indentfirst} diff --git a/pgloader.conf b/pgloader.conf index b100409..54df66b 100644 --- a/pgloader.conf +++ b/pgloader.conf @@ -1,3 +1,4 @@ +# $Id: pgloader.conf,v 1.3 2005-11-21 16:05:50 jpargudo Exp $ # ---- # Conversion parameter file for pgloader # @@ -13,10 +14,9 @@ # ---- table_name = my_table -table_columns = column1, column2, ... -file_format = COPY +table_columns = column1, column2, ... +file_format = COPY # COPY or CSV or MSCSV group_size = 1000 - -# file_sepchar = , -# nulls = '' - +# file_sepchar = ; # , (default) or ; or other +# nulls = '' # NULL or '' or other +# quote = " # how your strings are quoted in the CSV file