mirror of
https://github.com/dimitri/pgloader.git
synced 2026-05-04 10:31:02 +02:00
A new version of pgloader with:
- bugfixes - new options in pgloader.conf Added a small ReST documentation to help the common mortal to use pgloader, and Makefile + stylesheet.sty to generate pretty PDF / HTML output of that doc
This commit is contained in:
jpargudo
parent
e27ebba4bf
commit
0c1e04dfb3
43
doc/Makefile
Normal file
43
doc/Makefile
Normal file
@ -0,0 +1,43 @@
|
||||
# $Id: Makefile,v 1.1 2005-11-21 16:05:50 jpargudo Exp $
|
||||
|
||||
rest = $(wildcard *.rest)
|
||||
html = $(addsuffix .html, $(basename $(rest)))
|
||||
pdf = $(addsuffix .pdf, $(basename $(rest)))
|
||||
|
||||
pdf: $(pdf) clean
|
||||
|
||||
html: $(html)
|
||||
|
||||
dist-clean: clean
|
||||
@rm -f $(pdf) $(html)
|
||||
|
||||
clean:
|
||||
@rm -f *.aux *.log *~ *.tex *.out *.toc *.dvi
|
||||
|
||||
%.html: %.rest
|
||||
rest2html --stylesheet lib/stylesheet.sty \
|
||||
--no-section-numbering \
|
||||
--language=fr \
|
||||
$< > $@
|
||||
|
||||
%.pdf: %.dvi
|
||||
dvipdf $<
|
||||
|
||||
%.dvi: %.tex
|
||||
latex $< >> /dev/null
|
||||
latex $< >> /dev/null
|
||||
|
||||
%.tex: %.rest
|
||||
rest2latex --use-latex-toc \
|
||||
--stylesheet lib/stylesheet.sty \
|
||||
--use-latex-footnotes \
|
||||
--no-section-numbering \
|
||||
--language=fr \
|
||||
--input-encoding=iso-8859-15 \
|
||||
--table-style=booktabs \
|
||||
--output-encoding=iso-8859-15 \
|
||||
$< > $@
|
||||
help:
|
||||
@echo " Programmes nécessaires: docbook, latex, dvipdf, kpdf"
|
||||
|
||||
.PHONY: pdf
|
||||
34
doc/README
Normal file
34
doc/README
Normal file
@ -0,0 +1,34 @@
|
||||
======
|
||||
README
|
||||
======
|
||||
|
||||
How to compile documentation for pgloader
|
||||
-----------------------------------------
|
||||
|
||||
**doc_pgloader.rest**
|
||||
|
||||
Some documentation for pgloader project, in english.
|
||||
|
||||
ReST format (see
|
||||
http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html for more info
|
||||
on this format and tools).
|
||||
|
||||
Since its plain text, you don't need to compile anything to read it ;-)
|
||||
|
||||
But you can still get a PDF or an HTML file:
|
||||
|
||||
compile PDF version :
|
||||
|
||||
$ make doc_pgloader.pdf
|
||||
|
||||
compile HTML version :
|
||||
|
||||
$ make doc_pgloader.html
|
||||
|
||||
Contact the authors
|
||||
-------------------
|
||||
|
||||
Feel free to send me questions / comments / patches / whatever :
|
||||
|
||||
Jean-Paul Argudo <jean-paul.argudo@dalibo.com>
|
||||
|
||||
179
doc/doc_pgloader.rest
Normal file
179
doc/doc_pgloader.rest
Normal file
@ -0,0 +1,179 @@
|
||||
========
|
||||
pgloader
|
||||
========
|
||||
|
||||
:Author:
|
||||
Jean-Paul Argudo <jean-paul.argudo@dalibo.com>
|
||||
|
||||
:Version:
|
||||
$Id: doc_pgloader.rest,v 1.1 2005-11-21 16:05:50 jpargudo Exp $
|
||||
|
||||
:Comment:
|
||||
pgLoader v.1.x documentation (install, usage and example)
|
||||
|
||||
:Licence:
|
||||
BSD
|
||||
|
||||
About
|
||||
=====
|
||||
|
||||
pgloader (http://pgfoundry.org/projects/pgloader/) is a new project allowing
|
||||
you to import data in a PostgreSQL database.
|
||||
|
||||
You have to launch pgloader as many times you have tables. pgloader handles
|
||||
just one table at a time.
|
||||
|
||||
All bad records are put together in a file, with a logfile explaining origins
|
||||
of errors.
|
||||
|
||||
|
||||
Installation
|
||||
============
|
||||
|
||||
Under Debian, the current installation is a bit tricky (as per 200510xx): ::
|
||||
|
||||
wget http://debian.wow-vision.com.sg/debian/pool/main/p/postgresql libpgtcl_7.4.7-6sarge1_i386.deb
|
||||
dpgk -i libpgtcl_7.4.7-6sarge1_i386.deb
|
||||
apt-get install tcllib
|
||||
wget http://pgfoundry.org/frs/download.php/233/pgloader-1.0.tar.gz
|
||||
tar zxvf pgloader-1.0.tar.gz
|
||||
|
||||
Then you can eventually put the binary into /usr/local/bin to facilitate
|
||||
comandlines: ::
|
||||
|
||||
$ cp pgloader-1.0/pgloader /usr/local/bin
|
||||
|
||||
Principle
|
||||
=========
|
||||
|
||||
You must fill two files per table:
|
||||
|
||||
* a parameter file, let's call it <table>.conf
|
||||
* a datafile, let's call it <table>.data
|
||||
|
||||
You need also all necessary parameters to the db connexion you want to use:
|
||||
|
||||
Common ones are the following:
|
||||
|
||||
* host : name of the server where your PostgreSQL db lives (localhost ?)
|
||||
* user : username (you?)
|
||||
* password : username's password (mybigsecret)
|
||||
* dbname : name of the PostgreSQL db
|
||||
|
||||
This parameters are put together in a double-quoted string:
|
||||
|
||||
"host=localhost user=me password=mybigsecret dbname=mydatabase"
|
||||
|
||||
This string as the same type that PQconnectdb awaits for in the libpq. Its
|
||||
complete documentation can be read at:
|
||||
http://www.postgresql.org/docs/current/static/libpq.html#LIBPQ-CONNECT
|
||||
|
||||
You can for sure add much more parameters, depending your db configuration.
|
||||
|
||||
Example
|
||||
=======
|
||||
|
||||
We want to insert records in "foo" table: ::
|
||||
|
||||
test=> \d foo
|
||||
Table «public.foo»
|
||||
Colonne | Type | Modificateurs
|
||||
---------+---------+---------------
|
||||
a | integer | not null
|
||||
b | date |
|
||||
c | text |
|
||||
Index :
|
||||
«foo_pkey» PRIMARY KEY, btree (a)
|
||||
|
||||
The datafile
|
||||
------------
|
||||
|
||||
Our datafile "foo.data" as following records: ::
|
||||
|
||||
1;1987-12-04;"This is a test of data file"
|
||||
2;2005-03-02;"diziz'another test with som'o'lil'quotes"
|
||||
42;;"No need to date this"
|
||||
67;1999-01-02;Oops I didn't escape this string?!
|
||||
|
||||
Please note that:
|
||||
|
||||
* fields are separated with a semicolon
|
||||
* you can handle presence of empty data: the empty field is represented with
|
||||
two semicolons following
|
||||
* we have a record per line
|
||||
* theres is no other line separator excepted \n
|
||||
* dates are in ISO format: YYYY-MM-DD (a fix is coming to handle "set datestyle
|
||||
to" in the conf file)
|
||||
* you can escape strings, optionnaly, double quoting them
|
||||
|
||||
Configuration file
|
||||
------------------
|
||||
|
||||
The corresponding file "foo.conf" for the above datafile is the following: ::
|
||||
|
||||
# ----
|
||||
# Conversion parameter file for pgloader
|
||||
#
|
||||
# Possible file formats:
|
||||
# COPY native PostgreSQL COPY format (default)
|
||||
# CSV Comma separated variables
|
||||
# MSCSV Comma separated variables alternate format
|
||||
#
|
||||
# The COPY command is constructed from the table_name, the
|
||||
# table_columns and the eventual nulls string definition.
|
||||
#
|
||||
# The default column separator character is comma.
|
||||
# ----
|
||||
|
||||
table_name = foo
|
||||
table_columns = a,b,c
|
||||
file_format = CSV
|
||||
group_size = 1000
|
||||
file_sepchar = ;
|
||||
#nulls = NULL
|
||||
quote = "
|
||||
|
||||
Note that separation character is set to ";" and that quoting is specifyied
|
||||
with the character double-quote: "
|
||||
|
||||
Inserts will be commited each 1000, per blocks of 1000 rows at a time.
|
||||
|
||||
|
||||
pgloader execution
|
||||
------------------
|
||||
|
||||
The execution is quite simple: ::
|
||||
|
||||
$ pgloader foo.conf foo.data "host=localhost user=me password=mybigsecret \
|
||||
dbname=mydatabase"
|
||||
|
||||
4 row(s) loaded
|
||||
0 row(s) rejected
|
||||
|
||||
A simple verification of what has been inserted: ::
|
||||
|
||||
test=> select * from foo ;
|
||||
a | b | c
|
||||
----+------------+------------------------------------------
|
||||
1 | 1987-12-04 | This is a test of data file
|
||||
2 | 2005-03-02 | diziz'another test with som'o'lil'quotes
|
||||
42 | | No need to date this
|
||||
67 | 1999-01-02 | Oops I didn't escape this string?!
|
||||
(4 lines)
|
||||
|
||||
**Note**: You will find this example in the doc/example/ directory.
|
||||
|
||||
when errors occurs
|
||||
------------------
|
||||
|
||||
Check the following:
|
||||
|
||||
* if your configuration file is not okay, pgloader will tell you whats wrong
|
||||
|
||||
* if you have a problem with the data you try to import, you'll find in the
|
||||
.rej file data that have bee rejected. In the .rejlog file given problems
|
||||
will be explicited: a group of error messages per rejected row.
|
||||
|
||||
Then you'll have to correct errors in .rej file and import *that* file like all
|
||||
the others: don't reimport anything else, all the good data is already in the
|
||||
box :)
|
||||
2
doc/example/create_table.sql
Normal file
2
doc/example/create_table.sql
Normal file
@ -0,0 +1,2 @@
|
||||
drop table foo;
|
||||
create table foo (a integer primary key, b date, c text);
|
||||
21
doc/example/foo.conf
Normal file
21
doc/example/foo.conf
Normal file
@ -0,0 +1,21 @@
|
||||
# ----
|
||||
# Conversion parameter file for pgloader
|
||||
#
|
||||
# Possible file formats:
|
||||
# COPY native PostgreSQL COPY format (default)
|
||||
# CSV Comma separated variables
|
||||
# MSCSV Comma separated variables alternate format
|
||||
#
|
||||
# The COPY command is constructed from the table_name, the
|
||||
# table_columns and the eventual nulls string definition.
|
||||
#
|
||||
# The default column separator character is comma.
|
||||
# ----
|
||||
|
||||
table_name = foo
|
||||
table_columns = a,b,c
|
||||
file_format = CSV
|
||||
group_size = 1000
|
||||
file_sepchar = ;
|
||||
#nulls = NULL
|
||||
quote = "
|
||||
4
doc/example/foo.data
Normal file
4
doc/example/foo.data
Normal file
@ -0,0 +1,4 @@
|
||||
1;1987-12-04;"This is a test of data file"
|
||||
2;2005-03-02;"diziz'another test with som'o'lil'quotes"
|
||||
42;;"No need to date this"
|
||||
67;1999-01-02;Oops I didn't escape this string?!
|
||||
52
doc/lib/stylesheet.sty
Normal file
52
doc/lib/stylesheet.sty
Normal file
@ -0,0 +1,52 @@
|
||||
\let\oldAuthor\author
|
||||
\renewcommand{\author}[1]{\newcommand{\myAuthor}{#1}\oldAuthor{#1}}
|
||||
\let\oldTitle\title
|
||||
\renewcommand{\title}[1]{\newcommand{\myTitle}{#1}\oldTitle{#1}}
|
||||
|
||||
\usepackage{eurosym}
|
||||
\usepackage[latin9]{inputenc}
|
||||
\let ¤ = \euro
|
||||
|
||||
\usepackage{fancyhdr}
|
||||
\pagestyle{fancy}
|
||||
|
||||
\lhead{}
|
||||
\chead{}
|
||||
\rhead{\myTitle}
|
||||
\lfoot{\textsf{pgFoundry}
|
||||
}
|
||||
\cfoot{\small{pgloader documentation \\
|
||||
http://pgfoundry.org/projects/pgloader/}}
|
||||
\rfoot{\thepage\ / \pageref*{LastPage}}
|
||||
|
||||
\renewcommand{\headrulewidth}{0.4pt}
|
||||
\renewcommand{\footrulewidth}{0.4pt}
|
||||
|
||||
\usepackage{helvet}
|
||||
|
||||
\renewcommand{\familydefault}{phv}
|
||||
|
||||
%Parametrage pour une feuille A4 pleine (merci SBI)
|
||||
\evensidemargin = 30mm
|
||||
\oddsidemargin = 30mm
|
||||
\voffset=-1in
|
||||
\topmargin = 17mm
|
||||
\headheight = 14.5mm
|
||||
\headsep = 15mm
|
||||
\hoffset=-1in
|
||||
\marginparsep = 0pt
|
||||
\marginparwidth = 0pt
|
||||
\footskip = 20mm
|
||||
\textwidth=162mm
|
||||
\textheight=200mm
|
||||
\paperwidth=210mm
|
||||
\paperheight=297mm
|
||||
\parindent=0pt
|
||||
\parskip=5pt
|
||||
%fin parametrage A4 plein
|
||||
|
||||
\usepackage{lastpage}
|
||||
|
||||
\hypersetup{colorlinks=true}
|
||||
|
||||
\usepackage{indentfirst}
|
||||
@ -1,3 +1,4 @@
|
||||
# $Id: pgloader.conf,v 1.3 2005-11-21 16:05:50 jpargudo Exp $
|
||||
# ----
|
||||
# Conversion parameter file for pgloader
|
||||
#
|
||||
@ -13,10 +14,9 @@
|
||||
# ----
|
||||
|
||||
table_name = my_table
|
||||
table_columns = column1, column2, ...
|
||||
file_format = COPY
|
||||
table_columns = column1, column2, ...
|
||||
file_format = COPY # COPY or CSV or MSCSV
|
||||
group_size = 1000
|
||||
|
||||
# file_sepchar = ,
|
||||
# nulls = ''
|
||||
|
||||
# file_sepchar = ; # , (default) or ; or other
|
||||
# nulls = '' # NULL or '' or other
|
||||
# quote = " # how your strings are quoted in the CSV file
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user