diff --git a/.gitignore b/.gitignore index 5e84101..a2b163b 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ web/howto/quickstart.html web/howto/sqlite.html .DS_Store system-index.txt +docs/_build \ No newline at end of file diff --git a/docs/_build/doctrees/bugreport.doctree b/docs/_build/doctrees/bugreport.doctree deleted file mode 100644 index 18b937b..0000000 Binary files a/docs/_build/doctrees/bugreport.doctree and /dev/null differ diff --git a/docs/_build/doctrees/environment.pickle b/docs/_build/doctrees/environment.pickle deleted file mode 100644 index b970489..0000000 Binary files a/docs/_build/doctrees/environment.pickle and /dev/null differ diff --git a/docs/_build/doctrees/index.doctree b/docs/_build/doctrees/index.doctree deleted file mode 100644 index 98788c1..0000000 Binary files a/docs/_build/doctrees/index.doctree and /dev/null differ diff --git a/docs/_build/doctrees/intro.doctree b/docs/_build/doctrees/intro.doctree deleted file mode 100644 index 4e007ec..0000000 Binary files a/docs/_build/doctrees/intro.doctree and /dev/null differ diff --git a/docs/_build/doctrees/pgloader-usage-examples.doctree b/docs/_build/doctrees/pgloader-usage-examples.doctree deleted file mode 100644 index feb3882..0000000 Binary files a/docs/_build/doctrees/pgloader-usage-examples.doctree and /dev/null differ diff --git a/docs/_build/doctrees/pgloader.doctree b/docs/_build/doctrees/pgloader.doctree deleted file mode 100644 index e40b92a..0000000 Binary files a/docs/_build/doctrees/pgloader.doctree and /dev/null differ diff --git a/docs/_build/doctrees/ref/archive.doctree b/docs/_build/doctrees/ref/archive.doctree deleted file mode 100644 index bf65935..0000000 Binary files a/docs/_build/doctrees/ref/archive.doctree and /dev/null differ diff --git a/docs/_build/doctrees/ref/copy.doctree b/docs/_build/doctrees/ref/copy.doctree deleted file mode 100644 index 2c724f1..0000000 Binary files a/docs/_build/doctrees/ref/copy.doctree and /dev/null differ diff --git a/docs/_build/doctrees/ref/csv.doctree b/docs/_build/doctrees/ref/csv.doctree deleted file mode 100644 index 9b59c40..0000000 Binary files a/docs/_build/doctrees/ref/csv.doctree and /dev/null differ diff --git a/docs/_build/doctrees/ref/dbf.doctree b/docs/_build/doctrees/ref/dbf.doctree deleted file mode 100644 index 59b7d5b..0000000 Binary files a/docs/_build/doctrees/ref/dbf.doctree and /dev/null differ diff --git a/docs/_build/doctrees/ref/fixed.doctree b/docs/_build/doctrees/ref/fixed.doctree deleted file mode 100644 index b9c3e95..0000000 Binary files a/docs/_build/doctrees/ref/fixed.doctree and /dev/null differ diff --git a/docs/_build/doctrees/ref/ixf.doctree b/docs/_build/doctrees/ref/ixf.doctree deleted file mode 100644 index 2e33fce..0000000 Binary files a/docs/_build/doctrees/ref/ixf.doctree and /dev/null differ diff --git a/docs/_build/doctrees/ref/mssql.doctree b/docs/_build/doctrees/ref/mssql.doctree deleted file mode 100644 index e661d1a..0000000 Binary files a/docs/_build/doctrees/ref/mssql.doctree and /dev/null differ diff --git a/docs/_build/doctrees/ref/mysql.doctree b/docs/_build/doctrees/ref/mysql.doctree deleted file mode 100644 index c511810..0000000 Binary files a/docs/_build/doctrees/ref/mysql.doctree and /dev/null differ diff --git a/docs/_build/doctrees/ref/sqlite.doctree b/docs/_build/doctrees/ref/sqlite.doctree deleted file mode 100644 index 9a8a981..0000000 Binary files a/docs/_build/doctrees/ref/sqlite.doctree and /dev/null differ diff --git a/docs/_build/doctrees/ref/transforms.doctree b/docs/_build/doctrees/ref/transforms.doctree deleted file mode 100644 index 959692e..0000000 Binary files a/docs/_build/doctrees/ref/transforms.doctree and /dev/null differ diff --git a/docs/_build/doctrees/tutorial/csv.doctree b/docs/_build/doctrees/tutorial/csv.doctree deleted file mode 100644 index 094551b..0000000 Binary files a/docs/_build/doctrees/tutorial/csv.doctree and /dev/null differ diff --git a/docs/_build/doctrees/tutorial/dBase.doctree b/docs/_build/doctrees/tutorial/dBase.doctree deleted file mode 100644 index 0c5ba5d..0000000 Binary files a/docs/_build/doctrees/tutorial/dBase.doctree and /dev/null differ diff --git a/docs/_build/doctrees/tutorial/fixed.doctree b/docs/_build/doctrees/tutorial/fixed.doctree deleted file mode 100644 index 0f87ca3..0000000 Binary files a/docs/_build/doctrees/tutorial/fixed.doctree and /dev/null differ diff --git a/docs/_build/doctrees/tutorial/geolite.doctree b/docs/_build/doctrees/tutorial/geolite.doctree deleted file mode 100644 index c4d3e69..0000000 Binary files a/docs/_build/doctrees/tutorial/geolite.doctree and /dev/null differ diff --git a/docs/_build/doctrees/tutorial/mysql.doctree b/docs/_build/doctrees/tutorial/mysql.doctree deleted file mode 100644 index 65b4322..0000000 Binary files a/docs/_build/doctrees/tutorial/mysql.doctree and /dev/null differ diff --git a/docs/_build/doctrees/tutorial/quickstart.doctree b/docs/_build/doctrees/tutorial/quickstart.doctree deleted file mode 100644 index 35430fd..0000000 Binary files a/docs/_build/doctrees/tutorial/quickstart.doctree and /dev/null differ diff --git a/docs/_build/doctrees/tutorial/sqlite.doctree b/docs/_build/doctrees/tutorial/sqlite.doctree deleted file mode 100644 index 258a0e1..0000000 Binary files a/docs/_build/doctrees/tutorial/sqlite.doctree and /dev/null differ diff --git a/docs/_build/doctrees/tutorial/tutorial.doctree b/docs/_build/doctrees/tutorial/tutorial.doctree deleted file mode 100644 index 40878d4..0000000 Binary files a/docs/_build/doctrees/tutorial/tutorial.doctree and /dev/null differ diff --git a/docs/_build/html/.buildinfo b/docs/_build/html/.buildinfo deleted file mode 100644 index 6cd40d1..0000000 --- a/docs/_build/html/.buildinfo +++ /dev/null @@ -1,4 +0,0 @@ -# Sphinx build info version 1 -# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. -config: 59392f424cbb8621f1b34cf4df7fbec1 -tags: 645f666f9bcd5a90fca523b33c5a78b7 diff --git a/docs/_build/html/.nojekyll b/docs/_build/html/.nojekyll deleted file mode 100644 index e69de29..0000000 diff --git a/docs/_build/html/_sources/bugreport.rst.txt b/docs/_build/html/_sources/bugreport.rst.txt deleted file mode 100644 index 84726b5..0000000 --- a/docs/_build/html/_sources/bugreport.rst.txt +++ /dev/null @@ -1,49 +0,0 @@ -Reporting Bugs -============== - -pgloader is a software and as such contains bugs. Most bugs are easy to -solve and taken care of in a short delay. For this to be possible though, -bug reports need to follow those recommandations: - - - include pgloader version, - - include problematic input and output, - - include a description of the output you expected, - - explain the difference between the ouput you have and the one you expected, - - include a self-reproducing test-case - -Test Cases to Reproduce Bugs ----------------------------- - -Use the *inline* source type to help reproduce a bug, as in the pgloader tests:: - - LOAD CSV - FROM INLINE - INTO postgresql://dim@localhost/pgloader?public."HS" - - WITH truncate, - fields terminated by '\t', - fields not enclosed, - fields escaped by backslash-quote, - quote identifiers - - SET work_mem to '128MB', - standard_conforming_strings to 'on', - application_name to 'my app name' - - BEFORE LOAD DO - $$ create extension if not exists hstore; $$, - $$ drop table if exists "HS"; $$, - $$ CREATE TABLE "HS" - ( - id serial primary key, - kv hstore - ) - $$; - - - 1 email=>foo@example.com,a=>b - 2 test=>value - 3 a=>b,c=>"quoted hstore value",d=>other - 4 baddata - - diff --git a/docs/_build/html/_sources/index.rst.txt b/docs/_build/html/_sources/index.rst.txt deleted file mode 100644 index d69915e..0000000 --- a/docs/_build/html/_sources/index.rst.txt +++ /dev/null @@ -1,33 +0,0 @@ -.. pgloader documentation master file, created by - sphinx-quickstart on Tue Dec 5 19:23:32 2017. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to pgloader's documentation! -==================================== - -.. toctree:: - :maxdepth: 2 - :caption: Table Of Contents: - - intro - tutorial/tutorial - pgloader - ref/csv - ref/fixed - ref/copy - ref/dbf - ref/ixf - ref/archive - ref/mysql - ref/sqlite - ref/mssql - ref/transforms - bugreport - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/docs/_build/html/_sources/intro.rst.txt b/docs/_build/html/_sources/intro.rst.txt deleted file mode 100644 index 0dc75e2..0000000 --- a/docs/_build/html/_sources/intro.rst.txt +++ /dev/null @@ -1,63 +0,0 @@ -Introduction -============ - -pgloader loads data from various sources into PostgreSQL. It can -transform the data it reads on the fly and submit raw SQL before and -after the loading. It uses the `COPY` PostgreSQL protocol to stream -the data into the server, and manages errors by filling a pair of -*reject.dat* and *reject.log* files. - -pgloader knows how to read data from different kind of sources: - - * Files - * CSV - * Fixed Format - * DBF - * Databases - * SQLite - * MySQL - * MS SQL Server - -The level of automation provided by pgloader depends on the data source -type. In the case of CSV and Fixed Format files, a full description of the -expected input properties must be given to pgloader. In the case of a -database, pgloader connects to the live service and knows how to fetch the -metadata it needs directly from it. - -Continuous Migration --------------------- - -pgloader is meant to migrate a whole database in a single command line and -without any manual intervention. The goal is to be able to setup a -*Continuous Integration* environment as described in the `Project -Methodology `_ document of the `MySQL to -PostgreSQL `_ webpage. - - 1. Setup your target PostgreSQL Architecture - 2. Fork a Continuous Integration environment that uses PostgreSQL - 3. Migrate the data over and over again every night, from production - 4. As soon as the CI is all green using PostgreSQL, schedule the D-Day - 5. Migrate without suprise and enjoy! - -In order to be able to follow this great methodology, you need tooling to -implement the third step in a fully automated way. That's pgloader. - -Commands --------- - -pgloader implements its own *Command Language*, a DSL that allows to specify -every aspect of the data load and migration to implement. Some of the -features provided in the language are only available for a specific source -type. - -Command Line ------------- - -The pgloader command line accepts those two variants:: - - pgloader [] []... - pgloader [] SOURCE TARGET - -Either you have a *command-file* containing migration specifications in the -pgloader *Command Language*, or you can give a *Source* for the data and a -PostgreSQL database connection *Target* where to load the data into. diff --git a/docs/_build/html/_sources/pgloader-usage-examples.rst.txt b/docs/_build/html/_sources/pgloader-usage-examples.rst.txt deleted file mode 100644 index 72223b9..0000000 --- a/docs/_build/html/_sources/pgloader-usage-examples.rst.txt +++ /dev/null @@ -1,163 +0,0 @@ -Pgloader Usage Examples -======================= - -Currently not included, because redundant with the tutorial. - -Usage Examples --------------- - -Review the command line options and pgloader's version:: - - pgloader --help - pgloader --version - -Loading from a complex command -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Use the command file as the pgloader command argument, pgloader will parse -that file and execute the commands found in it:: - - pgloader --verbose ./test/csv-districts.load - -CSV -^^^ - -Load data from a CSV file into a pre-existing table in your database, having -pgloader guess the CSV properties (separator, quote and escape character):: - - pgloader ./test/data/matching-1.csv pgsql:///pgloader?tablename=matching - -Load data from a CSV file into a pre-existing table in your database, with -expanded options:: - - pgloader --type csv \ - --field id --field field \ - --with truncate \ - --with "fields terminated by ','" \ - ./test/data/matching-1.csv \ - postgres:///pgloader?tablename=matching - -In that example the whole loading is driven from the command line, bypassing -the need for writing a command in the pgloader command syntax entirely. As -there's no command though, the extra inforamtion needed must be provided on -the command line using the `--type` and `--field` and `--with` switches. - -For documentation about the available syntaxes for the `--field` and -`--with` switches, please refer to the CSV section later in the man page. - -Note also that the PostgreSQL URI includes the target *tablename*. - -Reading from STDIN -^^^^^^^^^^^^^^^^^^ - -File based pgloader sources can be loaded from the standard input, as in the -following example:: - - pgloader --type csv \ - --field "usps,geoid,aland,awater,aland_sqmi,awater_sqmi,intptlat,intptlong" \ - --with "skip header = 1" \ - --with "fields terminated by '\t'" \ - - \ - postgresql:///pgloader?districts_longlat \ - < test/data/2013_Gaz_113CDs_national.txt - -The dash (`-`) character as a source is used to mean *standard input*, as -usual in Unix command lines. It's possible to stream compressed content to -pgloader with this technique, using the Unix pipe: - - gunzip -c source.gz | pgloader --type csv ... - pgsql:///target?foo - -Loading from CSV available through HTTP -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The same command as just above can also be run if the CSV file happens to be -found on a remote HTTP location:: - - pgloader --type csv \ - --field "usps,geoid,aland,awater,aland_sqmi,awater_sqmi,intptlat,intptlong" \ - --with "skip header = 1" \ - --with "fields terminated by '\t'" \ - http://pgsql.tapoueh.org/temp/2013_Gaz_113CDs_national.txt \ - postgresql:///pgloader?districts_longlat - -Some more options have to be used in that case, as the file contains a -one-line header (most commonly that's column names, could be a copyright -notice). Also, in that case, we specify all the fields right into a single -`--field` option argument. - -Again, the PostgreSQL target connection string must contain the *tablename* -option and you have to ensure that the target table exists and may fit the -data. Here's the SQL command used in that example in case you want to try it -yourself:: - - create table districts_longlat - ( - usps text, - geoid text, - aland bigint, - awater bigint, - aland_sqmi double precision, - awater_sqmi double precision, - intptlat double precision, - intptlong double precision - ); - -Also notice that the same command will work against an archived version of -the same data, e.g. -http://pgsql.tapoueh.org/temp/2013_Gaz_113CDs_national.txt.gz. - -Finally, it's important to note that pgloader first fetches the content from -the HTTP URL it to a local file, then expand the archive when it's -recognized to be one, and only then processes the locally expanded file. - -In some cases, either because pgloader has no direct support for your -archive format or maybe because expanding the archive is not feasible in -your environment, you might want to *stream* the content straight from its -remote location into PostgreSQL. Here's how to do that, using the old battle -tested Unix Pipes trick:: - - curl http://pgsql.tapoueh.org/temp/2013_Gaz_113CDs_national.txt.gz \ - | gunzip -c \ - | pgloader --type csv \ - --field "usps,geoid,aland,awater,aland_sqmi,awater_sqmi,intptlat,intptlong" - --with "skip header = 1" \ - --with "fields terminated by '\t'" \ - - \ - postgresql:///pgloader?districts_longlat - -Now the OS will take care of the streaming and buffering between the network -and the commands and pgloader will take care of streaming the data down to -PostgreSQL. - -Migrating from SQLite -^^^^^^^^^^^^^^^^^^^^^ - -The following command will open the SQLite database, discover its tables -definitions including indexes and foreign keys, migrate those definitions -while *casting* the data type specifications to their PostgreSQL equivalent -and then migrate the data over:: - - createdb newdb - pgloader ./test/sqlite/sqlite.db postgresql:///newdb - -Migrating from MySQL -^^^^^^^^^^^^^^^^^^^^ - -Just create a database where to host the MySQL data and definitions and have -pgloader do the migration for you in a single command line:: - - createdb pagila - pgloader mysql://user@localhost/sakila postgresql:///pagila - -Fetching an archived DBF file from a HTTP remote location -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -It's possible for pgloader to download a file from HTTP, unarchive it, and -only then open it to discover the schema then load the data:: - - createdb foo - pgloader --type dbf http://www.insee.fr/fr/methodes/nomenclatures/cog/telechargement/2013/dbf/historiq2013.zip postgresql:///foo - -Here it's not possible for pgloader to guess the kind of data source it's -being given, so it's necessary to use the `--type` command line switch. - diff --git a/docs/_build/html/_sources/pgloader.rst.txt b/docs/_build/html/_sources/pgloader.rst.txt deleted file mode 100644 index fb51bdf..0000000 --- a/docs/_build/html/_sources/pgloader.rst.txt +++ /dev/null @@ -1,713 +0,0 @@ -PgLoader Reference Manual -========================= - -pgloader loads data from various sources into PostgreSQL. It can -transform the data it reads on the fly and submit raw SQL before and -after the loading. It uses the `COPY` PostgreSQL protocol to stream -the data into the server, and manages errors by filling a pair of -*reject.dat* and *reject.log* files. - -pgloader operates either using commands which are read from files:: - - pgloader commands.load - -or by using arguments and options all provided on the command line:: - - pgloader SOURCE TARGET - -Arguments ---------- - -The pgloader arguments can be as many load files as needed, or a couple of -connection strings to a specific input file. - -Source Connection String -^^^^^^^^^^^^^^^^^^^^^^^^ - -The source connection string format is as follows:: - - format:///absolute/path/to/file.ext - format://./relative/path/to/file.ext - -Where format might be one of `csv`, `fixed`, `copy`, `dbf`, `db3` or `ixf`.:: - - db://user:pass@host:port/dbname - -Where db might be of `sqlite`, `mysql` or `mssql`. - -When using a file based source format, pgloader also support natively -fetching the file from an http location and decompressing an archive if -needed. In that case it's necessary to use the `--type` option to specify -the expected format of the file. See the examples below. - -Also note that some file formats require describing some implementation -details such as columns to be read and delimiters and quoting when loading -from csv. - -For more complex loading scenarios, you will need to write a full fledge -load command in the syntax described later in this document. - -Target Connection String -^^^^^^^^^^^^^^^^^^^^^^^^ - -The target connection string format is described in details later in this -document, see Section Connection String. - -Options -------- - -Inquiry Options -^^^^^^^^^^^^^^^ - -Use these options when you want to know more about how to use `pgloader`, as -those options will cause `pgloader` not to load any data. - - * `-h`, `--help` - - Show command usage summary and exit. - - * `-V`, `--version` - - Show pgloader version string and exit. - - * `-E`, `--list-encodings` - - List known encodings in this version of pgloader. - - * `-U`, `--upgrade-config` - - Parse given files in the command line as `pgloader.conf` files with the - `INI` syntax that was in use in pgloader versions 2.x, and output the - new command syntax for pgloader on standard output. - - -General Options -^^^^^^^^^^^^^^^ - -Those options are meant to tweak `pgloader` behavior when loading data. - - * `-v`, `--verbose` - - Be verbose. - - * `-q`, `--quiet` - - Be quiet. - - * `-d`, `--debug` - - Show debug level information messages. - - * `-D`, `--root-dir` - - Set the root working directory (default to "/tmp/pgloader"). - - * `-L`, `--logfile` - - Set the pgloader log file (default to "/tmp/pgloader.log"). - - * `--log-min-messages` - - Minimum level of verbosity needed for log message to make it to the - logfile. One of critical, log, error, warning, notice, info or debug. - - * `--client-min-messages` - - Minimum level of verbosity needed for log message to make it to the - console. One of critical, log, error, warning, notice, info or debug. - - * `-S`, `--summary` - - A filename where to copy the summary output. When relative, the filename - is expanded into `*root-dir*`. - - The format of the filename defaults to being *human readable*. It is - possible to have the output in machine friendly formats such as *CSV*, - *COPY* (PostgreSQL's own COPY format) or *JSON* by specifying a filename - with the extension resp. `.csv`, `.copy` or `.json`. - - * `-l `, `--load-lisp-file ` - - Specify a lisp to compile and load into the pgloader image before - reading the commands, allowing to define extra transformation function. - Those functions should be defined in the `pgloader.transforms` package. - This option can appear more than once in the command line. - - * `--dry-run` - - Allow testing a `.load` file without actually trying to load any data. - It's useful to debug it until it's ok, in particular to fix connection - strings. - - * `--on-error-stop` - - Alter pgloader behavior: rather than trying to be smart about error - handling and continue loading good data, separating away the bad one, - just stop as soon as PostgreSQL refuses anything sent to it. Useful to - debug data processing, transformation function and specific type - casting. - - * `--self-upgrade ` - - Specify a where to find pgloader sources so that one of the - very first things it does is dynamically loading-in (and compiling to - machine code) another version of itself, usually a newer one like a very - recent git checkout. - -Command Line Only Operations -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Those options are meant to be used when using `pgloader` from the command -line only, rather than using a command file and the rich command clauses and -parser. In simple cases, it can be much easier to use the *SOURCE* and -*TARGET* directly on the command line, then tweak the loading with those -options: - - * `--with "option"` - - Allows setting options from the command line. You can use that option as - many times as you want. The option arguments must follow the *WITH* - clause for the source type of the `SOURCE` specification, as described - later in this document. - - * `--set "guc_name='value'"` - - Allows setting PostgreSQL configuration from the command line. Note that - the option parsing is the same as when used from the *SET* command - clause, in particular you must enclose the guc value with single-quotes. - - * `--field "..."` - - Allows setting a source field definition. Fields are accumulated in the - order given on the command line. It's possible to either use a `--field` - option per field in the source file, or to separate field definitions by - a comma, as you would do in the *HAVING FIELDS* clause. - - * `--cast "..."` - - Allows setting a specific casting rule for loading the data. - - * `--type csv|fixed|db3|ixf|sqlite|mysql|mssql` - - Allows forcing the source type, in case when the *SOURCE* parsing isn't - satisfying. - - * `--encoding ` - - Set the encoding of the source file to load data from. - - * `--before ` - - Parse given filename for SQL queries and run them against the target - database before loading the data from the source. The queries are parsed - by pgloader itself: they need to be terminated by a semi-colon (;) and - the file may include `\i` or `\ir` commands to *include* another file. - - * `--after ` - - Parse given filename for SQL queries and run them against the target - database after having loaded the data from the source. The queries are - parsed in the same way as with the `--before` option, see above. - -More Debug Information -^^^^^^^^^^^^^^^^^^^^^^ - -To get the maximum amount of debug information, you can use both the -`--verbose` and the `--debug` switches at the same time, which is equivalent -to saying `--client-min-messages data`. Then the log messages will show the -data being processed, in the cases where the code has explicit support for -it. - -Batches And Retry Behaviour ---------------------------- - -To load data to PostgreSQL, pgloader uses the `COPY` streaming protocol. -While this is the faster way to load data, `COPY` has an important drawback: -as soon as PostgreSQL emits an error with any bit of data sent to it, -whatever the problem is, the whole data set is rejected by PostgreSQL. - -To work around that, pgloader cuts the data into *batches* of 25000 rows -each, so that when a problem occurs it's only impacting that many rows of -data. Each batch is kept in memory while the `COPY` streaming happens, in -order to be able to handle errors should some happen. - -When PostgreSQL rejects the whole batch, pgloader logs the error message -then isolates the bad row(s) from the accepted ones by retrying the batched -rows in smaller batches. To do that, pgloader parses the *CONTEXT* error -message from the failed COPY, as the message contains the line number where -the error was found in the batch, as in the following example:: - - CONTEXT: COPY errors, line 3, column b: "2006-13-11" - -Using that information, pgloader will reload all rows in the batch before -the erroneous one, log the erroneous one as rejected, then try loading the -remaining of the batch in a single attempt, which may or may not contain -other erroneous data. - -At the end of a load containing rejected rows, you will find two files in -the *root-dir* location, under a directory named the same as the target -database of your setup. The filenames are the target table, and their -extensions are `.dat` for the rejected data and `.log` for the file -containing the full PostgreSQL client side logs about the rejected data. - -The `.dat` file is formatted in PostgreSQL the text COPY format as documented -in `http://www.postgresql.org/docs/9.2/static/sql-copy.html#AEN66609`. - -A Note About Performance ------------------------- - -pgloader has been developed with performance in mind, to be able to cope -with ever growing needs in loading large amounts of data into PostgreSQL. - -The basic architecture it uses is the old Unix pipe model, where a thread is -responsible for loading the data (reading a CSV file, querying MySQL, etc) -and fills pre-processed data into a queue. Another threads feeds from the -queue, apply some more *transformations* to the input data and stream the -end result to PostgreSQL using the COPY protocol. - -When given a file that the PostgreSQL `COPY` command knows how to parse, and -if the file contains no erroneous data, then pgloader will never be as fast -as just using the PostgreSQL `COPY` command. - -Note that while the `COPY` command is restricted to read either from its -standard input or from a local file on the server's file system, the command -line tool `psql` implements a `\copy` command that knows how to stream a -file local to the client over the network and into the PostgreSQL server, -using the same protocol as pgloader uses. - -A Note About Parallelism ------------------------- - -pgloader uses several concurrent tasks to process the data being loaded: - - - a reader task reads the data in and pushes it to a queue, - - - at last one write task feeds from the queue and formats the raw into the - PostgreSQL COPY format in batches (so that it's possible to then retry a - failed batch without reading the data from source again), and then sends - the data to PostgreSQL using the COPY protocol. - -The parameter *workers* allows to control how many worker threads are -allowed to be active at any time (that's the parallelism level); and the -parameter *concurrency* allows to control how many tasks are started to -handle the data (they may not all run at the same time, depending on the -*workers* setting). - -We allow *workers* simultaneous workers to be active at the same time in the -context of a single table. A single unit of work consist of several kinds of -workers: - - - a reader getting raw data from the source, - - N writers preparing and sending the data down to PostgreSQL. - -The N here is setup to the *concurrency* parameter: with a *CONCURRENCY* of -2, we start (+ 1 2) = 3 concurrent tasks, with a *concurrency* of 4 we start -(+ 1 4) = 9 concurrent tasks, of which only *workers* may be active -simultaneously. - -The defaults are `workers = 4, concurrency = 1` when loading from a database -source, and `workers = 8, concurrency = 2` when loading from something else -(currently, a file). Those defaults are arbitrary and waiting for feedback -from users, so please consider providing feedback if you play with the -settings. - -As the `CREATE INDEX` threads started by pgloader are only waiting until -PostgreSQL is done with the real work, those threads are *NOT* counted into -the concurrency levels as detailed here. - -By default, as many `CREATE INDEX` threads as the maximum number of indexes -per table are found in your source schema. It is possible to set the `max -parallel create index` *WITH* option to another number in case there's just -too many of them to create. - -Source Formats --------------- - -pgloader supports the following input formats: - - - csv, which includes also tsv and other common variants where you can - change the *separator* and the *quoting* rules and how to *escape* the - *quotes* themselves; - - - fixed columns file, where pgloader is flexible enough to accomodate with - source files missing columns (*ragged fixed length column files* do - exist); - - - PostgreSLQ COPY formatted files, following the COPY TEXT documentation - of PostgreSQL, such as the reject files prepared by pgloader; - - - dbase files known as db3 or dbf file; - - - ixf formated files, ixf being a binary storage format from IBM; - - - sqlite databases with fully automated discovery of the schema and - advanced cast rules; - - - mysql databases with fully automated discovery of the schema and - advanced cast rules; - - - MS SQL databases with fully automated discovery of the schema and - advanced cast rules. - -Pgloader Commands Syntax ------------------------- - -pgloader implements a Domain Specific Language allowing to setup complex -data loading scripts handling computed columns and on-the-fly sanitization -of the input data. For more complex data loading scenarios, you will be -required to learn that DSL's syntax. It's meant to look familiar to DBA by -being inspired by SQL where it makes sense, which is not that much after -all. - -The pgloader commands follow the same global grammar rules. Each of them -might support only a subset of the general options and provide specific -options. - -:: - - LOAD - FROM - [ HAVING FIELDS ] - INTO - [ TARGET TABLE [ "" ]."" ] - [ TARGET COLUMNS ] - - [ WITH ] - - [ SET ] - - [ BEFORE LOAD [ DO | EXECUTE ] ... ] - [ AFTER LOAD [ DO | EXECUTE ] ... ] - ; - -The main clauses are the `LOAD`, `FROM`, `INTO` and `WITH` clauses that each -command implements. Some command then implement the `SET` command, or some -specific clauses such as the `CAST` clause. - -Templating with Mustache ------------------------- - -pgloader implements the https://mustache.github.io/ templating system so -that you may have dynamic parts of your commands. See the documentation for -this template system online. - -A specific feature of pgloader is the ability to fetch a variable from the -OS environment of the pgloader process, making it possible to run pgloader -as in the following example:: - - $ DBPATH=sqlite/sqlite.db pgloader ./test/sqlite-env.load - -or in several steps:: - - $ export DBPATH=sqlite/sqlite.db - $ pgloader ./test/sqlite-env.load - -The variable can then be used in a typical mustache fashion:: - - load database - from '{{DBPATH}}' - into postgresql:///pgloader; - -It's also possible to prepare a INI file such as the following:: - - [pgloader] - - DBPATH = sqlite/sqlite.db - -And run the following command, feeding the INI values as a *context* for -pgloader templating system:: - - $ pgloader --context ./test/sqlite.ini ./test/sqlite-ini.load - -The mustache templates implementation with OS environment support replaces -former `GETENV` implementation, which didn't work anyway. - -Common Clauses --------------- - -Some clauses are common to all commands: - -FROM -^^^^ - -The *FROM* clause specifies where to read the data from, and each command -introduces its own variant of sources. For instance, the *CSV* source -supports `inline`, `stdin`, a filename, a quoted filename, and a *FILENAME -MATCHING* clause (see above); whereas the *MySQL* source only supports a -MySQL database URI specification. - -INTO -^^^^ - -The PostgreSQL connection URI must contains the name of the target table -where to load the data into. That table must have already been created in -PostgreSQL, and the name might be schema qualified. - -Then *INTO* option also supports an optional comma separated list of target -columns, which are either the name of an input *field* or the white space -separated list of the target column name, its PostgreSQL data type and a -*USING* expression. - -The *USING* expression can be any valid Common Lisp form and will be read -with the current package set to `pgloader.transforms`, so that you can use -functions defined in that package, such as functions loaded dynamically with -the `--load` command line parameter. - -Each *USING* expression is compiled at runtime to native code. - -This feature allows pgloader to load any number of fields in a CSV file into -a possibly different number of columns in the database, using custom code -for that projection. - -WITH -^^^^ - -Set of options to apply to the command, using a global syntax of either: - - - *key = value* - - *use option* - - *do not use option* - -See each specific command for details. - -All data sources specific commands support the following options: - - - *on error stop* - - *batch rows = R* - - *batch size = ... MB* - - *prefetch rows = ...* - -See the section BATCH BEHAVIOUR OPTIONS for more details. - -In addition, the following settings are available: - - - *workers = W* - - *concurrency = C* - - *max parallel create index = I* - -See section A NOTE ABOUT PARALLELISM for more details. - -SET -^^^ - -This clause allows to specify session parameters to be set for all the -sessions opened by pgloader. It expects a list of parameter name, the equal -sign, then the single-quoted value as a comma separated list. - -The names and values of the parameters are not validated by pgloader, they -are given as-is to PostgreSQL. - -BEFORE LOAD DO -^^^^^^^^^^^^^^ - -You can run SQL queries against the database before loading the data from -the `CSV` file. Most common SQL queries are `CREATE TABLE IF NOT EXISTS` so -that the data can be loaded. - -Each command must be *dollar-quoted*: it must begin and end with a double -dollar sign, `$$`. Dollar-quoted queries are then comma separated. No extra -punctuation is expected after the last SQL query. - -BEFORE LOAD EXECUTE -^^^^^^^^^^^^^^^^^^^ - -Same behaviour as in the *BEFORE LOAD DO* clause. Allows you to read the SQL -queries from a SQL file. Implements support for PostgreSQL dollar-quoting -and the `\i` and `\ir` include facilities as in `psql` batch mode (where -they are the same thing). - -AFTER LOAD DO -^^^^^^^^^^^^^ - -Same format as *BEFORE LOAD DO*, the dollar-quoted queries found in that -section are executed once the load is done. That's the right time to create -indexes and constraints, or re-enable triggers. - -AFTER LOAD EXECUTE -^^^^^^^^^^^^^^^^^^ - -Same behaviour as in the *AFTER LOAD DO* clause. Allows you to read the SQL -queries from a SQL file. Implements support for PostgreSQL dollar-quoting -and the `\i` and `\ir` include facilities as in `psql` batch mode (where -they are the same thing). - -Connection String -^^^^^^^^^^^^^^^^^ - -The `` parameter is expected to be given as a *Connection URI* -as documented in the PostgreSQL documentation at -http://www.postgresql.org/docs/9.3/static/libpq-connect.html#LIBPQ-CONNSTRING. - -:: - - postgresql://[user[:password]@][netloc][:port][/dbname][?option=value&...] - -Where: - - - *user* - - Can contain any character, including colon (`:`) which must then be - doubled (`::`) and at-sign (`@`) which must then be doubled (`@@`). - - When omitted, the *user* name defaults to the value of the `PGUSER` - environment variable, and if it is unset, the value of the `USER` - environment variable. - - - *password* - - Can contain any character, including the at sign (`@`) which must then - be doubled (`@@`). To leave the password empty, when the *user* name - ends with at at sign, you then have to use the syntax user:@. - - When omitted, the *password* defaults to the value of the `PGPASSWORD` - environment variable if it is set, otherwise the password is left - unset. - - When no *password* is found either in the connection URI nor in the - environment, then pgloader looks for a `.pgpass` file as documented at - https://www.postgresql.org/docs/current/static/libpq-pgpass.html. The - implementation is not that of `libpq` though. As with `libpq` you can - set the environment variable `PGPASSFILE` to point to a `.pgpass` file, - and pgloader defaults to `~/.pgpass` on unix like systems and - `%APPDATA%\postgresql\pgpass.conf` on windows. Matching rules and syntax - are the same as with `libpq`, refer to its documentation. - - - *netloc* - - Can be either a hostname in dotted notation, or an ipv4, or an Unix - domain socket path. Empty is the default network location, under a - system providing *unix domain socket* that method is preferred, otherwise - the *netloc* default to `localhost`. - - It's possible to force the *unix domain socket* path by using the syntax - `unix:/path/to/where/the/socket/file/is`, so to force a non default - socket path and a non default port, you would have: - - postgresql://unix:/tmp:54321/dbname - - The *netloc* defaults to the value of the `PGHOST` environment - variable, and if it is unset, to either the default `unix` socket path - when running on a Unix system, and `localhost` otherwise. - - Socket path containing colons are supported by doubling the colons - within the path, as in the following example: - - postgresql://unix:/tmp/project::region::instance:5432/dbname - - - *dbname* - - Should be a proper identifier (letter followed by a mix of letters, - digits and the punctuation signs comma (`,`), dash (`-`) and underscore - (`_`). - - When omitted, the *dbname* defaults to the value of the environment - variable `PGDATABASE`, and if that is unset, to the *user* value as - determined above. - - - *options* - - The optional parameters must be supplied with the form `name=value`, and - you may use several parameters by separating them away using an - ampersand (`&`) character. - - Only some options are supported here, *tablename* (which might be - qualified with a schema name) *sslmode*, *host*, *port*, *dbname*, - *user* and *password*. - - The *sslmode* parameter values can be one of `disable`, `allow`, - `prefer` or `require`. - - For backward compatibility reasons, it's possible to specify the - *tablename* option directly, without spelling out the `tablename=` - parts. - - The options override the main URI components when both are given, and - using the percent-encoded option parameters allow using passwords - starting with a colon and bypassing other URI components parsing - limitations. - -Regular Expressions -^^^^^^^^^^^^^^^^^^^ - -Several clauses listed in the following accept *regular expressions* with -the following input rules: - - - A regular expression begins with a tilde sign (`~`), - - - is then followed with an opening sign, - - - then any character is allowed and considered part of the regular - expression, except for the closing sign, - - - then a closing sign is expected. - -The opening and closing sign are allowed by pair, here's the complete list -of allowed delimiters:: - - ~// - ~[] - ~{} - ~() - ~<> - ~"" - ~'' - ~|| - ~## - -Pick the set of delimiters that don't collide with the *regular expression* -you're trying to input. If your expression is such that none of the -solutions allow you to enter it, the places where such expressions are -allowed should allow for a list of expressions. - -Comments -^^^^^^^^ - -Any command may contain comments, following those input rules: - - - the `--` delimiter begins a comment that ends with the end of the - current line, - - - the delimiters `/*` and `*/` respectively start and end a comment, which - can be found in the middle of a command or span several lines. - -Any place where you could enter a *whitespace* will accept a comment too. - -Batch behaviour options -^^^^^^^^^^^^^^^^^^^^^^^ - -All pgloader commands have support for a *WITH* clause that allows for -specifying options. Some options are generic and accepted by all commands, -such as the *batch behaviour options*, and some options are specific to a -data source kind, such as the CSV *skip header* option. - -The global batch behaviour options are: - - - *batch rows* - - Takes a numeric value as argument, used as the maximum number of rows - allowed in a batch. The default is `25 000` and can be changed to try - having better performance characteristics or to control pgloader memory - usage; - - - *batch size* - - Takes a memory unit as argument, such as *20 MB*, its default value. - Accepted multipliers are *kB*, *MB*, *GB*, *TB* and *PB*. The case is - important so as not to be confused about bits versus bytes, we're only - talking bytes here. - - - *prefetch rows* - - Takes a numeric value as argument, defaults to `100000`. That's the - number of rows that pgloader is allowed to read in memory in each reader - thread. See the *workers* setting for how many reader threads are - allowed to run at the same time. - -Other options are specific to each input source, please refer to specific -parts of the documentation for their listing and covering. - -A batch is then closed as soon as either the *batch rows* or the *batch -size* threshold is crossed, whichever comes first. In cases when a batch has -to be closed because of the *batch size* setting, a *debug* level log -message is printed with how many rows did fit in the *oversized* batch. - diff --git a/docs/_build/html/_sources/ref/archive.rst.txt b/docs/_build/html/_sources/ref/archive.rst.txt deleted file mode 100644 index 1cb95c2..0000000 --- a/docs/_build/html/_sources/ref/archive.rst.txt +++ /dev/null @@ -1,104 +0,0 @@ -Loading From an Archive -======================= - -This command instructs pgloader to load data from one or more files contained -in an archive. Currently the only supported archive format is *ZIP*, and the -archive might be downloaded from an *HTTP* URL. - -Here's an example:: - - LOAD ARCHIVE - FROM /Users/dim/Downloads/GeoLiteCity-latest.zip - INTO postgresql:///ip4r - - BEFORE LOAD - DO $$ create extension if not exists ip4r; $$, - $$ create schema if not exists geolite; $$, - - EXECUTE 'geolite.sql' - - LOAD CSV - FROM FILENAME MATCHING ~/GeoLiteCity-Location.csv/ - WITH ENCODING iso-8859-1 - ( - locId, - country, - region null if blanks, - city null if blanks, - postalCode null if blanks, - latitude, - longitude, - metroCode null if blanks, - areaCode null if blanks - ) - INTO postgresql:///ip4r?geolite.location - ( - locid,country,region,city,postalCode, - location point using (format nil "(~a,~a)" longitude latitude), - metroCode,areaCode - ) - WITH skip header = 2, - fields optionally enclosed by '"', - fields escaped by double-quote, - fields terminated by ',' - - AND LOAD CSV - FROM FILENAME MATCHING ~/GeoLiteCity-Blocks.csv/ - WITH ENCODING iso-8859-1 - ( - startIpNum, endIpNum, locId - ) - INTO postgresql:///ip4r?geolite.blocks - ( - iprange ip4r using (ip-range startIpNum endIpNum), - locId - ) - WITH skip header = 2, - fields optionally enclosed by '"', - fields escaped by double-quote, - fields terminated by ',' - - FINALLY DO - $$ create index blocks_ip4r_idx on geolite.blocks using gist(iprange); $$; - -The `archive` command accepts the following clauses and options. - -Archive Source Specification: FROM ----------------------------------- - -Filename or HTTP URI where to load the data from. When given an HTTP URL the -linked file will get downloaded locally before processing. - -If the file is a `zip` file, the command line utility `unzip` is used to -expand the archive into files in `$TMPDIR`, or `/tmp` if `$TMPDIR` is unset -or set to a non-existing directory. - -Then the following commands are used from the top level directory where the -archive has been expanded. - -Archive Sub Commands --------------------- - - - command [ *AND* command ... ] - - A series of commands against the contents of the archive, at the moment - only `CSV`,`'FIXED` and `DBF` commands are supported. - - Note that commands are supporting the clause *FROM FILENAME MATCHING* - which allows the pgloader command not to depend on the exact names of - the archive directories. - - The same clause can also be applied to several files with using the - spelling *FROM ALL FILENAMES MATCHING* and a regular expression. - - The whole *matching* clause must follow the following rule:: - - FROM [ ALL FILENAMES | [ FIRST ] FILENAME ] MATCHING - -Archive Final SQL Commands --------------------------- - - - *FINALLY DO* - - SQL Queries to run once the data is loaded, such as `CREATE INDEX`. - diff --git a/docs/_build/html/_sources/ref/copy.rst.txt b/docs/_build/html/_sources/ref/copy.rst.txt deleted file mode 100644 index e8b156d..0000000 --- a/docs/_build/html/_sources/ref/copy.rst.txt +++ /dev/null @@ -1,115 +0,0 @@ -Loading COPY Formatted Files -============================ - -This commands instructs pgloader to load from a file containing COPY TEXT -data as described in the PostgreSQL documentation. Here's an example:: - - LOAD COPY - FROM copy://./data/track.copy - ( - trackid, track, album, media, genre, composer, - milliseconds, bytes, unitprice - ) - INTO postgresql:///pgloader - TARGET TABLE track_full - - WITH truncate - - SET work_mem to '14MB', - standard_conforming_strings to 'on' - - BEFORE LOAD DO - $$ drop table if exists track_full; $$, - $$ create table track_full ( - trackid bigserial, - track text, - album text, - media text, - genre text, - composer text, - milliseconds bigint, - bytes bigint, - unitprice numeric - ); - $$; - -The `COPY` format command accepts the following clauses and options. - -COPY Formatted Files Source Specification: FROM ------------------------------------------------ - -Filename where to load the data from. This support local files, HTTP URLs -and zip files containing a single dbf file of the same name. Fetch such a -zip file from an HTTP address is of course supported. - - - *inline* - - The data is found after the end of the parsed commands. Any number of - empty lines between the end of the commands and the beginning of the - data is accepted. - - - *stdin* - - Reads the data from the standard input stream. - - - *FILENAMES MATCHING* - - The whole *matching* clause must follow the following rule:: - - [ ALL FILENAMES | [ FIRST ] FILENAME ] - MATCHING regexp - [ IN DIRECTORY '...' ] - - The *matching* clause applies given *regular expression* (see above for - exact syntax, several options can be used here) to filenames. It's then - possible to load data from only the first match of all of them. - - The optional *IN DIRECTORY* clause allows specifying which directory to - walk for finding the data files, and can be either relative to where the - command file is read from, or absolute. The given directory must exists. - -COPY Formatted File Options: WITH ---------------------------------- - - -When loading from a `COPY` file, the following options are supported: - - - *delimiter* - - Takes a single character as argument, which must be found inside single - quotes, and might be given as the printable character itself, the - special value \t to denote a tabulation character, or `0x` then an - hexadecimal value read as the ASCII code for the character. - - This character is used as the *delimiter* when reading the data, in a - similar way to the PostgreSQL `COPY` option. - - - *null* - - Takes a quoted string as an argument (quotes can be either double quotes - or single quotes) and uses that string as the `NULL` representation in - the data. - - This is similar to the *null* `COPY` option in PostgreSQL. - - - *truncate* - - When this option is listed, pgloader issues a `TRUNCATE` command against - the PostgreSQL target table before reading the data file. - - - *disable triggers* - - When this option is listed, pgloader issues an `ALTER TABLE ... DISABLE - TRIGGER ALL` command against the PostgreSQL target table before copying - the data, then the command `ALTER TABLE ... ENABLE TRIGGER ALL` once the - `COPY` is done. - - This option allows loading data into a pre-existing table ignoring the - *foreign key constraints* and user defined triggers and may result in - invalid *foreign key constraints* once the data is loaded. Use with - care. - - - *skip header* - - Takes a numeric value as argument. Instruct pgloader to skip that many - lines at the beginning of the input file. diff --git a/docs/_build/html/_sources/ref/csv.rst.txt b/docs/_build/html/_sources/ref/csv.rst.txt deleted file mode 100644 index 222e37e..0000000 --- a/docs/_build/html/_sources/ref/csv.rst.txt +++ /dev/null @@ -1,243 +0,0 @@ -Loading CSV data -================ - -This command instructs pgloader to load data from a `CSV` file. Here's an -example:: - - LOAD CSV - FROM 'GeoLiteCity-Blocks.csv' WITH ENCODING iso-646-us - HAVING FIELDS - ( - startIpNum, endIpNum, locId - ) - INTO postgresql://user@localhost:54393/dbname - TARGET TABLE geolite.blocks - TARGET COLUMNS - ( - iprange ip4r using (ip-range startIpNum endIpNum), - locId - ) - WITH truncate, - skip header = 2, - fields optionally enclosed by '"', - fields escaped by backslash-quote, - fields terminated by '\t' - - SET work_mem to '32 MB', maintenance_work_mem to '64 MB'; - -The `csv` format command accepts the following clauses and options. - -CSV Source Specification: FROM ------------------------------- - -Filename where to load the data from. Accepts an *ENCODING* option. Use the -`--list-encodings` option to know which encoding names are supported. - -The filename may be enclosed by single quotes, and could be one of the -following special values: - - - *inline* - - The data is found after the end of the parsed commands. Any number - of empty lines between the end of the commands and the beginning of - the data is accepted. - - - *stdin* - - Reads the data from the standard input stream. - - - *FILENAMES MATCHING* - - The whole *matching* clause must follow the following rule:: - - [ ALL FILENAMES | [ FIRST ] FILENAME ] - MATCHING regexp - [ IN DIRECTORY '...' ] - - The *matching* clause applies given *regular expression* (see above - for exact syntax, several options can be used here) to filenames. - It's then possible to load data from only the first match of all of - them. - - The optional *IN DIRECTORY* clause allows specifying which directory - to walk for finding the data files, and can be either relative to - where the command file is read from, or absolute. The given - directory must exists. - -Fields Specifications ---------------------- - -The *FROM* option also supports an optional comma separated list of *field* -names describing what is expected in the `CSV` data file, optionally -introduced by the clause `HAVING FIELDS`. - -Each field name can be either only one name or a name following with -specific reader options for that field, enclosed in square brackets and -comma-separated. Supported per-field reader options are: - - - *terminated by* - - See the description of *field terminated by* below. - - The processing of this option is not currently implemented. - - - *date format* - - When the field is expected of the date type, then this option allows - to specify the date format used in the file. - - Date format string are template strings modeled against the - PostgreSQL `to_char` template strings support, limited to the - following patterns: - - - YYYY, YYY, YY for the year part - - MM for the numeric month part - - DD for the numeric day part - - HH, HH12, HH24 for the hour part - - am, AM, a.m., A.M. - - pm, PM, p.m., P.M. - - MI for the minutes part - - SS for the seconds part - - MS for the milliseconds part (4 digits) - - US for the microseconds part (6 digits) - - unparsed punctuation signs: - . * # @ T / \ and space - - Here's an example of a *date format* specification:: - - column-name [date format 'YYYY-MM-DD HH24-MI-SS.US'] - - - *null if* - - This option takes an argument which is either the keyword *blanks* - or a double-quoted string. - - When *blanks* is used and the field value that is read contains - only space characters, then it's automatically converted to an SQL - `NULL` value. - - When a double-quoted string is used and that string is read as the - field value, then the field value is automatically converted to an - SQL `NULL` value. - - - *trim both whitespace*, *trim left whitespace*, *trim right whitespace* - - This option allows to trim whitespaces in the read data, either from - both sides of the data, or only the whitespace characters found on - the left of the streaing, or only those on the right of the string. - -CSV Loading Options: WITH -------------------------- - -When loading from a `CSV` file, the following options are supported: - - - *truncate* - - When this option is listed, pgloader issues a `TRUNCATE` command - against the PostgreSQL target table before reading the data file. - - - *drop indexes* - - When this option is listed, pgloader issues `DROP INDEX` commands - against all the indexes defined on the target table before copying - the data, then `CREATE INDEX` commands once the `COPY` is done. - - In order to get the best performance possible, all the indexes are - created in parallel and when done the primary keys are built again - from the unique indexes just created. This two step process allows - creating the primary key index in parallel with the other indexes, - as only the `ALTER TABLE` command needs an *access exclusive lock* - on the target table. - - - *disable triggers* - - When this option is listed, pgloader issues an `ALTER TABLE ... - DISABLE TRIGGER ALL` command against the PostgreSQL target table - before copying the data, then the command `ALTER TABLE ... ENABLE - TRIGGER ALL` once the `COPY` is done. - - This option allows loading data into a pre-existing table ignoring - the *foreign key constraints* and user defined triggers and may - result in invalid *foreign key constraints* once the data is loaded. - Use with care. - - - *skip header* - - Takes a numeric value as argument. Instruct pgloader to skip that - many lines at the beginning of the input file. - - - *csv header* - - Use the first line read after *skip header* as the list of csv field - names to be found in the CSV file, using the same CSV parameters as - for the CSV data. - - - *trim unquoted blanks* - - When reading unquoted values in the `CSV` file, remove the blanks - found in between the separator and the value. That behaviour is the - default. - - - *keep unquoted blanks* - - When reading unquoted values in the `CSV` file, keep blanks found in - between the separator and the value. - - - *fields optionally enclosed by* - - Takes a single character as argument, which must be found inside single - quotes, and might be given as the printable character itself, the - special value \t to denote a tabulation character, the special value \' - to denote a single-quote, or `0x` then an hexadecimal value read as the - ASCII code for the character. - - The following options specify the same enclosing character, a single quote:: - - fields optionally enclosed by '\'' - fields optionally enclosed by '0x27' - - This character is used as the quoting character in the `CSV` file, - and defaults to double-quote. - - - *fields not enclosed* - - By default, pgloader will use the double-quote character as the - enclosing character. If you have a CSV file where fields are not - enclosed and are using double-quote as an expected ordinary - character, then use the option *fields not enclosed* for the CSV - parser to accept those values. - - - *fields escaped by* - - Takes either the special value *backslash-quote* or *double-quote*, - or any value supported by the *fields terminated by* option (see - below). This value is used to recognize escaped field separators - when they are to be found within the data fields themselves. - Defaults to *double-quote*. - - - *csv escape mode* - - Takes either the special value *quote* (the default) or *following* - and allows the CSV parser to parse either only escaped field - separator or any character (including CSV data) when using the - *following* value. - - - *fields terminated by* - - Takes a single character as argument, which must be found inside - single quotes, and might be given as the printable character itself, - the special value \t to denote a tabulation character, or `0x` then - an hexadecimal value read as the ASCII code for the character. - - This character is used as the *field separator* when reading the - `CSV` data. - - - *lines terminated by* - - Takes a single character as argument, which must be found inside - single quotes, and might be given as the printable character itself, - the special value \t to denote a tabulation character, or `0x` then - an hexadecimal value read as the ASCII code for the character. - - This character is used to recognize *end-of-line* condition when - reading the `CSV` data. - diff --git a/docs/_build/html/_sources/ref/dbf.rst.txt b/docs/_build/html/_sources/ref/dbf.rst.txt deleted file mode 100644 index 28269f3..0000000 --- a/docs/_build/html/_sources/ref/dbf.rst.txt +++ /dev/null @@ -1,53 +0,0 @@ -Loading DBF data -================= - -This command instructs pgloader to load data from a `DBF` file. Here's an -example:: - - LOAD DBF - FROM http://www.insee.fr/fr/methodes/nomenclatures/cog/telechargement/2013/dbf/reg2013.dbf - INTO postgresql://user@localhost/dbname - WITH truncate, create table; - -The `dbf` format command accepts the following clauses and options. - -DBF Source Specification: FROM ------------------------------- - -Filename where to load the data from. This support local files, HTTP URLs -and zip files containing a single dbf file of the same name. Fetch such a -zip file from an HTTP address is of course supported. - -DBF Loading Options: WITH -------------------------- - -When loading from a `DBF` file, the following options are supported: - - - *truncate* - - When this option is listed, pgloader issues a `TRUNCATE` command against - the PostgreSQL target table before reading the data file. - - - *disable triggers* - - When this option is listed, pgloader issues an `ALTER TABLE ... DISABLE - TRIGGER ALL` command against the PostgreSQL target table before copying - the data, then the command `ALTER TABLE ... ENABLE TRIGGER ALL` once the - `COPY` is done. - - This option allows loading data into a pre-existing table ignoring the - *foreign key constraints* and user defined triggers and may result in - invalid *foreign key constraints* once the data is loaded. Use with - care. - - - *create table* - - When this option is listed, pgloader creates the table using the meta - data found in the `DBF` file, which must contain a list of fields with - their data type. A standard data type conversion from DBF to PostgreSQL - is done. - - - *table name* - - This options expects as its value the possibly qualified name of the - table to create. diff --git a/docs/_build/html/_sources/ref/fixed.rst.txt b/docs/_build/html/_sources/ref/fixed.rst.txt deleted file mode 100644 index dd567da..0000000 --- a/docs/_build/html/_sources/ref/fixed.rst.txt +++ /dev/null @@ -1,182 +0,0 @@ -Loading Fixed Cols File Formats -=============================== - -This command instructs pgloader to load data from a text file containing -columns arranged in a *fixed size* manner. Here's an example:: - - LOAD FIXED - FROM inline - ( - a from 0 for 10, - b from 10 for 8, - c from 18 for 8, - d from 26 for 17 [null if blanks, trim right whitespace] - ) - INTO postgresql:///pgloader - TARGET TABLE fixed - ( - a, b, - c time using (time-with-no-separator c), - d - ) - - WITH truncate - - SET work_mem to '14MB', - standard_conforming_strings to 'on' - - BEFORE LOAD DO - $$ drop table if exists fixed; $$, - $$ create table fixed ( - a integer, - b date, - c time, - d text - ); - $$; - - 01234567892008052011431250firstline - 01234562008052115182300left blank-padded - 12345678902008052208231560another line - 2345609872014092914371500 - 2345678902014092914371520 - -The `fixed` format command accepts the following clauses and options. - -Fixed File Format Source Specification: FROM --------------------------------------------- - -Filename where to load the data from. Accepts an *ENCODING* option. Use the -`--list-encodings` option to know which encoding names are supported. - -The filename may be enclosed by single quotes, and could be one of the -following special values: - - - *inline* - - The data is found after the end of the parsed commands. Any number - of empty lines between the end of the commands and the beginning of - the data is accepted. - - - *stdin* - - Reads the data from the standard input stream. - - - *FILENAMES MATCHING* - - The whole *matching* clause must follow the following rule:: - - [ ALL FILENAMES | [ FIRST ] FILENAME ] - MATCHING regexp - [ IN DIRECTORY '...' ] - - The *matching* clause applies given *regular expression* (see above - for exact syntax, several options can be used here) to filenames. - It's then possible to load data from only the first match of all of - them. - - The optional *IN DIRECTORY* clause allows specifying which directory - to walk for finding the data files, and can be either relative to - where the command file is read from, or absolute. The given - directory must exists. - -Fields Specifications ---------------------- - -The *FROM* option also supports an optional comma separated list of *field* -names describing what is expected in the `FIXED` data file. - -Each field name is composed of the field name followed with specific reader -options for that field. Supported per-field reader options are the -following, where only *start* and *length* are required. - - - *start* - - Position in the line where to start reading that field's value. Can - be entered with decimal digits or `0x` then hexadecimal digits. - - - *length* - - How many bytes to read from the *start* position to read that - field's value. Same format as *start*. - -Those optional parameters must be enclosed in square brackets and -comma-separated: - - - *terminated by* - - See the description of *field terminated by* below. - - The processing of this option is not currently implemented. - - - *date format* - - When the field is expected of the date type, then this option allows - to specify the date format used in the file. - - Date format string are template strings modeled against the - PostgreSQL `to_char` template strings support, limited to the - following patterns: - - - YYYY, YYY, YY for the year part - - MM for the numeric month part - - DD for the numeric day part - - HH, HH12, HH24 for the hour part - - am, AM, a.m., A.M. - - pm, PM, p.m., P.M. - - MI for the minutes part - - SS for the seconds part - - MS for the milliseconds part (4 digits) - - US for the microseconds part (6 digits) - - unparsed punctuation signs: - . * # @ T / \ and space - - Here's an example of a *date format* specification:: - - column-name [date format 'YYYY-MM-DD HH24-MI-SS.US'] - - - *null if* - - This option takes an argument which is either the keyword *blanks* - or a double-quoted string. - - When *blanks* is used and the field value that is read contains only - space characters, then it's automatically converted to an SQL `NULL` - value. - - When a double-quoted string is used and that string is read as the - field value, then the field value is automatically converted to an - SQL `NULL` value. - - - *trim both whitespace*, *trim left whitespace*, *trim right whitespace* - - This option allows to trim whitespaces in the read data, either from - both sides of the data, or only the whitespace characters found on - the left of the streaing, or only those on the right of the string. - -Fixed File Format Loading Options: WITH ---------------------------------------- - -When loading from a `FIXED` file, the following options are supported: - - - *truncate* - - When this option is listed, pgloader issues a `TRUNCATE` command - against the PostgreSQL target table before reading the data file. - - - *disable triggers* - - When this option is listed, pgloader issues an `ALTER TABLE ... - DISABLE TRIGGER ALL` command against the PostgreSQL target table - before copying the data, then the command `ALTER TABLE ... ENABLE - TRIGGER ALL` once the `COPY` is done. - - This option allows loading data into a pre-existing table ignoring - the *foreign key constraints* and user defined triggers and may - result in invalid *foreign key constraints* once the data is loaded. - Use with care. - - - *skip header* - - Takes a numeric value as argument. Instruct pgloader to skip that - many lines at the beginning of the input file. - diff --git a/docs/_build/html/_sources/ref/ixf.rst.txt b/docs/_build/html/_sources/ref/ixf.rst.txt deleted file mode 100644 index 3f698fa..0000000 --- a/docs/_build/html/_sources/ref/ixf.rst.txt +++ /dev/null @@ -1,66 +0,0 @@ -Loading IXF Data -================ - -This command instructs pgloader to load data from an IBM `IXF` file. Here's -an example:: - - LOAD IXF - FROM data/nsitra.test1.ixf - INTO postgresql:///pgloader - TARGET TABLE nsitra.test1 - WITH truncate, create table, timezone UTC - - BEFORE LOAD DO - $$ create schema if not exists nsitra; $$, - $$ drop table if exists nsitra.test1; $$; - -The `ixf` format command accepts the following clauses and options. - -IXF Source Specification: FROM ------------------------------- - -Filename where to load the data from. This support local files, HTTP URLs -and zip files containing a single ixf file of the same name. Fetch such a -zip file from an HTTP address is of course supported. - -IXF Loading Options: WITH -------------------------- - -When loading from a `IXF` file, the following options are supported: - - - *truncate* - - When this option is listed, pgloader issues a `TRUNCATE` command against - the PostgreSQL target table before reading the data file. - - - *disable triggers* - - When this option is listed, pgloader issues an `ALTER TABLE ... DISABLE - TRIGGER ALL` command against the PostgreSQL target table before copying - the data, then the command `ALTER TABLE ... ENABLE TRIGGER ALL` once the - `COPY` is done. - - This option allows loading data into a pre-existing table ignoring the - *foreign key constraints* and user defined triggers and may result in - invalid *foreign key constraints* once the data is loaded. Use with - care. - - - *create table* - - When this option is listed, pgloader creates the table using the meta - data found in the `DBF` file, which must contain a list of fields with - their data type. A standard data type conversion from DBF to PostgreSQL - is done. - - - *table name* - - This options expects as its value the possibly qualified name of the - table to create. - - - *timezone* - - This options allows to specify which timezone is used when parsing - timestamps from an IXF file, and defaults to *UTC*. Expected values are - either `UTC`, `GMT` or a single quoted location name such as - `'Universal'` or `'Europe/Paris'`. - diff --git a/docs/_build/html/_sources/ref/mssql.rst.txt b/docs/_build/html/_sources/ref/mssql.rst.txt deleted file mode 100644 index 89ff6f3..0000000 --- a/docs/_build/html/_sources/ref/mssql.rst.txt +++ /dev/null @@ -1,159 +0,0 @@ -Migrating a MS SQL Database to PostgreSQL -========================================= - -This command instructs pgloader to load data from a MS SQL database. -Automatic discovery of the schema is supported, including build of the -indexes, primary and foreign keys constraints. - -Here's an example:: - - load database - from mssql://user@host/dbname - into postgresql:///dbname - - including only table names like 'GlobalAccount' in schema 'dbo' - - set work_mem to '16MB', maintenance_work_mem to '512 MB' - - before load do $$ drop schema if exists dbo cascade; $$; - -The `mssql` command accepts the following clauses and options. - -MS SQL Database Source Specification: FROM ------------------------------------------- - -Connection string to an existing MS SQL database server that listens and -welcome external TCP/IP connection. As pgloader currently piggybacks on the -FreeTDS driver, to change the port of the server please export the `TDSPORT` -environment variable. - -MS SQL Database Migration Options: WITH ---------------------------------------- - -When loading from a `MS SQL` database, the same options as when loading a -`MySQL` database are supported. Please refer to the MySQL section. The -following options are added: - - - *create schemas* - - When this option is listed, pgloader creates the same schemas as found - on the MS SQL instance. This is the default. - - - *create no schemas* - - When this option is listed, pgloader refrains from creating any schemas - at all, you must then ensure that the target schema do exist. - -MS SQL Database Casting Rules ------------------------------ - -CAST -^^^^ - -The cast clause allows to specify custom casting rules, either to overload -the default casting rules or to amend them with special cases. - -Please refer to the MySQL CAST clause for details. - -MS SQL Partial Migration ------------------------- - - -INCLUDING ONLY TABLE NAMES LIKE -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Introduce a comma separated list of table name patterns used to limit the -tables to migrate to a sublist. More than one such clause may be used, they -will be accumulated together. - -Example:: - - including only table names lile 'GlobalAccount' in schema 'dbo' - -EXCLUDING TABLE NAMES LIKE -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Introduce a comma separated list of table name patterns used to exclude -table names from the migration. This filter only applies to the result of -the *INCLUDING* filter. - -:: - - excluding table names matching 'LocalAccount' in schema 'dbo' - -MS SQL Schema Transformations ------------------------------ - -ALTER SCHEMA '...' RENAME TO '...' -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Allows to rename a schema on the flight, so that for instance the tables -found in the schema 'dbo' in your source database will get migrated into the -schema 'public' in the target database with this command:: - - alter schema 'dbo' rename to 'public' - -ALTER TABLE NAMES MATCHING ... IN SCHEMA '...' -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -See the MySQL explanation for this clause above. It works the same in the -context of migrating from MS SQL, only with the added option to specify the -name of the schema where to find the definition of the target tables. - -The matching is done in pgloader itself, with a Common Lisp regular -expression lib, so doesn't depend on the *LIKE* implementation of MS SQL, -nor on the lack of support for regular expressions in the engine. - -MS SQL Driver setup and encoding --------------------------------- - -pgloader is using the `FreeTDS` driver, and internally expects the data to -be sent in utf-8. To achieve that, you can configure the FreeTDS driver with -those defaults, in the file `~/.freetds.conf`:: - - [global] - tds version = 7.4 - client charset = UTF-8 - -Default MS SQL Casting Rules ----------------------------- - -When migrating from MS SQL the following Casting Rules are provided: - -Numbers:: - - type tinyint to smallint - - type float to float using float-to-string - type real to real using float-to-string - type double to double precision using float-to-string - type numeric to numeric using float-to-string - type decimal to numeric using float-to-string - type money to numeric using float-to-string - type smallmoney to numeric using float-to-string - -Texts:: - - type char to text drop typemod - type nchat to text drop typemod - type varchar to text drop typemod - type nvarchar to text drop typemod - type xml to text drop typemod - -Binary:: - - type binary to bytea using byte-vector-to-bytea - type varbinary to bytea using byte-vector-to-bytea - -Date:: - - type datetime to timestamptz - type datetime2 to timestamptz - -Others:: - - type bit to boolean - type hierarchyid to bytea - type geography to bytea - type uniqueidentifier to uuid using sql-server-uniqueidentifier-to-uuid - diff --git a/docs/_build/html/_sources/ref/mysql.rst.txt b/docs/_build/html/_sources/ref/mysql.rst.txt deleted file mode 100644 index b484d5b..0000000 --- a/docs/_build/html/_sources/ref/mysql.rst.txt +++ /dev/null @@ -1,623 +0,0 @@ -Migrating a MySQL Database to PostgreSQL -======================================== - -This command instructs pgloader to load data from a database connection. The -only supported database source is currently *MySQL*, and pgloader supports -dynamically converting the schema of the source database and the indexes -building. - -A default set of casting rules are provided and might be overloaded and -appended to by the command. - -Here's an example using as many options as possible, some of them even being -defaults. Chances are you don't need that complex a setup, don't copy and -paste it, use it only as a reference! - -:: - - LOAD DATABASE - FROM mysql://root@localhost/sakila - INTO postgresql://localhost:54393/sakila - - WITH include drop, create tables, create indexes, reset sequences, - workers = 8, concurrency = 1, - multiple readers per thread, rows per range = 50000 - - SET PostgreSQL PARAMETERS - maintenance_work_mem to '128MB', - work_mem to '12MB', - search_path to 'sakila, public, "$user"' - - SET MySQL PARAMETERS - net_read_timeout = '120', - net_write_timeout = '120' - - CAST type bigint when (= precision 20) to bigserial drop typemod, - type date drop not null drop default using zero-dates-to-null, - -- type tinyint to boolean using tinyint-to-boolean, - type year to integer - - MATERIALIZE VIEWS film_list, staff_list - - -- INCLUDING ONLY TABLE NAMES MATCHING ~/film/, 'actor' - -- EXCLUDING TABLE NAMES MATCHING ~ - -- DECODING TABLE NAMES MATCHING ~/messed/, ~/encoding/ AS utf8 - -- ALTER TABLE NAMES MATCHING 'film' RENAME TO 'films' - -- ALTER TABLE NAMES MATCHING ~/_list$/ SET SCHEMA 'mv' - - ALTER TABLE NAMES MATCHING ~/_list$/, 'sales_by_store', ~/sales_by/ - SET SCHEMA 'mv' - - ALTER TABLE NAMES MATCHING 'film' RENAME TO 'films' - ALTER TABLE NAMES MATCHING ~/./ SET (fillfactor='40') - - ALTER SCHEMA 'sakila' RENAME TO 'pagila' - - BEFORE LOAD DO - $$ create schema if not exists pagila; $$, - $$ create schema if not exists mv; $$, - $$ alter database sakila set search_path to pagila, mv, public; $$; - - -The `database` command accepts the following clauses and options. - -MySQL Database Source Specification: FROM ------------------------------------------ - -Must be a connection URL pointing to a MySQL database. - -If the connection URI contains a table name, then only this table is -migrated from MySQL to PostgreSQL. - -See the `SOURCE CONNECTION STRING` section above for details on how to write -the connection string. The MySQL connection string accepts the same -parameter *sslmode* as the PostgreSQL connection string, but the *verify* -mode is not implemented (yet). - -Environment variables described in - can be -used as default values too. If the user is not provided, then it defaults to -`USER` environment variable value. The password can be provided with the -environment variable `MYSQL_PWD`. The host can be provided with the -environment variable `MYSQL_HOST` and otherwise defaults to `localhost`. The -port can be provided with the environment variable `MYSQL_TCP_PORT` and -otherwise defaults to `3306`. - -MySQL Database Migration Options: WITH --------------------------------------- - -When loading from a `MySQL` database, the following options are supported, -and the default *WITH* clause is: *no truncate*, *create schema*, *create -tables*, *include drop*, *create indexes*, *reset sequences*, *foreign -keys*, *downcase identifiers*, *uniquify index names*. - - - *include drop* - - When this option is listed, pgloader drops all the tables in the target - PostgreSQL database whose names appear in the MySQL database. This - option allows for using the same command several times in a row until - you figure out all the options, starting automatically from a clean - environment. Please note that `CASCADE` is used to ensure that tables - are dropped even if there are foreign keys pointing to them. This is - precisely what `include drop` is intended to do: drop all target tables - and recreate them. - - Great care needs to be taken when using `include drop`, as it will - cascade to *all* objects referencing the target tables, possibly - including other tables that are not being loaded from the source DB. - - - *include no drop* - - When this option is listed, pgloader will not include any `DROP` - statement when loading the data. - - - *truncate* - - When this option is listed, pgloader issue the `TRUNCATE` command - against each PostgreSQL table just before loading data into it. - - - *no truncate* - - When this option is listed, pgloader issues no `TRUNCATE` command. - - - *disable triggers* - - When this option is listed, pgloader issues an `ALTER TABLE ... DISABLE - TRIGGER ALL` command against the PostgreSQL target table before copying - the data, then the command `ALTER TABLE ... ENABLE TRIGGER ALL` once the - `COPY` is done. - - This option allows loading data into a pre-existing table ignoring the - *foreign key constraints* and user defined triggers and may result in - invalid *foreign key constraints* once the data is loaded. Use with - care. - - - *create tables* - - When this option is listed, pgloader creates the table using the meta - data found in the `MySQL` file, which must contain a list of fields with - their data type. A standard data type conversion from DBF to PostgreSQL - is done. - - - *create no tables* - - When this option is listed, pgloader skips the creation of table before - loading data, target tables must then already exist. - - Also, when using *create no tables* pgloader fetches the metadata from - the current target database and checks type casting, then will remove - constraints and indexes prior to loading the data and install them back - again once the loading is done. - - - *create indexes* - - When this option is listed, pgloader gets the definitions of all the - indexes found in the MySQL database and create the same set of index - definitions against the PostgreSQL database. - - - *create no indexes* - - When this option is listed, pgloader skips the creating indexes. - - - *drop indexes* - - When this option is listed, pgloader drops the indexes in the target - database before loading the data, and creates them again at the end - of the data copy. - - - *uniquify index names*, *preserve index names* - - MySQL index names are unique per-table whereas in PostgreSQL index names - have to be unique per-schema. The default for pgloader is to change the - index name by prefixing it with `idx_OID` where `OID` is the internal - numeric identifier of the table the index is built against. - - In somes cases like when the DDL are entirely left to a framework it - might be sensible for pgloader to refrain from handling index unique - names, that is achieved by using the *preserve index names* option. - - The default is to *uniquify index names*. - - Even when using the option *preserve index names*, MySQL primary key - indexes named "PRIMARY" will get their names uniquified. Failing to do - so would prevent the primary keys to be created again in PostgreSQL - where the index names must be unique per schema. - - - *drop schema* - - When this option is listed, pgloader drops the target schema in the - target PostgreSQL database before creating it again and all the objects - it contains. The default behavior doesn't drop the target schemas. - - - *foreign keys* - - When this option is listed, pgloader gets the definitions of all the - foreign keys found in the MySQL database and create the same set of - foreign key definitions against the PostgreSQL database. - - - *no foreign keys* - - When this option is listed, pgloader skips creating foreign keys. - - - *reset sequences* - - When this option is listed, at the end of the data loading and after the - indexes have all been created, pgloader resets all the PostgreSQL - sequences created to the current maximum value of the column they are - attached to. - - The options *schema only* and *data only* have no effects on this - option. - - - *reset no sequences* - - When this option is listed, pgloader skips resetting sequences after the - load. - - The options *schema only* and *data only* have no effects on this - option. - - - *downcase identifiers* - - When this option is listed, pgloader converts all MySQL identifiers - (table names, index names, column names) to *downcase*, except for - PostgreSQL *reserved* keywords. - - The PostgreSQL *reserved* keywords are determined dynamically by using - the system function `pg_get_keywords()`. - - - *quote identifiers* - - When this option is listed, pgloader quotes all MySQL identifiers so - that their case is respected. Note that you will then have to do the - same thing in your application code queries. - - - *schema only* - - When this option is listed pgloader refrains from migrating the data - over. Note that the schema in this context includes the indexes when the - option *create indexes* has been listed. - - - *data only* - - When this option is listed pgloader only issues the `COPY` statements, - without doing any other processing. - - - *single reader per thread*, *multiple readers per thread* - - The default is *single reader per thread* and it means that each - MySQL table is read by a single thread as a whole, with a single - `SELECT` statement using no `WHERE` clause. - - When using *multiple readers per thread* pgloader may be able to - divide the reading work into several threads, as many as the - *concurrency* setting, which needs to be greater than 1 for this - option to kick be activated. - - For each source table, pgloader searches for a primary key over a - single numeric column, or a multiple-column primary key index for - which the first column is of a numeric data type (one of `integer` - or `bigint`). When such an index exists, pgloader runs a query to - find the *min* and *max* values on this column, and then split that - range into many ranges containing a maximum of *rows per range*. - - When the range list we then obtain contains at least as many ranges - than our concurrency setting, then we distribute those ranges to - each reader thread. - - So when all the conditions are met, pgloader then starts as many - reader thread as the *concurrency* setting, and each reader thread - issues several queries with a `WHERE id >= x AND id < y`, where `y - - x = rows per range` or less (for the last range, depending on the - max value just obtained. - - - *rows per range* - - How many rows are fetched per `SELECT` query when using *multiple - readers per thread*, see above for details. - - - *SET MySQL PARAMETERS* - - The *SET MySQL PARAMETERS* allows setting MySQL parameters using the - MySQL `SET` command each time pgloader connects to it. - -MySQL Database Casting Rules ----------------------------- - -The command *CAST* introduces user-defined casting rules. - -The cast clause allows to specify custom casting rules, either to overload -the default casting rules or to amend them with special cases. - -A casting rule is expected to follow one of the forms:: - - type [ ... ] to [ or other required elements. - thead: [ 1, "
", "
" ], - col: [ 2, "", "
" ], - tr: [ 2, "", "
" ], - td: [ 3, "", "
" ], - - _default: [ 0, "", "" ] -}; - -// Support: IE <=9 only -wrapMap.optgroup = wrapMap.option; - -wrapMap.tbody = wrapMap.tfoot = wrapMap.colgroup = wrapMap.caption = wrapMap.thead; -wrapMap.th = wrapMap.td; - - -function getAll( context, tag ) { - - // Support: IE <=9 - 11 only - // Use typeof to avoid zero-argument method invocation on host objects (#15151) - var ret = typeof context.getElementsByTagName !== "undefined" ? - context.getElementsByTagName( tag || "*" ) : - typeof context.querySelectorAll !== "undefined" ? - context.querySelectorAll( tag || "*" ) : - []; - - return tag === undefined || tag && jQuery.nodeName( context, tag ) ? - jQuery.merge( [ context ], ret ) : - ret; -} - - -// Mark scripts as having already been evaluated -function setGlobalEval( elems, refElements ) { - var i = 0, - l = elems.length; - - for ( ; i < l; i++ ) { - dataPriv.set( - elems[ i ], - "globalEval", - !refElements || dataPriv.get( refElements[ i ], "globalEval" ) - ); - } -} - - -var rhtml = /<|&#?\w+;/; - -function buildFragment( elems, context, scripts, selection, ignored ) { - var elem, tmp, tag, wrap, contains, j, - fragment = context.createDocumentFragment(), - nodes = [], - i = 0, - l = elems.length; - - for ( ; i < l; i++ ) { - elem = elems[ i ]; - - if ( elem || elem === 0 ) { - - // Add nodes directly - if ( jQuery.type( elem ) === "object" ) { - - // Support: Android <=4.0 only, PhantomJS 1 only - // push.apply(_, arraylike) throws on ancient WebKit - jQuery.merge( nodes, elem.nodeType ? [ elem ] : elem ); - - // Convert non-html into a text node - } else if ( !rhtml.test( elem ) ) { - nodes.push( context.createTextNode( elem ) ); - - // Convert html into DOM nodes - } else { - tmp = tmp || fragment.appendChild( context.createElement( "div" ) ); - - // Deserialize a standard representation - tag = ( rtagName.exec( elem ) || [ "", "" ] )[ 1 ].toLowerCase(); - wrap = wrapMap[ tag ] || wrapMap._default; - tmp.innerHTML = wrap[ 1 ] + jQuery.htmlPrefilter( elem ) + wrap[ 2 ]; - - // Descend through wrappers to the right content - j = wrap[ 0 ]; - while ( j-- ) { - tmp = tmp.lastChild; - } - - // Support: Android <=4.0 only, PhantomJS 1 only - // push.apply(_, arraylike) throws on ancient WebKit - jQuery.merge( nodes, tmp.childNodes ); - - // Remember the top-level container - tmp = fragment.firstChild; - - // Ensure the created nodes are orphaned (#12392) - tmp.textContent = ""; - } - } - } - - // Remove wrapper from fragment - fragment.textContent = ""; - - i = 0; - while ( ( elem = nodes[ i++ ] ) ) { - - // Skip elements already in the context collection (trac-4087) - if ( selection && jQuery.inArray( elem, selection ) > -1 ) { - if ( ignored ) { - ignored.push( elem ); - } - continue; - } - - contains = jQuery.contains( elem.ownerDocument, elem ); - - // Append to fragment - tmp = getAll( fragment.appendChild( elem ), "script" ); - - // Preserve script evaluation history - if ( contains ) { - setGlobalEval( tmp ); - } - - // Capture executables - if ( scripts ) { - j = 0; - while ( ( elem = tmp[ j++ ] ) ) { - if ( rscriptType.test( elem.type || "" ) ) { - scripts.push( elem ); - } - } - } - } - - return fragment; -} - - -( function() { - var fragment = document.createDocumentFragment(), - div = fragment.appendChild( document.createElement( "div" ) ), - input = document.createElement( "input" ); - - // Support: Android 4.0 - 4.3 only - // Check state lost if the name is set (#11217) - // Support: Windows Web Apps (WWA) - // `name` and `type` must use .setAttribute for WWA (#14901) - input.setAttribute( "type", "radio" ); - input.setAttribute( "checked", "checked" ); - input.setAttribute( "name", "t" ); - - div.appendChild( input ); - - // Support: Android <=4.1 only - // Older WebKit doesn't clone checked state correctly in fragments - support.checkClone = div.cloneNode( true ).cloneNode( true ).lastChild.checked; - - // Support: IE <=11 only - // Make sure textarea (and checkbox) defaultValue is properly cloned - div.innerHTML = ""; - support.noCloneChecked = !!div.cloneNode( true ).lastChild.defaultValue; -} )(); -var documentElement = document.documentElement; - - - -var - rkeyEvent = /^key/, - rmouseEvent = /^(?:mouse|pointer|contextmenu|drag|drop)|click/, - rtypenamespace = /^([^.]*)(?:\.(.+)|)/; - -function returnTrue() { - return true; -} - -function returnFalse() { - return false; -} - -// Support: IE <=9 only -// See #13393 for more info -function safeActiveElement() { - try { - return document.activeElement; - } catch ( err ) { } -} - -function on( elem, types, selector, data, fn, one ) { - var origFn, type; - - // Types can be a map of types/handlers - if ( typeof types === "object" ) { - - // ( types-Object, selector, data ) - if ( typeof selector !== "string" ) { - - // ( types-Object, data ) - data = data || selector; - selector = undefined; - } - for ( type in types ) { - on( elem, type, selector, data, types[ type ], one ); - } - return elem; - } - - if ( data == null && fn == null ) { - - // ( types, fn ) - fn = selector; - data = selector = undefined; - } else if ( fn == null ) { - if ( typeof selector === "string" ) { - - // ( types, selector, fn ) - fn = data; - data = undefined; - } else { - - // ( types, data, fn ) - fn = data; - data = selector; - selector = undefined; - } - } - if ( fn === false ) { - fn = returnFalse; - } else if ( !fn ) { - return elem; - } - - if ( one === 1 ) { - origFn = fn; - fn = function( event ) { - - // Can use an empty set, since event contains the info - jQuery().off( event ); - return origFn.apply( this, arguments ); - }; - - // Use same guid so caller can remove using origFn - fn.guid = origFn.guid || ( origFn.guid = jQuery.guid++ ); - } - return elem.each( function() { - jQuery.event.add( this, types, fn, data, selector ); - } ); -} - -/* - * Helper functions for managing events -- not part of the public interface. - * Props to Dean Edwards' addEvent library for many of the ideas. - */ -jQuery.event = { - - global: {}, - - add: function( elem, types, handler, data, selector ) { - - var handleObjIn, eventHandle, tmp, - events, t, handleObj, - special, handlers, type, namespaces, origType, - elemData = dataPriv.get( elem ); - - // Don't attach events to noData or text/comment nodes (but allow plain objects) - if ( !elemData ) { - return; - } - - // Caller can pass in an object of custom data in lieu of the handler - if ( handler.handler ) { - handleObjIn = handler; - handler = handleObjIn.handler; - selector = handleObjIn.selector; - } - - // Ensure that invalid selectors throw exceptions at attach time - // Evaluate against documentElement in case elem is a non-element node (e.g., document) - if ( selector ) { - jQuery.find.matchesSelector( documentElement, selector ); - } - - // Make sure that the handler has a unique ID, used to find/remove it later - if ( !handler.guid ) { - handler.guid = jQuery.guid++; - } - - // Init the element's event structure and main handler, if this is the first - if ( !( events = elemData.events ) ) { - events = elemData.events = {}; - } - if ( !( eventHandle = elemData.handle ) ) { - eventHandle = elemData.handle = function( e ) { - - // Discard the second event of a jQuery.event.trigger() and - // when an event is called after a page has unloaded - return typeof jQuery !== "undefined" && jQuery.event.triggered !== e.type ? - jQuery.event.dispatch.apply( elem, arguments ) : undefined; - }; - } - - // Handle multiple events separated by a space - types = ( types || "" ).match( rnotwhite ) || [ "" ]; - t = types.length; - while ( t-- ) { - tmp = rtypenamespace.exec( types[ t ] ) || []; - type = origType = tmp[ 1 ]; - namespaces = ( tmp[ 2 ] || "" ).split( "." ).sort(); - - // There *must* be a type, no attaching namespace-only handlers - if ( !type ) { - continue; - } - - // If event changes its type, use the special event handlers for the changed type - special = jQuery.event.special[ type ] || {}; - - // If selector defined, determine special event api type, otherwise given type - type = ( selector ? special.delegateType : special.bindType ) || type; - - // Update special based on newly reset type - special = jQuery.event.special[ type ] || {}; - - // handleObj is passed to all event handlers - handleObj = jQuery.extend( { - type: type, - origType: origType, - data: data, - handler: handler, - guid: handler.guid, - selector: selector, - needsContext: selector && jQuery.expr.match.needsContext.test( selector ), - namespace: namespaces.join( "." ) - }, handleObjIn ); - - // Init the event handler queue if we're the first - if ( !( handlers = events[ type ] ) ) { - handlers = events[ type ] = []; - handlers.delegateCount = 0; - - // Only use addEventListener if the special events handler returns false - if ( !special.setup || - special.setup.call( elem, data, namespaces, eventHandle ) === false ) { - - if ( elem.addEventListener ) { - elem.addEventListener( type, eventHandle ); - } - } - } - - if ( special.add ) { - special.add.call( elem, handleObj ); - - if ( !handleObj.handler.guid ) { - handleObj.handler.guid = handler.guid; - } - } - - // Add to the element's handler list, delegates in front - if ( selector ) { - handlers.splice( handlers.delegateCount++, 0, handleObj ); - } else { - handlers.push( handleObj ); - } - - // Keep track of which events have ever been used, for event optimization - jQuery.event.global[ type ] = true; - } - - }, - - // Detach an event or set of events from an element - remove: function( elem, types, handler, selector, mappedTypes ) { - - var j, origCount, tmp, - events, t, handleObj, - special, handlers, type, namespaces, origType, - elemData = dataPriv.hasData( elem ) && dataPriv.get( elem ); - - if ( !elemData || !( events = elemData.events ) ) { - return; - } - - // Once for each type.namespace in types; type may be omitted - types = ( types || "" ).match( rnotwhite ) || [ "" ]; - t = types.length; - while ( t-- ) { - tmp = rtypenamespace.exec( types[ t ] ) || []; - type = origType = tmp[ 1 ]; - namespaces = ( tmp[ 2 ] || "" ).split( "." ).sort(); - - // Unbind all events (on this namespace, if provided) for the element - if ( !type ) { - for ( type in events ) { - jQuery.event.remove( elem, type + types[ t ], handler, selector, true ); - } - continue; - } - - special = jQuery.event.special[ type ] || {}; - type = ( selector ? special.delegateType : special.bindType ) || type; - handlers = events[ type ] || []; - tmp = tmp[ 2 ] && - new RegExp( "(^|\\.)" + namespaces.join( "\\.(?:.*\\.|)" ) + "(\\.|$)" ); - - // Remove matching events - origCount = j = handlers.length; - while ( j-- ) { - handleObj = handlers[ j ]; - - if ( ( mappedTypes || origType === handleObj.origType ) && - ( !handler || handler.guid === handleObj.guid ) && - ( !tmp || tmp.test( handleObj.namespace ) ) && - ( !selector || selector === handleObj.selector || - selector === "**" && handleObj.selector ) ) { - handlers.splice( j, 1 ); - - if ( handleObj.selector ) { - handlers.delegateCount--; - } - if ( special.remove ) { - special.remove.call( elem, handleObj ); - } - } - } - - // Remove generic event handler if we removed something and no more handlers exist - // (avoids potential for endless recursion during removal of special event handlers) - if ( origCount && !handlers.length ) { - if ( !special.teardown || - special.teardown.call( elem, namespaces, elemData.handle ) === false ) { - - jQuery.removeEvent( elem, type, elemData.handle ); - } - - delete events[ type ]; - } - } - - // Remove data and the expando if it's no longer used - if ( jQuery.isEmptyObject( events ) ) { - dataPriv.remove( elem, "handle events" ); - } - }, - - dispatch: function( nativeEvent ) { - - // Make a writable jQuery.Event from the native event object - var event = jQuery.event.fix( nativeEvent ); - - var i, j, ret, matched, handleObj, handlerQueue, - args = new Array( arguments.length ), - handlers = ( dataPriv.get( this, "events" ) || {} )[ event.type ] || [], - special = jQuery.event.special[ event.type ] || {}; - - // Use the fix-ed jQuery.Event rather than the (read-only) native event - args[ 0 ] = event; - - for ( i = 1; i < arguments.length; i++ ) { - args[ i ] = arguments[ i ]; - } - - event.delegateTarget = this; - - // Call the preDispatch hook for the mapped type, and let it bail if desired - if ( special.preDispatch && special.preDispatch.call( this, event ) === false ) { - return; - } - - // Determine handlers - handlerQueue = jQuery.event.handlers.call( this, event, handlers ); - - // Run delegates first; they may want to stop propagation beneath us - i = 0; - while ( ( matched = handlerQueue[ i++ ] ) && !event.isPropagationStopped() ) { - event.currentTarget = matched.elem; - - j = 0; - while ( ( handleObj = matched.handlers[ j++ ] ) && - !event.isImmediatePropagationStopped() ) { - - // Triggered event must either 1) have no namespace, or 2) have namespace(s) - // a subset or equal to those in the bound event (both can have no namespace). - if ( !event.rnamespace || event.rnamespace.test( handleObj.namespace ) ) { - - event.handleObj = handleObj; - event.data = handleObj.data; - - ret = ( ( jQuery.event.special[ handleObj.origType ] || {} ).handle || - handleObj.handler ).apply( matched.elem, args ); - - if ( ret !== undefined ) { - if ( ( event.result = ret ) === false ) { - event.preventDefault(); - event.stopPropagation(); - } - } - } - } - } - - // Call the postDispatch hook for the mapped type - if ( special.postDispatch ) { - special.postDispatch.call( this, event ); - } - - return event.result; - }, - - handlers: function( event, handlers ) { - var i, matches, sel, handleObj, - handlerQueue = [], - delegateCount = handlers.delegateCount, - cur = event.target; - - // Support: IE <=9 - // Find delegate handlers - // Black-hole SVG instance trees (#13180) - // - // Support: Firefox <=42 - // Avoid non-left-click in FF but don't block IE radio events (#3861, gh-2343) - if ( delegateCount && cur.nodeType && - ( event.type !== "click" || isNaN( event.button ) || event.button < 1 ) ) { - - for ( ; cur !== this; cur = cur.parentNode || this ) { - - // Don't check non-elements (#13208) - // Don't process clicks on disabled elements (#6911, #8165, #11382, #11764) - if ( cur.nodeType === 1 && ( cur.disabled !== true || event.type !== "click" ) ) { - matches = []; - for ( i = 0; i < delegateCount; i++ ) { - handleObj = handlers[ i ]; - - // Don't conflict with Object.prototype properties (#13203) - sel = handleObj.selector + " "; - - if ( matches[ sel ] === undefined ) { - matches[ sel ] = handleObj.needsContext ? - jQuery( sel, this ).index( cur ) > -1 : - jQuery.find( sel, this, null, [ cur ] ).length; - } - if ( matches[ sel ] ) { - matches.push( handleObj ); - } - } - if ( matches.length ) { - handlerQueue.push( { elem: cur, handlers: matches } ); - } - } - } - } - - // Add the remaining (directly-bound) handlers - if ( delegateCount < handlers.length ) { - handlerQueue.push( { elem: this, handlers: handlers.slice( delegateCount ) } ); - } - - return handlerQueue; - }, - - addProp: function( name, hook ) { - Object.defineProperty( jQuery.Event.prototype, name, { - enumerable: true, - configurable: true, - - get: jQuery.isFunction( hook ) ? - function() { - if ( this.originalEvent ) { - return hook( this.originalEvent ); - } - } : - function() { - if ( this.originalEvent ) { - return this.originalEvent[ name ]; - } - }, - - set: function( value ) { - Object.defineProperty( this, name, { - enumerable: true, - configurable: true, - writable: true, - value: value - } ); - } - } ); - }, - - fix: function( originalEvent ) { - return originalEvent[ jQuery.expando ] ? - originalEvent : - new jQuery.Event( originalEvent ); - }, - - special: { - load: { - - // Prevent triggered image.load events from bubbling to window.load - noBubble: true - }, - focus: { - - // Fire native event if possible so blur/focus sequence is correct - trigger: function() { - if ( this !== safeActiveElement() && this.focus ) { - this.focus(); - return false; - } - }, - delegateType: "focusin" - }, - blur: { - trigger: function() { - if ( this === safeActiveElement() && this.blur ) { - this.blur(); - return false; - } - }, - delegateType: "focusout" - }, - click: { - - // For checkbox, fire native event so checked state will be right - trigger: function() { - if ( this.type === "checkbox" && this.click && jQuery.nodeName( this, "input" ) ) { - this.click(); - return false; - } - }, - - // For cross-browser consistency, don't fire native .click() on links - _default: function( event ) { - return jQuery.nodeName( event.target, "a" ); - } - }, - - beforeunload: { - postDispatch: function( event ) { - - // Support: Firefox 20+ - // Firefox doesn't alert if the returnValue field is not set. - if ( event.result !== undefined && event.originalEvent ) { - event.originalEvent.returnValue = event.result; - } - } - } - } -}; - -jQuery.removeEvent = function( elem, type, handle ) { - - // This "if" is needed for plain objects - if ( elem.removeEventListener ) { - elem.removeEventListener( type, handle ); - } -}; - -jQuery.Event = function( src, props ) { - - // Allow instantiation without the 'new' keyword - if ( !( this instanceof jQuery.Event ) ) { - return new jQuery.Event( src, props ); - } - - // Event object - if ( src && src.type ) { - this.originalEvent = src; - this.type = src.type; - - // Events bubbling up the document may have been marked as prevented - // by a handler lower down the tree; reflect the correct value. - this.isDefaultPrevented = src.defaultPrevented || - src.defaultPrevented === undefined && - - // Support: Android <=2.3 only - src.returnValue === false ? - returnTrue : - returnFalse; - - // Create target properties - // Support: Safari <=6 - 7 only - // Target should not be a text node (#504, #13143) - this.target = ( src.target && src.target.nodeType === 3 ) ? - src.target.parentNode : - src.target; - - this.currentTarget = src.currentTarget; - this.relatedTarget = src.relatedTarget; - - // Event type - } else { - this.type = src; - } - - // Put explicitly provided properties onto the event object - if ( props ) { - jQuery.extend( this, props ); - } - - // Create a timestamp if incoming event doesn't have one - this.timeStamp = src && src.timeStamp || jQuery.now(); - - // Mark it as fixed - this[ jQuery.expando ] = true; -}; - -// jQuery.Event is based on DOM3 Events as specified by the ECMAScript Language Binding -// https://www.w3.org/TR/2003/WD-DOM-Level-3-Events-20030331/ecma-script-binding.html -jQuery.Event.prototype = { - constructor: jQuery.Event, - isDefaultPrevented: returnFalse, - isPropagationStopped: returnFalse, - isImmediatePropagationStopped: returnFalse, - isSimulated: false, - - preventDefault: function() { - var e = this.originalEvent; - - this.isDefaultPrevented = returnTrue; - - if ( e && !this.isSimulated ) { - e.preventDefault(); - } - }, - stopPropagation: function() { - var e = this.originalEvent; - - this.isPropagationStopped = returnTrue; - - if ( e && !this.isSimulated ) { - e.stopPropagation(); - } - }, - stopImmediatePropagation: function() { - var e = this.originalEvent; - - this.isImmediatePropagationStopped = returnTrue; - - if ( e && !this.isSimulated ) { - e.stopImmediatePropagation(); - } - - this.stopPropagation(); - } -}; - -// Includes all common event props including KeyEvent and MouseEvent specific props -jQuery.each( { - altKey: true, - bubbles: true, - cancelable: true, - changedTouches: true, - ctrlKey: true, - detail: true, - eventPhase: true, - metaKey: true, - pageX: true, - pageY: true, - shiftKey: true, - view: true, - "char": true, - charCode: true, - key: true, - keyCode: true, - button: true, - buttons: true, - clientX: true, - clientY: true, - offsetX: true, - offsetY: true, - pointerId: true, - pointerType: true, - screenX: true, - screenY: true, - targetTouches: true, - toElement: true, - touches: true, - - which: function( event ) { - var button = event.button; - - // Add which for key events - if ( event.which == null && rkeyEvent.test( event.type ) ) { - return event.charCode != null ? event.charCode : event.keyCode; - } - - // Add which for click: 1 === left; 2 === middle; 3 === right - if ( !event.which && button !== undefined && rmouseEvent.test( event.type ) ) { - return ( button & 1 ? 1 : ( button & 2 ? 3 : ( button & 4 ? 2 : 0 ) ) ); - } - - return event.which; - } -}, jQuery.event.addProp ); - -// Create mouseenter/leave events using mouseover/out and event-time checks -// so that event delegation works in jQuery. -// Do the same for pointerenter/pointerleave and pointerover/pointerout -// -// Support: Safari 7 only -// Safari sends mouseenter too often; see: -// https://bugs.chromium.org/p/chromium/issues/detail?id=470258 -// for the description of the bug (it existed in older Chrome versions as well). -jQuery.each( { - mouseenter: "mouseover", - mouseleave: "mouseout", - pointerenter: "pointerover", - pointerleave: "pointerout" -}, function( orig, fix ) { - jQuery.event.special[ orig ] = { - delegateType: fix, - bindType: fix, - - handle: function( event ) { - var ret, - target = this, - related = event.relatedTarget, - handleObj = event.handleObj; - - // For mouseenter/leave call the handler if related is outside the target. - // NB: No relatedTarget if the mouse left/entered the browser window - if ( !related || ( related !== target && !jQuery.contains( target, related ) ) ) { - event.type = handleObj.origType; - ret = handleObj.handler.apply( this, arguments ); - event.type = fix; - } - return ret; - } - }; -} ); - -jQuery.fn.extend( { - - on: function( types, selector, data, fn ) { - return on( this, types, selector, data, fn ); - }, - one: function( types, selector, data, fn ) { - return on( this, types, selector, data, fn, 1 ); - }, - off: function( types, selector, fn ) { - var handleObj, type; - if ( types && types.preventDefault && types.handleObj ) { - - // ( event ) dispatched jQuery.Event - handleObj = types.handleObj; - jQuery( types.delegateTarget ).off( - handleObj.namespace ? - handleObj.origType + "." + handleObj.namespace : - handleObj.origType, - handleObj.selector, - handleObj.handler - ); - return this; - } - if ( typeof types === "object" ) { - - // ( types-object [, selector] ) - for ( type in types ) { - this.off( type, selector, types[ type ] ); - } - return this; - } - if ( selector === false || typeof selector === "function" ) { - - // ( types [, fn] ) - fn = selector; - selector = undefined; - } - if ( fn === false ) { - fn = returnFalse; - } - return this.each( function() { - jQuery.event.remove( this, types, fn, selector ); - } ); - } -} ); - - -var - - /* eslint-disable max-len */ - - // See https://github.com/eslint/eslint/issues/3229 - rxhtmlTag = /<(?!area|br|col|embed|hr|img|input|link|meta|param)(([a-z][^\/\0>\x20\t\r\n\f]*)[^>]*)\/>/gi, - - /* eslint-enable */ - - // Support: IE <=10 - 11, Edge 12 - 13 - // In IE/Edge using regex groups here causes severe slowdowns. - // See https://connect.microsoft.com/IE/feedback/details/1736512/ - rnoInnerhtml = /\s*$/g; - -function manipulationTarget( elem, content ) { - if ( jQuery.nodeName( elem, "table" ) && - jQuery.nodeName( content.nodeType !== 11 ? content : content.firstChild, "tr" ) ) { - - return elem.getElementsByTagName( "tbody" )[ 0 ] || elem; - } - - return elem; -} - -// Replace/restore the type attribute of script elements for safe DOM manipulation -function disableScript( elem ) { - elem.type = ( elem.getAttribute( "type" ) !== null ) + "/" + elem.type; - return elem; -} -function restoreScript( elem ) { - var match = rscriptTypeMasked.exec( elem.type ); - - if ( match ) { - elem.type = match[ 1 ]; - } else { - elem.removeAttribute( "type" ); - } - - return elem; -} - -function cloneCopyEvent( src, dest ) { - var i, l, type, pdataOld, pdataCur, udataOld, udataCur, events; - - if ( dest.nodeType !== 1 ) { - return; - } - - // 1. Copy private data: events, handlers, etc. - if ( dataPriv.hasData( src ) ) { - pdataOld = dataPriv.access( src ); - pdataCur = dataPriv.set( dest, pdataOld ); - events = pdataOld.events; - - if ( events ) { - delete pdataCur.handle; - pdataCur.events = {}; - - for ( type in events ) { - for ( i = 0, l = events[ type ].length; i < l; i++ ) { - jQuery.event.add( dest, type, events[ type ][ i ] ); - } - } - } - } - - // 2. Copy user data - if ( dataUser.hasData( src ) ) { - udataOld = dataUser.access( src ); - udataCur = jQuery.extend( {}, udataOld ); - - dataUser.set( dest, udataCur ); - } -} - -// Fix IE bugs, see support tests -function fixInput( src, dest ) { - var nodeName = dest.nodeName.toLowerCase(); - - // Fails to persist the checked state of a cloned checkbox or radio button. - if ( nodeName === "input" && rcheckableType.test( src.type ) ) { - dest.checked = src.checked; - - // Fails to return the selected option to the default selected state when cloning options - } else if ( nodeName === "input" || nodeName === "textarea" ) { - dest.defaultValue = src.defaultValue; - } -} - -function domManip( collection, args, callback, ignored ) { - - // Flatten any nested arrays - args = concat.apply( [], args ); - - var fragment, first, scripts, hasScripts, node, doc, - i = 0, - l = collection.length, - iNoClone = l - 1, - value = args[ 0 ], - isFunction = jQuery.isFunction( value ); - - // We can't cloneNode fragments that contain checked, in WebKit - if ( isFunction || - ( l > 1 && typeof value === "string" && - !support.checkClone && rchecked.test( value ) ) ) { - return collection.each( function( index ) { - var self = collection.eq( index ); - if ( isFunction ) { - args[ 0 ] = value.call( this, index, self.html() ); - } - domManip( self, args, callback, ignored ); - } ); - } - - if ( l ) { - fragment = buildFragment( args, collection[ 0 ].ownerDocument, false, collection, ignored ); - first = fragment.firstChild; - - if ( fragment.childNodes.length === 1 ) { - fragment = first; - } - - // Require either new content or an interest in ignored elements to invoke the callback - if ( first || ignored ) { - scripts = jQuery.map( getAll( fragment, "script" ), disableScript ); - hasScripts = scripts.length; - - // Use the original fragment for the last item - // instead of the first because it can end up - // being emptied incorrectly in certain situations (#8070). - for ( ; i < l; i++ ) { - node = fragment; - - if ( i !== iNoClone ) { - node = jQuery.clone( node, true, true ); - - // Keep references to cloned scripts for later restoration - if ( hasScripts ) { - - // Support: Android <=4.0 only, PhantomJS 1 only - // push.apply(_, arraylike) throws on ancient WebKit - jQuery.merge( scripts, getAll( node, "script" ) ); - } - } - - callback.call( collection[ i ], node, i ); - } - - if ( hasScripts ) { - doc = scripts[ scripts.length - 1 ].ownerDocument; - - // Reenable scripts - jQuery.map( scripts, restoreScript ); - - // Evaluate executable scripts on first document insertion - for ( i = 0; i < hasScripts; i++ ) { - node = scripts[ i ]; - if ( rscriptType.test( node.type || "" ) && - !dataPriv.access( node, "globalEval" ) && - jQuery.contains( doc, node ) ) { - - if ( node.src ) { - - // Optional AJAX dependency, but won't run scripts if not present - if ( jQuery._evalUrl ) { - jQuery._evalUrl( node.src ); - } - } else { - DOMEval( node.textContent.replace( rcleanScript, "" ), doc ); - } - } - } - } - } - } - - return collection; -} - -function remove( elem, selector, keepData ) { - var node, - nodes = selector ? jQuery.filter( selector, elem ) : elem, - i = 0; - - for ( ; ( node = nodes[ i ] ) != null; i++ ) { - if ( !keepData && node.nodeType === 1 ) { - jQuery.cleanData( getAll( node ) ); - } - - if ( node.parentNode ) { - if ( keepData && jQuery.contains( node.ownerDocument, node ) ) { - setGlobalEval( getAll( node, "script" ) ); - } - node.parentNode.removeChild( node ); - } - } - - return elem; -} - -jQuery.extend( { - htmlPrefilter: function( html ) { - return html.replace( rxhtmlTag, "<$1>" ); - }, - - clone: function( elem, dataAndEvents, deepDataAndEvents ) { - var i, l, srcElements, destElements, - clone = elem.cloneNode( true ), - inPage = jQuery.contains( elem.ownerDocument, elem ); - - // Fix IE cloning issues - if ( !support.noCloneChecked && ( elem.nodeType === 1 || elem.nodeType === 11 ) && - !jQuery.isXMLDoc( elem ) ) { - - // We eschew Sizzle here for performance reasons: https://jsperf.com/getall-vs-sizzle/2 - destElements = getAll( clone ); - srcElements = getAll( elem ); - - for ( i = 0, l = srcElements.length; i < l; i++ ) { - fixInput( srcElements[ i ], destElements[ i ] ); - } - } - - // Copy the events from the original to the clone - if ( dataAndEvents ) { - if ( deepDataAndEvents ) { - srcElements = srcElements || getAll( elem ); - destElements = destElements || getAll( clone ); - - for ( i = 0, l = srcElements.length; i < l; i++ ) { - cloneCopyEvent( srcElements[ i ], destElements[ i ] ); - } - } else { - cloneCopyEvent( elem, clone ); - } - } - - // Preserve script evaluation history - destElements = getAll( clone, "script" ); - if ( destElements.length > 0 ) { - setGlobalEval( destElements, !inPage && getAll( elem, "script" ) ); - } - - // Return the cloned set - return clone; - }, - - cleanData: function( elems ) { - var data, elem, type, - special = jQuery.event.special, - i = 0; - - for ( ; ( elem = elems[ i ] ) !== undefined; i++ ) { - if ( acceptData( elem ) ) { - if ( ( data = elem[ dataPriv.expando ] ) ) { - if ( data.events ) { - for ( type in data.events ) { - if ( special[ type ] ) { - jQuery.event.remove( elem, type ); - - // This is a shortcut to avoid jQuery.event.remove's overhead - } else { - jQuery.removeEvent( elem, type, data.handle ); - } - } - } - - // Support: Chrome <=35 - 45+ - // Assign undefined instead of using delete, see Data#remove - elem[ dataPriv.expando ] = undefined; - } - if ( elem[ dataUser.expando ] ) { - - // Support: Chrome <=35 - 45+ - // Assign undefined instead of using delete, see Data#remove - elem[ dataUser.expando ] = undefined; - } - } - } - } -} ); - -jQuery.fn.extend( { - detach: function( selector ) { - return remove( this, selector, true ); - }, - - remove: function( selector ) { - return remove( this, selector ); - }, - - text: function( value ) { - return access( this, function( value ) { - return value === undefined ? - jQuery.text( this ) : - this.empty().each( function() { - if ( this.nodeType === 1 || this.nodeType === 11 || this.nodeType === 9 ) { - this.textContent = value; - } - } ); - }, null, value, arguments.length ); - }, - - append: function() { - return domManip( this, arguments, function( elem ) { - if ( this.nodeType === 1 || this.nodeType === 11 || this.nodeType === 9 ) { - var target = manipulationTarget( this, elem ); - target.appendChild( elem ); - } - } ); - }, - - prepend: function() { - return domManip( this, arguments, function( elem ) { - if ( this.nodeType === 1 || this.nodeType === 11 || this.nodeType === 9 ) { - var target = manipulationTarget( this, elem ); - target.insertBefore( elem, target.firstChild ); - } - } ); - }, - - before: function() { - return domManip( this, arguments, function( elem ) { - if ( this.parentNode ) { - this.parentNode.insertBefore( elem, this ); - } - } ); - }, - - after: function() { - return domManip( this, arguments, function( elem ) { - if ( this.parentNode ) { - this.parentNode.insertBefore( elem, this.nextSibling ); - } - } ); - }, - - empty: function() { - var elem, - i = 0; - - for ( ; ( elem = this[ i ] ) != null; i++ ) { - if ( elem.nodeType === 1 ) { - - // Prevent memory leaks - jQuery.cleanData( getAll( elem, false ) ); - - // Remove any remaining nodes - elem.textContent = ""; - } - } - - return this; - }, - - clone: function( dataAndEvents, deepDataAndEvents ) { - dataAndEvents = dataAndEvents == null ? false : dataAndEvents; - deepDataAndEvents = deepDataAndEvents == null ? dataAndEvents : deepDataAndEvents; - - return this.map( function() { - return jQuery.clone( this, dataAndEvents, deepDataAndEvents ); - } ); - }, - - html: function( value ) { - return access( this, function( value ) { - var elem = this[ 0 ] || {}, - i = 0, - l = this.length; - - if ( value === undefined && elem.nodeType === 1 ) { - return elem.innerHTML; - } - - // See if we can take a shortcut and just use innerHTML - if ( typeof value === "string" && !rnoInnerhtml.test( value ) && - !wrapMap[ ( rtagName.exec( value ) || [ "", "" ] )[ 1 ].toLowerCase() ] ) { - - value = jQuery.htmlPrefilter( value ); - - try { - for ( ; i < l; i++ ) { - elem = this[ i ] || {}; - - // Remove element nodes and prevent memory leaks - if ( elem.nodeType === 1 ) { - jQuery.cleanData( getAll( elem, false ) ); - elem.innerHTML = value; - } - } - - elem = 0; - - // If using innerHTML throws an exception, use the fallback method - } catch ( e ) {} - } - - if ( elem ) { - this.empty().append( value ); - } - }, null, value, arguments.length ); - }, - - replaceWith: function() { - var ignored = []; - - // Make the changes, replacing each non-ignored context element with the new content - return domManip( this, arguments, function( elem ) { - var parent = this.parentNode; - - if ( jQuery.inArray( this, ignored ) < 0 ) { - jQuery.cleanData( getAll( this ) ); - if ( parent ) { - parent.replaceChild( elem, this ); - } - } - - // Force callback invocation - }, ignored ); - } -} ); - -jQuery.each( { - appendTo: "append", - prependTo: "prepend", - insertBefore: "before", - insertAfter: "after", - replaceAll: "replaceWith" -}, function( name, original ) { - jQuery.fn[ name ] = function( selector ) { - var elems, - ret = [], - insert = jQuery( selector ), - last = insert.length - 1, - i = 0; - - for ( ; i <= last; i++ ) { - elems = i === last ? this : this.clone( true ); - jQuery( insert[ i ] )[ original ]( elems ); - - // Support: Android <=4.0 only, PhantomJS 1 only - // .get() because push.apply(_, arraylike) throws on ancient WebKit - push.apply( ret, elems.get() ); - } - - return this.pushStack( ret ); - }; -} ); -var rmargin = ( /^margin/ ); - -var rnumnonpx = new RegExp( "^(" + pnum + ")(?!px)[a-z%]+$", "i" ); - -var getStyles = function( elem ) { - - // Support: IE <=11 only, Firefox <=30 (#15098, #14150) - // IE throws on elements created in popups - // FF meanwhile throws on frame elements through "defaultView.getComputedStyle" - var view = elem.ownerDocument.defaultView; - - if ( !view || !view.opener ) { - view = window; - } - - return view.getComputedStyle( elem ); - }; - - - -( function() { - - // Executing both pixelPosition & boxSizingReliable tests require only one layout - // so they're executed at the same time to save the second computation. - function computeStyleTests() { - - // This is a singleton, we need to execute it only once - if ( !div ) { - return; - } - - div.style.cssText = - "box-sizing:border-box;" + - "position:relative;display:block;" + - "margin:auto;border:1px;padding:1px;" + - "top:1%;width:50%"; - div.innerHTML = ""; - documentElement.appendChild( container ); - - var divStyle = window.getComputedStyle( div ); - pixelPositionVal = divStyle.top !== "1%"; - - // Support: Android 4.0 - 4.3 only, Firefox <=3 - 44 - reliableMarginLeftVal = divStyle.marginLeft === "2px"; - boxSizingReliableVal = divStyle.width === "4px"; - - // Support: Android 4.0 - 4.3 only - // Some styles come back with percentage values, even though they shouldn't - div.style.marginRight = "50%"; - pixelMarginRightVal = divStyle.marginRight === "4px"; - - documentElement.removeChild( container ); - - // Nullify the div so it wouldn't be stored in the memory and - // it will also be a sign that checks already performed - div = null; - } - - var pixelPositionVal, boxSizingReliableVal, pixelMarginRightVal, reliableMarginLeftVal, - container = document.createElement( "div" ), - div = document.createElement( "div" ); - - // Finish early in limited (non-browser) environments - if ( !div.style ) { - return; - } - - // Support: IE <=9 - 11 only - // Style of cloned element affects source element cloned (#8908) - div.style.backgroundClip = "content-box"; - div.cloneNode( true ).style.backgroundClip = ""; - support.clearCloneStyle = div.style.backgroundClip === "content-box"; - - container.style.cssText = "border:0;width:8px;height:0;top:0;left:-9999px;" + - "padding:0;margin-top:1px;position:absolute"; - container.appendChild( div ); - - jQuery.extend( support, { - pixelPosition: function() { - computeStyleTests(); - return pixelPositionVal; - }, - boxSizingReliable: function() { - computeStyleTests(); - return boxSizingReliableVal; - }, - pixelMarginRight: function() { - computeStyleTests(); - return pixelMarginRightVal; - }, - reliableMarginLeft: function() { - computeStyleTests(); - return reliableMarginLeftVal; - } - } ); -} )(); - - -function curCSS( elem, name, computed ) { - var width, minWidth, maxWidth, ret, - style = elem.style; - - computed = computed || getStyles( elem ); - - // Support: IE <=9 only - // getPropertyValue is only needed for .css('filter') (#12537) - if ( computed ) { - ret = computed.getPropertyValue( name ) || computed[ name ]; - - if ( ret === "" && !jQuery.contains( elem.ownerDocument, elem ) ) { - ret = jQuery.style( elem, name ); - } - - // A tribute to the "awesome hack by Dean Edwards" - // Android Browser returns percentage for some values, - // but width seems to be reliably pixels. - // This is against the CSSOM draft spec: - // https://drafts.csswg.org/cssom/#resolved-values - if ( !support.pixelMarginRight() && rnumnonpx.test( ret ) && rmargin.test( name ) ) { - - // Remember the original values - width = style.width; - minWidth = style.minWidth; - maxWidth = style.maxWidth; - - // Put in the new values to get a computed value out - style.minWidth = style.maxWidth = style.width = ret; - ret = computed.width; - - // Revert the changed values - style.width = width; - style.minWidth = minWidth; - style.maxWidth = maxWidth; - } - } - - return ret !== undefined ? - - // Support: IE <=9 - 11 only - // IE returns zIndex value as an integer. - ret + "" : - ret; -} - - -function addGetHookIf( conditionFn, hookFn ) { - - // Define the hook, we'll check on the first run if it's really needed. - return { - get: function() { - if ( conditionFn() ) { - - // Hook not needed (or it's not possible to use it due - // to missing dependency), remove it. - delete this.get; - return; - } - - // Hook needed; redefine it so that the support test is not executed again. - return ( this.get = hookFn ).apply( this, arguments ); - } - }; -} - - -var - - // Swappable if display is none or starts with table - // except "table", "table-cell", or "table-caption" - // See here for display values: https://developer.mozilla.org/en-US/docs/CSS/display - rdisplayswap = /^(none|table(?!-c[ea]).+)/, - cssShow = { position: "absolute", visibility: "hidden", display: "block" }, - cssNormalTransform = { - letterSpacing: "0", - fontWeight: "400" - }, - - cssPrefixes = [ "Webkit", "Moz", "ms" ], - emptyStyle = document.createElement( "div" ).style; - -// Return a css property mapped to a potentially vendor prefixed property -function vendorPropName( name ) { - - // Shortcut for names that are not vendor prefixed - if ( name in emptyStyle ) { - return name; - } - - // Check for vendor prefixed names - var capName = name[ 0 ].toUpperCase() + name.slice( 1 ), - i = cssPrefixes.length; - - while ( i-- ) { - name = cssPrefixes[ i ] + capName; - if ( name in emptyStyle ) { - return name; - } - } -} - -function setPositiveNumber( elem, value, subtract ) { - - // Any relative (+/-) values have already been - // normalized at this point - var matches = rcssNum.exec( value ); - return matches ? - - // Guard against undefined "subtract", e.g., when used as in cssHooks - Math.max( 0, matches[ 2 ] - ( subtract || 0 ) ) + ( matches[ 3 ] || "px" ) : - value; -} - -function augmentWidthOrHeight( elem, name, extra, isBorderBox, styles ) { - var i = extra === ( isBorderBox ? "border" : "content" ) ? - - // If we already have the right measurement, avoid augmentation - 4 : - - // Otherwise initialize for horizontal or vertical properties - name === "width" ? 1 : 0, - - val = 0; - - for ( ; i < 4; i += 2 ) { - - // Both box models exclude margin, so add it if we want it - if ( extra === "margin" ) { - val += jQuery.css( elem, extra + cssExpand[ i ], true, styles ); - } - - if ( isBorderBox ) { - - // border-box includes padding, so remove it if we want content - if ( extra === "content" ) { - val -= jQuery.css( elem, "padding" + cssExpand[ i ], true, styles ); - } - - // At this point, extra isn't border nor margin, so remove border - if ( extra !== "margin" ) { - val -= jQuery.css( elem, "border" + cssExpand[ i ] + "Width", true, styles ); - } - } else { - - // At this point, extra isn't content, so add padding - val += jQuery.css( elem, "padding" + cssExpand[ i ], true, styles ); - - // At this point, extra isn't content nor padding, so add border - if ( extra !== "padding" ) { - val += jQuery.css( elem, "border" + cssExpand[ i ] + "Width", true, styles ); - } - } - } - - return val; -} - -function getWidthOrHeight( elem, name, extra ) { - - // Start with offset property, which is equivalent to the border-box value - var val, - valueIsBorderBox = true, - styles = getStyles( elem ), - isBorderBox = jQuery.css( elem, "boxSizing", false, styles ) === "border-box"; - - // Support: IE <=11 only - // Running getBoundingClientRect on a disconnected node - // in IE throws an error. - if ( elem.getClientRects().length ) { - val = elem.getBoundingClientRect()[ name ]; - } - - // Some non-html elements return undefined for offsetWidth, so check for null/undefined - // svg - https://bugzilla.mozilla.org/show_bug.cgi?id=649285 - // MathML - https://bugzilla.mozilla.org/show_bug.cgi?id=491668 - if ( val <= 0 || val == null ) { - - // Fall back to computed then uncomputed css if necessary - val = curCSS( elem, name, styles ); - if ( val < 0 || val == null ) { - val = elem.style[ name ]; - } - - // Computed unit is not pixels. Stop here and return. - if ( rnumnonpx.test( val ) ) { - return val; - } - - // Check for style in case a browser which returns unreliable values - // for getComputedStyle silently falls back to the reliable elem.style - valueIsBorderBox = isBorderBox && - ( support.boxSizingReliable() || val === elem.style[ name ] ); - - // Normalize "", auto, and prepare for extra - val = parseFloat( val ) || 0; - } - - // Use the active box-sizing model to add/subtract irrelevant styles - return ( val + - augmentWidthOrHeight( - elem, - name, - extra || ( isBorderBox ? "border" : "content" ), - valueIsBorderBox, - styles - ) - ) + "px"; -} - -jQuery.extend( { - - // Add in style property hooks for overriding the default - // behavior of getting and setting a style property - cssHooks: { - opacity: { - get: function( elem, computed ) { - if ( computed ) { - - // We should always get a number back from opacity - var ret = curCSS( elem, "opacity" ); - return ret === "" ? "1" : ret; - } - } - } - }, - - // Don't automatically add "px" to these possibly-unitless properties - cssNumber: { - "animationIterationCount": true, - "columnCount": true, - "fillOpacity": true, - "flexGrow": true, - "flexShrink": true, - "fontWeight": true, - "lineHeight": true, - "opacity": true, - "order": true, - "orphans": true, - "widows": true, - "zIndex": true, - "zoom": true - }, - - // Add in properties whose names you wish to fix before - // setting or getting the value - cssProps: { - "float": "cssFloat" - }, - - // Get and set the style property on a DOM Node - style: function( elem, name, value, extra ) { - - // Don't set styles on text and comment nodes - if ( !elem || elem.nodeType === 3 || elem.nodeType === 8 || !elem.style ) { - return; - } - - // Make sure that we're working with the right name - var ret, type, hooks, - origName = jQuery.camelCase( name ), - style = elem.style; - - name = jQuery.cssProps[ origName ] || - ( jQuery.cssProps[ origName ] = vendorPropName( origName ) || origName ); - - // Gets hook for the prefixed version, then unprefixed version - hooks = jQuery.cssHooks[ name ] || jQuery.cssHooks[ origName ]; - - // Check if we're setting a value - if ( value !== undefined ) { - type = typeof value; - - // Convert "+=" or "-=" to relative numbers (#7345) - if ( type === "string" && ( ret = rcssNum.exec( value ) ) && ret[ 1 ] ) { - value = adjustCSS( elem, name, ret ); - - // Fixes bug #9237 - type = "number"; - } - - // Make sure that null and NaN values aren't set (#7116) - if ( value == null || value !== value ) { - return; - } - - // If a number was passed in, add the unit (except for certain CSS properties) - if ( type === "number" ) { - value += ret && ret[ 3 ] || ( jQuery.cssNumber[ origName ] ? "" : "px" ); - } - - // background-* props affect original clone's values - if ( !support.clearCloneStyle && value === "" && name.indexOf( "background" ) === 0 ) { - style[ name ] = "inherit"; - } - - // If a hook was provided, use that value, otherwise just set the specified value - if ( !hooks || !( "set" in hooks ) || - ( value = hooks.set( elem, value, extra ) ) !== undefined ) { - - style[ name ] = value; - } - - } else { - - // If a hook was provided get the non-computed value from there - if ( hooks && "get" in hooks && - ( ret = hooks.get( elem, false, extra ) ) !== undefined ) { - - return ret; - } - - // Otherwise just get the value from the style object - return style[ name ]; - } - }, - - css: function( elem, name, extra, styles ) { - var val, num, hooks, - origName = jQuery.camelCase( name ); - - // Make sure that we're working with the right name - name = jQuery.cssProps[ origName ] || - ( jQuery.cssProps[ origName ] = vendorPropName( origName ) || origName ); - - // Try prefixed name followed by the unprefixed name - hooks = jQuery.cssHooks[ name ] || jQuery.cssHooks[ origName ]; - - // If a hook was provided get the computed value from there - if ( hooks && "get" in hooks ) { - val = hooks.get( elem, true, extra ); - } - - // Otherwise, if a way to get the computed value exists, use that - if ( val === undefined ) { - val = curCSS( elem, name, styles ); - } - - // Convert "normal" to computed value - if ( val === "normal" && name in cssNormalTransform ) { - val = cssNormalTransform[ name ]; - } - - // Make numeric if forced or a qualifier was provided and val looks numeric - if ( extra === "" || extra ) { - num = parseFloat( val ); - return extra === true || isFinite( num ) ? num || 0 : val; - } - return val; - } -} ); - -jQuery.each( [ "height", "width" ], function( i, name ) { - jQuery.cssHooks[ name ] = { - get: function( elem, computed, extra ) { - if ( computed ) { - - // Certain elements can have dimension info if we invisibly show them - // but it must have a current display style that would benefit - return rdisplayswap.test( jQuery.css( elem, "display" ) ) && - - // Support: Safari 8+ - // Table columns in Safari have non-zero offsetWidth & zero - // getBoundingClientRect().width unless display is changed. - // Support: IE <=11 only - // Running getBoundingClientRect on a disconnected node - // in IE throws an error. - ( !elem.getClientRects().length || !elem.getBoundingClientRect().width ) ? - swap( elem, cssShow, function() { - return getWidthOrHeight( elem, name, extra ); - } ) : - getWidthOrHeight( elem, name, extra ); - } - }, - - set: function( elem, value, extra ) { - var matches, - styles = extra && getStyles( elem ), - subtract = extra && augmentWidthOrHeight( - elem, - name, - extra, - jQuery.css( elem, "boxSizing", false, styles ) === "border-box", - styles - ); - - // Convert to pixels if value adjustment is needed - if ( subtract && ( matches = rcssNum.exec( value ) ) && - ( matches[ 3 ] || "px" ) !== "px" ) { - - elem.style[ name ] = value; - value = jQuery.css( elem, name ); - } - - return setPositiveNumber( elem, value, subtract ); - } - }; -} ); - -jQuery.cssHooks.marginLeft = addGetHookIf( support.reliableMarginLeft, - function( elem, computed ) { - if ( computed ) { - return ( parseFloat( curCSS( elem, "marginLeft" ) ) || - elem.getBoundingClientRect().left - - swap( elem, { marginLeft: 0 }, function() { - return elem.getBoundingClientRect().left; - } ) - ) + "px"; - } - } -); - -// These hooks are used by animate to expand properties -jQuery.each( { - margin: "", - padding: "", - border: "Width" -}, function( prefix, suffix ) { - jQuery.cssHooks[ prefix + suffix ] = { - expand: function( value ) { - var i = 0, - expanded = {}, - - // Assumes a single number if not a string - parts = typeof value === "string" ? value.split( " " ) : [ value ]; - - for ( ; i < 4; i++ ) { - expanded[ prefix + cssExpand[ i ] + suffix ] = - parts[ i ] || parts[ i - 2 ] || parts[ 0 ]; - } - - return expanded; - } - }; - - if ( !rmargin.test( prefix ) ) { - jQuery.cssHooks[ prefix + suffix ].set = setPositiveNumber; - } -} ); - -jQuery.fn.extend( { - css: function( name, value ) { - return access( this, function( elem, name, value ) { - var styles, len, - map = {}, - i = 0; - - if ( jQuery.isArray( name ) ) { - styles = getStyles( elem ); - len = name.length; - - for ( ; i < len; i++ ) { - map[ name[ i ] ] = jQuery.css( elem, name[ i ], false, styles ); - } - - return map; - } - - return value !== undefined ? - jQuery.style( elem, name, value ) : - jQuery.css( elem, name ); - }, name, value, arguments.length > 1 ); - } -} ); - - -function Tween( elem, options, prop, end, easing ) { - return new Tween.prototype.init( elem, options, prop, end, easing ); -} -jQuery.Tween = Tween; - -Tween.prototype = { - constructor: Tween, - init: function( elem, options, prop, end, easing, unit ) { - this.elem = elem; - this.prop = prop; - this.easing = easing || jQuery.easing._default; - this.options = options; - this.start = this.now = this.cur(); - this.end = end; - this.unit = unit || ( jQuery.cssNumber[ prop ] ? "" : "px" ); - }, - cur: function() { - var hooks = Tween.propHooks[ this.prop ]; - - return hooks && hooks.get ? - hooks.get( this ) : - Tween.propHooks._default.get( this ); - }, - run: function( percent ) { - var eased, - hooks = Tween.propHooks[ this.prop ]; - - if ( this.options.duration ) { - this.pos = eased = jQuery.easing[ this.easing ]( - percent, this.options.duration * percent, 0, 1, this.options.duration - ); - } else { - this.pos = eased = percent; - } - this.now = ( this.end - this.start ) * eased + this.start; - - if ( this.options.step ) { - this.options.step.call( this.elem, this.now, this ); - } - - if ( hooks && hooks.set ) { - hooks.set( this ); - } else { - Tween.propHooks._default.set( this ); - } - return this; - } -}; - -Tween.prototype.init.prototype = Tween.prototype; - -Tween.propHooks = { - _default: { - get: function( tween ) { - var result; - - // Use a property on the element directly when it is not a DOM element, - // or when there is no matching style property that exists. - if ( tween.elem.nodeType !== 1 || - tween.elem[ tween.prop ] != null && tween.elem.style[ tween.prop ] == null ) { - return tween.elem[ tween.prop ]; - } - - // Passing an empty string as a 3rd parameter to .css will automatically - // attempt a parseFloat and fallback to a string if the parse fails. - // Simple values such as "10px" are parsed to Float; - // complex values such as "rotate(1rad)" are returned as-is. - result = jQuery.css( tween.elem, tween.prop, "" ); - - // Empty strings, null, undefined and "auto" are converted to 0. - return !result || result === "auto" ? 0 : result; - }, - set: function( tween ) { - - // Use step hook for back compat. - // Use cssHook if its there. - // Use .style if available and use plain properties where available. - if ( jQuery.fx.step[ tween.prop ] ) { - jQuery.fx.step[ tween.prop ]( tween ); - } else if ( tween.elem.nodeType === 1 && - ( tween.elem.style[ jQuery.cssProps[ tween.prop ] ] != null || - jQuery.cssHooks[ tween.prop ] ) ) { - jQuery.style( tween.elem, tween.prop, tween.now + tween.unit ); - } else { - tween.elem[ tween.prop ] = tween.now; - } - } - } -}; - -// Support: IE <=9 only -// Panic based approach to setting things on disconnected nodes -Tween.propHooks.scrollTop = Tween.propHooks.scrollLeft = { - set: function( tween ) { - if ( tween.elem.nodeType && tween.elem.parentNode ) { - tween.elem[ tween.prop ] = tween.now; - } - } -}; - -jQuery.easing = { - linear: function( p ) { - return p; - }, - swing: function( p ) { - return 0.5 - Math.cos( p * Math.PI ) / 2; - }, - _default: "swing" -}; - -jQuery.fx = Tween.prototype.init; - -// Back compat <1.8 extension point -jQuery.fx.step = {}; - - - - -var - fxNow, timerId, - rfxtypes = /^(?:toggle|show|hide)$/, - rrun = /queueHooks$/; - -function raf() { - if ( timerId ) { - window.requestAnimationFrame( raf ); - jQuery.fx.tick(); - } -} - -// Animations created synchronously will run synchronously -function createFxNow() { - window.setTimeout( function() { - fxNow = undefined; - } ); - return ( fxNow = jQuery.now() ); -} - -// Generate parameters to create a standard animation -function genFx( type, includeWidth ) { - var which, - i = 0, - attrs = { height: type }; - - // If we include width, step value is 1 to do all cssExpand values, - // otherwise step value is 2 to skip over Left and Right - includeWidth = includeWidth ? 1 : 0; - for ( ; i < 4; i += 2 - includeWidth ) { - which = cssExpand[ i ]; - attrs[ "margin" + which ] = attrs[ "padding" + which ] = type; - } - - if ( includeWidth ) { - attrs.opacity = attrs.width = type; - } - - return attrs; -} - -function createTween( value, prop, animation ) { - var tween, - collection = ( Animation.tweeners[ prop ] || [] ).concat( Animation.tweeners[ "*" ] ), - index = 0, - length = collection.length; - for ( ; index < length; index++ ) { - if ( ( tween = collection[ index ].call( animation, prop, value ) ) ) { - - // We're done with this property - return tween; - } - } -} - -function defaultPrefilter( elem, props, opts ) { - var prop, value, toggle, hooks, oldfire, propTween, restoreDisplay, display, - isBox = "width" in props || "height" in props, - anim = this, - orig = {}, - style = elem.style, - hidden = elem.nodeType && isHiddenWithinTree( elem ), - dataShow = dataPriv.get( elem, "fxshow" ); - - // Queue-skipping animations hijack the fx hooks - if ( !opts.queue ) { - hooks = jQuery._queueHooks( elem, "fx" ); - if ( hooks.unqueued == null ) { - hooks.unqueued = 0; - oldfire = hooks.empty.fire; - hooks.empty.fire = function() { - if ( !hooks.unqueued ) { - oldfire(); - } - }; - } - hooks.unqueued++; - - anim.always( function() { - - // Ensure the complete handler is called before this completes - anim.always( function() { - hooks.unqueued--; - if ( !jQuery.queue( elem, "fx" ).length ) { - hooks.empty.fire(); - } - } ); - } ); - } - - // Detect show/hide animations - for ( prop in props ) { - value = props[ prop ]; - if ( rfxtypes.test( value ) ) { - delete props[ prop ]; - toggle = toggle || value === "toggle"; - if ( value === ( hidden ? "hide" : "show" ) ) { - - // Pretend to be hidden if this is a "show" and - // there is still data from a stopped show/hide - if ( value === "show" && dataShow && dataShow[ prop ] !== undefined ) { - hidden = true; - - // Ignore all other no-op show/hide data - } else { - continue; - } - } - orig[ prop ] = dataShow && dataShow[ prop ] || jQuery.style( elem, prop ); - } - } - - // Bail out if this is a no-op like .hide().hide() - propTween = !jQuery.isEmptyObject( props ); - if ( !propTween && jQuery.isEmptyObject( orig ) ) { - return; - } - - // Restrict "overflow" and "display" styles during box animations - if ( isBox && elem.nodeType === 1 ) { - - // Support: IE <=9 - 11, Edge 12 - 13 - // Record all 3 overflow attributes because IE does not infer the shorthand - // from identically-valued overflowX and overflowY - opts.overflow = [ style.overflow, style.overflowX, style.overflowY ]; - - // Identify a display type, preferring old show/hide data over the CSS cascade - restoreDisplay = dataShow && dataShow.display; - if ( restoreDisplay == null ) { - restoreDisplay = dataPriv.get( elem, "display" ); - } - display = jQuery.css( elem, "display" ); - if ( display === "none" ) { - if ( restoreDisplay ) { - display = restoreDisplay; - } else { - - // Get nonempty value(s) by temporarily forcing visibility - showHide( [ elem ], true ); - restoreDisplay = elem.style.display || restoreDisplay; - display = jQuery.css( elem, "display" ); - showHide( [ elem ] ); - } - } - - // Animate inline elements as inline-block - if ( display === "inline" || display === "inline-block" && restoreDisplay != null ) { - if ( jQuery.css( elem, "float" ) === "none" ) { - - // Restore the original display value at the end of pure show/hide animations - if ( !propTween ) { - anim.done( function() { - style.display = restoreDisplay; - } ); - if ( restoreDisplay == null ) { - display = style.display; - restoreDisplay = display === "none" ? "" : display; - } - } - style.display = "inline-block"; - } - } - } - - if ( opts.overflow ) { - style.overflow = "hidden"; - anim.always( function() { - style.overflow = opts.overflow[ 0 ]; - style.overflowX = opts.overflow[ 1 ]; - style.overflowY = opts.overflow[ 2 ]; - } ); - } - - // Implement show/hide animations - propTween = false; - for ( prop in orig ) { - - // General show/hide setup for this element animation - if ( !propTween ) { - if ( dataShow ) { - if ( "hidden" in dataShow ) { - hidden = dataShow.hidden; - } - } else { - dataShow = dataPriv.access( elem, "fxshow", { display: restoreDisplay } ); - } - - // Store hidden/visible for toggle so `.stop().toggle()` "reverses" - if ( toggle ) { - dataShow.hidden = !hidden; - } - - // Show elements before animating them - if ( hidden ) { - showHide( [ elem ], true ); - } - - /* eslint-disable no-loop-func */ - - anim.done( function() { - - /* eslint-enable no-loop-func */ - - // The final step of a "hide" animation is actually hiding the element - if ( !hidden ) { - showHide( [ elem ] ); - } - dataPriv.remove( elem, "fxshow" ); - for ( prop in orig ) { - jQuery.style( elem, prop, orig[ prop ] ); - } - } ); - } - - // Per-property setup - propTween = createTween( hidden ? dataShow[ prop ] : 0, prop, anim ); - if ( !( prop in dataShow ) ) { - dataShow[ prop ] = propTween.start; - if ( hidden ) { - propTween.end = propTween.start; - propTween.start = 0; - } - } - } -} - -function propFilter( props, specialEasing ) { - var index, name, easing, value, hooks; - - // camelCase, specialEasing and expand cssHook pass - for ( index in props ) { - name = jQuery.camelCase( index ); - easing = specialEasing[ name ]; - value = props[ index ]; - if ( jQuery.isArray( value ) ) { - easing = value[ 1 ]; - value = props[ index ] = value[ 0 ]; - } - - if ( index !== name ) { - props[ name ] = value; - delete props[ index ]; - } - - hooks = jQuery.cssHooks[ name ]; - if ( hooks && "expand" in hooks ) { - value = hooks.expand( value ); - delete props[ name ]; - - // Not quite $.extend, this won't overwrite existing keys. - // Reusing 'index' because we have the correct "name" - for ( index in value ) { - if ( !( index in props ) ) { - props[ index ] = value[ index ]; - specialEasing[ index ] = easing; - } - } - } else { - specialEasing[ name ] = easing; - } - } -} - -function Animation( elem, properties, options ) { - var result, - stopped, - index = 0, - length = Animation.prefilters.length, - deferred = jQuery.Deferred().always( function() { - - // Don't match elem in the :animated selector - delete tick.elem; - } ), - tick = function() { - if ( stopped ) { - return false; - } - var currentTime = fxNow || createFxNow(), - remaining = Math.max( 0, animation.startTime + animation.duration - currentTime ), - - // Support: Android 2.3 only - // Archaic crash bug won't allow us to use `1 - ( 0.5 || 0 )` (#12497) - temp = remaining / animation.duration || 0, - percent = 1 - temp, - index = 0, - length = animation.tweens.length; - - for ( ; index < length; index++ ) { - animation.tweens[ index ].run( percent ); - } - - deferred.notifyWith( elem, [ animation, percent, remaining ] ); - - if ( percent < 1 && length ) { - return remaining; - } else { - deferred.resolveWith( elem, [ animation ] ); - return false; - } - }, - animation = deferred.promise( { - elem: elem, - props: jQuery.extend( {}, properties ), - opts: jQuery.extend( true, { - specialEasing: {}, - easing: jQuery.easing._default - }, options ), - originalProperties: properties, - originalOptions: options, - startTime: fxNow || createFxNow(), - duration: options.duration, - tweens: [], - createTween: function( prop, end ) { - var tween = jQuery.Tween( elem, animation.opts, prop, end, - animation.opts.specialEasing[ prop ] || animation.opts.easing ); - animation.tweens.push( tween ); - return tween; - }, - stop: function( gotoEnd ) { - var index = 0, - - // If we are going to the end, we want to run all the tweens - // otherwise we skip this part - length = gotoEnd ? animation.tweens.length : 0; - if ( stopped ) { - return this; - } - stopped = true; - for ( ; index < length; index++ ) { - animation.tweens[ index ].run( 1 ); - } - - // Resolve when we played the last frame; otherwise, reject - if ( gotoEnd ) { - deferred.notifyWith( elem, [ animation, 1, 0 ] ); - deferred.resolveWith( elem, [ animation, gotoEnd ] ); - } else { - deferred.rejectWith( elem, [ animation, gotoEnd ] ); - } - return this; - } - } ), - props = animation.props; - - propFilter( props, animation.opts.specialEasing ); - - for ( ; index < length; index++ ) { - result = Animation.prefilters[ index ].call( animation, elem, props, animation.opts ); - if ( result ) { - if ( jQuery.isFunction( result.stop ) ) { - jQuery._queueHooks( animation.elem, animation.opts.queue ).stop = - jQuery.proxy( result.stop, result ); - } - return result; - } - } - - jQuery.map( props, createTween, animation ); - - if ( jQuery.isFunction( animation.opts.start ) ) { - animation.opts.start.call( elem, animation ); - } - - jQuery.fx.timer( - jQuery.extend( tick, { - elem: elem, - anim: animation, - queue: animation.opts.queue - } ) - ); - - // attach callbacks from options - return animation.progress( animation.opts.progress ) - .done( animation.opts.done, animation.opts.complete ) - .fail( animation.opts.fail ) - .always( animation.opts.always ); -} - -jQuery.Animation = jQuery.extend( Animation, { - - tweeners: { - "*": [ function( prop, value ) { - var tween = this.createTween( prop, value ); - adjustCSS( tween.elem, prop, rcssNum.exec( value ), tween ); - return tween; - } ] - }, - - tweener: function( props, callback ) { - if ( jQuery.isFunction( props ) ) { - callback = props; - props = [ "*" ]; - } else { - props = props.match( rnotwhite ); - } - - var prop, - index = 0, - length = props.length; - - for ( ; index < length; index++ ) { - prop = props[ index ]; - Animation.tweeners[ prop ] = Animation.tweeners[ prop ] || []; - Animation.tweeners[ prop ].unshift( callback ); - } - }, - - prefilters: [ defaultPrefilter ], - - prefilter: function( callback, prepend ) { - if ( prepend ) { - Animation.prefilters.unshift( callback ); - } else { - Animation.prefilters.push( callback ); - } - } -} ); - -jQuery.speed = function( speed, easing, fn ) { - var opt = speed && typeof speed === "object" ? jQuery.extend( {}, speed ) : { - complete: fn || !fn && easing || - jQuery.isFunction( speed ) && speed, - duration: speed, - easing: fn && easing || easing && !jQuery.isFunction( easing ) && easing - }; - - // Go to the end state if fx are off or if document is hidden - if ( jQuery.fx.off || document.hidden ) { - opt.duration = 0; - - } else { - opt.duration = typeof opt.duration === "number" ? - opt.duration : opt.duration in jQuery.fx.speeds ? - jQuery.fx.speeds[ opt.duration ] : jQuery.fx.speeds._default; - } - - // Normalize opt.queue - true/undefined/null -> "fx" - if ( opt.queue == null || opt.queue === true ) { - opt.queue = "fx"; - } - - // Queueing - opt.old = opt.complete; - - opt.complete = function() { - if ( jQuery.isFunction( opt.old ) ) { - opt.old.call( this ); - } - - if ( opt.queue ) { - jQuery.dequeue( this, opt.queue ); - } - }; - - return opt; -}; - -jQuery.fn.extend( { - fadeTo: function( speed, to, easing, callback ) { - - // Show any hidden elements after setting opacity to 0 - return this.filter( isHiddenWithinTree ).css( "opacity", 0 ).show() - - // Animate to the value specified - .end().animate( { opacity: to }, speed, easing, callback ); - }, - animate: function( prop, speed, easing, callback ) { - var empty = jQuery.isEmptyObject( prop ), - optall = jQuery.speed( speed, easing, callback ), - doAnimation = function() { - - // Operate on a copy of prop so per-property easing won't be lost - var anim = Animation( this, jQuery.extend( {}, prop ), optall ); - - // Empty animations, or finishing resolves immediately - if ( empty || dataPriv.get( this, "finish" ) ) { - anim.stop( true ); - } - }; - doAnimation.finish = doAnimation; - - return empty || optall.queue === false ? - this.each( doAnimation ) : - this.queue( optall.queue, doAnimation ); - }, - stop: function( type, clearQueue, gotoEnd ) { - var stopQueue = function( hooks ) { - var stop = hooks.stop; - delete hooks.stop; - stop( gotoEnd ); - }; - - if ( typeof type !== "string" ) { - gotoEnd = clearQueue; - clearQueue = type; - type = undefined; - } - if ( clearQueue && type !== false ) { - this.queue( type || "fx", [] ); - } - - return this.each( function() { - var dequeue = true, - index = type != null && type + "queueHooks", - timers = jQuery.timers, - data = dataPriv.get( this ); - - if ( index ) { - if ( data[ index ] && data[ index ].stop ) { - stopQueue( data[ index ] ); - } - } else { - for ( index in data ) { - if ( data[ index ] && data[ index ].stop && rrun.test( index ) ) { - stopQueue( data[ index ] ); - } - } - } - - for ( index = timers.length; index--; ) { - if ( timers[ index ].elem === this && - ( type == null || timers[ index ].queue === type ) ) { - - timers[ index ].anim.stop( gotoEnd ); - dequeue = false; - timers.splice( index, 1 ); - } - } - - // Start the next in the queue if the last step wasn't forced. - // Timers currently will call their complete callbacks, which - // will dequeue but only if they were gotoEnd. - if ( dequeue || !gotoEnd ) { - jQuery.dequeue( this, type ); - } - } ); - }, - finish: function( type ) { - if ( type !== false ) { - type = type || "fx"; - } - return this.each( function() { - var index, - data = dataPriv.get( this ), - queue = data[ type + "queue" ], - hooks = data[ type + "queueHooks" ], - timers = jQuery.timers, - length = queue ? queue.length : 0; - - // Enable finishing flag on private data - data.finish = true; - - // Empty the queue first - jQuery.queue( this, type, [] ); - - if ( hooks && hooks.stop ) { - hooks.stop.call( this, true ); - } - - // Look for any active animations, and finish them - for ( index = timers.length; index--; ) { - if ( timers[ index ].elem === this && timers[ index ].queue === type ) { - timers[ index ].anim.stop( true ); - timers.splice( index, 1 ); - } - } - - // Look for any animations in the old queue and finish them - for ( index = 0; index < length; index++ ) { - if ( queue[ index ] && queue[ index ].finish ) { - queue[ index ].finish.call( this ); - } - } - - // Turn off finishing flag - delete data.finish; - } ); - } -} ); - -jQuery.each( [ "toggle", "show", "hide" ], function( i, name ) { - var cssFn = jQuery.fn[ name ]; - jQuery.fn[ name ] = function( speed, easing, callback ) { - return speed == null || typeof speed === "boolean" ? - cssFn.apply( this, arguments ) : - this.animate( genFx( name, true ), speed, easing, callback ); - }; -} ); - -// Generate shortcuts for custom animations -jQuery.each( { - slideDown: genFx( "show" ), - slideUp: genFx( "hide" ), - slideToggle: genFx( "toggle" ), - fadeIn: { opacity: "show" }, - fadeOut: { opacity: "hide" }, - fadeToggle: { opacity: "toggle" } -}, function( name, props ) { - jQuery.fn[ name ] = function( speed, easing, callback ) { - return this.animate( props, speed, easing, callback ); - }; -} ); - -jQuery.timers = []; -jQuery.fx.tick = function() { - var timer, - i = 0, - timers = jQuery.timers; - - fxNow = jQuery.now(); - - for ( ; i < timers.length; i++ ) { - timer = timers[ i ]; - - // Checks the timer has not already been removed - if ( !timer() && timers[ i ] === timer ) { - timers.splice( i--, 1 ); - } - } - - if ( !timers.length ) { - jQuery.fx.stop(); - } - fxNow = undefined; -}; - -jQuery.fx.timer = function( timer ) { - jQuery.timers.push( timer ); - if ( timer() ) { - jQuery.fx.start(); - } else { - jQuery.timers.pop(); - } -}; - -jQuery.fx.interval = 13; -jQuery.fx.start = function() { - if ( !timerId ) { - timerId = window.requestAnimationFrame ? - window.requestAnimationFrame( raf ) : - window.setInterval( jQuery.fx.tick, jQuery.fx.interval ); - } -}; - -jQuery.fx.stop = function() { - if ( window.cancelAnimationFrame ) { - window.cancelAnimationFrame( timerId ); - } else { - window.clearInterval( timerId ); - } - - timerId = null; -}; - -jQuery.fx.speeds = { - slow: 600, - fast: 200, - - // Default speed - _default: 400 -}; - - -// Based off of the plugin by Clint Helfers, with permission. -// https://web.archive.org/web/20100324014747/http://blindsignals.com/index.php/2009/07/jquery-delay/ -jQuery.fn.delay = function( time, type ) { - time = jQuery.fx ? jQuery.fx.speeds[ time ] || time : time; - type = type || "fx"; - - return this.queue( type, function( next, hooks ) { - var timeout = window.setTimeout( next, time ); - hooks.stop = function() { - window.clearTimeout( timeout ); - }; - } ); -}; - - -( function() { - var input = document.createElement( "input" ), - select = document.createElement( "select" ), - opt = select.appendChild( document.createElement( "option" ) ); - - input.type = "checkbox"; - - // Support: Android <=4.3 only - // Default value for a checkbox should be "on" - support.checkOn = input.value !== ""; - - // Support: IE <=11 only - // Must access selectedIndex to make default options select - support.optSelected = opt.selected; - - // Support: IE <=11 only - // An input loses its value after becoming a radio - input = document.createElement( "input" ); - input.value = "t"; - input.type = "radio"; - support.radioValue = input.value === "t"; -} )(); - - -var boolHook, - attrHandle = jQuery.expr.attrHandle; - -jQuery.fn.extend( { - attr: function( name, value ) { - return access( this, jQuery.attr, name, value, arguments.length > 1 ); - }, - - removeAttr: function( name ) { - return this.each( function() { - jQuery.removeAttr( this, name ); - } ); - } -} ); - -jQuery.extend( { - attr: function( elem, name, value ) { - var ret, hooks, - nType = elem.nodeType; - - // Don't get/set attributes on text, comment and attribute nodes - if ( nType === 3 || nType === 8 || nType === 2 ) { - return; - } - - // Fallback to prop when attributes are not supported - if ( typeof elem.getAttribute === "undefined" ) { - return jQuery.prop( elem, name, value ); - } - - // Attribute hooks are determined by the lowercase version - // Grab necessary hook if one is defined - if ( nType !== 1 || !jQuery.isXMLDoc( elem ) ) { - hooks = jQuery.attrHooks[ name.toLowerCase() ] || - ( jQuery.expr.match.bool.test( name ) ? boolHook : undefined ); - } - - if ( value !== undefined ) { - if ( value === null ) { - jQuery.removeAttr( elem, name ); - return; - } - - if ( hooks && "set" in hooks && - ( ret = hooks.set( elem, value, name ) ) !== undefined ) { - return ret; - } - - elem.setAttribute( name, value + "" ); - return value; - } - - if ( hooks && "get" in hooks && ( ret = hooks.get( elem, name ) ) !== null ) { - return ret; - } - - ret = jQuery.find.attr( elem, name ); - - // Non-existent attributes return null, we normalize to undefined - return ret == null ? undefined : ret; - }, - - attrHooks: { - type: { - set: function( elem, value ) { - if ( !support.radioValue && value === "radio" && - jQuery.nodeName( elem, "input" ) ) { - var val = elem.value; - elem.setAttribute( "type", value ); - if ( val ) { - elem.value = val; - } - return value; - } - } - } - }, - - removeAttr: function( elem, value ) { - var name, - i = 0, - attrNames = value && value.match( rnotwhite ); - - if ( attrNames && elem.nodeType === 1 ) { - while ( ( name = attrNames[ i++ ] ) ) { - elem.removeAttribute( name ); - } - } - } -} ); - -// Hooks for boolean attributes -boolHook = { - set: function( elem, value, name ) { - if ( value === false ) { - - // Remove boolean attributes when set to false - jQuery.removeAttr( elem, name ); - } else { - elem.setAttribute( name, name ); - } - return name; - } -}; - -jQuery.each( jQuery.expr.match.bool.source.match( /\w+/g ), function( i, name ) { - var getter = attrHandle[ name ] || jQuery.find.attr; - - attrHandle[ name ] = function( elem, name, isXML ) { - var ret, handle, - lowercaseName = name.toLowerCase(); - - if ( !isXML ) { - - // Avoid an infinite loop by temporarily removing this function from the getter - handle = attrHandle[ lowercaseName ]; - attrHandle[ lowercaseName ] = ret; - ret = getter( elem, name, isXML ) != null ? - lowercaseName : - null; - attrHandle[ lowercaseName ] = handle; - } - return ret; - }; -} ); - - - - -var rfocusable = /^(?:input|select|textarea|button)$/i, - rclickable = /^(?:a|area)$/i; - -jQuery.fn.extend( { - prop: function( name, value ) { - return access( this, jQuery.prop, name, value, arguments.length > 1 ); - }, - - removeProp: function( name ) { - return this.each( function() { - delete this[ jQuery.propFix[ name ] || name ]; - } ); - } -} ); - -jQuery.extend( { - prop: function( elem, name, value ) { - var ret, hooks, - nType = elem.nodeType; - - // Don't get/set properties on text, comment and attribute nodes - if ( nType === 3 || nType === 8 || nType === 2 ) { - return; - } - - if ( nType !== 1 || !jQuery.isXMLDoc( elem ) ) { - - // Fix name and attach hooks - name = jQuery.propFix[ name ] || name; - hooks = jQuery.propHooks[ name ]; - } - - if ( value !== undefined ) { - if ( hooks && "set" in hooks && - ( ret = hooks.set( elem, value, name ) ) !== undefined ) { - return ret; - } - - return ( elem[ name ] = value ); - } - - if ( hooks && "get" in hooks && ( ret = hooks.get( elem, name ) ) !== null ) { - return ret; - } - - return elem[ name ]; - }, - - propHooks: { - tabIndex: { - get: function( elem ) { - - // Support: IE <=9 - 11 only - // elem.tabIndex doesn't always return the - // correct value when it hasn't been explicitly set - // https://web.archive.org/web/20141116233347/http://fluidproject.org/blog/2008/01/09/getting-setting-and-removing-tabindex-values-with-javascript/ - // Use proper attribute retrieval(#12072) - var tabindex = jQuery.find.attr( elem, "tabindex" ); - - return tabindex ? - parseInt( tabindex, 10 ) : - rfocusable.test( elem.nodeName ) || - rclickable.test( elem.nodeName ) && elem.href ? - 0 : - -1; - } - } - }, - - propFix: { - "for": "htmlFor", - "class": "className" - } -} ); - -// Support: IE <=11 only -// Accessing the selectedIndex property -// forces the browser to respect setting selected -// on the option -// The getter ensures a default option is selected -// when in an optgroup -if ( !support.optSelected ) { - jQuery.propHooks.selected = { - get: function( elem ) { - var parent = elem.parentNode; - if ( parent && parent.parentNode ) { - parent.parentNode.selectedIndex; - } - return null; - }, - set: function( elem ) { - var parent = elem.parentNode; - if ( parent ) { - parent.selectedIndex; - - if ( parent.parentNode ) { - parent.parentNode.selectedIndex; - } - } - } - }; -} - -jQuery.each( [ - "tabIndex", - "readOnly", - "maxLength", - "cellSpacing", - "cellPadding", - "rowSpan", - "colSpan", - "useMap", - "frameBorder", - "contentEditable" -], function() { - jQuery.propFix[ this.toLowerCase() ] = this; -} ); - - - - -var rclass = /[\t\r\n\f]/g; - -function getClass( elem ) { - return elem.getAttribute && elem.getAttribute( "class" ) || ""; -} - -jQuery.fn.extend( { - addClass: function( value ) { - var classes, elem, cur, curValue, clazz, j, finalValue, - i = 0; - - if ( jQuery.isFunction( value ) ) { - return this.each( function( j ) { - jQuery( this ).addClass( value.call( this, j, getClass( this ) ) ); - } ); - } - - if ( typeof value === "string" && value ) { - classes = value.match( rnotwhite ) || []; - - while ( ( elem = this[ i++ ] ) ) { - curValue = getClass( elem ); - cur = elem.nodeType === 1 && - ( " " + curValue + " " ).replace( rclass, " " ); - - if ( cur ) { - j = 0; - while ( ( clazz = classes[ j++ ] ) ) { - if ( cur.indexOf( " " + clazz + " " ) < 0 ) { - cur += clazz + " "; - } - } - - // Only assign if different to avoid unneeded rendering. - finalValue = jQuery.trim( cur ); - if ( curValue !== finalValue ) { - elem.setAttribute( "class", finalValue ); - } - } - } - } - - return this; - }, - - removeClass: function( value ) { - var classes, elem, cur, curValue, clazz, j, finalValue, - i = 0; - - if ( jQuery.isFunction( value ) ) { - return this.each( function( j ) { - jQuery( this ).removeClass( value.call( this, j, getClass( this ) ) ); - } ); - } - - if ( !arguments.length ) { - return this.attr( "class", "" ); - } - - if ( typeof value === "string" && value ) { - classes = value.match( rnotwhite ) || []; - - while ( ( elem = this[ i++ ] ) ) { - curValue = getClass( elem ); - - // This expression is here for better compressibility (see addClass) - cur = elem.nodeType === 1 && - ( " " + curValue + " " ).replace( rclass, " " ); - - if ( cur ) { - j = 0; - while ( ( clazz = classes[ j++ ] ) ) { - - // Remove *all* instances - while ( cur.indexOf( " " + clazz + " " ) > -1 ) { - cur = cur.replace( " " + clazz + " ", " " ); - } - } - - // Only assign if different to avoid unneeded rendering. - finalValue = jQuery.trim( cur ); - if ( curValue !== finalValue ) { - elem.setAttribute( "class", finalValue ); - } - } - } - } - - return this; - }, - - toggleClass: function( value, stateVal ) { - var type = typeof value; - - if ( typeof stateVal === "boolean" && type === "string" ) { - return stateVal ? this.addClass( value ) : this.removeClass( value ); - } - - if ( jQuery.isFunction( value ) ) { - return this.each( function( i ) { - jQuery( this ).toggleClass( - value.call( this, i, getClass( this ), stateVal ), - stateVal - ); - } ); - } - - return this.each( function() { - var className, i, self, classNames; - - if ( type === "string" ) { - - // Toggle individual class names - i = 0; - self = jQuery( this ); - classNames = value.match( rnotwhite ) || []; - - while ( ( className = classNames[ i++ ] ) ) { - - // Check each className given, space separated list - if ( self.hasClass( className ) ) { - self.removeClass( className ); - } else { - self.addClass( className ); - } - } - - // Toggle whole class name - } else if ( value === undefined || type === "boolean" ) { - className = getClass( this ); - if ( className ) { - - // Store className if set - dataPriv.set( this, "__className__", className ); - } - - // If the element has a class name or if we're passed `false`, - // then remove the whole classname (if there was one, the above saved it). - // Otherwise bring back whatever was previously saved (if anything), - // falling back to the empty string if nothing was stored. - if ( this.setAttribute ) { - this.setAttribute( "class", - className || value === false ? - "" : - dataPriv.get( this, "__className__" ) || "" - ); - } - } - } ); - }, - - hasClass: function( selector ) { - var className, elem, - i = 0; - - className = " " + selector + " "; - while ( ( elem = this[ i++ ] ) ) { - if ( elem.nodeType === 1 && - ( " " + getClass( elem ) + " " ).replace( rclass, " " ) - .indexOf( className ) > -1 - ) { - return true; - } - } - - return false; - } -} ); - - - - -var rreturn = /\r/g, - rspaces = /[\x20\t\r\n\f]+/g; - -jQuery.fn.extend( { - val: function( value ) { - var hooks, ret, isFunction, - elem = this[ 0 ]; - - if ( !arguments.length ) { - if ( elem ) { - hooks = jQuery.valHooks[ elem.type ] || - jQuery.valHooks[ elem.nodeName.toLowerCase() ]; - - if ( hooks && - "get" in hooks && - ( ret = hooks.get( elem, "value" ) ) !== undefined - ) { - return ret; - } - - ret = elem.value; - - return typeof ret === "string" ? - - // Handle most common string cases - ret.replace( rreturn, "" ) : - - // Handle cases where value is null/undef or number - ret == null ? "" : ret; - } - - return; - } - - isFunction = jQuery.isFunction( value ); - - return this.each( function( i ) { - var val; - - if ( this.nodeType !== 1 ) { - return; - } - - if ( isFunction ) { - val = value.call( this, i, jQuery( this ).val() ); - } else { - val = value; - } - - // Treat null/undefined as ""; convert numbers to string - if ( val == null ) { - val = ""; - - } else if ( typeof val === "number" ) { - val += ""; - - } else if ( jQuery.isArray( val ) ) { - val = jQuery.map( val, function( value ) { - return value == null ? "" : value + ""; - } ); - } - - hooks = jQuery.valHooks[ this.type ] || jQuery.valHooks[ this.nodeName.toLowerCase() ]; - - // If set returns undefined, fall back to normal setting - if ( !hooks || !( "set" in hooks ) || hooks.set( this, val, "value" ) === undefined ) { - this.value = val; - } - } ); - } -} ); - -jQuery.extend( { - valHooks: { - option: { - get: function( elem ) { - - var val = jQuery.find.attr( elem, "value" ); - return val != null ? - val : - - // Support: IE <=10 - 11 only - // option.text throws exceptions (#14686, #14858) - // Strip and collapse whitespace - // https://html.spec.whatwg.org/#strip-and-collapse-whitespace - jQuery.trim( jQuery.text( elem ) ).replace( rspaces, " " ); - } - }, - select: { - get: function( elem ) { - var value, option, - options = elem.options, - index = elem.selectedIndex, - one = elem.type === "select-one", - values = one ? null : [], - max = one ? index + 1 : options.length, - i = index < 0 ? - max : - one ? index : 0; - - // Loop through all the selected options - for ( ; i < max; i++ ) { - option = options[ i ]; - - // Support: IE <=9 only - // IE8-9 doesn't update selected after form reset (#2551) - if ( ( option.selected || i === index ) && - - // Don't return options that are disabled or in a disabled optgroup - !option.disabled && - ( !option.parentNode.disabled || - !jQuery.nodeName( option.parentNode, "optgroup" ) ) ) { - - // Get the specific value for the option - value = jQuery( option ).val(); - - // We don't need an array for one selects - if ( one ) { - return value; - } - - // Multi-Selects return an array - values.push( value ); - } - } - - return values; - }, - - set: function( elem, value ) { - var optionSet, option, - options = elem.options, - values = jQuery.makeArray( value ), - i = options.length; - - while ( i-- ) { - option = options[ i ]; - - /* eslint-disable no-cond-assign */ - - if ( option.selected = - jQuery.inArray( jQuery.valHooks.option.get( option ), values ) > -1 - ) { - optionSet = true; - } - - /* eslint-enable no-cond-assign */ - } - - // Force browsers to behave consistently when non-matching value is set - if ( !optionSet ) { - elem.selectedIndex = -1; - } - return values; - } - } - } -} ); - -// Radios and checkboxes getter/setter -jQuery.each( [ "radio", "checkbox" ], function() { - jQuery.valHooks[ this ] = { - set: function( elem, value ) { - if ( jQuery.isArray( value ) ) { - return ( elem.checked = jQuery.inArray( jQuery( elem ).val(), value ) > -1 ); - } - } - }; - if ( !support.checkOn ) { - jQuery.valHooks[ this ].get = function( elem ) { - return elem.getAttribute( "value" ) === null ? "on" : elem.value; - }; - } -} ); - - - - -// Return jQuery for attributes-only inclusion - - -var rfocusMorph = /^(?:focusinfocus|focusoutblur)$/; - -jQuery.extend( jQuery.event, { - - trigger: function( event, data, elem, onlyHandlers ) { - - var i, cur, tmp, bubbleType, ontype, handle, special, - eventPath = [ elem || document ], - type = hasOwn.call( event, "type" ) ? event.type : event, - namespaces = hasOwn.call( event, "namespace" ) ? event.namespace.split( "." ) : []; - - cur = tmp = elem = elem || document; - - // Don't do events on text and comment nodes - if ( elem.nodeType === 3 || elem.nodeType === 8 ) { - return; - } - - // focus/blur morphs to focusin/out; ensure we're not firing them right now - if ( rfocusMorph.test( type + jQuery.event.triggered ) ) { - return; - } - - if ( type.indexOf( "." ) > -1 ) { - - // Namespaced trigger; create a regexp to match event type in handle() - namespaces = type.split( "." ); - type = namespaces.shift(); - namespaces.sort(); - } - ontype = type.indexOf( ":" ) < 0 && "on" + type; - - // Caller can pass in a jQuery.Event object, Object, or just an event type string - event = event[ jQuery.expando ] ? - event : - new jQuery.Event( type, typeof event === "object" && event ); - - // Trigger bitmask: & 1 for native handlers; & 2 for jQuery (always true) - event.isTrigger = onlyHandlers ? 2 : 3; - event.namespace = namespaces.join( "." ); - event.rnamespace = event.namespace ? - new RegExp( "(^|\\.)" + namespaces.join( "\\.(?:.*\\.|)" ) + "(\\.|$)" ) : - null; - - // Clean up the event in case it is being reused - event.result = undefined; - if ( !event.target ) { - event.target = elem; - } - - // Clone any incoming data and prepend the event, creating the handler arg list - data = data == null ? - [ event ] : - jQuery.makeArray( data, [ event ] ); - - // Allow special events to draw outside the lines - special = jQuery.event.special[ type ] || {}; - if ( !onlyHandlers && special.trigger && special.trigger.apply( elem, data ) === false ) { - return; - } - - // Determine event propagation path in advance, per W3C events spec (#9951) - // Bubble up to document, then to window; watch for a global ownerDocument var (#9724) - if ( !onlyHandlers && !special.noBubble && !jQuery.isWindow( elem ) ) { - - bubbleType = special.delegateType || type; - if ( !rfocusMorph.test( bubbleType + type ) ) { - cur = cur.parentNode; - } - for ( ; cur; cur = cur.parentNode ) { - eventPath.push( cur ); - tmp = cur; - } - - // Only add window if we got to document (e.g., not plain obj or detached DOM) - if ( tmp === ( elem.ownerDocument || document ) ) { - eventPath.push( tmp.defaultView || tmp.parentWindow || window ); - } - } - - // Fire handlers on the event path - i = 0; - while ( ( cur = eventPath[ i++ ] ) && !event.isPropagationStopped() ) { - - event.type = i > 1 ? - bubbleType : - special.bindType || type; - - // jQuery handler - handle = ( dataPriv.get( cur, "events" ) || {} )[ event.type ] && - dataPriv.get( cur, "handle" ); - if ( handle ) { - handle.apply( cur, data ); - } - - // Native handler - handle = ontype && cur[ ontype ]; - if ( handle && handle.apply && acceptData( cur ) ) { - event.result = handle.apply( cur, data ); - if ( event.result === false ) { - event.preventDefault(); - } - } - } - event.type = type; - - // If nobody prevented the default action, do it now - if ( !onlyHandlers && !event.isDefaultPrevented() ) { - - if ( ( !special._default || - special._default.apply( eventPath.pop(), data ) === false ) && - acceptData( elem ) ) { - - // Call a native DOM method on the target with the same name as the event. - // Don't do default actions on window, that's where global variables be (#6170) - if ( ontype && jQuery.isFunction( elem[ type ] ) && !jQuery.isWindow( elem ) ) { - - // Don't re-trigger an onFOO event when we call its FOO() method - tmp = elem[ ontype ]; - - if ( tmp ) { - elem[ ontype ] = null; - } - - // Prevent re-triggering of the same event, since we already bubbled it above - jQuery.event.triggered = type; - elem[ type ](); - jQuery.event.triggered = undefined; - - if ( tmp ) { - elem[ ontype ] = tmp; - } - } - } - } - - return event.result; - }, - - // Piggyback on a donor event to simulate a different one - // Used only for `focus(in | out)` events - simulate: function( type, elem, event ) { - var e = jQuery.extend( - new jQuery.Event(), - event, - { - type: type, - isSimulated: true - } - ); - - jQuery.event.trigger( e, null, elem ); - } - -} ); - -jQuery.fn.extend( { - - trigger: function( type, data ) { - return this.each( function() { - jQuery.event.trigger( type, data, this ); - } ); - }, - triggerHandler: function( type, data ) { - var elem = this[ 0 ]; - if ( elem ) { - return jQuery.event.trigger( type, data, elem, true ); - } - } -} ); - - -jQuery.each( ( "blur focus focusin focusout resize scroll click dblclick " + - "mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave " + - "change select submit keydown keypress keyup contextmenu" ).split( " " ), - function( i, name ) { - - // Handle event binding - jQuery.fn[ name ] = function( data, fn ) { - return arguments.length > 0 ? - this.on( name, null, data, fn ) : - this.trigger( name ); - }; -} ); - -jQuery.fn.extend( { - hover: function( fnOver, fnOut ) { - return this.mouseenter( fnOver ).mouseleave( fnOut || fnOver ); - } -} ); - - - - -support.focusin = "onfocusin" in window; - - -// Support: Firefox <=44 -// Firefox doesn't have focus(in | out) events -// Related ticket - https://bugzilla.mozilla.org/show_bug.cgi?id=687787 -// -// Support: Chrome <=48 - 49, Safari <=9.0 - 9.1 -// focus(in | out) events fire after focus & blur events, -// which is spec violation - http://www.w3.org/TR/DOM-Level-3-Events/#events-focusevent-event-order -// Related ticket - https://bugs.chromium.org/p/chromium/issues/detail?id=449857 -if ( !support.focusin ) { - jQuery.each( { focus: "focusin", blur: "focusout" }, function( orig, fix ) { - - // Attach a single capturing handler on the document while someone wants focusin/focusout - var handler = function( event ) { - jQuery.event.simulate( fix, event.target, jQuery.event.fix( event ) ); - }; - - jQuery.event.special[ fix ] = { - setup: function() { - var doc = this.ownerDocument || this, - attaches = dataPriv.access( doc, fix ); - - if ( !attaches ) { - doc.addEventListener( orig, handler, true ); - } - dataPriv.access( doc, fix, ( attaches || 0 ) + 1 ); - }, - teardown: function() { - var doc = this.ownerDocument || this, - attaches = dataPriv.access( doc, fix ) - 1; - - if ( !attaches ) { - doc.removeEventListener( orig, handler, true ); - dataPriv.remove( doc, fix ); - - } else { - dataPriv.access( doc, fix, attaches ); - } - } - }; - } ); -} -var location = window.location; - -var nonce = jQuery.now(); - -var rquery = ( /\?/ ); - - - -// Cross-browser xml parsing -jQuery.parseXML = function( data ) { - var xml; - if ( !data || typeof data !== "string" ) { - return null; - } - - // Support: IE 9 - 11 only - // IE throws on parseFromString with invalid input. - try { - xml = ( new window.DOMParser() ).parseFromString( data, "text/xml" ); - } catch ( e ) { - xml = undefined; - } - - if ( !xml || xml.getElementsByTagName( "parsererror" ).length ) { - jQuery.error( "Invalid XML: " + data ); - } - return xml; -}; - - -var - rbracket = /\[\]$/, - rCRLF = /\r?\n/g, - rsubmitterTypes = /^(?:submit|button|image|reset|file)$/i, - rsubmittable = /^(?:input|select|textarea|keygen)/i; - -function buildParams( prefix, obj, traditional, add ) { - var name; - - if ( jQuery.isArray( obj ) ) { - - // Serialize array item. - jQuery.each( obj, function( i, v ) { - if ( traditional || rbracket.test( prefix ) ) { - - // Treat each array item as a scalar. - add( prefix, v ); - - } else { - - // Item is non-scalar (array or object), encode its numeric index. - buildParams( - prefix + "[" + ( typeof v === "object" && v != null ? i : "" ) + "]", - v, - traditional, - add - ); - } - } ); - - } else if ( !traditional && jQuery.type( obj ) === "object" ) { - - // Serialize object item. - for ( name in obj ) { - buildParams( prefix + "[" + name + "]", obj[ name ], traditional, add ); - } - - } else { - - // Serialize scalar item. - add( prefix, obj ); - } -} - -// Serialize an array of form elements or a set of -// key/values into a query string -jQuery.param = function( a, traditional ) { - var prefix, - s = [], - add = function( key, valueOrFunction ) { - - // If value is a function, invoke it and use its return value - var value = jQuery.isFunction( valueOrFunction ) ? - valueOrFunction() : - valueOrFunction; - - s[ s.length ] = encodeURIComponent( key ) + "=" + - encodeURIComponent( value == null ? "" : value ); - }; - - // If an array was passed in, assume that it is an array of form elements. - if ( jQuery.isArray( a ) || ( a.jquery && !jQuery.isPlainObject( a ) ) ) { - - // Serialize the form elements - jQuery.each( a, function() { - add( this.name, this.value ); - } ); - - } else { - - // If traditional, encode the "old" way (the way 1.3.2 or older - // did it), otherwise encode params recursively. - for ( prefix in a ) { - buildParams( prefix, a[ prefix ], traditional, add ); - } - } - - // Return the resulting serialization - return s.join( "&" ); -}; - -jQuery.fn.extend( { - serialize: function() { - return jQuery.param( this.serializeArray() ); - }, - serializeArray: function() { - return this.map( function() { - - // Can add propHook for "elements" to filter or add form elements - var elements = jQuery.prop( this, "elements" ); - return elements ? jQuery.makeArray( elements ) : this; - } ) - .filter( function() { - var type = this.type; - - // Use .is( ":disabled" ) so that fieldset[disabled] works - return this.name && !jQuery( this ).is( ":disabled" ) && - rsubmittable.test( this.nodeName ) && !rsubmitterTypes.test( type ) && - ( this.checked || !rcheckableType.test( type ) ); - } ) - .map( function( i, elem ) { - var val = jQuery( this ).val(); - - return val == null ? - null : - jQuery.isArray( val ) ? - jQuery.map( val, function( val ) { - return { name: elem.name, value: val.replace( rCRLF, "\r\n" ) }; - } ) : - { name: elem.name, value: val.replace( rCRLF, "\r\n" ) }; - } ).get(); - } -} ); - - -var - r20 = /%20/g, - rhash = /#.*$/, - rts = /([?&])_=[^&]*/, - rheaders = /^(.*?):[ \t]*([^\r\n]*)$/mg, - - // #7653, #8125, #8152: local protocol detection - rlocalProtocol = /^(?:about|app|app-storage|.+-extension|file|res|widget):$/, - rnoContent = /^(?:GET|HEAD)$/, - rprotocol = /^\/\//, - - /* Prefilters - * 1) They are useful to introduce custom dataTypes (see ajax/jsonp.js for an example) - * 2) These are called: - * - BEFORE asking for a transport - * - AFTER param serialization (s.data is a string if s.processData is true) - * 3) key is the dataType - * 4) the catchall symbol "*" can be used - * 5) execution will start with transport dataType and THEN continue down to "*" if needed - */ - prefilters = {}, - - /* Transports bindings - * 1) key is the dataType - * 2) the catchall symbol "*" can be used - * 3) selection will start with transport dataType and THEN go to "*" if needed - */ - transports = {}, - - // Avoid comment-prolog char sequence (#10098); must appease lint and evade compression - allTypes = "*/".concat( "*" ), - - // Anchor tag for parsing the document origin - originAnchor = document.createElement( "a" ); - originAnchor.href = location.href; - -// Base "constructor" for jQuery.ajaxPrefilter and jQuery.ajaxTransport -function addToPrefiltersOrTransports( structure ) { - - // dataTypeExpression is optional and defaults to "*" - return function( dataTypeExpression, func ) { - - if ( typeof dataTypeExpression !== "string" ) { - func = dataTypeExpression; - dataTypeExpression = "*"; - } - - var dataType, - i = 0, - dataTypes = dataTypeExpression.toLowerCase().match( rnotwhite ) || []; - - if ( jQuery.isFunction( func ) ) { - - // For each dataType in the dataTypeExpression - while ( ( dataType = dataTypes[ i++ ] ) ) { - - // Prepend if requested - if ( dataType[ 0 ] === "+" ) { - dataType = dataType.slice( 1 ) || "*"; - ( structure[ dataType ] = structure[ dataType ] || [] ).unshift( func ); - - // Otherwise append - } else { - ( structure[ dataType ] = structure[ dataType ] || [] ).push( func ); - } - } - } - }; -} - -// Base inspection function for prefilters and transports -function inspectPrefiltersOrTransports( structure, options, originalOptions, jqXHR ) { - - var inspected = {}, - seekingTransport = ( structure === transports ); - - function inspect( dataType ) { - var selected; - inspected[ dataType ] = true; - jQuery.each( structure[ dataType ] || [], function( _, prefilterOrFactory ) { - var dataTypeOrTransport = prefilterOrFactory( options, originalOptions, jqXHR ); - if ( typeof dataTypeOrTransport === "string" && - !seekingTransport && !inspected[ dataTypeOrTransport ] ) { - - options.dataTypes.unshift( dataTypeOrTransport ); - inspect( dataTypeOrTransport ); - return false; - } else if ( seekingTransport ) { - return !( selected = dataTypeOrTransport ); - } - } ); - return selected; - } - - return inspect( options.dataTypes[ 0 ] ) || !inspected[ "*" ] && inspect( "*" ); -} - -// A special extend for ajax options -// that takes "flat" options (not to be deep extended) -// Fixes #9887 -function ajaxExtend( target, src ) { - var key, deep, - flatOptions = jQuery.ajaxSettings.flatOptions || {}; - - for ( key in src ) { - if ( src[ key ] !== undefined ) { - ( flatOptions[ key ] ? target : ( deep || ( deep = {} ) ) )[ key ] = src[ key ]; - } - } - if ( deep ) { - jQuery.extend( true, target, deep ); - } - - return target; -} - -/* Handles responses to an ajax request: - * - finds the right dataType (mediates between content-type and expected dataType) - * - returns the corresponding response - */ -function ajaxHandleResponses( s, jqXHR, responses ) { - - var ct, type, finalDataType, firstDataType, - contents = s.contents, - dataTypes = s.dataTypes; - - // Remove auto dataType and get content-type in the process - while ( dataTypes[ 0 ] === "*" ) { - dataTypes.shift(); - if ( ct === undefined ) { - ct = s.mimeType || jqXHR.getResponseHeader( "Content-Type" ); - } - } - - // Check if we're dealing with a known content-type - if ( ct ) { - for ( type in contents ) { - if ( contents[ type ] && contents[ type ].test( ct ) ) { - dataTypes.unshift( type ); - break; - } - } - } - - // Check to see if we have a response for the expected dataType - if ( dataTypes[ 0 ] in responses ) { - finalDataType = dataTypes[ 0 ]; - } else { - - // Try convertible dataTypes - for ( type in responses ) { - if ( !dataTypes[ 0 ] || s.converters[ type + " " + dataTypes[ 0 ] ] ) { - finalDataType = type; - break; - } - if ( !firstDataType ) { - firstDataType = type; - } - } - - // Or just use first one - finalDataType = finalDataType || firstDataType; - } - - // If we found a dataType - // We add the dataType to the list if needed - // and return the corresponding response - if ( finalDataType ) { - if ( finalDataType !== dataTypes[ 0 ] ) { - dataTypes.unshift( finalDataType ); - } - return responses[ finalDataType ]; - } -} - -/* Chain conversions given the request and the original response - * Also sets the responseXXX fields on the jqXHR instance - */ -function ajaxConvert( s, response, jqXHR, isSuccess ) { - var conv2, current, conv, tmp, prev, - converters = {}, - - // Work with a copy of dataTypes in case we need to modify it for conversion - dataTypes = s.dataTypes.slice(); - - // Create converters map with lowercased keys - if ( dataTypes[ 1 ] ) { - for ( conv in s.converters ) { - converters[ conv.toLowerCase() ] = s.converters[ conv ]; - } - } - - current = dataTypes.shift(); - - // Convert to each sequential dataType - while ( current ) { - - if ( s.responseFields[ current ] ) { - jqXHR[ s.responseFields[ current ] ] = response; - } - - // Apply the dataFilter if provided - if ( !prev && isSuccess && s.dataFilter ) { - response = s.dataFilter( response, s.dataType ); - } - - prev = current; - current = dataTypes.shift(); - - if ( current ) { - - // There's only work to do if current dataType is non-auto - if ( current === "*" ) { - - current = prev; - - // Convert response if prev dataType is non-auto and differs from current - } else if ( prev !== "*" && prev !== current ) { - - // Seek a direct converter - conv = converters[ prev + " " + current ] || converters[ "* " + current ]; - - // If none found, seek a pair - if ( !conv ) { - for ( conv2 in converters ) { - - // If conv2 outputs current - tmp = conv2.split( " " ); - if ( tmp[ 1 ] === current ) { - - // If prev can be converted to accepted input - conv = converters[ prev + " " + tmp[ 0 ] ] || - converters[ "* " + tmp[ 0 ] ]; - if ( conv ) { - - // Condense equivalence converters - if ( conv === true ) { - conv = converters[ conv2 ]; - - // Otherwise, insert the intermediate dataType - } else if ( converters[ conv2 ] !== true ) { - current = tmp[ 0 ]; - dataTypes.unshift( tmp[ 1 ] ); - } - break; - } - } - } - } - - // Apply converter (if not an equivalence) - if ( conv !== true ) { - - // Unless errors are allowed to bubble, catch and return them - if ( conv && s.throws ) { - response = conv( response ); - } else { - try { - response = conv( response ); - } catch ( e ) { - return { - state: "parsererror", - error: conv ? e : "No conversion from " + prev + " to " + current - }; - } - } - } - } - } - } - - return { state: "success", data: response }; -} - -jQuery.extend( { - - // Counter for holding the number of active queries - active: 0, - - // Last-Modified header cache for next request - lastModified: {}, - etag: {}, - - ajaxSettings: { - url: location.href, - type: "GET", - isLocal: rlocalProtocol.test( location.protocol ), - global: true, - processData: true, - async: true, - contentType: "application/x-www-form-urlencoded; charset=UTF-8", - - /* - timeout: 0, - data: null, - dataType: null, - username: null, - password: null, - cache: null, - throws: false, - traditional: false, - headers: {}, - */ - - accepts: { - "*": allTypes, - text: "text/plain", - html: "text/html", - xml: "application/xml, text/xml", - json: "application/json, text/javascript" - }, - - contents: { - xml: /\bxml\b/, - html: /\bhtml/, - json: /\bjson\b/ - }, - - responseFields: { - xml: "responseXML", - text: "responseText", - json: "responseJSON" - }, - - // Data converters - // Keys separate source (or catchall "*") and destination types with a single space - converters: { - - // Convert anything to text - "* text": String, - - // Text to html (true = no transformation) - "text html": true, - - // Evaluate text as a json expression - "text json": JSON.parse, - - // Parse text as xml - "text xml": jQuery.parseXML - }, - - // For options that shouldn't be deep extended: - // you can add your own custom options here if - // and when you create one that shouldn't be - // deep extended (see ajaxExtend) - flatOptions: { - url: true, - context: true - } - }, - - // Creates a full fledged settings object into target - // with both ajaxSettings and settings fields. - // If target is omitted, writes into ajaxSettings. - ajaxSetup: function( target, settings ) { - return settings ? - - // Building a settings object - ajaxExtend( ajaxExtend( target, jQuery.ajaxSettings ), settings ) : - - // Extending ajaxSettings - ajaxExtend( jQuery.ajaxSettings, target ); - }, - - ajaxPrefilter: addToPrefiltersOrTransports( prefilters ), - ajaxTransport: addToPrefiltersOrTransports( transports ), - - // Main method - ajax: function( url, options ) { - - // If url is an object, simulate pre-1.5 signature - if ( typeof url === "object" ) { - options = url; - url = undefined; - } - - // Force options to be an object - options = options || {}; - - var transport, - - // URL without anti-cache param - cacheURL, - - // Response headers - responseHeadersString, - responseHeaders, - - // timeout handle - timeoutTimer, - - // Url cleanup var - urlAnchor, - - // Request state (becomes false upon send and true upon completion) - completed, - - // To know if global events are to be dispatched - fireGlobals, - - // Loop variable - i, - - // uncached part of the url - uncached, - - // Create the final options object - s = jQuery.ajaxSetup( {}, options ), - - // Callbacks context - callbackContext = s.context || s, - - // Context for global events is callbackContext if it is a DOM node or jQuery collection - globalEventContext = s.context && - ( callbackContext.nodeType || callbackContext.jquery ) ? - jQuery( callbackContext ) : - jQuery.event, - - // Deferreds - deferred = jQuery.Deferred(), - completeDeferred = jQuery.Callbacks( "once memory" ), - - // Status-dependent callbacks - statusCode = s.statusCode || {}, - - // Headers (they are sent all at once) - requestHeaders = {}, - requestHeadersNames = {}, - - // Default abort message - strAbort = "canceled", - - // Fake xhr - jqXHR = { - readyState: 0, - - // Builds headers hashtable if needed - getResponseHeader: function( key ) { - var match; - if ( completed ) { - if ( !responseHeaders ) { - responseHeaders = {}; - while ( ( match = rheaders.exec( responseHeadersString ) ) ) { - responseHeaders[ match[ 1 ].toLowerCase() ] = match[ 2 ]; - } - } - match = responseHeaders[ key.toLowerCase() ]; - } - return match == null ? null : match; - }, - - // Raw string - getAllResponseHeaders: function() { - return completed ? responseHeadersString : null; - }, - - // Caches the header - setRequestHeader: function( name, value ) { - if ( completed == null ) { - name = requestHeadersNames[ name.toLowerCase() ] = - requestHeadersNames[ name.toLowerCase() ] || name; - requestHeaders[ name ] = value; - } - return this; - }, - - // Overrides response content-type header - overrideMimeType: function( type ) { - if ( completed == null ) { - s.mimeType = type; - } - return this; - }, - - // Status-dependent callbacks - statusCode: function( map ) { - var code; - if ( map ) { - if ( completed ) { - - // Execute the appropriate callbacks - jqXHR.always( map[ jqXHR.status ] ); - } else { - - // Lazy-add the new callbacks in a way that preserves old ones - for ( code in map ) { - statusCode[ code ] = [ statusCode[ code ], map[ code ] ]; - } - } - } - return this; - }, - - // Cancel the request - abort: function( statusText ) { - var finalText = statusText || strAbort; - if ( transport ) { - transport.abort( finalText ); - } - done( 0, finalText ); - return this; - } - }; - - // Attach deferreds - deferred.promise( jqXHR ); - - // Add protocol if not provided (prefilters might expect it) - // Handle falsy url in the settings object (#10093: consistency with old signature) - // We also use the url parameter if available - s.url = ( ( url || s.url || location.href ) + "" ) - .replace( rprotocol, location.protocol + "//" ); - - // Alias method option to type as per ticket #12004 - s.type = options.method || options.type || s.method || s.type; - - // Extract dataTypes list - s.dataTypes = ( s.dataType || "*" ).toLowerCase().match( rnotwhite ) || [ "" ]; - - // A cross-domain request is in order when the origin doesn't match the current origin. - if ( s.crossDomain == null ) { - urlAnchor = document.createElement( "a" ); - - // Support: IE <=8 - 11, Edge 12 - 13 - // IE throws exception on accessing the href property if url is malformed, - // e.g. http://example.com:80x/ - try { - urlAnchor.href = s.url; - - // Support: IE <=8 - 11 only - // Anchor's host property isn't correctly set when s.url is relative - urlAnchor.href = urlAnchor.href; - s.crossDomain = originAnchor.protocol + "//" + originAnchor.host !== - urlAnchor.protocol + "//" + urlAnchor.host; - } catch ( e ) { - - // If there is an error parsing the URL, assume it is crossDomain, - // it can be rejected by the transport if it is invalid - s.crossDomain = true; - } - } - - // Convert data if not already a string - if ( s.data && s.processData && typeof s.data !== "string" ) { - s.data = jQuery.param( s.data, s.traditional ); - } - - // Apply prefilters - inspectPrefiltersOrTransports( prefilters, s, options, jqXHR ); - - // If request was aborted inside a prefilter, stop there - if ( completed ) { - return jqXHR; - } - - // We can fire global events as of now if asked to - // Don't fire events if jQuery.event is undefined in an AMD-usage scenario (#15118) - fireGlobals = jQuery.event && s.global; - - // Watch for a new set of requests - if ( fireGlobals && jQuery.active++ === 0 ) { - jQuery.event.trigger( "ajaxStart" ); - } - - // Uppercase the type - s.type = s.type.toUpperCase(); - - // Determine if request has content - s.hasContent = !rnoContent.test( s.type ); - - // Save the URL in case we're toying with the If-Modified-Since - // and/or If-None-Match header later on - // Remove hash to simplify url manipulation - cacheURL = s.url.replace( rhash, "" ); - - // More options handling for requests with no content - if ( !s.hasContent ) { - - // Remember the hash so we can put it back - uncached = s.url.slice( cacheURL.length ); - - // If data is available, append data to url - if ( s.data ) { - cacheURL += ( rquery.test( cacheURL ) ? "&" : "?" ) + s.data; - - // #9682: remove data so that it's not used in an eventual retry - delete s.data; - } - - // Add anti-cache in uncached url if needed - if ( s.cache === false ) { - cacheURL = cacheURL.replace( rts, "" ); - uncached = ( rquery.test( cacheURL ) ? "&" : "?" ) + "_=" + ( nonce++ ) + uncached; - } - - // Put hash and anti-cache on the URL that will be requested (gh-1732) - s.url = cacheURL + uncached; - - // Change '%20' to '+' if this is encoded form body content (gh-2658) - } else if ( s.data && s.processData && - ( s.contentType || "" ).indexOf( "application/x-www-form-urlencoded" ) === 0 ) { - s.data = s.data.replace( r20, "+" ); - } - - // Set the If-Modified-Since and/or If-None-Match header, if in ifModified mode. - if ( s.ifModified ) { - if ( jQuery.lastModified[ cacheURL ] ) { - jqXHR.setRequestHeader( "If-Modified-Since", jQuery.lastModified[ cacheURL ] ); - } - if ( jQuery.etag[ cacheURL ] ) { - jqXHR.setRequestHeader( "If-None-Match", jQuery.etag[ cacheURL ] ); - } - } - - // Set the correct header, if data is being sent - if ( s.data && s.hasContent && s.contentType !== false || options.contentType ) { - jqXHR.setRequestHeader( "Content-Type", s.contentType ); - } - - // Set the Accepts header for the server, depending on the dataType - jqXHR.setRequestHeader( - "Accept", - s.dataTypes[ 0 ] && s.accepts[ s.dataTypes[ 0 ] ] ? - s.accepts[ s.dataTypes[ 0 ] ] + - ( s.dataTypes[ 0 ] !== "*" ? ", " + allTypes + "; q=0.01" : "" ) : - s.accepts[ "*" ] - ); - - // Check for headers option - for ( i in s.headers ) { - jqXHR.setRequestHeader( i, s.headers[ i ] ); - } - - // Allow custom headers/mimetypes and early abort - if ( s.beforeSend && - ( s.beforeSend.call( callbackContext, jqXHR, s ) === false || completed ) ) { - - // Abort if not done already and return - return jqXHR.abort(); - } - - // Aborting is no longer a cancellation - strAbort = "abort"; - - // Install callbacks on deferreds - completeDeferred.add( s.complete ); - jqXHR.done( s.success ); - jqXHR.fail( s.error ); - - // Get transport - transport = inspectPrefiltersOrTransports( transports, s, options, jqXHR ); - - // If no transport, we auto-abort - if ( !transport ) { - done( -1, "No Transport" ); - } else { - jqXHR.readyState = 1; - - // Send global event - if ( fireGlobals ) { - globalEventContext.trigger( "ajaxSend", [ jqXHR, s ] ); - } - - // If request was aborted inside ajaxSend, stop there - if ( completed ) { - return jqXHR; - } - - // Timeout - if ( s.async && s.timeout > 0 ) { - timeoutTimer = window.setTimeout( function() { - jqXHR.abort( "timeout" ); - }, s.timeout ); - } - - try { - completed = false; - transport.send( requestHeaders, done ); - } catch ( e ) { - - // Rethrow post-completion exceptions - if ( completed ) { - throw e; - } - - // Propagate others as results - done( -1, e ); - } - } - - // Callback for when everything is done - function done( status, nativeStatusText, responses, headers ) { - var isSuccess, success, error, response, modified, - statusText = nativeStatusText; - - // Ignore repeat invocations - if ( completed ) { - return; - } - - completed = true; - - // Clear timeout if it exists - if ( timeoutTimer ) { - window.clearTimeout( timeoutTimer ); - } - - // Dereference transport for early garbage collection - // (no matter how long the jqXHR object will be used) - transport = undefined; - - // Cache response headers - responseHeadersString = headers || ""; - - // Set readyState - jqXHR.readyState = status > 0 ? 4 : 0; - - // Determine if successful - isSuccess = status >= 200 && status < 300 || status === 304; - - // Get response data - if ( responses ) { - response = ajaxHandleResponses( s, jqXHR, responses ); - } - - // Convert no matter what (that way responseXXX fields are always set) - response = ajaxConvert( s, response, jqXHR, isSuccess ); - - // If successful, handle type chaining - if ( isSuccess ) { - - // Set the If-Modified-Since and/or If-None-Match header, if in ifModified mode. - if ( s.ifModified ) { - modified = jqXHR.getResponseHeader( "Last-Modified" ); - if ( modified ) { - jQuery.lastModified[ cacheURL ] = modified; - } - modified = jqXHR.getResponseHeader( "etag" ); - if ( modified ) { - jQuery.etag[ cacheURL ] = modified; - } - } - - // if no content - if ( status === 204 || s.type === "HEAD" ) { - statusText = "nocontent"; - - // if not modified - } else if ( status === 304 ) { - statusText = "notmodified"; - - // If we have data, let's convert it - } else { - statusText = response.state; - success = response.data; - error = response.error; - isSuccess = !error; - } - } else { - - // Extract error from statusText and normalize for non-aborts - error = statusText; - if ( status || !statusText ) { - statusText = "error"; - if ( status < 0 ) { - status = 0; - } - } - } - - // Set data for the fake xhr object - jqXHR.status = status; - jqXHR.statusText = ( nativeStatusText || statusText ) + ""; - - // Success/Error - if ( isSuccess ) { - deferred.resolveWith( callbackContext, [ success, statusText, jqXHR ] ); - } else { - deferred.rejectWith( callbackContext, [ jqXHR, statusText, error ] ); - } - - // Status-dependent callbacks - jqXHR.statusCode( statusCode ); - statusCode = undefined; - - if ( fireGlobals ) { - globalEventContext.trigger( isSuccess ? "ajaxSuccess" : "ajaxError", - [ jqXHR, s, isSuccess ? success : error ] ); - } - - // Complete - completeDeferred.fireWith( callbackContext, [ jqXHR, statusText ] ); - - if ( fireGlobals ) { - globalEventContext.trigger( "ajaxComplete", [ jqXHR, s ] ); - - // Handle the global AJAX counter - if ( !( --jQuery.active ) ) { - jQuery.event.trigger( "ajaxStop" ); - } - } - } - - return jqXHR; - }, - - getJSON: function( url, data, callback ) { - return jQuery.get( url, data, callback, "json" ); - }, - - getScript: function( url, callback ) { - return jQuery.get( url, undefined, callback, "script" ); - } -} ); - -jQuery.each( [ "get", "post" ], function( i, method ) { - jQuery[ method ] = function( url, data, callback, type ) { - - // Shift arguments if data argument was omitted - if ( jQuery.isFunction( data ) ) { - type = type || callback; - callback = data; - data = undefined; - } - - // The url can be an options object (which then must have .url) - return jQuery.ajax( jQuery.extend( { - url: url, - type: method, - dataType: type, - data: data, - success: callback - }, jQuery.isPlainObject( url ) && url ) ); - }; -} ); - - -jQuery._evalUrl = function( url ) { - return jQuery.ajax( { - url: url, - - // Make this explicit, since user can override this through ajaxSetup (#11264) - type: "GET", - dataType: "script", - cache: true, - async: false, - global: false, - "throws": true - } ); -}; - - -jQuery.fn.extend( { - wrapAll: function( html ) { - var wrap; - - if ( this[ 0 ] ) { - if ( jQuery.isFunction( html ) ) { - html = html.call( this[ 0 ] ); - } - - // The elements to wrap the target around - wrap = jQuery( html, this[ 0 ].ownerDocument ).eq( 0 ).clone( true ); - - if ( this[ 0 ].parentNode ) { - wrap.insertBefore( this[ 0 ] ); - } - - wrap.map( function() { - var elem = this; - - while ( elem.firstElementChild ) { - elem = elem.firstElementChild; - } - - return elem; - } ).append( this ); - } - - return this; - }, - - wrapInner: function( html ) { - if ( jQuery.isFunction( html ) ) { - return this.each( function( i ) { - jQuery( this ).wrapInner( html.call( this, i ) ); - } ); - } - - return this.each( function() { - var self = jQuery( this ), - contents = self.contents(); - - if ( contents.length ) { - contents.wrapAll( html ); - - } else { - self.append( html ); - } - } ); - }, - - wrap: function( html ) { - var isFunction = jQuery.isFunction( html ); - - return this.each( function( i ) { - jQuery( this ).wrapAll( isFunction ? html.call( this, i ) : html ); - } ); - }, - - unwrap: function( selector ) { - this.parent( selector ).not( "body" ).each( function() { - jQuery( this ).replaceWith( this.childNodes ); - } ); - return this; - } -} ); - - -jQuery.expr.pseudos.hidden = function( elem ) { - return !jQuery.expr.pseudos.visible( elem ); -}; -jQuery.expr.pseudos.visible = function( elem ) { - return !!( elem.offsetWidth || elem.offsetHeight || elem.getClientRects().length ); -}; - - - - -jQuery.ajaxSettings.xhr = function() { - try { - return new window.XMLHttpRequest(); - } catch ( e ) {} -}; - -var xhrSuccessStatus = { - - // File protocol always yields status code 0, assume 200 - 0: 200, - - // Support: IE <=9 only - // #1450: sometimes IE returns 1223 when it should be 204 - 1223: 204 - }, - xhrSupported = jQuery.ajaxSettings.xhr(); - -support.cors = !!xhrSupported && ( "withCredentials" in xhrSupported ); -support.ajax = xhrSupported = !!xhrSupported; - -jQuery.ajaxTransport( function( options ) { - var callback, errorCallback; - - // Cross domain only allowed if supported through XMLHttpRequest - if ( support.cors || xhrSupported && !options.crossDomain ) { - return { - send: function( headers, complete ) { - var i, - xhr = options.xhr(); - - xhr.open( - options.type, - options.url, - options.async, - options.username, - options.password - ); - - // Apply custom fields if provided - if ( options.xhrFields ) { - for ( i in options.xhrFields ) { - xhr[ i ] = options.xhrFields[ i ]; - } - } - - // Override mime type if needed - if ( options.mimeType && xhr.overrideMimeType ) { - xhr.overrideMimeType( options.mimeType ); - } - - // X-Requested-With header - // For cross-domain requests, seeing as conditions for a preflight are - // akin to a jigsaw puzzle, we simply never set it to be sure. - // (it can always be set on a per-request basis or even using ajaxSetup) - // For same-domain requests, won't change header if already provided. - if ( !options.crossDomain && !headers[ "X-Requested-With" ] ) { - headers[ "X-Requested-With" ] = "XMLHttpRequest"; - } - - // Set headers - for ( i in headers ) { - xhr.setRequestHeader( i, headers[ i ] ); - } - - // Callback - callback = function( type ) { - return function() { - if ( callback ) { - callback = errorCallback = xhr.onload = - xhr.onerror = xhr.onabort = xhr.onreadystatechange = null; - - if ( type === "abort" ) { - xhr.abort(); - } else if ( type === "error" ) { - - // Support: IE <=9 only - // On a manual native abort, IE9 throws - // errors on any property access that is not readyState - if ( typeof xhr.status !== "number" ) { - complete( 0, "error" ); - } else { - complete( - - // File: protocol always yields status 0; see #8605, #14207 - xhr.status, - xhr.statusText - ); - } - } else { - complete( - xhrSuccessStatus[ xhr.status ] || xhr.status, - xhr.statusText, - - // Support: IE <=9 only - // IE9 has no XHR2 but throws on binary (trac-11426) - // For XHR2 non-text, let the caller handle it (gh-2498) - ( xhr.responseType || "text" ) !== "text" || - typeof xhr.responseText !== "string" ? - { binary: xhr.response } : - { text: xhr.responseText }, - xhr.getAllResponseHeaders() - ); - } - } - }; - }; - - // Listen to events - xhr.onload = callback(); - errorCallback = xhr.onerror = callback( "error" ); - - // Support: IE 9 only - // Use onreadystatechange to replace onabort - // to handle uncaught aborts - if ( xhr.onabort !== undefined ) { - xhr.onabort = errorCallback; - } else { - xhr.onreadystatechange = function() { - - // Check readyState before timeout as it changes - if ( xhr.readyState === 4 ) { - - // Allow onerror to be called first, - // but that will not handle a native abort - // Also, save errorCallback to a variable - // as xhr.onerror cannot be accessed - window.setTimeout( function() { - if ( callback ) { - errorCallback(); - } - } ); - } - }; - } - - // Create the abort callback - callback = callback( "abort" ); - - try { - - // Do send the request (this may raise an exception) - xhr.send( options.hasContent && options.data || null ); - } catch ( e ) { - - // #14683: Only rethrow if this hasn't been notified as an error yet - if ( callback ) { - throw e; - } - } - }, - - abort: function() { - if ( callback ) { - callback(); - } - } - }; - } -} ); - - - - -// Prevent auto-execution of scripts when no explicit dataType was provided (See gh-2432) -jQuery.ajaxPrefilter( function( s ) { - if ( s.crossDomain ) { - s.contents.script = false; - } -} ); - -// Install script dataType -jQuery.ajaxSetup( { - accepts: { - script: "text/javascript, application/javascript, " + - "application/ecmascript, application/x-ecmascript" - }, - contents: { - script: /\b(?:java|ecma)script\b/ - }, - converters: { - "text script": function( text ) { - jQuery.globalEval( text ); - return text; - } - } -} ); - -// Handle cache's special case and crossDomain -jQuery.ajaxPrefilter( "script", function( s ) { - if ( s.cache === undefined ) { - s.cache = false; - } - if ( s.crossDomain ) { - s.type = "GET"; - } -} ); - -// Bind script tag hack transport -jQuery.ajaxTransport( "script", function( s ) { - - // This transport only deals with cross domain requests - if ( s.crossDomain ) { - var script, callback; - return { - send: function( _, complete ) { - script = jQuery( " - - - - - - - - - - - - - - - - -
-
-
-
- -
-

Reporting Bugs

-

pgloader is a software and as such contains bugs. Most bugs are easy to -solve and taken care of in a short delay. For this to be possible though, -bug reports need to follow those recommandations:

-
-
    -
  • include pgloader version,
    • -
  • include problematic input and output,
    • -
  • include a description of the output you expected,
    • -
  • explain the difference between the ouput you have and the one you expected,
    • -
  • include a self-reproducing test-case
    • -
-
-
-

Test Cases to Reproduce Bugs

-

Use the inline source type to help reproduce a bug, as in the pgloader tests:

-
LOAD CSV
-   FROM INLINE
-   INTO postgresql://dim@localhost/pgloader?public."HS"
-
-   WITH truncate,
-        fields terminated by '\t',
-        fields not enclosed,
-        fields escaped by backslash-quote,
-        quote identifiers
-
-    SET work_mem to '128MB',
-        standard_conforming_strings to 'on',
-        application_name to 'my app name'
-
- BEFORE LOAD DO
-  $$ create extension if not exists hstore; $$,
-  $$ drop table if exists "HS"; $$,
-  $$ CREATE TABLE "HS"
-     (
-        id     serial primary key,
-        kv     hstore
-     )
-  $$;
-
-
-1     email=>foo@example.com,a=>b
-2     test=>value
-3     a=>b,c=>"quoted hstore value",d=>other
-4     baddata
-
-
-
-
- - -
-
-
- -
-
-
- ©2017, Dimitri Fontaine. - - | - Powered by Sphinx 1.6.5 - & Alabaster 0.7.10 - - | - Page source -
- - - - - - \ No newline at end of file diff --git a/docs/_build/html/genindex.html b/docs/_build/html/genindex.html deleted file mode 100644 index 9e8d443..0000000 --- a/docs/_build/html/genindex.html +++ /dev/null @@ -1,88 +0,0 @@ - - - - - - - - Index — pgloader 3.4.1 documentation - - - - - - - - - - - - - - - - - - -
-
-
-
- - -

Index

- -
- -
- - -
-
-
- -
-
-
- ©2017, Dimitri Fontaine. - - | - Powered by Sphinx 1.6.5 - & Alabaster 0.7.10 - -
- - - - - - \ No newline at end of file diff --git a/docs/_build/html/index.html b/docs/_build/html/index.html deleted file mode 100644 index 14daa3e..0000000 --- a/docs/_build/html/index.html +++ /dev/null @@ -1,198 +0,0 @@ - - - - - - - Welcome to pgloader’s documentation! — pgloader 3.4.1 documentation - - - - - - - - - - - - - - - - - - - -
-
-
-
- -
-

Welcome to pgloader’s documentation!

-
-

Table Of Contents:

- -
-
-
-

Indices and tables

- -
- - -
-
-
- -
-
-
- ©2017, Dimitri Fontaine. - - | - Powered by Sphinx 1.6.5 - & Alabaster 0.7.10 - - | - Page source -
- - - - - - \ No newline at end of file diff --git a/docs/_build/html/intro.html b/docs/_build/html/intro.html deleted file mode 100644 index 09b664f..0000000 --- a/docs/_build/html/intro.html +++ /dev/null @@ -1,151 +0,0 @@ - - - - - - - Introduction — pgloader 3.4.1 documentation - - - - - - - - - - - - - - - - - - - - -
-
-
-
- -
-

Introduction

-

pgloader loads data from various sources into PostgreSQL. It can -transform the data it reads on the fly and submit raw SQL before and -after the loading. It uses the COPY PostgreSQL protocol to stream -the data into the server, and manages errors by filling a pair of -reject.dat and reject.log files.

-

pgloader knows how to read data from different kind of sources:

-
-
    -
  • Files -* CSV -* Fixed Format -* DBF
    • -
  • Databases -* SQLite -* MySQL -* MS SQL Server
    • -
-
-

The level of automation provided by pgloader depends on the data source -type. In the case of CSV and Fixed Format files, a full description of the -expected input properties must be given to pgloader. In the case of a -database, pgloader connects to the live service and knows how to fetch the -metadata it needs directly from it.

-
-

Continuous Migration

-

pgloader is meant to migrate a whole database in a single command line and -without any manual intervention. The goal is to be able to setup a -Continuous Integration environment as described in the Project -Methodology document of the MySQL to -PostgreSQL webpage.

-
-
    -
  1. Setup your target PostgreSQL Architecture
    2. -
  2. Fork a Continuous Integration environment that uses PostgreSQL
    3. -
  3. Migrate the data over and over again every night, from production
    4. -
  4. As soon as the CI is all green using PostgreSQL, schedule the D-Day
    5. -
  5. Migrate without suprise and enjoy!
    6. -
-
-

In order to be able to follow this great methodology, you need tooling to -implement the third step in a fully automated way. That’s pgloader.

-
-
-

Commands

-

pgloader implements its own Command Language, a DSL that allows to specify -every aspect of the data load and migration to implement. Some of the -features provided in the language are only available for a specific source -type.

-
-
-

Command Line

-

The pgloader command line accepts those two variants:

-
pgloader [<options>] [<command-file>]...
-pgloader [<options>] SOURCE TARGET
-
-
-

Either you have a command-file containing migration specifications in the -pgloader Command Language, or you can give a Source for the data and a -PostgreSQL database connection Target where to load the data into.

-
-
- - -
-
-
- -
-
-
- ©2017, Dimitri Fontaine. - - | - Powered by Sphinx 1.6.5 - & Alabaster 0.7.10 - - | - Page source -
- - - - - - \ No newline at end of file diff --git a/docs/_build/html/objects.inv b/docs/_build/html/objects.inv deleted file mode 100644 index e887187..0000000 Binary files a/docs/_build/html/objects.inv and /dev/null differ diff --git a/docs/_build/html/pgloader-usage-examples.html b/docs/_build/html/pgloader-usage-examples.html deleted file mode 100644 index ee73f61..0000000 --- a/docs/_build/html/pgloader-usage-examples.html +++ /dev/null @@ -1,237 +0,0 @@ - - - - - - - Pgloader Usage Examples — pgloader 3.4.1 documentation - - - - - - - - - - - - - - - - - - -
-
-
-
- -
-

Pgloader Usage Examples

-

Currently not included, because redundant with the tutorial.

-
-

Usage Examples

-

Review the command line options and pgloader’s version:

-
pgloader --help
-pgloader --version
-
-
-
-

Loading from a complex command

-

Use the command file as the pgloader command argument, pgloader will parse -that file and execute the commands found in it:

-
pgloader --verbose ./test/csv-districts.load
-
-
-
-
-

CSV

-

Load data from a CSV file into a pre-existing table in your database, having -pgloader guess the CSV properties (separator, quote and escape character):

-
pgloader ./test/data/matching-1.csv pgsql:///pgloader?tablename=matching
-
-
-

Load data from a CSV file into a pre-existing table in your database, with -expanded options:

-
pgloader --type csv                                   \
-         --field id --field field                     \
-         --with truncate                              \
-         --with "fields terminated by ','"            \
-         ./test/data/matching-1.csv                   \
-         postgres:///pgloader?tablename=matching
-
-
-

In that example the whole loading is driven from the command line, bypassing -the need for writing a command in the pgloader command syntax entirely. As -there’s no command though, the extra inforamtion needed must be provided on -the command line using the –type and –field and –with switches.

-

For documentation about the available syntaxes for the –field and -–with switches, please refer to the CSV section later in the man page.

-

Note also that the PostgreSQL URI includes the target tablename.

-
-
-

Reading from STDIN

-

File based pgloader sources can be loaded from the standard input, as in the -following example:

-
pgloader --type csv                                         \
-         --field "usps,geoid,aland,awater,aland_sqmi,awater_sqmi,intptlat,intptlong" \
-         --with "skip header = 1"                          \
-         --with "fields terminated by '\t'"                \
-         -                                                 \
-         postgresql:///pgloader?districts_longlat          \
-         < test/data/2013_Gaz_113CDs_national.txt
-
-
-

The dash (-) character as a source is used to mean standard input, as -usual in Unix command lines. It’s possible to stream compressed content to -pgloader with this technique, using the Unix pipe:

-
-
gunzip -c source.gz | pgloader –type csv … - pgsql:///target?foo
-
-
-

Loading from CSV available through HTTP

-

The same command as just above can also be run if the CSV file happens to be -found on a remote HTTP location:

-
pgloader --type csv                                                     \
-         --field "usps,geoid,aland,awater,aland_sqmi,awater_sqmi,intptlat,intptlong" \
-         --with "skip header = 1"                                       \
-         --with "fields terminated by '\t'"                             \
-         http://pgsql.tapoueh.org/temp/2013_Gaz_113CDs_national.txt     \
-         postgresql:///pgloader?districts_longlat
-
-
-

Some more options have to be used in that case, as the file contains a -one-line header (most commonly that’s column names, could be a copyright -notice). Also, in that case, we specify all the fields right into a single -–field option argument.

-

Again, the PostgreSQL target connection string must contain the tablename -option and you have to ensure that the target table exists and may fit the -data. Here’s the SQL command used in that example in case you want to try it -yourself:

-
create table districts_longlat
-(
-         usps        text,
-         geoid       text,
-         aland       bigint,
-         awater      bigint,
-         aland_sqmi  double precision,
-         awater_sqmi double precision,
-         intptlat    double precision,
-         intptlong   double precision
-);
-
-
-

Also notice that the same command will work against an archived version of -the same data, e.g. -http://pgsql.tapoueh.org/temp/2013_Gaz_113CDs_national.txt.gz.

-

Finally, it’s important to note that pgloader first fetches the content from -the HTTP URL it to a local file, then expand the archive when it’s -recognized to be one, and only then processes the locally expanded file.

-

In some cases, either because pgloader has no direct support for your -archive format or maybe because expanding the archive is not feasible in -your environment, you might want to stream the content straight from its -remote location into PostgreSQL. Here’s how to do that, using the old battle -tested Unix Pipes trick:

-
curl http://pgsql.tapoueh.org/temp/2013_Gaz_113CDs_national.txt.gz \
-| gunzip -c                                                        \
-| pgloader --type csv                                              \
-           --field "usps,geoid,aland,awater,aland_sqmi,awater_sqmi,intptlat,intptlong"
-           --with "skip header = 1"                                \
-           --with "fields terminated by '\t'"                      \
-           -                                                       \
-           postgresql:///pgloader?districts_longlat
-
-
-

Now the OS will take care of the streaming and buffering between the network -and the commands and pgloader will take care of streaming the data down to -PostgreSQL.

-
-
-

Migrating from SQLite

-

The following command will open the SQLite database, discover its tables -definitions including indexes and foreign keys, migrate those definitions -while casting the data type specifications to their PostgreSQL equivalent -and then migrate the data over:

-
createdb newdb
-pgloader ./test/sqlite/sqlite.db postgresql:///newdb
-
-
-
-
-

Migrating from MySQL

-

Just create a database where to host the MySQL data and definitions and have -pgloader do the migration for you in a single command line:

-
createdb pagila
-pgloader mysql://user@localhost/sakila postgresql:///pagila
-
-
-
-
-

Fetching an archived DBF file from a HTTP remote location

-

It’s possible for pgloader to download a file from HTTP, unarchive it, and -only then open it to discover the schema then load the data:

-
createdb foo
-pgloader --type dbf http://www.insee.fr/fr/methodes/nomenclatures/cog/telechargement/2013/dbf/historiq2013.zip postgresql:///foo
-
-
-

Here it’s not possible for pgloader to guess the kind of data source it’s -being given, so it’s necessary to use the –type command line switch.

-
-
-
- - -
-
-
- -
-
-
- ©2017, Dimitri Fontaine. - - | - Powered by Sphinx 1.6.5 - & Alabaster 0.7.10 - - | - Page source -
- - - - - - \ No newline at end of file diff --git a/docs/_build/html/ref/archive.html b/docs/_build/html/ref/archive.html deleted file mode 100644 index 7bab0bf..0000000 --- a/docs/_build/html/ref/archive.html +++ /dev/null @@ -1,191 +0,0 @@ - - - - - - - Loading From an Archive — pgloader 3.4.1 documentation - - - - - - - - - - - - - - - - - - - - -
-
-
-
- -
-

Loading From an Archive

-

This command instructs pgloader to load data from one or more files contained -in an archive. Currently the only supported archive format is ZIP, and the -archive might be downloaded from an HTTP URL.

-

Here’s an example:

-
LOAD ARCHIVE
-   FROM /Users/dim/Downloads/GeoLiteCity-latest.zip
-   INTO postgresql:///ip4r
-
-   BEFORE LOAD
-     DO $$ create extension if not exists ip4r; $$,
-        $$ create schema if not exists geolite; $$,
-
-     EXECUTE 'geolite.sql'
-
-   LOAD CSV
-        FROM FILENAME MATCHING ~/GeoLiteCity-Location.csv/
-             WITH ENCODING iso-8859-1
-             (
-                locId,
-                country,
-                region     null if blanks,
-                city       null if blanks,
-                postalCode null if blanks,
-                latitude,
-                longitude,
-                metroCode  null if blanks,
-                areaCode   null if blanks
-             )
-        INTO postgresql:///ip4r?geolite.location
-             (
-                locid,country,region,city,postalCode,
-                location point using (format nil "(~a,~a)" longitude latitude),
-                metroCode,areaCode
-             )
-        WITH skip header = 2,
-             fields optionally enclosed by '"',
-             fields escaped by double-quote,
-             fields terminated by ','
-
-  AND LOAD CSV
-        FROM FILENAME MATCHING ~/GeoLiteCity-Blocks.csv/
-             WITH ENCODING iso-8859-1
-             (
-                startIpNum, endIpNum, locId
-             )
-        INTO postgresql:///ip4r?geolite.blocks
-             (
-                iprange ip4r using (ip-range startIpNum endIpNum),
-                locId
-             )
-        WITH skip header = 2,
-             fields optionally enclosed by '"',
-             fields escaped by double-quote,
-             fields terminated by ','
-
-   FINALLY DO
-     $$ create index blocks_ip4r_idx on geolite.blocks using gist(iprange); $$;
-
-
-

The archive command accepts the following clauses and options.

-
-

Archive Source Specification: FROM

-

Filename or HTTP URI where to load the data from. When given an HTTP URL the -linked file will get downloaded locally before processing.

-

If the file is a zip file, the command line utility unzip is used to -expand the archive into files in $TMPDIR, or /tmp if $TMPDIR is unset -or set to a non-existing directory.

-

Then the following commands are used from the top level directory where the -archive has been expanded.

-
-
-

Archive Sub Commands

-
-
    -

  • command [ AND command … ]

    -

    A series of commands against the contents of the archive, at the moment -only CSV,`’FIXED` and DBF commands are supported.

    -

    Note that commands are supporting the clause FROM FILENAME MATCHING -which allows the pgloader command not to depend on the exact names of -the archive directories.

    -

    The same clause can also be applied to several files with using the -spelling FROM ALL FILENAMES MATCHING and a regular expression.

    -

    The whole matching clause must follow the following rule:

    -
    FROM [ ALL FILENAMES | [ FIRST ] FILENAME ] MATCHING
-
    -
    -
    • -
-
-
-
-

Archive Final SQL Commands

-
-
    -

  • FINALLY DO

    -

    SQL Queries to run once the data is loaded, such as CREATE INDEX.

    -
    • -
-
-
-
- - -
-
-
- -
-
-
- ©2017, Dimitri Fontaine. - - | - Powered by Sphinx 1.6.5 - & Alabaster 0.7.10 - - | - Page source -
- - - - - - \ No newline at end of file diff --git a/docs/_build/html/ref/copy.html b/docs/_build/html/ref/copy.html deleted file mode 100644 index c03182f..0000000 --- a/docs/_build/html/ref/copy.html +++ /dev/null @@ -1,196 +0,0 @@ - - - - - - - Loading COPY Formatted Files — pgloader 3.4.1 documentation - - - - - - - - - - - - - - - - - - - - -
-
-
-
- -
-

Loading COPY Formatted Files

-

This commands instructs pgloader to load from a file containing COPY TEXT -data as described in the PostgreSQL documentation. Here’s an example:

-
LOAD COPY
-     FROM copy://./data/track.copy
-          (
-            trackid, track, album, media, genre, composer,
-            milliseconds, bytes, unitprice
-          )
-     INTO postgresql:///pgloader
-   TARGET TABLE track_full
-
-     WITH truncate
-
-      SET work_mem to '14MB',
-          standard_conforming_strings to 'on'
-
-BEFORE LOAD DO
-     $$ drop table if exists track_full; $$,
-     $$ create table track_full (
-          trackid      bigserial,
-          track        text,
-          album        text,
-          media        text,
-          genre        text,
-          composer     text,
-          milliseconds bigint,
-          bytes        bigint,
-          unitprice    numeric
-        );
-     $$;
-
-
-

The COPY format command accepts the following clauses and options.

-
-

COPY Formatted Files Source Specification: FROM

-

Filename where to load the data from. This support local files, HTTP URLs -and zip files containing a single dbf file of the same name. Fetch such a -zip file from an HTTP address is of course supported.

-
-
    -

  • inline

    -

    The data is found after the end of the parsed commands. Any number of -empty lines between the end of the commands and the beginning of the -data is accepted.

    -
    • -

  • stdin

    -

    Reads the data from the standard input stream.

    -
    • -

  • FILENAMES MATCHING

    -

    The whole matching clause must follow the following rule:

    -
    [ ALL FILENAMES | [ FIRST ] FILENAME ]
-MATCHING regexp
-[ IN DIRECTORY '...' ]
-
    -
    -

    The matching clause applies given regular expression (see above for -exact syntax, several options can be used here) to filenames. It’s then -possible to load data from only the first match of all of them.

    -

    The optional IN DIRECTORY clause allows specifying which directory to -walk for finding the data files, and can be either relative to where the -command file is read from, or absolute. The given directory must exists.

    -
    • -
-
-
-
-

COPY Formatted File Options: WITH

-

When loading from a COPY file, the following options are supported:

-
-
    -

  • delimiter

    -

    Takes a single character as argument, which must be found inside single -quotes, and might be given as the printable character itself, the -special value t to denote a tabulation character, or 0x then an -hexadecimal value read as the ASCII code for the character.

    -

    This character is used as the delimiter when reading the data, in a -similar way to the PostgreSQL COPY option.

    -
    • -

  • null

    -

    Takes a quoted string as an argument (quotes can be either double quotes -or single quotes) and uses that string as the NULL representation in -the data.

    -

    This is similar to the null COPY option in PostgreSQL.

    -
    • -

  • truncate

    -

    When this option is listed, pgloader issues a TRUNCATE command against -the PostgreSQL target table before reading the data file.

    -
    • -

  • disable triggers

    -

    When this option is listed, pgloader issues an ALTER TABLE … DISABLE -TRIGGER ALL command against the PostgreSQL target table before copying -the data, then the command ALTER TABLE … ENABLE TRIGGER ALL once the -COPY is done.

    -

    This option allows loading data into a pre-existing table ignoring the -foreign key constraints and user defined triggers and may result in -invalid foreign key constraints once the data is loaded. Use with -care.

    -
    • -

  • skip header

    -

    Takes a numeric value as argument. Instruct pgloader to skip that many -lines at the beginning of the input file.

    -
    • -
-
-
-
- - -
-
-
- -
-
-
- ©2017, Dimitri Fontaine. - - | - Powered by Sphinx 1.6.5 - & Alabaster 0.7.10 - - | - Page source -
- - - - - - \ No newline at end of file diff --git a/docs/_build/html/ref/csv.html b/docs/_build/html/ref/csv.html deleted file mode 100644 index df6eb14..0000000 --- a/docs/_build/html/ref/csv.html +++ /dev/null @@ -1,316 +0,0 @@ - - - - - - - Loading CSV data — pgloader 3.4.1 documentation - - - - - - - - - - - - - - - - - - - - -
-
-
-
- -
-

Loading CSV data

-

This command instructs pgloader to load data from a CSV file. Here’s an -example:

-
LOAD CSV
-   FROM 'GeoLiteCity-Blocks.csv' WITH ENCODING iso-646-us
-        HAVING FIELDS
-        (
-           startIpNum, endIpNum, locId
-        )
-   INTO postgresql://user@localhost:54393/dbname
-        TARGET TABLE geolite.blocks
-        TARGET COLUMNS
-        (
-           iprange ip4r using (ip-range startIpNum endIpNum),
-           locId
-        )
-   WITH truncate,
-        skip header = 2,
-        fields optionally enclosed by '"',
-        fields escaped by backslash-quote,
-        fields terminated by '\t'
-
-    SET work_mem to '32 MB', maintenance_work_mem to '64 MB';
-
-
-

The csv format command accepts the following clauses and options.

-
-

CSV Source Specification: FROM

-

Filename where to load the data from. Accepts an ENCODING option. Use the -–list-encodings option to know which encoding names are supported.

-

The filename may be enclosed by single quotes, and could be one of the -following special values:

-
-
    -

  • inline

    -

    The data is found after the end of the parsed commands. Any number -of empty lines between the end of the commands and the beginning of -the data is accepted.

    -
    • -

  • stdin

    -

    Reads the data from the standard input stream.

    -
    • -

  • FILENAMES MATCHING

    -

    The whole matching clause must follow the following rule:

    -
    [ ALL FILENAMES | [ FIRST ] FILENAME ]
-MATCHING regexp
-[ IN DIRECTORY '...' ]
-
    -
    -

    The matching clause applies given regular expression (see above -for exact syntax, several options can be used here) to filenames. -It’s then possible to load data from only the first match of all of -them.

    -

    The optional IN DIRECTORY clause allows specifying which directory -to walk for finding the data files, and can be either relative to -where the command file is read from, or absolute. The given -directory must exists.

    -
    • -
-
-
-
-

Fields Specifications

-

The FROM option also supports an optional comma separated list of field -names describing what is expected in the CSV data file, optionally -introduced by the clause HAVING FIELDS.

-

Each field name can be either only one name or a name following with -specific reader options for that field, enclosed in square brackets and -comma-separated. Supported per-field reader options are:

-
-
    -

  • terminated by

    -
    -

    See the description of field terminated by below.

    -

    The processing of this option is not currently implemented.

    -
    -
    • -

  • date format

    -

    When the field is expected of the date type, then this option allows -to specify the date format used in the file.

    -

    Date format string are template strings modeled against the -PostgreSQL to_char template strings support, limited to the -following patterns:

    -
    -
      -
    • YYYY, YYY, YY for the year part
      • -
    • MM for the numeric month part
      • -
    • DD for the numeric day part
      • -
    • HH, HH12, HH24 for the hour part
      • -
    • am, AM, a.m., A.M.
      • -
    • pm, PM, p.m., P.M.
      • -
    • MI for the minutes part
      • -
    • SS for the seconds part
      • -
    • MS for the milliseconds part (4 digits)
      • -
    • US for the microseconds part (6 digits)
      • -
    • unparsed punctuation signs: - . * # @ T / and space
      • -
    -
    -

    Here’s an example of a date format specification:

    -
    column-name [date format 'YYYY-MM-DD HH24-MI-SS.US']
-
    -
    -
    • -

  • null if

    -
    -

    This option takes an argument which is either the keyword blanks -or a double-quoted string.

    -

    When blanks is used and the field value that is read contains -only space characters, then it’s automatically converted to an SQL -NULL value.

    -

    When a double-quoted string is used and that string is read as the -field value, then the field value is automatically converted to an -SQL NULL value.

    -
    -
    • -

  • trim both whitespace, trim left whitespace, trim right whitespace

    -

    This option allows to trim whitespaces in the read data, either from -both sides of the data, or only the whitespace characters found on -the left of the streaing, or only those on the right of the string.

    -
    • -
-
-
-
-

CSV Loading Options: WITH

-

When loading from a CSV file, the following options are supported:

-
-
    -

  • truncate

    -
    -

    When this option is listed, pgloader issues a TRUNCATE command -against the PostgreSQL target table before reading the data file.

    -
    -
    • -

  • drop indexes

    -

    When this option is listed, pgloader issues DROP INDEX commands -against all the indexes defined on the target table before copying -the data, then CREATE INDEX commands once the COPY is done.

    -

    In order to get the best performance possible, all the indexes are -created in parallel and when done the primary keys are built again -from the unique indexes just created. This two step process allows -creating the primary key index in parallel with the other indexes, -as only the ALTER TABLE command needs an access exclusive lock -on the target table.

    -
    • -

  • disable triggers

    -

    When this option is listed, pgloader issues an ALTER TABLE … -DISABLE TRIGGER ALL command against the PostgreSQL target table -before copying the data, then the command ALTER TABLE … ENABLE -TRIGGER ALL once the COPY is done.

    -

    This option allows loading data into a pre-existing table ignoring -the foreign key constraints and user defined triggers and may -result in invalid foreign key constraints once the data is loaded. -Use with care.

    -
    • -

  • skip header

    -

    Takes a numeric value as argument. Instruct pgloader to skip that -many lines at the beginning of the input file.

    -
    • -

  • csv header

    -

    Use the first line read after skip header as the list of csv field -names to be found in the CSV file, using the same CSV parameters as -for the CSV data.

    -
    • -

  • trim unquoted blanks

    -

    When reading unquoted values in the CSV file, remove the blanks -found in between the separator and the value. That behaviour is the -default.

    -
    • -

  • keep unquoted blanks

    -

    When reading unquoted values in the CSV file, keep blanks found in -between the separator and the value.

    -
    • -

  • fields optionally enclosed by

    -

    Takes a single character as argument, which must be found inside single -quotes, and might be given as the printable character itself, the -special value t to denote a tabulation character, the special value ‘ -to denote a single-quote, or 0x then an hexadecimal value read as the -ASCII code for the character.

    -

    The following options specify the same enclosing character, a single quote:

    -
    fields optionally enclosed by '\''
-fields optionally enclosed by '0x27'
-
    -
    -

    This character is used as the quoting character in the CSV file, -and defaults to double-quote.

    -
    • -

  • fields not enclosed

    -

    By default, pgloader will use the double-quote character as the -enclosing character. If you have a CSV file where fields are not -enclosed and are using double-quote as an expected ordinary -character, then use the option fields not enclosed for the CSV -parser to accept those values.

    -
    • -

  • fields escaped by

    -

    Takes either the special value backslash-quote or double-quote, -or any value supported by the fields terminated by option (see -below). This value is used to recognize escaped field separators -when they are to be found within the data fields themselves. -Defaults to double-quote.

    -
    • -

  • csv escape mode

    -

    Takes either the special value quote (the default) or following -and allows the CSV parser to parse either only escaped field -separator or any character (including CSV data) when using the -following value.

    -
    • -

  • fields terminated by

    -

    Takes a single character as argument, which must be found inside -single quotes, and might be given as the printable character itself, -the special value t to denote a tabulation character, or 0x then -an hexadecimal value read as the ASCII code for the character.

    -

    This character is used as the field separator when reading the -CSV data.

    -
    • -

  • lines terminated by

    -

    Takes a single character as argument, which must be found inside -single quotes, and might be given as the printable character itself, -the special value t to denote a tabulation character, or 0x then -an hexadecimal value read as the ASCII code for the character.

    -

    This character is used to recognize end-of-line condition when -reading the CSV data.

    -
    • -
-
-
-
- - -
-
-
- -
-
-
- ©2017, Dimitri Fontaine. - - | - Powered by Sphinx 1.6.5 - & Alabaster 0.7.10 - - | - Page source -
- - - - - - \ No newline at end of file diff --git a/docs/_build/html/ref/dbf.html b/docs/_build/html/ref/dbf.html deleted file mode 100644 index 7d54a68..0000000 --- a/docs/_build/html/ref/dbf.html +++ /dev/null @@ -1,138 +0,0 @@ - - - - - - - Loading DBF data — pgloader 3.4.1 documentation - - - - - - - - - - - - - - - - - - - - -
-
-
-
- -
-

Loading DBF data

-

This command instructs pgloader to load data from a DBF file. Here’s an -example:

-
LOAD DBF
-        FROM http://www.insee.fr/fr/methodes/nomenclatures/cog/telechargement/2013/dbf/reg2013.dbf
-    INTO postgresql://user@localhost/dbname
-    WITH truncate, create table;
-
-
-

The dbf format command accepts the following clauses and options.

-
-

DBF Source Specification: FROM

-

Filename where to load the data from. This support local files, HTTP URLs -and zip files containing a single dbf file of the same name. Fetch such a -zip file from an HTTP address is of course supported.

-
-
-

DBF Loading Options: WITH

-

When loading from a DBF file, the following options are supported:

-
-
    -

  • truncate

    -

    When this option is listed, pgloader issues a TRUNCATE command against -the PostgreSQL target table before reading the data file.

    -
    • -

  • disable triggers

    -

    When this option is listed, pgloader issues an ALTER TABLE … DISABLE -TRIGGER ALL command against the PostgreSQL target table before copying -the data, then the command ALTER TABLE … ENABLE TRIGGER ALL once the -COPY is done.

    -

    This option allows loading data into a pre-existing table ignoring the -foreign key constraints and user defined triggers and may result in -invalid foreign key constraints once the data is loaded. Use with -care.

    -
    • -

  • create table

    -

    When this option is listed, pgloader creates the table using the meta -data found in the DBF file, which must contain a list of fields with -their data type. A standard data type conversion from DBF to PostgreSQL -is done.

    -
    • -

  • table name

    -

    This options expects as its value the possibly qualified name of the -table to create.

    -
    • -
-
-
-
- - -
-
-
- -
-
-
- ©2017, Dimitri Fontaine. - - | - Powered by Sphinx 1.6.5 - & Alabaster 0.7.10 - - | - Page source -
- - - - - - \ No newline at end of file diff --git a/docs/_build/html/ref/fixed.html b/docs/_build/html/ref/fixed.html deleted file mode 100644 index e5b6c9a..0000000 --- a/docs/_build/html/ref/fixed.html +++ /dev/null @@ -1,270 +0,0 @@ - - - - - - - Loading Fixed Cols File Formats — pgloader 3.4.1 documentation - - - - - - - - - - - - - - - - - - - - -
-
-
-
- -
-

Loading Fixed Cols File Formats

-

This command instructs pgloader to load data from a text file containing -columns arranged in a fixed size manner. Here’s an example:

-
LOAD FIXED
-     FROM inline
-          (
-           a from  0 for 10,
-           b from 10 for  8,
-           c from 18 for  8,
-           d from 26 for 17 [null if blanks, trim right whitespace]
-          )
-     INTO postgresql:///pgloader
-   TARGET TABLE fixed
-          (
-             a, b,
-             c time using (time-with-no-separator c),
-             d
-          )
-
-     WITH truncate
-
-      SET work_mem to '14MB',
-          standard_conforming_strings to 'on'
-
-BEFORE LOAD DO
-     $$ drop table if exists fixed; $$,
-     $$ create table fixed (
-         a integer,
-         b date,
-         c time,
-         d text
-        );
-     $$;
-
- 01234567892008052011431250firstline
-    01234562008052115182300left blank-padded
- 12345678902008052208231560another line
-  2345609872014092914371500
-  2345678902014092914371520
-
-
-

The fixed format command accepts the following clauses and options.

-
-

Fixed File Format Source Specification: FROM

-

Filename where to load the data from. Accepts an ENCODING option. Use the -–list-encodings option to know which encoding names are supported.

-

The filename may be enclosed by single quotes, and could be one of the -following special values:

-
-
    -

  • inline

    -
    -

    The data is found after the end of the parsed commands. Any number -of empty lines between the end of the commands and the beginning of -the data is accepted.

    -
    -
    • -

  • stdin

    -
    -

    Reads the data from the standard input stream.

    -
    -
    • -

  • FILENAMES MATCHING

    -

    The whole matching clause must follow the following rule:

    -
    [ ALL FILENAMES | [ FIRST ] FILENAME ]
-MATCHING regexp
-[ IN DIRECTORY '...' ]
-
    -
    -

    The matching clause applies given regular expression (see above -for exact syntax, several options can be used here) to filenames. -It’s then possible to load data from only the first match of all of -them.

    -

    The optional IN DIRECTORY clause allows specifying which directory -to walk for finding the data files, and can be either relative to -where the command file is read from, or absolute. The given -directory must exists.

    -
    • -
-
-
-
-

Fields Specifications

-

The FROM option also supports an optional comma separated list of field -names describing what is expected in the FIXED data file.

-

Each field name is composed of the field name followed with specific reader -options for that field. Supported per-field reader options are the -following, where only start and length are required.

-
-
    -

  • start

    -

    Position in the line where to start reading that field’s value. Can -be entered with decimal digits or 0x then hexadecimal digits.

    -
    • -

  • length

    -

    How many bytes to read from the start position to read that -field’s value. Same format as start.

    -
    • -
-
-

Those optional parameters must be enclosed in square brackets and -comma-separated:

-
-
    -

  • terminated by

    -
    -

    See the description of field terminated by below.

    -

    The processing of this option is not currently implemented.

    -
    -
    • -

  • date format

    -

    When the field is expected of the date type, then this option allows -to specify the date format used in the file.

    -

    Date format string are template strings modeled against the -PostgreSQL to_char template strings support, limited to the -following patterns:

    -
    -
      -
    • YYYY, YYY, YY for the year part
      • -
    • MM for the numeric month part
      • -
    • DD for the numeric day part
      • -
    • HH, HH12, HH24 for the hour part
      • -
    • am, AM, a.m., A.M.
      • -
    • pm, PM, p.m., P.M.
      • -
    • MI for the minutes part
      • -
    • SS for the seconds part
      • -
    • MS for the milliseconds part (4 digits)
      • -
    • US for the microseconds part (6 digits)
      • -
    • unparsed punctuation signs: - . * # @ T / and space
      • -
    -
    -

    Here’s an example of a date format specification:

    -
    column-name [date format 'YYYY-MM-DD HH24-MI-SS.US']
-
    -
    -
    • -

  • null if

    -

    This option takes an argument which is either the keyword blanks -or a double-quoted string.

    -

    When blanks is used and the field value that is read contains only -space characters, then it’s automatically converted to an SQL NULL -value.

    -

    When a double-quoted string is used and that string is read as the -field value, then the field value is automatically converted to an -SQL NULL value.

    -
    • -

  • trim both whitespace, trim left whitespace, trim right whitespace

    -

    This option allows to trim whitespaces in the read data, either from -both sides of the data, or only the whitespace characters found on -the left of the streaing, or only those on the right of the string.

    -
    • -
-
-
-
-

Fixed File Format Loading Options: WITH

-

When loading from a FIXED file, the following options are supported:

-
-
    -

  • truncate

    -

    When this option is listed, pgloader issues a TRUNCATE command -against the PostgreSQL target table before reading the data file.

    -
    • -

  • disable triggers

    -

    When this option is listed, pgloader issues an ALTER TABLE … -DISABLE TRIGGER ALL command against the PostgreSQL target table -before copying the data, then the command ALTER TABLE … ENABLE -TRIGGER ALL once the COPY is done.

    -

    This option allows loading data into a pre-existing table ignoring -the foreign key constraints and user defined triggers and may -result in invalid foreign key constraints once the data is loaded. -Use with care.

    -
    • -

  • skip header

    -

    Takes a numeric value as argument. Instruct pgloader to skip that -many lines at the beginning of the input file.

    -
    • -
-
-
-
- - -
-
-
- -
-
-
- ©2017, Dimitri Fontaine. - - | - Powered by Sphinx 1.6.5 - & Alabaster 0.7.10 - - | - Page source -
- - - - - - \ No newline at end of file diff --git a/docs/_build/html/ref/ixf.html b/docs/_build/html/ref/ixf.html deleted file mode 100644 index eb568f1..0000000 --- a/docs/_build/html/ref/ixf.html +++ /dev/null @@ -1,149 +0,0 @@ - - - - - - - Loading IXF Data — pgloader 3.4.1 documentation - - - - - - - - - - - - - - - - - - - - -
-
-
-
- -
-

Loading IXF Data

-

This command instructs pgloader to load data from an IBM IXF file. Here’s -an example:

-
LOAD IXF
-    FROM data/nsitra.test1.ixf
-    INTO postgresql:///pgloader
-  TARGET TABLE nsitra.test1
-    WITH truncate, create table, timezone UTC
-
-  BEFORE LOAD DO
-   $$ create schema if not exists nsitra; $$,
-   $$ drop table if exists nsitra.test1; $$;
-
-
-

The ixf format command accepts the following clauses and options.

-
-

IXF Source Specification: FROM

-

Filename where to load the data from. This support local files, HTTP URLs -and zip files containing a single ixf file of the same name. Fetch such a -zip file from an HTTP address is of course supported.

-
-
-

IXF Loading Options: WITH

-

When loading from a IXF file, the following options are supported:

-
-
    -

  • truncate

    -

    When this option is listed, pgloader issues a TRUNCATE command against -the PostgreSQL target table before reading the data file.

    -
    • -

  • disable triggers

    -

    When this option is listed, pgloader issues an ALTER TABLE … DISABLE -TRIGGER ALL command against the PostgreSQL target table before copying -the data, then the command ALTER TABLE … ENABLE TRIGGER ALL once the -COPY is done.

    -

    This option allows loading data into a pre-existing table ignoring the -foreign key constraints and user defined triggers and may result in -invalid foreign key constraints once the data is loaded. Use with -care.

    -
    • -

  • create table

    -

    When this option is listed, pgloader creates the table using the meta -data found in the DBF file, which must contain a list of fields with -their data type. A standard data type conversion from DBF to PostgreSQL -is done.

    -
    • -

  • table name

    -

    This options expects as its value the possibly qualified name of the -table to create.

    -
    • -

  • timezone

    -

    This options allows to specify which timezone is used when parsing -timestamps from an IXF file, and defaults to UTC. Expected values are -either UTC, GMT or a single quoted location name such as -‘Universal’ or ‘Europe/Paris’.

    -
    • -
-
-
-
- - -
-
-
- -
-
-
- ©2017, Dimitri Fontaine. - - | - Powered by Sphinx 1.6.5 - & Alabaster 0.7.10 - - | - Page source -
- - - - - - \ No newline at end of file diff --git a/docs/_build/html/ref/mssql.html b/docs/_build/html/ref/mssql.html deleted file mode 100644 index 0dac5ad..0000000 --- a/docs/_build/html/ref/mssql.html +++ /dev/null @@ -1,236 +0,0 @@ - - - - - - - Migrating a MS SQL Database to PostgreSQL — pgloader 3.4.1 documentation - - - - - - - - - - - - - - - - - - - - -
-
-
-
- -
-

Migrating a MS SQL Database to PostgreSQL

-

This command instructs pgloader to load data from a MS SQL database. -Automatic discovery of the schema is supported, including build of the -indexes, primary and foreign keys constraints.

-

Here’s an example:

-
load database
-     from mssql://user@host/dbname
-     into postgresql:///dbname
-
-including only table names like 'GlobalAccount' in schema 'dbo'
-
-set work_mem to '16MB', maintenance_work_mem to '512 MB'
-
-before load do $$ drop schema if exists dbo cascade; $$;
-
-
-

The mssql command accepts the following clauses and options.

-
-

MS SQL Database Source Specification: FROM

-

Connection string to an existing MS SQL database server that listens and -welcome external TCP/IP connection. As pgloader currently piggybacks on the -FreeTDS driver, to change the port of the server please export the TDSPORT -environment variable.

-
-
-

MS SQL Database Migration Options: WITH

-

When loading from a MS SQL database, the same options as when loading a -MySQL database are supported. Please refer to the MySQL section. The -following options are added:

-
-
    -

  • create schemas

    -

    When this option is listed, pgloader creates the same schemas as found -on the MS SQL instance. This is the default.

    -
    • -

  • create no schemas

    -

    When this option is listed, pgloader refrains from creating any schemas -at all, you must then ensure that the target schema do exist.

    -
    • -
-
-
-
-

MS SQL Database Casting Rules

-
-

CAST

-

The cast clause allows to specify custom casting rules, either to overload -the default casting rules or to amend them with special cases.

-

Please refer to the MySQL CAST clause for details.

-
-
-
-

MS SQL Partial Migration

-
-

INCLUDING ONLY TABLE NAMES LIKE

-

Introduce a comma separated list of table name patterns used to limit the -tables to migrate to a sublist. More than one such clause may be used, they -will be accumulated together.

-

Example:

-
including only table names lile 'GlobalAccount' in schema 'dbo'
-
-
-
-
-

EXCLUDING TABLE NAMES LIKE

-

Introduce a comma separated list of table name patterns used to exclude -table names from the migration. This filter only applies to the result of -the INCLUDING filter.

-
excluding table names matching 'LocalAccount' in schema 'dbo'
-
-
-
-
-
-

MS SQL Schema Transformations

-
-

ALTER SCHEMA ‘…’ RENAME TO ‘…’

-

Allows to rename a schema on the flight, so that for instance the tables -found in the schema ‘dbo’ in your source database will get migrated into the -schema ‘public’ in the target database with this command:

-
alter schema 'dbo' rename to 'public'
-
-
-
-
-

ALTER TABLE NAMES MATCHING … IN SCHEMA ‘…’

-

See the MySQL explanation for this clause above. It works the same in the -context of migrating from MS SQL, only with the added option to specify the -name of the schema where to find the definition of the target tables.

-

The matching is done in pgloader itself, with a Common Lisp regular -expression lib, so doesn’t depend on the LIKE implementation of MS SQL, -nor on the lack of support for regular expressions in the engine.

-
-
-
-

MS SQL Driver setup and encoding

-

pgloader is using the FreeTDS driver, and internally expects the data to -be sent in utf-8. To achieve that, you can configure the FreeTDS driver with -those defaults, in the file ~/.freetds.conf:

-
[global]
-    tds version = 7.4
-    client charset = UTF-8
-
-
-
-
-

Default MS SQL Casting Rules

-

When migrating from MS SQL the following Casting Rules are provided:

-

Numbers:

-
type tinyint to smallint
-
-type float to float   using float-to-string
-type real to real     using float-to-string
-type double to double precision     using float-to-string
-type numeric to numeric     using float-to-string
-type decimal to numeric     using float-to-string
-type money to numeric     using float-to-string
-type smallmoney to numeric     using float-to-string
-
-
-

Texts:

-
type char      to text drop typemod
-type nchat     to text drop typemod
-type varchar   to text drop typemod
-type nvarchar  to text drop typemod
-type xml       to text drop typemod
-
-
-

Binary:

-
type binary    to bytea using byte-vector-to-bytea
-type varbinary to bytea using byte-vector-to-bytea
-
-
-

Date:

-
type datetime    to timestamptz
-type datetime2   to timestamptz
-
-
-

Others:

-
type bit to boolean
-type hierarchyid to bytea
-type geography to bytea
-type uniqueidentifier to uuid using sql-server-uniqueidentifier-to-uuid
-
-
-
-
- - -
-
-
- -
-
-
- ©2017, Dimitri Fontaine. - - | - Powered by Sphinx 1.6.5 - & Alabaster 0.7.10 - - | - Page source -
- - - - - - \ No newline at end of file diff --git a/docs/_build/html/ref/mysql.html b/docs/_build/html/ref/mysql.html deleted file mode 100644 index 9e7c2cc..0000000 --- a/docs/_build/html/ref/mysql.html +++ /dev/null @@ -1,639 +0,0 @@ - - - - - - - Migrating a MySQL Database to PostgreSQL — pgloader 3.4.1 documentation - - - - - - - - - - - - - - - - - - - - -
-
-
-
- -
-

Migrating a MySQL Database to PostgreSQL

-

This command instructs pgloader to load data from a database connection. The -only supported database source is currently MySQL, and pgloader supports -dynamically converting the schema of the source database and the indexes -building.

-

A default set of casting rules are provided and might be overloaded and -appended to by the command.

-

Here’s an example using as many options as possible, some of them even being -defaults. Chances are you don’t need that complex a setup, don’t copy and -paste it, use it only as a reference!

-
LOAD DATABASE
-     FROM      mysql://root@localhost/sakila
-     INTO postgresql://localhost:54393/sakila
-
- WITH include drop, create tables, create indexes, reset sequences,
-      workers = 8, concurrency = 1,
-      multiple readers per thread, rows per range = 50000
-
-  SET PostgreSQL PARAMETERS
-      maintenance_work_mem to '128MB',
-      work_mem to '12MB',
-      search_path to 'sakila, public, "$user"'
-
-  SET MySQL PARAMETERS
-      net_read_timeout  = '120',
-      net_write_timeout = '120'
-
- CAST type bigint when (= precision 20) to bigserial drop typemod,
-      type date drop not null drop default using zero-dates-to-null,
-      -- type tinyint to boolean using tinyint-to-boolean,
-      type year to integer
-
- MATERIALIZE VIEWS film_list, staff_list
-
- -- INCLUDING ONLY TABLE NAMES MATCHING ~/film/, 'actor'
- -- EXCLUDING TABLE NAMES MATCHING ~<ory>
- -- DECODING TABLE NAMES MATCHING ~/messed/, ~/encoding/ AS utf8
- -- ALTER TABLE NAMES MATCHING 'film' RENAME TO 'films'
- -- ALTER TABLE NAMES MATCHING ~/_list$/ SET SCHEMA 'mv'
-
- ALTER TABLE NAMES MATCHING ~/_list$/, 'sales_by_store', ~/sales_by/
-  SET SCHEMA 'mv'
-
- ALTER TABLE NAMES MATCHING 'film' RENAME TO 'films'
- ALTER TABLE NAMES MATCHING ~/./ SET (fillfactor='40')
-
- ALTER SCHEMA 'sakila' RENAME TO 'pagila'
-
- BEFORE LOAD DO
-   $$ create schema if not exists pagila; $$,
-   $$ create schema if not exists mv;     $$,
-   $$ alter database sakila set search_path to pagila, mv, public; $$;
-
-
-

The database command accepts the following clauses and options.

-
-

MySQL Database Source Specification: FROM

-

Must be a connection URL pointing to a MySQL database.

-

If the connection URI contains a table name, then only this table is -migrated from MySQL to PostgreSQL.

-

See the SOURCE CONNECTION STRING section above for details on how to write -the connection string. The MySQL connection string accepts the same -parameter sslmode as the PostgreSQL connection string, but the verify -mode is not implemented (yet).

-

Environment variables described in -<http://dev.mysql.com/doc/refman/5.0/en/environment-variables.html> can be -used as default values too. If the user is not provided, then it defaults to -USER environment variable value. The password can be provided with the -environment variable MYSQL_PWD. The host can be provided with the -environment variable MYSQL_HOST and otherwise defaults to localhost. The -port can be provided with the environment variable MYSQL_TCP_PORT and -otherwise defaults to 3306.

-
-
-

MySQL Database Migration Options: WITH

-

When loading from a MySQL database, the following options are supported, -and the default WITH clause is: no truncate, create schema, create -tables, include drop, create indexes, reset sequences, foreign -keys, downcase identifiers, uniquify index names.

-
-
    -

  • include drop

    -

    When this option is listed, pgloader drops all the tables in the target -PostgreSQL database whose names appear in the MySQL database. This -option allows for using the same command several times in a row until -you figure out all the options, starting automatically from a clean -environment. Please note that CASCADE is used to ensure that tables -are dropped even if there are foreign keys pointing to them. This is -precisely what include drop is intended to do: drop all target tables -and recreate them.

    -

    Great care needs to be taken when using include drop, as it will -cascade to all objects referencing the target tables, possibly -including other tables that are not being loaded from the source DB.

    -
    • -

  • include no drop

    -

    When this option is listed, pgloader will not include any DROP -statement when loading the data.

    -
    • -

  • truncate

    -

    When this option is listed, pgloader issue the TRUNCATE command -against each PostgreSQL table just before loading data into it.

    -
    • -

  • no truncate

    -

    When this option is listed, pgloader issues no TRUNCATE command.

    -
    • -

  • disable triggers

    -

    When this option is listed, pgloader issues an ALTER TABLE … DISABLE -TRIGGER ALL command against the PostgreSQL target table before copying -the data, then the command ALTER TABLE … ENABLE TRIGGER ALL once the -COPY is done.

    -

    This option allows loading data into a pre-existing table ignoring the -foreign key constraints and user defined triggers and may result in -invalid foreign key constraints once the data is loaded. Use with -care.

    -
    • -

  • create tables

    -

    When this option is listed, pgloader creates the table using the meta -data found in the MySQL file, which must contain a list of fields with -their data type. A standard data type conversion from DBF to PostgreSQL -is done.

    -
    • -

  • create no tables

    -

    When this option is listed, pgloader skips the creation of table before -loading data, target tables must then already exist.

    -

    Also, when using create no tables pgloader fetches the metadata from -the current target database and checks type casting, then will remove -constraints and indexes prior to loading the data and install them back -again once the loading is done.

    -
    • -

  • create indexes

    -

    When this option is listed, pgloader gets the definitions of all the -indexes found in the MySQL database and create the same set of index -definitions against the PostgreSQL database.

    -
    • -

  • create no indexes

    -

    When this option is listed, pgloader skips the creating indexes.

    -
    • -

  • drop indexes

    -

    When this option is listed, pgloader drops the indexes in the target -database before loading the data, and creates them again at the end -of the data copy.

    -
    • -

  • uniquify index names, preserve index names

    -

    MySQL index names are unique per-table whereas in PostgreSQL index names -have to be unique per-schema. The default for pgloader is to change the -index name by prefixing it with idx_OID where OID is the internal -numeric identifier of the table the index is built against.

    -

    In somes cases like when the DDL are entirely left to a framework it -might be sensible for pgloader to refrain from handling index unique -names, that is achieved by using the preserve index names option.

    -

    The default is to uniquify index names.

    -

    Even when using the option preserve index names, MySQL primary key -indexes named “PRIMARY” will get their names uniquified. Failing to do -so would prevent the primary keys to be created again in PostgreSQL -where the index names must be unique per schema.

    -
    • -

  • drop schema

    -

    When this option is listed, pgloader drops the target schema in the -target PostgreSQL database before creating it again and all the objects -it contains. The default behavior doesn’t drop the target schemas.

    -
    • -

  • foreign keys

    -

    When this option is listed, pgloader gets the definitions of all the -foreign keys found in the MySQL database and create the same set of -foreign key definitions against the PostgreSQL database.

    -
    • -

  • no foreign keys

    -

    When this option is listed, pgloader skips creating foreign keys.

    -
    • -

  • reset sequences

    -

    When this option is listed, at the end of the data loading and after the -indexes have all been created, pgloader resets all the PostgreSQL -sequences created to the current maximum value of the column they are -attached to.

    -

    The options schema only and data only have no effects on this -option.

    -
    • -

  • reset no sequences

    -

    When this option is listed, pgloader skips resetting sequences after the -load.

    -

    The options schema only and data only have no effects on this -option.

    -
    • -

  • downcase identifiers

    -

    When this option is listed, pgloader converts all MySQL identifiers -(table names, index names, column names) to downcase, except for -PostgreSQL reserved keywords.

    -

    The PostgreSQL reserved keywords are determined dynamically by using -the system function pg_get_keywords().

    -
    • -

  • quote identifiers

    -

    When this option is listed, pgloader quotes all MySQL identifiers so -that their case is respected. Note that you will then have to do the -same thing in your application code queries.

    -
    • -

  • schema only

    -

    When this option is listed pgloader refrains from migrating the data -over. Note that the schema in this context includes the indexes when the -option create indexes has been listed.

    -
    • -

  • data only

    -

    When this option is listed pgloader only issues the COPY statements, -without doing any other processing.

    -
    • -

  • single reader per thread, multiple readers per thread

    -

    The default is single reader per thread and it means that each -MySQL table is read by a single thread as a whole, with a single -SELECT statement using no WHERE clause.

    -

    When using multiple readers per thread pgloader may be able to -divide the reading work into several threads, as many as the -concurrency setting, which needs to be greater than 1 for this -option to kick be activated.

    -

    For each source table, pgloader searches for a primary key over a -single numeric column, or a multiple-column primary key index for -which the first column is of a numeric data type (one of integer -or bigint). When such an index exists, pgloader runs a query to -find the min and max values on this column, and then split that -range into many ranges containing a maximum of rows per range.

    -

    When the range list we then obtain contains at least as many ranges -than our concurrency setting, then we distribute those ranges to -each reader thread.

    -

    So when all the conditions are met, pgloader then starts as many -reader thread as the concurrency setting, and each reader thread -issues several queries with a WHERE id >= x AND id < y, where y - -x = rows per range or less (for the last range, depending on the -max value just obtained.

    -
    • -

  • rows per range

    -

    How many rows are fetched per SELECT query when using multiple -readers per thread, see above for details.

    -
    • -

  • SET MySQL PARAMETERS

    -

    The SET MySQL PARAMETERS allows setting MySQL parameters using the -MySQL SET command each time pgloader connects to it.

    -
    • -
-
-
-
-

MySQL Database Casting Rules

-

The command CAST introduces user-defined casting rules.

-

The cast clause allows to specify custom casting rules, either to overload -the default casting rules or to amend them with special cases.

-

A casting rule is expected to follow one of the forms:

-
type <mysql-type-name> [ <guard> ... ] to <pgsql-type-name> [ <option> ... ]
-column <table-name>.<column-name> [ <guards> ] to ...
-
-
-

It’s possible for a casting rule to either match against a MySQL data type -or against a given column name in a given table name. That flexibility -allows to cope with cases where the type tinyint might have been used as a -boolean in some cases but as a smallint in others.

-

The casting rules are applied in order, the first match prevents following -rules to be applied, and user defined rules are evaluated first.

-

The supported guards are:

-
-
    -

  • when unsigned

    -

    The casting rule is only applied against MySQL columns of the source -type that have the keyword unsigned in their data type definition.

    -

    Example of a casting rule using a unsigned guard:

    -
    type smallint when unsigned to integer drop typemod
-
    -
    -
    • -

  • when default ‘value’

    -

    The casting rule is only applied against MySQL columns of the source -type that have given value, which must be a single-quoted or a -double-quoted string.

    -
    • -

  • when typemod expression

    -

    The casting rule is only applied against MySQL columns of the source -type that have a typemod value matching the given typemod -expression. The typemod is separated into its precision and scale -components.

    -

    Example of a cast rule using a typemod guard:

    -
    type char when (= precision 1) to char keep typemod
-
    -
    -

    This expression casts MySQL char(1) column to a PostgreSQL column of -type char(1) while allowing for the general case char(N) will be -converted by the default cast rule into a PostgreSQL type varchar(N).

    -
    • -

  • with extra auto_increment

    -

    The casting rule is only applied against MySQL columns having the -extra column auto_increment option set, so that it’s possible to -target e.g. serial rather than integer.

    -

    The default matching behavior, when this option isn’t set, is to match -both columns with the extra definition and without.

    -

    This means that if you want to implement a casting rule that target -either serial or integer from a smallint definition depending on -the auto_increment extra bit of information from MySQL, then you need -to spell out two casting rules as following:

    -
    type smallint  with extra auto_increment
-  to serial drop typemod keep default keep not null,
-
-type smallint
-  to integer drop typemod keep default keep not null
-
    -
    -
    • -
-
-

The supported casting options are:

-
-
    -

  • drop default, keep default

    -

    When the option drop default is listed, pgloader drops any -existing default expression in the MySQL database for columns of the -source type from the CREATE TABLE statement it generates.

    -

    The spelling keep default explicitly prevents that behaviour and -can be used to overload the default casting rules.

    -
    • -

  • drop not null, keep not null, set not null

    -

    When the option drop not null is listed, pgloader drops any -existing NOT NULL constraint associated with the given source -MySQL datatype when it creates the tables in the PostgreSQL -database.

    -

    The spelling keep not null explicitly prevents that behaviour and -can be used to overload the default casting rules.

    -

    When the option set not null is listed, pgloader sets a NOT NULL -constraint on the target column regardless whether it has been set -in the source MySQL column.

    -
    • -

  • drop typemod, keep typemod

    -

    When the option drop typemod is listed, pgloader drops any -existing typemod definition (e.g. precision and scale) from -the datatype definition found in the MySQL columns of the source -type when it created the tables in the PostgreSQL database.

    -

    The spelling keep typemod explicitly prevents that behaviour and -can be used to overload the default casting rules.

    -
    • -

  • using

    -

    This option takes as its single argument the name of a function to -be found in the pgloader.transforms Common Lisp package. See above -for details.

    -

    It’s possible to augment a default cast rule (such as one that -applies against ENUM data type for example) with a transformation -function by omitting entirely the type parts of the casting rule, -as in the following example:

    -
    column enumerate.foo using empty-string-to-null
-
    -
    -
    • -
-
-
-
-

MySQL Views Support

-

MySQL views support allows pgloader to migrate view as if they were base -tables. This feature then allows for on-the-fly transformation from MySQL to -PostgreSQL, as the view definition is used rather than the base data.

-
-

MATERIALIZE VIEWS

-

This clause allows you to implement custom data processing at the data -source by providing a view definition against which pgloader will query -the data. It’s not possible to just allow for plain SQL because we want to -know a lot about the exact data types of each column involved in the query -output.

-

This clause expect a comma separated list of view definitions, each one -being either the name of an existing view in your database or the following -expression:

-
*name* `AS` `$$` *sql query* `$$`
-
-
-

The name and the sql query will be used in a CREATE VIEW statement at -the beginning of the data loading, and the resulting view will then be -dropped at the end of the data loading.

-
-
-

MATERIALIZE ALL VIEWS

-

Same behaviour as MATERIALIZE VIEWS using the dynamic list of views as -returned by MySQL rather than asking the user to specify the list.

-
-
-
-

MySQL Partial Migration

-
-

INCLUDING ONLY TABLE NAMES MATCHING

-

Introduce a comma separated list of table names or regular expression used -to limit the tables to migrate to a sublist.

-

Example:

-
including only table names matching ~/film/, 'actor'
-
-
-
-
-

EXCLUDING TABLE NAMES MATCHING

-

Introduce a comma separated list of table names or regular expression used -to exclude table names from the migration. This filter only applies to the -result of the INCLUDING filter.

-
excluding table names matching ~<ory>
-
-
-
-
-
-

MySQL Encoding Support

-
-

DECODING TABLE NAMES MATCHING

-

Introduce a comma separated list of table names or regular expressions -used to force the encoding to use when processing data from MySQL. If the -data encoding known to you is different from MySQL’s idea about it, this is -the option to use.

-
decoding table names matching ~/messed/, ~/encoding/ AS utf8
-
-
-

You can use as many such rules as you need, all with possibly different -encodings.

-
-
-
-

MySQL Schema Transformations

-
-

ALTER TABLE NAMES MATCHING

-

Introduce a comma separated list of table names or regular expressions -that you want to target in the pgloader ALTER TABLE command. The only two -available actions are SET SCHEMA and RENAME TO, both take a quoted -string as parameter:

-
ALTER TABLE NAMES MATCHING ~/_list$/, 'sales_by_store', ~/sales_by/
- SET SCHEMA 'mv'
-
-ALTER TABLE NAMES MATCHING 'film' RENAME TO 'films'
-
-ALTER TABLE NAMES MATCHING ~/./ SET (fillfactor='40')
-
-
-

You can use as many such rules as you need. The list of tables to be -migrated is searched in pgloader memory against the ALTER TABLE matching -rules, and for each command pgloader stops at the first matching criteria -(regexp or string).

-

No ALTER TABLE command is sent to PostgreSQL, the modification happens at -the level of the pgloader in-memory representation of your source database -schema. In case of a name change, the mapping is kept and reused in the -foreign key and index support.

-

The SET () action takes effect as a WITH clause for the CREATE TABLE -command that pgloader will run when it has to create a table.

-
-
-
-

MySQL Migration: limitations

-

The database command currently only supports MySQL source database and has -the following limitations:

-
-
    -

  • Views are not migrated,

    -

    Supporting views might require implementing a full SQL parser for the -MySQL dialect with a porting engine to rewrite the SQL against -PostgreSQL, including renaming functions and changing some constructs.

    -

    While it’s not theoretically impossible, don’t hold your breath.

    -
    • -

  • Triggers are not migrated

    -

    The difficulty of doing so is not yet assessed.

    -
    • -

  • Of the geometric datatypes, only the POINT database has been covered. -The other ones should be easy enough to implement now, it’s just not -done yet.

    -
    • -
-
-
-
-

Default MySQL Casting Rules

-

When migrating from MySQL the following Casting Rules are provided:

-

Numbers:

-
type int with extra auto_increment to serial when (< precision 10)
-type int with extra auto_increment to bigserial when (<= 10 precision)
-type int to int       when  (< precision 10)
-type int to bigint    when  (<= 10 precision)
-type tinyint   with extra auto_increment to serial
-type smallint  with extra auto_increment to serial
-type mediumint with extra auto_increment to serial
-type bigint    with extra auto_increment to bigserial
-
-type tinyint to boolean when (= 1 precision) using tinyint-to-boolean
-
-type tinyint when unsigned to smallint   drop typemod
-type smallint when unsigned to integer  drop typemod
-type mediumint when unsigned to integer  drop typemod
-type integer when unsigned to bigint    drop typemod
-
-type tinyint to smallint   drop typemod
-type smallint to smallint  drop typemod
-type mediumint to integer  drop typemod
-type integer to integer    drop typemod
-type bigint to bigint      drop typemod
-
-type float to float        drop typemod
-type double to double precision drop typemod
-
-type numeric to numeric keep typemod
-type decimal to decimal keep typemod
-
-
-

Texts:

-
type char       to char keep typemod using remove-null-characters
-type varchar    to varchar keep typemod using remove-null-characters
-type tinytext   to text using remove-null-characters
-type text       to text using remove-null-characters
-type mediumtext to text using remove-null-characters
-type longtext   to text using remove-null-characters
-
-
-

Binary:

-
-
type binary to bytea -type varbinary to bytea -type tinyblob to bytea -type blob to bytea -type mediumblob to bytea -type longblob to bytea
-

Date:

-
type datetime when default "0000-00-00 00:00:00" and not null
-  to timestamptz drop not null drop default
-      using zero-dates-to-null
-
-type datetime when default "0000-00-00 00:00:00"
-  to timestamptz drop default
-      using zero-dates-to-null
-
-type timestamp when default "0000-00-00 00:00:00" and not null
-  to timestamptz drop not null drop default
-      using zero-dates-to-null
-
-type timestamp when default "0000-00-00 00:00:00"
-  to timestamptz drop default
-      using zero-dates-to-null
-
-type date when default "0000-00-00" to date drop default
-      using zero-dates-to-null
-
-type date to date
-type datetime to timestamptz
-type timestamp to timestamptz
-type year to integer drop typemod
-
-
-

Geometric:

-
type point to point using pgloader.transforms::convert-mysql-point
-
-
-

Enum types are declared inline in MySQL and separately with a CREATE TYPE -command in PostgreSQL, so each column of Enum Type is converted to a type -named after the table and column names defined with the same labels in the -same order.

-

When the source type definition is not matched in the default casting rules -nor in the casting rules provided in the command, then the type name with -the typemod is used.

-
-
- - -
-
-
- -
-
-
- ©2017, Dimitri Fontaine. - - | - Powered by Sphinx 1.6.5 - & Alabaster 0.7.10 - - | - Page source -
- - - - - - \ No newline at end of file diff --git a/docs/_build/html/ref/sqlite.html b/docs/_build/html/ref/sqlite.html deleted file mode 100644 index 4da185d..0000000 --- a/docs/_build/html/ref/sqlite.html +++ /dev/null @@ -1,268 +0,0 @@ - - - - - - - Migrating a SQLite database to PostgreSQL — pgloader 3.4.1 documentation - - - - - - - - - - - - - - - - - - - - -
-
-
-
- -
-

Migrating a SQLite database to PostgreSQL

-

This command instructs pgloader to load data from a SQLite file. Automatic -discovery of the schema is supported, including build of the indexes.

-

Here’s an example:

-
load database
-     from sqlite:///Users/dim/Downloads/lastfm_tags.db
-     into postgresql:///tags
-
- with include drop, create tables, create indexes, reset sequences
-
-  set work_mem to '16MB', maintenance_work_mem to '512 MB';
-
-
-

The sqlite command accepts the following clauses and options.

-
-

SQLite Database Source Specification: FROM

-

Path or HTTP URL to a SQLite file, might be a .zip file.

-
-
-

SQLite Database Migration Options: WITH

-

When loading from a SQLite database, the following options are -supported:

-

When loading from a SQLite database, the following options are -supported, and the default WITH clause is: no truncate, create -tables, include drop, create indexes, reset sequences, downcase -identifiers, encoding ‘utf-8’.

-
-
    -

  • include drop

    -

    When this option is listed, pgloader drops all the tables in the target -PostgreSQL database whose names appear in the SQLite database. This -option allows for using the same command several times in a row until -you figure out all the options, starting automatically from a clean -environment. Please note that CASCADE is used to ensure that tables -are dropped even if there are foreign keys pointing to them. This is -precisely what include drop is intended to do: drop all target tables -and recreate them.

    -

    Great care needs to be taken when using include drop, as it will -cascade to all objects referencing the target tables, possibly -including other tables that are not being loaded from the source DB.

    -
    • -

  • include no drop

    -

    When this option is listed, pgloader will not include any DROP -statement when loading the data.

    -
    • -

  • truncate

    -

    When this option is listed, pgloader issue the TRUNCATE command -against each PostgreSQL table just before loading data into it.

    -
    • -

  • no truncate

    -

    When this option is listed, pgloader issues no TRUNCATE command.

    -
    • -

  • disable triggers

    -

    When this option is listed, pgloader issues an ALTER TABLE … DISABLE -TRIGGER ALL command against the PostgreSQL target table before copying -the data, then the command ALTER TABLE … ENABLE TRIGGER ALL once the -COPY is done.

    -

    This option allows loading data into a pre-existing table ignoring -the foreign key constraints and user defined triggers and may -result in invalid foreign key constraints once the data is loaded. -Use with care.

    -
    • -

  • create tables

    -

    When this option is listed, pgloader creates the table using the meta -data found in the SQLite file, which must contain a list of fields -with their data type. A standard data type conversion from SQLite to -PostgreSQL is done.

    -
    • -

  • create no tables

    -

    When this option is listed, pgloader skips the creation of table before -loading data, target tables must then already exist.

    -

    Also, when using create no tables pgloader fetches the metadata -from the current target database and checks type casting, then will -remove constraints and indexes prior to loading the data and install -them back again once the loading is done.

    -
    • -

  • create indexes

    -

    When this option is listed, pgloader gets the definitions of all the -indexes found in the SQLite database and create the same set of index -definitions against the PostgreSQL database.

    -
    • -

  • create no indexes

    -

    When this option is listed, pgloader skips the creating indexes.

    -
    • -

  • drop indexes

    -

    When this option is listed, pgloader drops the indexes in the target -database before loading the data, and creates them again at the end -of the data copy.

    -
    • -

  • reset sequences

    -

    When this option is listed, at the end of the data loading and after -the indexes have all been created, pgloader resets all the -PostgreSQL sequences created to the current maximum value of the -column they are attached to.

    -
    • -

  • reset no sequences

    -

    When this option is listed, pgloader skips resetting sequences after the -load.

    -

    The options schema only and data only have no effects on this -option.

    -
    • -

  • schema only

    -

    When this option is listed pgloader will refrain from migrating the data -over. Note that the schema in this context includes the indexes when the -option create indexes has been listed.

    -
    • -

  • data only

    -

    When this option is listed pgloader only issues the COPY statements, -without doing any other processing.

    -
    • -

  • encoding

    -

    This option allows to control which encoding to parse the SQLite text -data with. Defaults to UTF-8.

    -
    • -
-
-
-
-

SQLite Database Casting Rules

-

The command CAST introduces user-defined casting rules.

-

The cast clause allows to specify custom casting rules, either to overload -the default casting rules or to amend them with special cases.

-
-
-

SQlite Database Partial Migrations

-
-

INCLUDING ONLY TABLE NAMES LIKE

-

Introduce a comma separated list of table name patterns used to limit the -tables to migrate to a sublist.

-

Example:

-
including only table names like 'Invoice%'
-
-
-
-
-

EXCLUDING TABLE NAMES LIKE

-

Introduce a comma separated list of table name patterns used to exclude -table names from the migration. This filter only applies to the result of -the INCLUDING filter.

-
excluding table names like 'appointments'
-
-
-
-
-
-

Default SQLite Casting Rules

-

When migrating from SQLite the following Casting Rules are provided:

-

Numbers:

-
type tinyint to smallint using integer-to-string
-type integer to bigint   using integer-to-string
-
-type float to float   using float-to-string
-type real to real     using float-to-string
-type double to double precision     using float-to-string
-type numeric to numeric     using float-to-string
-
-
-

Texts:

-
type character  to text drop typemod
-type varchar    to text drop typemod
-type nvarchar   to text drop typemod
-type char       to text drop typemod
-type nchar      to text drop typemod
-type nvarchar   to text drop typemod
-type clob       to text drop typemod
-
-
-

Binary:

-
type blob       to bytea
-
-
-

Date:

-
type datetime    to timestamptz using sqlite-timestamp-to-timestamp
-type timestamp   to timestamptz using sqlite-timestamp-to-timestamp
-type timestamptz to timestamptz using sqlite-timestamp-to-timestamp
-
-
-
-
- - -
-
-
- -
-
-
- ©2017, Dimitri Fontaine. - - | - Powered by Sphinx 1.6.5 - & Alabaster 0.7.10 - - | - Page source -
- - - - - - \ No newline at end of file diff --git a/docs/_build/html/ref/transforms.html b/docs/_build/html/ref/transforms.html deleted file mode 100644 index e3cffd8..0000000 --- a/docs/_build/html/ref/transforms.html +++ /dev/null @@ -1,214 +0,0 @@ - - - - - - - Transformation Functions — pgloader 3.4.1 documentation - - - - - - - - - - - - - - - - - - - - -
-
-
-
- -
-

Transformation Functions

-

Some data types are implemented in a different enough way that a -transformation function is necessary. This function must be written in -Common lisp and is searched in the pgloader.transforms package.

-

Some default transformation function are provided with pgloader, and you can -use the –load command line option to load and compile your own lisp file -into pgloader at runtime. For your functions to be found, remember to begin -your lisp file with the following form:

-
(in-package #:pgloader.transforms)
-
-
-

The provided transformation functions are:

-
-
    -

  • zero-dates-to-null

    -

    When the input date is all zeroes, return nil, which gets loaded as a -PostgreSQL NULL value.

    -
    • -

  • date-with-no-separator

    -

    Applies zero-dates-to-null then transform the given date into a format -that PostgreSQL will actually process:

    -
    In:  "20041002152952"
-Out: "2004-10-02 15:29:52"
-
    -
    -
    • -

  • time-with-no-separator

    -

    Transform the given time into a format that PostgreSQL will actually -process:

    -
    In:  "08231560"
-Out: "08:23:15.60"
-
    -
    -
    • -

  • tinyint-to-boolean

    -

    As MySQL lacks a proper boolean type, tinyint is often used to -implement that. This function transforms 0 to ‘false’ and anything -else to ‘true’.

    -
    • -

  • bits-to-boolean

    -

    As MySQL lacks a proper boolean type, BIT is often used to implement -that. This function transforms 1-bit bit vectors from 0 to f and any -other value to t..

    -
    • -

  • int-to-ip

    -

    Convert an integer into a dotted representation of an ip4.

    -
    In:  18435761
-Out: "1.25.78.177"
-
    -
    -
    • -

  • ip-range

    -

    Converts a couple of integers given as strings into a range of ip4.

    -
    In:  "16825344" "16825599"
-Out: "1.0.188.0-1.0.188.255"
-
    -
    -
    • -

  • convert-mysql-point

    -

    Converts from the astext representation of points in MySQL to the -PostgreSQL representation.

    -
    In:  "POINT(48.5513589 7.6926827)"
-Out: "(48.5513589,7.6926827)"
-
    -
    -
    • -

  • integer-to-string

    -

    Converts a integer string or a Common Lisp integer into a string -suitable for a PostgreSQL integer. Takes care of quoted integers.

    -
    In:  "\"0\""
-Out: "0"
-
    -
    -
    • -

  • float-to-string

    -

    Converts a Common Lisp float into a string suitable for a PostgreSQL float:

    -
    In:  100.0d0
-Out: "100.0"
-
    -
    -
    • -

  • set-to-enum-array

    -

    Converts a string representing a MySQL SET into a PostgreSQL Array of -Enum values from the set.

    -
    In: "foo,bar"
-Out: "{foo,bar}"
-
    -
    -
    • -

  • empty-string-to-null

    -

    Convert an empty string to a null.

    -
    • -

  • right-trim

    -

    Remove whitespace at end of string.

    -
    • -

  • remove-null-characters

    -

    Remove NUL characters (0x0) from given strings.

    -
    • -

  • byte-vector-to-bytea

    -

    Transform a simple array of unsigned bytes to the PostgreSQL bytea Hex -Format representation as documented at -http://www.postgresql.org/docs/9.3/interactive/datatype-binary.html

    -
    • -

  • sqlite-timestamp-to-timestamp

    -

    SQLite type system is quite interesting, so cope with it here to produce -timestamp literals as expected by PostgreSQL. That covers year only on 4 -digits, 0 dates to null, and proper date strings.

    -
    • -

  • sql-server-uniqueidentifier-to-uuid

    -

    The SQL Server driver receives data fo type uniqueidentifier as byte -vector that we then need to convert to an UUID string for PostgreSQL -COPY input format to process.

    -
    • -

  • unix-timestamp-to-timestamptz

    -

    Converts a unix timestamp (number of seconds elapsed since beginning of -1970) into a proper PostgreSQL timestamp format.

    -
    • -

  • varbinary-to-string

    -

    Converts binary encoded string (such as a MySQL varbinary entry) to a -decoded text, using the table’s encoding that may be overloaded with the -DECODING TABLE NAMES MATCHING clause.

    -
    • -
-
-
- - -
-
-
- -
-
-
- ©2017, Dimitri Fontaine. - - | - Powered by Sphinx 1.6.5 - & Alabaster 0.7.10 - - | - Page source -
- - - - - - \ No newline at end of file diff --git a/docs/_build/html/search.html b/docs/_build/html/search.html deleted file mode 100644 index 35dce69..0000000 --- a/docs/_build/html/search.html +++ /dev/null @@ -1,101 +0,0 @@ - - - - - - - Search — pgloader 3.4.1 documentation - - - - - - - - - - - - - - - - - - - - - - - - -
-
-
-
- -

Search

-
- -

- Please activate JavaScript to enable the search - functionality. -

-
-

- From here you can search these documents. Enter your search - words into the box below and click "search". Note that the search - function will automatically search for all of the words. Pages - containing fewer words won't appear in the result list. -

-
- - - -
- -
- -
- -
-
-
- -
-
-
- ©2017, Dimitri Fontaine. - - | - Powered by Sphinx 1.6.5 - & Alabaster 0.7.10 - -
- - - - - - \ No newline at end of file diff --git a/docs/_build/html/searchindex.js b/docs/_build/html/searchindex.js deleted file mode 100644 index 14a0261..0000000 --- a/docs/_build/html/searchindex.js +++ /dev/null @@ -1 +0,0 @@ -Search.setIndex({docnames:["bugreport","index","intro","pgloader","pgloader-usage-examples","ref/archive","ref/copy","ref/csv","ref/dbf","ref/fixed","ref/ixf","ref/mssql","ref/mysql","ref/sqlite","ref/transforms","tutorial/csv","tutorial/dBase","tutorial/fixed","tutorial/geolite","tutorial/mysql","tutorial/quickstart","tutorial/sqlite","tutorial/tutorial"],envversion:53,filenames:["bugreport.rst","index.rst","intro.rst","pgloader.rst","pgloader-usage-examples.rst","ref/archive.rst","ref/copy.rst","ref/csv.rst","ref/dbf.rst","ref/fixed.rst","ref/ixf.rst","ref/mssql.rst","ref/mysql.rst","ref/sqlite.rst","ref/transforms.rst","tutorial/csv.rst","tutorial/dBase.rst","tutorial/fixed.rst","tutorial/geolite.rst","tutorial/mysql.rst","tutorial/quickstart.rst","tutorial/sqlite.rst","tutorial/tutorial.rst"],objects:{},objnames:{},objtypes:{},terms:{"000s":[19,21,22],"001s":[19,22],"003s":[21,22],"006s":[19,21,22],"007s":[19,22],"008s":[21,22],"010s":[16,22],"01234562008052115182300left":9,"01234567892008052011431250firstlin":9,"012s":[18,19,21,22],"013s":[17,21,22],"014s":[21,22],"016s":[21,22],"017s":[21,22],"018s":[19,21,22],"019s":[15,17,18,22],"021s":[21,22],"023s":[19,21,22],"025s":[21,22],"027s":[21,22],"031s":[21,22],"032s":[21,22],"033s":[21,22],"034s":[21,22],"035s":[19,22],"036s":[19,22],"037s":[19,22],"039s":[15,22],"040s":[21,22],"041s":[19,22],"043s":[21,22],"052s":[21,22],"058s":[15,19,22],"059s":[19,22],"068s":[19,22],"070s":[19,21,22],"071s":[16,19,21,22],"073s":[19,22],"077s":[21,22],"080s":[19,22],"0d0":14,"0x0":14,"0x27":7,"105s":[21,22],"111s":[21,22],"12345678902008052208231560anoth":9,"128mb":[0,12,19,22],"12mb":[12,15,19,22],"130s":[19,22],"135s":[19,22],"136s":[19,22],"14mb":[6,9],"150s":[19,22],"154s":[19,22],"160s":[21,22],"167s":[16,22],"16mb":[11,13,21,22],"16t08":[19,22],"1m8":[18,22],"2013_gaz_113cds_nat":[4,20,22],"205s":[19,22],"208s":[19,22],"20t16":[21,22],"214s":[19,22],"227s":[19,22],"228s":[19,22],"236s":[19,22],"247s":[19,22],"254s":[19,22],"268s":[19,22],"279s":[21,22],"303s":[19,22],"313s":[21,22],"332s":[18,22],"365s":[19,22],"390s":[18,22],"402s":[19,22],"42p16":[21,22],"476s":[21,22],"494s":[17,22],"497s":[19,22],"499s":[17,22],"502s":[19,22],"527s":[19,22],"539s":[19,22],"547s":[19,22],"583s":[19,22],"592s":[18,22],"648s":[19,22],"658s":[16,19,22],"669s":[21,22],"692s":[18,22],"743s":[18,22],"877s":[21,22],"892s":[19,22],"906s":[16,22],"964s":[19,22],"9n8v8pw54t1gngfff0lj16040000gn":[16,17,18,22],"boolean":[11,12,14,19,22],"byte":[3,6,9,11,14],"case":[1,2,3,4,11,12,13,19,20,21,22],"char":[11,12,13,15,17,22],"default":[1,3,7,10,14,19,22],"enum":[12,14],"export":[3,11],"final":[1,4,18,20,22],"float":[11,12,13,14,18,22],"function":[1,3,12],"import":[3,4,15,16,17,18,19,20,21,22],"int":[12,14],"long":[17,22],"new":3,"null":[5,6,7,9,12,14,17,18,19,22],"public":[0,11,12],"return":[12,14],"short":0,"static":3,"switch":[3,4,20,22],"true":14,"try":[3,4,20,22],"var":[16,17,18,22],"while":[3,4,12,18,19,20,22],AND:[5,12,18,22],And:[1,18,22],For:[0,3,4,12,14,20,22],INTO:[0,5,6,7,8,9,10,12,15,16,17,18,22],Its:[21,22],NOT:[3,12],One:3,That:[2,3,7,12,14,18,22],The:[2,3,4,5,6,7,8,9,10,11,12,13,14,20],Then:[3,5],USING:[3,18,21,22],Use:[0,3,4,6,7,8,9,10,12,13],Useful:3,Using:[3,18,22],WITH:[0,1,5,15,16,17,18,19,22],_list:12,abil:3,abl:[2,3,12],about:[1,4,12,20,22],abov:[3,4,6,7,9,11,12,15,20,22],absolut:[3,6,7,9],accept:[2,3,5,6,7,8,9,10,11,12,13,21,22],access:7,accomod:3,accumul:[3,11],achiev:[11,12],action:12,activ:[3,12],actor:[12,19,22],actual:[3,14,18,22],add:[19,21,22],added:11,addit:3,address:[6,8,10,18,19,22],administr:[16,22],advanc:[3,19,21,22],aen66609:3,after:[2,6,7,9,12,13],again:[2,3,4,7,12,13,19,20,22],against:[3,4,5,6,7,8,9,10,12,13,20,21,22],al0100124abbevil:[17,22],al0100460adamsvil:[17,22],al0100484addison:[17,22],al0100676akron:[17,22],al0100820alabast:[17,22],al0100988albertvil:[17,22],al0101132alexand:[17,22],aland:[4,20,22],aland_sqmi:[4,20,22],album:[6,21,22],all:[2,3,4,5,6,7,8,9,10,11,13,14,19,20,21,22],allow:[2,3,5,6,7,8,9,10,11,12,13,15,17,19,21,22],alon:[19,22],alreadi:[3,12,13],also:[3,4,5,7,9,12,13,18,20,21,22],alter:[3,6,7,8,9,10,13,21,22],amend:[11,12,13],amount:3,ampersand:3,ani:[2,3,6,7,9,11,12,13,14],anoth:3,anyth:[3,14],anywai:3,api:[19,22],app:0,appdata:3,appear:[3,12,13],append:12,appli:[3,5,6,7,9,11,12,13,14],applic:12,application_nam:0,appoint:13,appropri:[18,22],arbitrari:3,architectur:[2,3],archiv:[1,3,16,17,18],areacod:[5,18,22],argument:[1,4,6,7,9,12,20,22],around:[3,19,22],arrai:14,arrang:9,artist:[21,22],ascii:[6,7],ask:12,aspect:2,assess:12,associ:12,astext:14,attach:[12,13],attempt:3,augment:12,auto_incr:[12,19,22],autom:[2,3],automat:[7,9,11,12,13,19,22],avail:[2,3,12],awai:[3,19,22],awat:[4,20,22],awater_sqmi:[4,20,22],back:[12,13],backslash:[0,7],backward:3,bad:3,baddata:0,bar:14,base:[3,4,12,20,22],basic:3,batch:1,battl:[4,20,22],becaus:[3,4,12,20,22],been:[3,5,12,13,15,16,17,18,19,21,22],befor:[0,2,5,6,7,8,9,10,11,12,13,15,16,17,18,19,21,22],begin:[3,6,7,9,12,14],behavior:[3,12],behaviour:[1,7,12],being:[3,4,12,13,20,22],below:[3,7,9],benefit:[16,19,21,22],best:7,better:3,between:[0,4,6,7,9,20,22],bigint:[4,6,12,13,15,20,22],bigseri:[6,12,19,22],binari:[3,11,12,13,14],bit:[3,11,12,14],blank:[5,7,9,17,18,22],blob:[12,13],block:[5,7,18,22],blocks_ip4r_idx:[5,18,22],both:[3,7,9,12,19,22],bracket:[7,9],breath:12,brows:[15,16,17,18,19,21,22],buffer:[4,20,22],bug:1,build:[11,12,13,19,21,22],built:[7,12],bypass:[3,4,20,22],bytea:[11,12,13,14],calendar:[19,22],can:[2,3,4,5,6,7,9,11,12,14,15,16,17,18,19,20,21,22],canada:[15,22],capabl:[16,19,21,22],care:[0,4,6,7,8,9,10,12,13,14,19,20,21,22],cascad:[11,12,13,18,22],cast:[1,3,4,19,20,22],categori:[19,22],caus:3,censu:[17,22],chanc:12,chang:[3,11,12],charact:[3,4,6,7,9,12,13,14,20,22],characterist:3,charset:11,check:[12,13],checkout:3,chinook:[21,22],chinook_sqlite_autoincrementpk:[21,22],chinookdatabas:[21,22],choic:[19,22],circuit:[19,22],citi:[5,17,18,19,22],claus:[1,5,6,7,8,9,10,11,12,13,14,19,22],clean:[12,13],client:[3,11],client_encod:[15,16,22],clob:13,close:3,code:[3,6,7,12],cog:[4,8,16,20,22],col:1,collid:3,colon:3,column:[3,4,7,9,12,13,15,17,18,19,20,22],com:[0,12,18,19,21,22],come:3,comma:[3,7,9,11,12,13,15,22],command:[1,6,7,8,9,10,11,12,13,14,20],comment:[19,21,22],commerci:[19,22],common:[1,11,12,14],commonli:[4,20,22],compat:3,compil:[3,14],complet:[3,19,21,22],complex:[3,12],compon:[3,12],compos:[6,9],compress:4,comput:3,concurr:[3,12,21,22],condit:[7,12],conf:[3,11],config:3,configur:[3,11],confus:3,connect:[2,4,11,12,20,22],connstr:3,consid:3,consist:[3,18,22],consol:3,constraint:[3,6,7,8,9,10,11,12,13,19,21,22],construct:12,constructor:[19,22],constructorresult:[19,22],constructorstand:[19,22],contain:[0,2,3,4,5,6,7,8,9,10,12,13,18,20,21,22],content:[1,4,5,15,17,18,19,20,22],context:[3,11,12,13],continu:[1,3],control:[3,13],convers:[8,10,12,13],convert:[7,9,12,14],cope:[3,12,14,19,22],copi:[1,2,3,7,8,9,10,12,13,14,19,21,22],copyright:[4,20,22],could:[3,4,7,9,20,22],count:3,countri:[5,18,19,22],coupl:[3,14,17,22],cours:[6,8,10,18,22],cover:[3,12,14],creat:[0,3,4,5,6,7,8,9,10,11,12,13,15,16,17,18,19,20,21,22],createdb:[4,19,20,21,22],creation:[12,13],criteria:12,critic:3,cross:3,csv:[0,1,2,3,5,18],curl:[4,20,22],current:[3,4,5,7,9,11,12,13],current_timestamp:[19,22],custom:[3,11,12,13,19,21,22],cut:3,dai:[2,7,9],dash:[3,4,20,22],dat:[2,3],data:[1,2,3,4,5,6,9,11,12,13,14,19],databas:[1,2,3,4,15,18,19,20,21,22],dataset:[18,22],datasourc:[21,22],datatyp:[12,14,18,22],date:[7,9,11,12,13,14,19,22],datetim:[11,12,13,19,22],datetime2:11,db3:3,dba:3,dbase:[1,3],dbf:[1,2,3,5,6,10,12,16],dbname:[3,7,8,11],dbo:11,dbpath:3,ddl:12,decim:[9,11,12],declar:12,decod:14,decompress:3,defin:[3,6,7,8,9,10,12,13,15,16,17,18,19,21,22],definit:[3,4,11,12,13,19,20,21,22],delai:0,delimit:[3,6],denot:[6,7],depend:[2,3,5,11,12],describ:[2,3,6,7,9,12,15,16,19,21,22],descript:[0,2,7,9],detail:[3,11,12,15,16,17,18,19,21,22],determin:[3,12],dev:[12,15,16,17,18,19,21,22],develop:3,dialect:[12,19,22],did:[3,16,17,21,22],didn:3,differ:[0,2,3,12,14],difficulti:12,digit:[3,7,9,14],dim:[0,5,13,15,16,17,18,19,21,22],dir:3,direct:[4,20,22],directli:[2,3,18,22],directori:[3,5,6,7,9],disabl:[3,6,7,8,9,10,12,13],discov:[4,20,22],discoveri:[3,11,13],distribut:12,district:4,districts_longlat:[4,20,22],divid:12,doc:[3,12,14,17,22],document:[2,3,4,6,14,20,22],doe:[3,19,21,22],doesn:[11,12],doing:[12,13,19,22],dollar:3,domain:3,don:[3,12],done:[3,6,7,8,9,10,11,12,13,18,19,21,22],dot:[3,14],doubl:[3,4,5,6,7,9,11,12,13,15,18,20,22],down:[3,4,20,22],downcas:[12,13],download:[4,5,13,16,17,18,19,20,21,22],drawback:3,driven:[4,20,22],driver:[1,14,19,22],driverstand:[19,22],drop:[0,6,7,9,10,11,12,13,15,17,18,19,21,22],dry:3,dsl:[2,3],dynam:[3,12,18,22],each:[3,7,9,12,13,17,18,22],easi:[0,12,20,22],easier:3,edit:[15,16,17,18,19,21,22],effect:[12,13],either:[2,3,4,6,7,9,10,11,12,13,19,20,22],elaps:14,els:[3,14],email:0,embed:[21,22],emit:3,employe:[21,22],empti:[3,6,7,9,12,14,19,21,22],enabl:[3,6,7,8,9,10,12,13],enclos:[0,3,5,7,9,15,18,22],encod:[1,3,5,7,9,13,14,17,18,22],end:[3,6,7,9,12,13,14,18,19,22],endipnum:[5,7,18,22],engin:[11,12],enjoi:2,enough:[3,12,14],ensur:[4,11,12,13,20,22],enter:[3,9],entir:[4,12,20,22],entri:14,enumer:12,env:3,environ:[2,3,4,11,12,13,20,22],equal:3,equival:[3,4,20,22],ergast:[19,22],erron:3,error:[2,3,15,16,17,18,19,21,22],escap:[0,3,4,5,7,15,18,22],etc:[3,21,22],europ:10,evalu:12,even:[12,13],ever:3,everi:2,exact:[5,6,7,9,12],exampl:[0,3,5,6,7,8,9,10,11,12,13,15,16,17,18,19,20,22],excel:[16,22],except:[3,12],exclud:[19,22],exclus:7,execut:[4,5],exist:[0,3,4,5,6,7,8,9,10,11,12,13,15,17,18,19,20,21,22],exit:3,expand:[3,4,5,20,22],expect:[0,2,3,7,8,9,10,11,12,14,19,21,22],explain:0,explan:11,explicit:3,explicitli:12,express:[5,6,7,9,11,12],ext:3,extens:[0,3,5,18,22],extern:11,extra:[3,4,12,20,22],extract:[16,17,18,22],f1db:[19,22],facil:3,facilit:[15,16,17,18,19,21,22],fact:[19,22],fail:[3,12],fals:14,familiar:3,fashion:3,fast:3,faster:3,feasibl:[4,20,22],featur:[2,3,12],feed:3,feedback:3,fetch:[2,3,6,8,10,12,13,16,17,18,19,21],field:[0,1,3,4,5,8,10,12,13,15,18,20,22],figur:[12,13],file:[1,2,3,5,7,8,10,11,12,13,14,15,18,19],filemak:[16,22],filenam:[3,5,6,7,8,9,10,17,18,22],fill:[2,3],fillfactor:12,film:[12,19,22],film_actor:[19,22],film_categori:[19,22],film_list:[12,19,22],film_text:[19,22],filter:[11,12,13,15,22],finali:[19,22],find:[3,6,7,9,11,12,16,17,18,22],fip:[17,22],fips_cod:[17,22],first:[3,4,5,6,7,9,12,17,20,22],fit:[3,4,20,22],fix:[1,2,3,5],fledg:3,flexibl:[3,12],flight:11,fly:[2,3,12,18,22],folder:[16,17,18,22],follow:[0,2,3,4,5,6,7,8,9,10,11,12,13,14,15,17,20,22],foo:[0,4,12,14,18,20,22],forc:[3,12],foreign:[4,6,7,8,9,10,11,12,13,19,20,21,22],fork:2,form:[3,12,14],format:[1,2,4,5,7,8,10,14,16,17,18,20,22],former:3,found:[3,4,6,7,8,9,10,11,12,13,14,15,16,18,19,20,21,22],framework:12,free:[18,22],freetd:11,french:[16,22],friendli:3,from:[0,1,2,14,15,16,17,18,21],full:[2,3,12,17,18,22],fulli:[2,3],gazett:[17,22],gazetteer2000:[17,22],gener:12,genr:[6,21,22],geo:[17,22],geographi:11,geoid:[4,20,22],geoip:[18,22],geolit:[1,5,7],geolitec:[5,7,18,22],geolitecity_csv:[18,22],geoloc:[18,22],geometr:12,get:[3,5,7,11,12,13,14,15,22],getenv:3,gist:[5,18,22],git:3,github:[3,21,22],give:[2,19,22],given:[2,3,4,5,6,7,9,12,14,17,18,19,20,21,22],global:[3,11],globalaccount:11,gmt:10,goal:2,good:[3,21,22],gov:[17,22],grammar:3,great:[2,12,13],greater:12,green:2,grow:3,guard:12,guc:3,guc_nam:3,guess:[4,20,22],gunzip:[4,20,22],had:[19,22],handl:[3,12,21,22],happen:[3,4,12,20,22],has:[3,4,5,12,13,15,16,17,18,19,20,21,22],have:[0,2,3,4,7,12,13,19,20,21,22],header:[3,4,5,6,7,9,15,18,20,22],help:[0,3,4,21,22],here:[3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22],hex:14,hexadecim:[6,7,9],hh12:[7,9],hh24:[7,9],hierarchyid:11,histor:[19,22],historiq2013:[4,16,20,22],hold:12,host:[3,4,11,12,16,19,20,21,22],hostnam:3,hour:[7,9],how:[2,3,4,9,12,15,18,20,22],hstore:0,html:[3,12,14,17,22],http:[3,5,6,8,10,12,13,14,15,16,17,18,19,21],human:3,ibm:[3,10],idea:12,identifi:[0,3,12,13],idx_66873_sqlite_autoindex_playlisttrack_1:[21,22],idx_oid:12,ignor:[6,7,8,9,10,12,13],imag:3,impact:3,implement:[2,3,7,9,11,12,14],imposs:12,includ:[0,3,4,7,18,19,20,21,22],index:[1,3,4,5,7,11,12,13,17,18,19,20,21,22],info:3,inforamt:4,inform:[12,16,19,20,21,22],ini:3,inlin:[0,3,6,7,9,12],input:[0,2,3,4,6,7,9,14,15,18,20,22],inse:[4,8,16,20,22],insid:[6,7],inspir:3,instal:[12,13,19,21,22],instanc:[3,11],instead:[18,22],instruct:[5,6,7,8,9,10,11,12,13],integ:[9,12,13,14,18,22],integr:2,intend:[12,13],interact:14,interest:14,intern:[11,12],intervent:2,intptlat:[4,20,22],intptlong:[4,20,22],introduc:[3,7,11,12,13],introduct:1,invalid:[6,7,8,9,10,12,13],inventori:[19,22],invoic:[13,21,22],invoicelin:[21,22],involv:12,ip4:14,ip4r:[5,7,18,22],iprang:[5,7,18,22],ipv4:[3,18,22],isn:[3,12],iso:[5,7,18,22],isol:3,issu:[6,7,8,9,10,12,13],its:[2,3,4,8,10,12,15,16,17,18,19,20,21,22],itself:[3,6,7,11,16,17,22],ixf:[1,3],join:[18,22],json:3,just:[3,4,7,12,13,20,22],keep:[7,12],kei:[0,3,4,6,7,8,9,10,11,12,13,18,19,20,21,22],kept:[3,12],keyword:[7,9,12],kick:12,kind:[2,3,4,20,22],kingdom:[15,22],know:[2,3,7,9,12],known:[3,12],label:12,lack:[11,14],land:[17,22],languag:[2,3,19,22],laptim:[19,22],larg:3,last:[3,12],lastest:[18,22],lastfm_tag:13,lat:[17,22],later:[3,4,20,22],latest:[5,18,22],latin1:[15,16,17,22],latitud:[5,18,22],ldm:[17,22],learn:3,least:12,leav:[3,19,22],left:[3,7,9,12],length:[3,9,17,22],lerocha:[21,22],less:12,let:[16,17,19,21,22],letter:3,level:[2,3,5,12,16,22],lib:11,libpq:3,light:[18,22],like:[3,12],lile:11,limit:[1,3,7,9,11,13],line:[1,4,5,6,7,9,14,17,20],link:5,lisp:[3,11,12,14],list:[3,6,7,8,9,10,11,12,13,16,22],listen:11,liter:14,live:2,load:[0,1,2,11,12,13,14,19],loc_nam:[17,22],local:[3,4,5,6,8,10,20,22],localaccount:11,localhost:[0,3,4,7,8,12,19,20,22],locat:[3,5,10,16,17,18,21],locationnam:[17,22],locid:[5,7,18,22],lock:7,log:[2,3,15,16,17,18,19,21,22],logfil:3,longblob:12,longitud:[5,18,22],longtext:12,look:3,lot:12,machin:3,mai:[3,4,6,7,8,9,10,11,12,13,14,19,20,21,22],main:[3,18,19,22],maintenance_work_mem:[7,11,12,13,19,21,22],make:[3,21,22],man:[4,20,22],manag:[2,3,21,22],mani:[3,6,7,9,12],manner:9,manual:[1,2],map:[12,17,22],master:[21,22],match:[3,4,5,6,7,9,14,17,18,19,20,22],materi:[19,22],max:[3,12],maximum:[3,12,13],maxmind:1,mayb:[4,20,22],mean:[4,12,15,20,22],meant:[2,3],media:6,mediatyp:[21,22],mediumblob:12,mediumint:12,mediumtext:12,memori:[3,12],mess:12,messag:[3,19,21,22],met:12,meta:[8,10,12,13,16,19,21,22],metadata:[2,12,13],method:[3,4,8,16,20,22],methodolog:2,metrocod:[5,18,22],microsecond:[7,9],middl:3,might:[3,4,5,6,7,12,13,20,22],migrat:[1,21],million:[18,22],millisecond:[6,7,9],min:[3,12],mind:3,minimum:3,minut:[7,9],miss:3,mix:3,mode:[3,7,12],model:[3,7,9],modern:[16,22],modif:12,modul:1,moment:5,monei:11,month:[7,9],more:[4,5,11,16,17,20,21,22],most:[0,3,4,19,20,22],motor:[19,22],mrd:[19,22],mssql:[3,11],much:3,multipl:[12,21,22],multipli:3,must:[2,3,4,5,6,7,8,9,10,11,12,13,14,20,22],mustach:1,mysql:[1,2,3,11,14],mysql_host:12,mysql_pwd:12,mysql_tcp_port:12,name:[0,3,4,5,6,7,8,9,10,14,15,16,17,18,19,20,21,22],nativ:3,natur:[21,22],nchar:13,nchat:11,necessari:[3,4,14,20,22],need:[0,2,3,4,7,12,13,14,15,16,17,18,19,20,21,22],net_read_timeout:12,net_write_timeout:12,netloc:3,network:[3,4,20,22],never:3,newdb:[4,20,22],newer:3,night:2,nil:[5,14,18,22],nomenclatur:[4,8,16,20,22],non:[3,5,19,22],none:3,nor:[3,11,12],notat:3,note:[1,4,5,12,13,15,16,17,18,19,20,21,22],notic:[3,4,20,22],now:[4,12,19,20,21,22],nsitra:10,nul:14,number:[3,6,7,9,11,12,13,14],numer:[3,6,7,9,11,12,13],nvarchar:[11,13],object:[12,13],obtain:12,occur:3,offer:[16,18,22],often:[14,15,22],oid:[12,19,21,22],old:[3,4,20,22],omit:[3,12],onc:[3,5,6,7,8,9,10,12,13,19,22],one:[0,3,4,5,7,9,11,12,20,22],ones:[3,12],onli:[2,4,5,6,7,9,14,15,17,19,20,22],onlin:[3,15,16,17,18,19,21,22],open:[3,4,20,22],oper:[15,16,17,18,19,21,22],option:[1,2,4,5,14,15,18,19,20,21,22],order:[2,3,7,12],ordinari:7,org:[3,4,14,18,20,21,22],ori:[12,19,22],other:[0,3,7,11,12,13,14,19,22],otherwis:[3,12],ouput:[0,15,18,22],our:[12,15,16,17,18,19,21,22],out:[3,12,13,14],output:[0,3,12,16,17,19,21,22],over:[2,3,4,12,13,19,20,22],overload:[11,12,13,14],overrid:3,overs:3,own:[2,3,14],packag:[3,12,14],pad:[9,17,22],page:[1,4,20,22],pagila:[4,12,20,22],pair:[2,3,18,22],parallel:[1,7],paramet:[3,7,9,12],pari:10,pars:[3,4,6,7,9,10,13,15,16,17,18,19,21,22],parser:[3,7,12],part:[3,7,9,12],partial:1,particular:[3,19,22],pass:3,password:[3,12],past:12,path:[3,13,15,22],pattern:[7,9,11,13],payment:[19,22],per:[3,7,9,12,19,22],percent:3,perform:[1,7],pg_get_keyword:12,pgdatabas:3,pghost:3,pgloader:[0,2,5,6,7,8,9,10,11,12,13,14,19],pgpass:3,pgpassfil:3,pgpassword:3,pgsql:[4,12,18,19,20,21,22],pguser:3,pick:3,piggyback:11,pipe:[3,4,20,22],pitstop:[19,22],place:[3,16,17,22],places2k:[17,22],plai:3,plain:12,playlist:[21,22],playlisttrack:[21,22],pleas:[3,4,11,12,13,20,22],point:[3,5,12,13,14,18,22],popular:[18,22],port:[3,11,12],posit:[9,17,22],possibl:[0,3,4,6,7,8,9,10,12,13,19,20,22],postalcod:[5,18,22],postgr:[4,20,21,22],postgreslq:3,postgresql:[0,1,2,3,4,5,6,7,8,9,10,14,15,16,17,18,20,21],postgresqlpgpass:3,pow:[18,22],pre:[3,4,6,7,8,9,10,12,13,20,22],precis:[4,11,12,13,20,22],prefer:3,prefetch:3,prefix:12,prepar:3,preserv:12,prevent:12,primari:[0,7,11,12,18,19,21,22],print:3,printabl:[6,7],prior:[12,13],privat:[16,17,18,19,21,22],problem:[3,19,22],problemat:0,process:[3,4,5,7,9,12,13,14,18,20,22],produc:14,product:2,project:[2,3,21,22],proper:[3,14],properti:[2,4],protocol:[2,3],provid:[2,3,4,11,12,13,14,16,17,18,19,20,22],psql:3,punctuat:[3,7,9],purpos:[19,22],push:3,qualifi:[3,8,10,19,22],queri:[3,5,12,18,21,22],queue:3,quick:[1,18],quiet:3,quit:[14,15,18,22],quot:[0,3,4,5,6,7,9,10,12,14,15,18,22],race:[19,22],rag:3,rang:[5,7,12,14,17,18,22],rather:[3,12],raw:[2,3,21,22],read:[2,3,6,7,8,9,10,12,15,16,17,18,19,21],readabl:3,reader:[3,7,9,12],readi:[15,16,17,18,19,21,22],real:[3,11,13],reason:3,receiv:14,recent:3,recogn:[4,7,20,22],recommand:0,record:[18,19,22],recreat:[12,13],redund:4,refer:[1,4,11,12,20,22],referenc:[12,13],refman:12,refrain:[11,12,13],refus:3,reg2013:8,regardless:12,regexp:[6,7,9,12],region:[3,5,18,22],regular:[5,6,7,9,11,12],reject:[2,3,19,22],rel:[3,6,7,9],reload:3,remain:3,rememb:14,remov:[7,12,13,14],renam:12,rental:[19,22],replac:3,report:[1,19,21,22],repres:[14,18,22],represent:[6,12,14,18,22],reproduc:1,requir:[3,9,12],reserv:[12,17,22],reset:[12,13,19,21,22],resp:3,respect:[3,12,21,22],respons:3,restrict:3,result:[3,6,7,8,9,10,11,12,13,19],retri:1,reus:12,review:4,rewrit:12,rich:3,right:[3,4,7,9,14,16,17,20,22],root:[3,12,19,22],row:[3,12,13,15,22],rule:[1,3,5,6,7,9,19,22],run:[3,4,5,12,19,20,21,22],runtim:[3,14],sai:3,sakila:[4,12,19,20,22],sales_bi:12,sales_by_stor:12,same:[3,4,5,6,7,8,9,10,11,12,13,20,21,22],sampl:[19,22],sanit:[3,19,22],satisfi:3,scale:12,scenario:3,schedul:2,schema:[1,3,4,5,10,13,18,19,20,21,22],script:3,search:[1,12,14],search_path:[12,19,22],season:[19,22],second:[7,9,14],section:[3,4,11,12,20,22],see:[3,6,7,9,11,12,15,16,17,19,21,22],select:[12,18,19,22],self:[0,3],semi:3,send:3,sens:3,sensibl:12,sent:[3,11,12],separ:[3,4,7,9,11,12,13,14,15,22],sequenc:[12,13,19,21,22],seri:5,serial:[0,12],server:[2,3,11,14],servic:2,session:3,set:[0,5,6,7,9,11,12,13,14,15,16,18,19,21,22],setup:[1,2,3,12],sever:[3,5,6,7,9,12,13,21,22],shorter:[17,22],should:[3,12],show:[3,17,22],side:[3,7,9],sign:[3,7,9,15,22],similar:6,simpl:[3,14,20,22],simultan:3,sinc:14,singl:[2,3,4,6,7,8,9,10,12,18,20],size:[3,9],skip:[3,4,5,6,7,9,12,13,15,18,19,20,21,22],smaller:3,smallint:[11,12,13,19,22],smallmonei:11,smart:3,socket:3,softwar:0,solut:[3,21,22],solv:0,some:[2,3,4,12,14,15,16,17,18,19,20,21,22],someth:3,soon:[2,3],sourc:[0,1,2,4,19,20,21,22],space:[3,7,9],span:3,speak:[16,22],spec:[15,22],special:[6,7,9,11,12,13,18,19,21,22],specif:[1,2,3,4,15,18,19,20,22],specifi:[2,3,4,6,7,9,10,11,12,13,17,20,22],speed:[18,22],spell:[3,5,12],split:12,sql:[1,2,3,4,7,9,12,14,19,20,21,22],sqlite:[1,2,3,14],squar:[7,9],sslmode:[3,12],staff:[19,22],staff_list:[12,19,22],standard:[3,4,6,7,8,9,10,12,13,20,22],standard_conforming_str:[0,6,9,15,22],start:[1,3,9,12,13,15,16,17,18,19,21],startipnum:[5,7,18,22],state:[15,22],statement:[3,12,13],statu:[19,22],stdin:[3,6,7,9],step:[2,3,7,18,22],still:[16,17,22],stop:[3,12],storag:3,store:[19,22],straight:[4,20,22],strea:[7,9],stream:[2,3,4,6,7,9,21],string:[4,6,7,9,11,12,13,14,20,22],sub:1,sublist:[11,12,13],submit:[2,3],subset:3,suitabl:14,summari:[3,19,21,22],suppli:3,support:[1,3,4,5,6,7,8,9,10,11,13,16,20,22],supris:2,syntax:[1,4,6,7,9,20,22],system:[3,12,14,15,16,17,18,19,21,22],tabl:[0,3,4,6,7,8,9,10,14,15,16,17,18,19,20,21,22],tablenam:[3,4,20,22],tabul:[6,7],tag:13,take:[3,4,6,7,9,12,14,19,20,21,22],taken:[0,12,13],talk:3,tapoueh:[4,18,20,22],target:[2,4,6,7,8,9,10,11,12,13,20,22],task:3,tcp:11,tds:11,tdsport:11,techniqu:[4,20,22],telecharg:[4,8,16,20,22],temp:[4,18,20,22],templat:[1,7,9],termin:[0,3,4,5,7,9,15,18,20,22],test1:10,test:[1,3,4,15,16,17,18,19,20,21,22],text:[3,4,6,9,11,12,13,14,15,18,20,22],than:[3,11,12,17,22],thei:[3,7,11,12,13,19,22],them:[3,6,7,9,11,12,13,16,17,22],themselv:[3,7],theoret:12,thi:[0,2,3,4,5,6,7,8,9,10,11,12,13,14,15,17,18,19,20,22],thing:[3,12],third:2,tho:[19,21,22],those:[0,2,3,4,7,9,11,12,15,19,20,22],though:[0,3,4,20,22],thread:[3,12,19,21,22],threshold:3,tild:3,time:[3,9,12,13,14,15,16,17,18,19,21,22],timestamp:[10,12,13,14],timestamptz:[11,12,13,14,19,22],timezon:10,tinyblob:12,tinyint:[11,12,13,14,19,22],tinytext:12,tmp:[3,5,19,21,22],tmpdir:5,to_char:[7,9],togeth:11,too:[3,12,16,19,22],tool:[2,3,16,19,22],top:5,topic:[19,22],total:[15,16,17,18,19,21,22],town:[17,22],track:[6,21,22],track_ful:6,trackid:6,transform:[1,2,3,18,22],translat:[19,22],trick:[4,20,22],trigger:[3,6,7,8,9,10,12,13,19,21,22],trim:[7,9,14,17,22],truncat:[0,4,6,7,8,9,10,12,13,15,16,18,19,20,21,22],tsv:3,turn:[19,22],tutori:[1,4],tweak:3,two:[2,3,7,12],txt:[4,17,20,22],type:[0,2,3,4,7,8,9,10,11,12,13,14,18,19,20,21,22],typemod:[11,12,13],typic:3,unarch:[4,20,22],under:3,underscor:3,uniqu:[7,12],uniqueidentifi:[11,14],uniquifi:12,unit:[3,15,22],unitpric:6,univers:10,unix:[3,4,14,20,22],unpars:[7,9],unquot:7,unset:[3,5],unsign:[12,14],until:[3,12,13],unzip:[5,16,17,21,22],updat:[19,22],upgrad:3,uri:[3,4,5,12,20,22],url:[3,4,5,6,8,10,12,13,16,17,20,21,22],usag:3,use:[3,4,7,12,14,16,17,18,19,20,21,22],used:[3,4,5,6,7,9,10,11,12,13,14,18,20,22],useful:[3,19,22],user:[3,4,5,6,7,8,9,10,11,12,13,15,16,17,18,19,20,21,22],uses:[2,3,6,19,21,22],using:[2,3,4,5,7,8,9,10,11,12,13,14,16,17,18,19,20,21,22],usp:[4,17,20,22],usual:[3,4,17,20,22],utc:10,utf8:12,utf:[11,13],util:5,uuid:[11,14],valid:3,valu:[0,3,6,7,8,9,10,12,13,14,15,19,22],varbinari:[11,12,14],varchar:[11,12,13,17,22],vari:[15,22],variabl:[3,11,12],variant:[2,3],variou:[2,3],vector:[11,14],verbos:[3,4],veri:[3,18,20,21,22],verifi:12,version:[0,3,4,11,18,20,22],versu:3,view:[1,19,22],wai:[2,3,6,14],wait:3,walk:[6,7,9],want:[3,4,12,18,19,20,22],warn:[3,19,21,22],water:[17,22],webpag:2,websit:[16,22],welcom:11,were:12,what:[7,9,12,13,15,22],whatev:3,when:[3,4,5,6,7,8,9,10,11,12,13,14,17,19,20,21,22],where:[2,3,4,5,6,7,8,9,10,11,12,17,18,20,22],wherea:[3,12],whether:12,which:[3,5,6,7,8,9,10,12,13,14,18,19,21,22],whichev:3,white:3,whitespac:[3,7,9,14,17,22],whole:[2,3,4,5,6,7,9,12,19,20,22],whose:[12,13],width:1,window:3,within:[3,7],without:[2,3,12,13],work:[3,4,11,12,20,22],work_mem:[0,6,7,9,11,12,13,15,19,21,22],worker:[3,12],would:[3,12,19,22],write:[3,4,12,20,22],writer:3,written:14,wtm:[17,22],www:[3,4,8,14,16,17,20,21,22],xml:11,xxx:[19,22],year:[7,9,12,14],yet:12,you:[0,2,3,4,7,11,12,13,14,15,16,17,18,19,20,21,22],your:[2,3,4,11,12,14,18,19,20,21,22],yourself:[4,20,22],yyi:[7,9],yyyi:[7,9],zero:[12,14,19,22],zip:[4,5,6,8,10,13,16,17,18,20,22]},titles:["Reporting Bugs","Welcome to pgloader\u2019s documentation!","Introduction","PgLoader Reference Manual","Pgloader Usage Examples","Loading From an Archive","Loading COPY Formatted Files","Loading CSV data","Loading DBF data","Loading Fixed Cols File Formats","Loading IXF Data","Migrating a MS SQL Database to PostgreSQL","Migrating a MySQL Database to PostgreSQL","Migrating a SQLite database to PostgreSQL","Transformation Functions","Loading CSV Data with pgloader","Loading dBase files with pgloader","Loading Fixed Width Data File with pgloader","Loading MaxMind Geolite Data with pgloader","Migrating from MySQL to PostgreSQL","PgLoader Quick Start","Loading SQLite files with pgloader","PgLoader Tutorial"],titleterms:{"case":0,"default":[11,12,13],"final":5,"function":14,And:3,INTO:3,The:[15,16,17,18,19,21,22],WITH:[3,6,7,8,9,10,11,12,13],about:3,after:3,all:12,alter:[11,12],archiv:[4,5,20,22],argument:3,avail:[4,20,22],batch:3,befor:3,behaviour:3,bug:0,cast:[11,12,13],claus:3,col:9,command:[2,3,4,5,15,16,17,18,19,21,22],comment:3,common:3,complex:4,compress:[20,22],connect:3,continu:2,copi:6,csv:[4,7,15,20,22],data:[7,8,10,15,16,17,18,20,21,22],databas:[11,12,13],dbase:[16,22],dbf:[4,8,20,22],debug:3,decod:12,document:1,driver:11,encod:[11,12],exampl:4,exclud:[11,12,13],execut:3,express:3,fetch:[4,20,22],field:[7,9],file:[4,6,9,16,17,20,21,22],fix:[9,17,22],format:[3,6,9],from:[3,4,5,6,7,8,9,10,11,12,13,19,20,22],gener:3,geolit:[18,22],http:[4,20,22],includ:[11,12,13],indic:1,inform:3,inquiri:3,introduct:2,ixf:10,like:[11,13],limit:12,line:[2,3,19,21,22],load:[3,4,5,6,7,8,9,10,15,16,17,18,20,21,22],locat:[4,20,22],manual:3,match:[11,12],materi:12,maxmind:[18,22],migrat:[2,4,11,12,13,19,20,22],more:3,mustach:3,mysql:[4,12,19,20,22],name:[11,12,13],note:3,onli:[3,11,12,13],oper:3,option:[3,6,7,8,9,10,11,12,13],parallel:3,partial:[11,12,13],perform:3,pgloader:[1,3,4,15,16,17,18,20,21,22],postgresql:[11,12,13,19,22],quick:[20,22],read:[4,20,22],refer:3,regular:3,remot:[4,20,22],renam:11,report:0,reproduc:0,result:[15,22],retri:3,rule:[11,12,13],schema:[11,12],set:3,setup:11,singl:[19,21,22],sourc:[3,5,6,7,8,9,10,11,12,13],specif:[5,6,7,8,9,10,11,12,13],sql:[5,11],sqlite:[4,13,20,21,22],start:[20,22],stdin:[4,20,22],stream:[20,22],string:3,sub:5,support:12,syntax:3,tabl:[1,11,12,13],target:3,templat:3,test:0,through:[4,20,22],transform:[11,12,14],tutori:22,usag:4,view:12,welcom:1,width:[17,22]}}) \ No newline at end of file diff --git a/docs/_build/html/tutorial/csv.html b/docs/_build/html/tutorial/csv.html deleted file mode 100644 index bbec8ef..0000000 --- a/docs/_build/html/tutorial/csv.html +++ /dev/null @@ -1,169 +0,0 @@ - - - - - - - Loading CSV Data with pgloader — pgloader 3.4.1 documentation - - - - - - - - - - - - - - - - - - -
-
-
-
- -
-

Loading CSV Data with pgloader

-

CSV means comma separated values and is often found with quite varying -specifications. pgloader allows you to describe those specs in its command.

-
-

The Command

-

To load data with [pgloader](http://pgloader.io/) you need to define in a -command the operations in some details. Here’s our example for loading CSV -data:

-
LOAD CSV
-     FROM 'path/to/file.csv' (x, y, a, b, c, d)
-     INTO postgresql:///pgloader?csv (a, b, d, c)
-
-     WITH truncate,
-          skip header = 1,
-          fields optionally enclosed by '"',
-          fields escaped by double-quote,
-          fields terminated by ','
-
-      SET client_encoding to 'latin1',
-          work_mem to '12MB',
-          standard_conforming_strings to 'on'
-
-   BEFORE LOAD DO
-    $$ drop table if exists csv; $$,
-    $$ create table csv (
-        a bigint,
-        b bigint,
-        c char(2),
-        d text
-       );
-  $$;
-
-
-
-
-

The Data

-

This command allows loading the following CSV file content:

-
Header, with a © sign
-"2.6.190.56","2.6.190.63","33996344","33996351","GB","United Kingdom"
-"3.0.0.0","4.17.135.31","50331648","68257567","US","United States"
-"4.17.135.32","4.17.135.63","68257568","68257599","CA","Canada"
-"4.17.135.64","4.17.142.255","68257600","68259583","US","United States"
-"4.17.143.0","4.17.143.15","68259584","68259599","CA","Canada"
-"4.17.143.16","4.18.32.71","68259600","68296775","US","United States"
-
-
-
-
-

Loading the data

-

Here’s how to start loading the data. Note that the ouput here has been -edited so as to facilitate its browsing online:

-
$ pgloader csv.load
-... LOG Starting pgloader, log system is ready.
-... LOG Parsing commands from file "/Users/dim/dev/pgloader/test/csv.load"
-
-       table name       read   imported     errors            time
------------------  ---------  ---------  ---------  --------------
-      before load          2          2          0          0.039s
------------------  ---------  ---------  ---------  --------------
-              csv          6          6          0          0.019s
------------------  ---------  ---------  ---------  --------------
-Total import time          6          6          0          0.058s
-
-
-
-
-

The result

-

As you can see, the command described above is filtering the input and only -importing some of the columns from the example data file. Here’s what gets -loaded in the PostgreSQL database:

-
pgloader# table csv;
-    a     |    b     | c  |       d
-----------+----------+----+----------------
- 33996344 | 33996351 | GB | United Kingdom
- 50331648 | 68257567 | US | United States
- 68257568 | 68257599 | CA | Canada
- 68257600 | 68259583 | US | United States
- 68259584 | 68259599 | CA | Canada
- 68259600 | 68296775 | US | United States
-(6 rows)
-
-
-
-
- - -
-
-
- -
-
-
- ©2017, Dimitri Fontaine. - - | - Powered by Sphinx 1.6.5 - & Alabaster 0.7.10 - - | - Page source -
- - - - - - \ No newline at end of file diff --git a/docs/_build/html/tutorial/dBase.html b/docs/_build/html/tutorial/dBase.html deleted file mode 100644 index ae47be7..0000000 --- a/docs/_build/html/tutorial/dBase.html +++ /dev/null @@ -1,134 +0,0 @@ - - - - - - - Loading dBase files with pgloader — pgloader 3.4.1 documentation - - - - - - - - - - - - - - - - - - -
-
-
-
- -
-

Loading dBase files with pgloader

-

The dBase format is still in use in some places as modern tools such as -Filemaker and Excel offer some level of support for it. Speaking of -support in modern tools, pgloader is right there on the list too!

-
-

The Command

-

To load data with [pgloader](http://pgloader.io/) you need to define in a -command the operations in some details. Here’s our example for loading a -dBase file, using a file provided by the french administration.

-

You can find more files from them at the Insee -website.

-

Here’s our command:

-
LOAD DBF
-    FROM http://www.insee.fr/fr/methodes/nomenclatures/cog/telechargement/2013/dbf/historiq2013.zip
-    INTO postgresql:///pgloader
-    WITH truncate, create table
-     SET client_encoding TO 'latin1';
-
-
-

Note that here pgloader will benefit from the meta-data information found in -the dBase file to create a PostgreSQL table capable of hosting the data as -described, then load the data.

-
-
-

Loading the data

-

Let’s start the pgloader command with our dbf-zip.load command file:

-
$ pgloader dbf-zip.load
-... LOG Starting pgloader, log system is ready.
-... LOG Parsing commands from file "/Users/dim/dev/pgloader/test/dbf-zip.load"
-... LOG Fetching 'http://www.insee.fr/fr/methodes/nomenclatures/cog/telechargement/2013/dbf/historiq2013.zip'
-... LOG Extracting files from archive '//private/var/folders/w7/9n8v8pw54t1gngfff0lj16040000gn/T/pgloader//historiq2013.zip'
-
-       table name       read   imported     errors            time
------------------  ---------  ---------  ---------  --------------
-         download          0          0          0          0.167s
-          extract          0          0          0          1.010s
- create, truncate          0          0          0          0.071s
------------------  ---------  ---------  ---------  --------------
-     historiq2013       9181       9181          0          0.658s
------------------  ---------  ---------  ---------  --------------
-Total import time       9181       9181          0          1.906s
-
-
-

We can see that pgloader did download the file from -its HTTP URL location then unziped it before the loading itself.

-

Note that the output of the command has been edited to facilitate its -browsing online.

-
-
- - -
-
-
- -
-
-
- ©2017, Dimitri Fontaine. - - | - Powered by Sphinx 1.6.5 - & Alabaster 0.7.10 - - | - Page source -
- - - - - - \ No newline at end of file diff --git a/docs/_build/html/tutorial/fixed.html b/docs/_build/html/tutorial/fixed.html deleted file mode 100644 index 1a38f88..0000000 --- a/docs/_build/html/tutorial/fixed.html +++ /dev/null @@ -1,176 +0,0 @@ - - - - - - - Loading Fixed Width Data File with pgloader — pgloader 3.4.1 documentation - - - - - - - - - - - - - - - - - - -
-
-
-
- -
-

Loading Fixed Width Data File with pgloader

-

Some data providers still use a format where each column is specified with a -starting index position and a given length. Usually the columns are -blank-padded when the data is shorter than the full reserved range.

-
-

The Command

-

To load data with [pgloader](http://pgloader.io/) you need to define in a -command the operations in some details. Here’s our example for loading -Fixed Width Data, using a file provided by the US census.

-

You can find more files from them at the -[Census 2000 Gazetteer Files](http://www.census.gov/geo/maps-data/data/gazetteer2000.html).

-

Here’s our command:

-
LOAD ARCHIVE
-   FROM http://www.census.gov/geo/maps-data/data/docs/gazetteer/places2k.zip
-   INTO postgresql:///pgloader
-
-   BEFORE LOAD DO
-     $$ drop table if exists places; $$,
-     $$ create table places
-       (
-          usps      char(2)  not null,
-          fips      char(2)  not null,
-          fips_code char(5),
-          loc_name  varchar(64)
-       );
-     $$
-
-   LOAD FIXED
-        FROM FILENAME MATCHING ~/places2k.txt/
-             WITH ENCODING latin1
-             (
-                usps           from   0 for  2,
-                fips           from   2 for  2,
-                fips_code      from   4 for  5,
-                "LocationName" from   9 for 64 [trim right whitespace],
-                p              from  73 for  9,
-                h              from  82 for  9,
-                land           from  91 for 14,
-                water          from 105 for 14,
-                ldm            from 119 for 14,
-                wtm            from 131 for 14,
-                lat            from 143 for 10,
-                long           from 153 for 11
-             )
-        INTO postgresql:///pgloader?places
-             (
-            usps, fips, fips_code, "LocationName"
-             );
-
-
-
-
-

The Data

-

This command allows loading the following file content, where we are only -showing the first couple of lines:

-
AL0100124Abbeville city                                                       2987     1353      40301945        120383   15.560669    0.046480 31.566367 -85.251300
-AL0100460Adamsville city                                                      4965     2042      50779330         14126   19.606010    0.005454 33.590411 -86.949166
-AL0100484Addison town                                                          723      339       9101325             0    3.514041    0.000000 34.200042 -87.177851
-AL0100676Akron town                                                            521      239       1436797             0    0.554750    0.000000 32.876425 -87.740978
-AL0100820Alabaster city                                                      22619     8594      53023800        141711   20.472605    0.054715 33.231162 -86.823829
-AL0100988Albertville city                                                    17247     7090      67212867        258738   25.951034    0.099899 34.265362 -86.211261
-AL0101132Alexander City city                                                 15008     6855     100534344        433413   38.816529    0.167342 32.933157 -85.936008
-
-
-
-
-

Loading the data

-

Let’s start the pgloader command with our census-places.load command file:

-
$ pgloader census-places.load
-... LOG Starting pgloader, log system is ready.
-... LOG Parsing commands from file "/Users/dim/dev/pgloader/test/census-places.load"
-... LOG Fetching 'http://www.census.gov/geo/maps-data/data/docs/gazetteer/places2k.zip'
-... LOG Extracting files from archive '//private/var/folders/w7/9n8v8pw54t1gngfff0lj16040000gn/T/pgloader//places2k.zip'
-
-       table name       read   imported     errors            time
------------------  ---------  ---------  ---------  --------------
-         download          0          0          0          1.494s
-          extract          0          0          0          1.013s
-      before load          2          2          0          0.013s
------------------  ---------  ---------  ---------  --------------
-           places      25375      25375          0          0.499s
------------------  ---------  ---------  ---------  --------------
-Total import time      25375      25375          0          3.019s
-
-
-

We can see that pgloader did download the file from its HTTP URL location -then unziped it before the loading itself.

-

Note that the output of the command has been edited to facilitate its -browsing online.

-
-
- - -
-
-
- -
-
-
- ©2017, Dimitri Fontaine. - - | - Powered by Sphinx 1.6.5 - & Alabaster 0.7.10 - - | - Page source -
- - - - - - \ No newline at end of file diff --git a/docs/_build/html/tutorial/geolite.html b/docs/_build/html/tutorial/geolite.html deleted file mode 100644 index a7da1cf..0000000 --- a/docs/_build/html/tutorial/geolite.html +++ /dev/null @@ -1,239 +0,0 @@ - - - - - - - Loading MaxMind Geolite Data with pgloader — pgloader 3.4.1 documentation - - - - - - - - - - - - - - - - - - -
-
-
-
- -
-

Loading MaxMind Geolite Data with pgloader

-

MaxMind provides a free dataset for -geolocation, which is quite popular. Using pgloader you can download the -lastest version of it, extract the CSV files from the archive and load their -content into your database directly.

-
-

The Command

-

To load data with pgloader you need to define in a command the operations -in some details. Here’s our example for loading the Geolite data:

-
/*
- * Loading from a ZIP archive containing CSV files. The full test can be
- * done with using the archive found at
- * http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.zip
- *
- * And a very light version of this data set is found at
- * http://pgsql.tapoueh.org/temp/foo.zip for quick testing.
- */
-
-LOAD ARCHIVE
-   FROM http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.zip
-   INTO postgresql:///ip4r
-
-   BEFORE LOAD DO
-     $$ create extension if not exists ip4r; $$,
-     $$ create schema if not exists geolite; $$,
-     $$ create table if not exists geolite.location
-       (
-          locid      integer primary key,
-          country    text,
-          region     text,
-          city       text,
-          postalcode text,
-          location   point,
-          metrocode  text,
-          areacode   text
-       );
-     $$,
-     $$ create table if not exists geolite.blocks
-       (
-          iprange    ip4r,
-          locid      integer
-       );
-     $$,
-     $$ drop index if exists geolite.blocks_ip4r_idx; $$,
-     $$ truncate table geolite.blocks, geolite.location cascade; $$
-
-   LOAD CSV
-        FROM FILENAME MATCHING ~/GeoLiteCity-Location.csv/
-             WITH ENCODING iso-8859-1
-             (
-                locId,
-                country,
-                region     null if blanks,
-                city       null if blanks,
-                postalCode null if blanks,
-                latitude,
-                longitude,
-                metroCode  null if blanks,
-                areaCode   null if blanks
-             )
-        INTO postgresql:///ip4r?geolite.location
-             (
-                locid,country,region,city,postalCode,
-                location point using (format nil "(~a,~a)" longitude latitude),
-                metroCode,areaCode
-             )
-        WITH skip header = 2,
-             fields optionally enclosed by '"',
-             fields escaped by double-quote,
-             fields terminated by ','
-
-  AND LOAD CSV
-        FROM FILENAME MATCHING ~/GeoLiteCity-Blocks.csv/
-             WITH ENCODING iso-8859-1
-             (
-                startIpNum, endIpNum, locId
-             )
-        INTO postgresql:///ip4r?geolite.blocks
-             (
-                iprange ip4r using (ip-range startIpNum endIpNum),
-                locId
-             )
-        WITH skip header = 2,
-             fields optionally enclosed by '"',
-             fields escaped by double-quote,
-             fields terminated by ','
-
-   FINALLY DO
-     $$ create index blocks_ip4r_idx on geolite.blocks using gist(iprange); $$;
-
-
-

Note that while the Geolite data is using a pair of integers (start, -end) to represent ipv4 data, we use the very poweful ip4r PostgreSQL Extension instead.

-

The transformation from a pair of integers into an IP is done dynamically by -the pgloader process.

-

Also, the location is given as a pair of float columns for the longitude -and the latitude where PostgreSQL offers the -point -datatype, so the pgloader command here will actually transform the data on -the fly to use the appropriate data type and its input representation.

-
-
-

Loading the data

-

Here’s how to start loading the data. Note that the ouput here has been -edited so as to facilitate its browsing online:

-
$ pgloader archive.load
-... LOG Starting pgloader, log system is ready.
-... LOG Parsing commands from file "/Users/dim/dev/pgloader/test/archive.load"
-... LOG Fetching 'http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.zip'
-... LOG Extracting files from archive '//private/var/folders/w7/9n8v8pw54t1gngfff0lj16040000gn/T/pgloader//GeoLiteCity-latest.zip'
-
-       table name       read   imported     errors            time
------------------  ---------  ---------  ---------  --------------
-         download          0          0          0         11.592s
-          extract          0          0          0          1.012s
-      before load          6          6          0          0.019s
------------------  ---------  ---------  ---------  --------------
- geolite.location     470387     470387          0          7.743s
-   geolite.blocks    1903155    1903155          0         16.332s
------------------  ---------  ---------  ---------  --------------
-          finally          1          1          0         31.692s
------------------  ---------  ---------  ---------  --------------
-Total import time    2373542    2373542          0        1m8.390s
-
-
-

The timing of course includes the transformation of the 1.9 million pairs -of integer into a single ipv4 range each. The finally step consists of -creating the GiST specialized index as given in the main command:

-
CREATE INDEX blocks_ip4r_idx ON geolite.blocks USING gist(iprange);
-
-
-

That index will then be used to speed up queries wanting to find which -recorded geolocation contains a specific IP address:

-
ip4r> select *
-        from      geolite.location l
-             join geolite.blocks b using(locid)
-       where iprange >>= '8.8.8.8';
-
--[ RECORD 1 ]------------------
-locid      | 223
-country    | US
-region     |
-city       |
-postalcode |
-location   | (-97,38)
-metrocode  |
-areacode   |
-iprange    | 8.8.8.8-8.8.37.255
-
-Time: 0.747 ms
-
-
-
-
- - -
-
-
- -
-
-
- ©2017, Dimitri Fontaine. - - | - Powered by Sphinx 1.6.5 - & Alabaster 0.7.10 - - | - Page source -
- - - - - - \ No newline at end of file diff --git a/docs/_build/html/tutorial/mysql.html b/docs/_build/html/tutorial/mysql.html deleted file mode 100644 index e854de6..0000000 --- a/docs/_build/html/tutorial/mysql.html +++ /dev/null @@ -1,246 +0,0 @@ - - - - - - - Migrating from MySQL to PostgreSQL — pgloader 3.4.1 documentation - - - - - - - - - - - - - - - - - - -
-
-
-
- -
-

Migrating from MySQL to PostgreSQL

-

If you want to migrate your data over to PostgreSQL from MySQL then pgloader is the tool of -choice!

-

Most tools around are skipping the main problem with migrating from MySQL, -which is to do with the type casting and data sanitizing that needs to be -done. pgloader will not leave you alone on those topics.

-
-

In a Single Command Line

-

As an example, we will use the f1db database from <http://ergast.com/mrd/> -which which provides a historical record of motor racing data for -non-commercial purposes. You can either use their API or download the whole -database at http://ergast.com/downloads/f1db.sql.gz. Once you’ve done that load the -database in MySQL:

-
$ mysql -u root
-> create database f1db;
-> source f1db.sql
-
-
-

Now let’s migrate this database into PostgreSQL in a single command line:

-
$ createdb f1db
-$ pgloader mysql://root@localhost/f1db pgsql:///f1db
-
-
-

Done! All with schema, table definitions, constraints, indexes, primary -keys, auto_increment columns turned into bigserial , foreign keys, -comments, and if you had some MySQL default values such as ON UPDATE -CURRENT_TIMESTAMP they would have been translated to a PostgreSQL before -update trigger -automatically.

-
$ pgloader mysql://root@localhost/f1db pgsql:///f1db
-2017-06-16T08:56:14.064000+02:00 LOG Main logs in '/private/tmp/pgloader/pgloader.log'
-2017-06-16T08:56:14.068000+02:00 LOG Data errors in '/private/tmp/pgloader/'
-2017-06-16T08:56:19.542000+02:00 LOG report summary reset
-               table name       read   imported     errors      total time
--------------------------  ---------  ---------  ---------  --------------
-          fetch meta data         33         33          0          0.365s
-           Create Schemas          0          0          0          0.007s
-         Create SQL Types          0          0          0          0.006s
-            Create tables         26         26          0          0.068s
-           Set Table OIDs         13         13          0          0.012s
--------------------------  ---------  ---------  ---------  --------------
-  f1db.constructorresults      11011      11011          0          0.205s
-            f1db.circuits         73         73          0          0.150s
-        f1db.constructors        208        208          0          0.059s
-f1db.constructorstandings      11766      11766          0          0.365s
-             f1db.drivers        841        841          0          0.268s
-            f1db.laptimes     413578     413578          0          2.892s
-     f1db.driverstandings      31420      31420          0          0.583s
-            f1db.pitstops       5796       5796          0          2.154s
-               f1db.races        976        976          0          0.227s
-          f1db.qualifying       7257       7257          0          0.228s
-             f1db.seasons         68         68          0          0.527s
-             f1db.results      23514      23514          0          0.658s
-              f1db.status        133        133          0          0.130s
--------------------------  ---------  ---------  ---------  --------------
-  COPY Threads Completion         39         39          0          4.303s
-           Create Indexes         20         20          0          1.497s
-   Index Build Completion         20         20          0          0.214s
-          Reset Sequences          0         10          0          0.058s
-             Primary Keys         13         13          0          0.012s
-      Create Foreign Keys          0          0          0          0.000s
-          Create Triggers          0          0          0          0.001s
-         Install Comments          0          0          0          0.000s
--------------------------  ---------  ---------  ---------  --------------
-        Total import time     506641     506641          0          5.547s
-
-
-

You may need to have special cases to take care of tho, or views that you -want to materialize while doing the migration. In advanced case you can use -the pgloader command.

-
-
-

The Command

-

To load data with pgloader you need to define in a command the operations -in some details. Here’s our example for loading the MySQL Sakila Sample -Database.

-

Here’s our command:

-
load database
-     from      mysql://root@localhost/sakila
-     into postgresql:///sakila
-
- WITH include drop, create tables, no truncate,
-      create indexes, reset sequences, foreign keys
-
-  SET maintenance_work_mem to '128MB', work_mem to '12MB', search_path to 'sakila'
-
- CAST type datetime to timestamptz
-                drop default drop not null using zero-dates-to-null,
-      type date drop not null drop default using zero-dates-to-null
-
- MATERIALIZE VIEWS film_list, staff_list
-
- -- INCLUDING ONLY TABLE NAMES MATCHING ~/film/, 'actor'
- -- EXCLUDING TABLE NAMES MATCHING ~<ory>
-
- BEFORE LOAD DO
- $$ create schema if not exists sakila; $$;
-
-
-

Note that here pgloader will benefit from the meta-data information found in -the MySQL database to create a PostgreSQL database capable of hosting the -data as described, then load the data.

-

In particular, some specific casting rules are given here, to cope with -date values such as 0000-00-00 that MySQL allows and PostgreSQL rejects -for not existing in our calendar. It’s possible to add per-column casting -rules too, which is useful is some of your tinyint are in fact smallint -while some others are in fact boolean values.

-

Finaly note that we are using the MATERIALIZE VIEWS clause of pgloader: -the selected views here will be migrated over to PostgreSQL with their -contents.

-

It’s possible to use the MATERIALIZE VIEWS clause and give both the name -and the SQL (in MySQL dialect) definition of view, then pgloader creates the -view before loading the data, then drops it again at the end.

-

## Loading the data

-

Let’s start the pgloader command with our sakila.load command file:

-
$ pgloader sakila.load
-... LOG Starting pgloader, log system is ready.
-... LOG Parsing commands from file "/Users/dim/dev/pgloader/test/sakila.load"
-   <WARNING: table "xxx" does not exists have been edited away>
-
-            table name       read   imported     errors            time
-----------------------  ---------  ---------  ---------  --------------
-           before load          1          1          0          0.007s
-       fetch meta data         45         45          0          0.402s
-          create, drop          0         36          0          0.208s
-----------------------  ---------  ---------  ---------  --------------
-                 actor        200        200          0          0.071s
-               address        603        603          0          0.035s
-              category         16         16          0          0.018s
-                  city        600        600          0          0.037s
-               country        109        109          0          0.023s
-              customer        599        599          0          0.073s
-                  film       1000       1000          0          0.135s
-            film_actor       5462       5462          0          0.236s
-         film_category       1000       1000          0          0.070s
-             film_text       1000       1000          0          0.080s
-             inventory       4581       4581          0          0.136s
-              language          6          6          0          0.036s
-               payment      16049      16049          0          0.539s
-                rental      16044      16044          0          0.648s
-                 staff          2          2          0          0.041s
-                 store          2          2          0          0.036s
-             film_list        997        997          0          0.247s
-            staff_list          2          2          0          0.135s
-Index Build Completion          0          0          0          0.000s
-----------------------  ---------  ---------  ---------  --------------
-        Create Indexes         41         41          0          0.964s
-       Reset Sequences          0          1          0          0.035s
-          Foreign Keys         22         22          0          0.254s
-----------------------  ---------  ---------  ---------  --------------
-     Total import time      48272      48272          0          3.502s
-
-
-

The WARNING messages we see here are expected as the PostgreSQL database -is empty when running the command, and pgloader is using the SQL commands -DROP TABLE IF EXISTS when the given command uses the include drop -option.

-

Note that the output of the command has been edited to facilitate its -browsing online.

-
-
- - -
-
-
- -
-
-
- ©2017, Dimitri Fontaine. - - | - Powered by Sphinx 1.6.5 - & Alabaster 0.7.10 - - | - Page source -
- - - - - - \ No newline at end of file diff --git a/docs/_build/html/tutorial/quickstart.html b/docs/_build/html/tutorial/quickstart.html deleted file mode 100644 index 3608181..0000000 --- a/docs/_build/html/tutorial/quickstart.html +++ /dev/null @@ -1,218 +0,0 @@ - - - - - - - PgLoader Quick Start — pgloader 3.4.1 documentation - - - - - - - - - - - - - - - - - - -
-
-
-
- -
-

PgLoader Quick Start

-

In simple cases, pgloader is very easy to use.

-
-

CSV

-

Load data from a CSV file into a pre-existing table in your database:

-
pgloader --type csv                                   \
-         --field id --field field                     \
-         --with truncate                              \
-         --with "fields terminated by ','"            \
-         ./test/data/matching-1.csv                   \
-         postgres:///pgloader?tablename=matching
-
-
-

In that example the whole loading is driven from the command line, bypassing -the need for writing a command in the pgloader command syntax entirely. As -there’s no command though, the extra information needed must be provided on -the command line using the –type and –field and –with switches.

-

For documentation about the available syntaxes for the –field and -–with switches, please refer to the CSV section later in the man page.

-

Note also that the PostgreSQL URI includes the target tablename.

-
-
-

Reading from STDIN

-

File based pgloader sources can be loaded from the standard input, as in the -following example:

-
pgloader --type csv                                         \
-         --field "usps,geoid,aland,awater,aland_sqmi,awater_sqmi,intptlat,intptlong" \
-         --with "skip header = 1"                          \
-         --with "fields terminated by '\t'"                \
-         -                                                 \
-         postgresql:///pgloader?districts_longlat          \
-         < test/data/2013_Gaz_113CDs_national.txt
-
-
-

The dash (-) character as a source is used to mean standard input, as -usual in Unix command lines. It’s possible to stream compressed content to -pgloader with this technique, using the Unix pipe:

-
gunzip -c source.gz | pgloader --type csv ... - pgsql:///target?foo
-
-
-
-
-

Loading from CSV available through HTTP

-

The same command as just above can also be run if the CSV file happens to be -found on a remote HTTP location:

-
pgloader --type csv                                                     \
-         --field "usps,geoid,aland,awater,aland_sqmi,awater_sqmi,intptlat,intptlong" \
-         --with "skip header = 1"                                       \
-         --with "fields terminated by '\t'"                             \
-         http://pgsql.tapoueh.org/temp/2013_Gaz_113CDs_national.txt     \
-         postgresql:///pgloader?districts_longlat
-
-
-

Some more options have to be used in that case, as the file contains a -one-line header (most commonly that’s column names, could be a copyright -notice). Also, in that case, we specify all the fields right into a single -–field option argument.

-

Again, the PostgreSQL target connection string must contain the tablename -option and you have to ensure that the target table exists and may fit the -data. Here’s the SQL command used in that example in case you want to try it -yourself:

-
create table districts_longlat
-(
-         usps        text,
-         geoid       text,
-         aland       bigint,
-         awater      bigint,
-         aland_sqmi  double precision,
-         awater_sqmi double precision,
-         intptlat    double precision,
-         intptlong   double precision
-);
-
-
-

Also notice that the same command will work against an archived version of -the same data.

-
-
-

Streaming CSV data from an HTTP compressed file

-

Finally, it’s important to note that pgloader first fetches the content from -the HTTP URL it to a local file, then expand the archive when it’s -recognized to be one, and only then processes the locally expanded file.

-

In some cases, either because pgloader has no direct support for your -archive format or maybe because expanding the archive is not feasible in -your environment, you might want to stream the content straight from its -remote location into PostgreSQL. Here’s how to do that, using the old battle -tested Unix Pipes trick:

-
curl http://pgsql.tapoueh.org/temp/2013_Gaz_113CDs_national.txt.gz \
-| gunzip -c                                                        \
-| pgloader --type csv                                              \
-           --field "usps,geoid,aland,awater,aland_sqmi,awater_sqmi,intptlat,intptlong"
-           --with "skip header = 1"                                \
-           --with "fields terminated by '\t'"                      \
-           -                                                       \
-           postgresql:///pgloader?districts_longlat
-
-
-

Now the OS will take care of the streaming and buffering between the network -and the commands and pgloader will take care of streaming the data down to -PostgreSQL.

-
-
-

Migrating from SQLite

-

The following command will open the SQLite database, discover its tables -definitions including indexes and foreign keys, migrate those definitions -while casting the data type specifications to their PostgreSQL equivalent -and then migrate the data over:

-
createdb newdb
-pgloader ./test/sqlite/sqlite.db postgresql:///newdb
-
-
-
-
-

Migrating from MySQL

-

Just create a database where to host the MySQL data and definitions and have -pgloader do the migration for you in a single command line:

-
createdb pagila
-pgloader mysql://user@localhost/sakila postgresql:///pagila
-
-
-
-
-

Fetching an archived DBF file from a HTTP remote location

-

It’s possible for pgloader to download a file from HTTP, unarchive it, and -only then open it to discover the schema then load the data:

-
createdb foo
-pgloader --type dbf http://www.insee.fr/fr/methodes/nomenclatures/cog/telechargement/2013/dbf/historiq2013.zip postgresql:///foo
-
-
-

Here it’s not possible for pgloader to guess the kind of data source it’s -being given, so it’s necessary to use the –type command line switch.

-
-
- - -
-
-
- -
-
-
- ©2017, Dimitri Fontaine. - - | - Powered by Sphinx 1.6.5 - & Alabaster 0.7.10 - - | - Page source -
- - - - - - \ No newline at end of file diff --git a/docs/_build/html/tutorial/sqlite.html b/docs/_build/html/tutorial/sqlite.html deleted file mode 100644 index 7bcc290..0000000 --- a/docs/_build/html/tutorial/sqlite.html +++ /dev/null @@ -1,208 +0,0 @@ - - - - - - - Loading SQLite files with pgloader — pgloader 3.4.1 documentation - - - - - - - - - - - - - - - - - - -
-
-
-
- -
-

Loading SQLite files with pgloader

-

The SQLite database is a respected solution to manage your data with. Its -embeded nature makes it a source of migrations when a projects now needs to -handle more concurrency, which [PostgreSQL](http://www.postgresql.org/) is -very good at. pgloader can help you there.

-
-

In a Single Command Line

-

You can

-
$ createdb chinook
-$ pgloader https://github.com/lerocha/chinook-database/raw/master/ChinookDatabase/DataSources/Chinook_Sqlite_AutoIncrementPKs.sqlite pgsql:///chinook
-
-
-

Done! All with the schema, data, constraints, primary keys and foreign keys, -etc. We also see an error with the Chinook schema that contains several -primary key definitions against the same table, which is not accepted by -PostgreSQL:

-
2017-06-20T16:18:59.019000+02:00 LOG Data errors in '/private/tmp/pgloader/'
-2017-06-20T16:18:59.236000+02:00 LOG Fetching 'https://github.com/lerocha/chinook-database/raw/master/ChinookDatabase/DataSources/Chinook_Sqlite_AutoIncrementPKs.sqlite'
-2017-06-20T16:19:00.664000+02:00 ERROR Database error 42P16: multiple primary keys for table "playlisttrack" are not allowed
-QUERY: ALTER TABLE playlisttrack ADD PRIMARY KEY USING INDEX idx_66873_sqlite_autoindex_playlisttrack_1;
-2017-06-20T16:19:00.665000+02:00 LOG report summary reset
-             table name       read   imported     errors      total time
------------------------  ---------  ---------  ---------  --------------
-                  fetch          0          0          0          0.877s
-        fetch meta data         33         33          0          0.033s
-         Create Schemas          0          0          0          0.003s
-       Create SQL Types          0          0          0          0.006s
-          Create tables         22         22          0          0.043s
-         Set Table OIDs         11         11          0          0.012s
------------------------  ---------  ---------  ---------  --------------
-                  album        347        347          0          0.023s
-                 artist        275        275          0          0.023s
-               customer         59         59          0          0.021s
-               employee          8          8          0          0.018s
-                invoice        412        412          0          0.031s
-                  genre         25         25          0          0.021s
-            invoiceline       2240       2240          0          0.034s
-              mediatype          5          5          0          0.025s
-          playlisttrack       8715       8715          0          0.040s
-               playlist         18         18          0          0.016s
-                  track       3503       3503          0          0.111s
------------------------  ---------  ---------  ---------  --------------
-COPY Threads Completion         33         33          0          0.313s
-         Create Indexes         22         22          0          0.160s
- Index Build Completion         22         22          0          0.027s
-        Reset Sequences          0          0          0          0.017s
-           Primary Keys         12          0          1          0.013s
-    Create Foreign Keys         11         11          0          0.040s
-        Create Triggers          0          0          0          0.000s
-       Install Comments          0          0          0          0.000s
------------------------  ---------  ---------  ---------  --------------
-      Total import time      15607      15607          0          1.669s
-
-
-

You may need to have special cases to take care of tho. In advanced case you -can use the pgloader command.

-
-
-

The Command

-

To load data with [pgloader](http://pgloader.io/) you need to define in a -command the operations in some details. Here’s our command:

-
load database
-     from 'sqlite/Chinook_Sqlite_AutoIncrementPKs.sqlite'
-     into postgresql:///pgloader
-
- with include drop, create tables, create indexes, reset sequences
-
-  set work_mem to '16MB', maintenance_work_mem to '512 MB';
-
-
-

Note that here pgloader will benefit from the meta-data information found in -the SQLite file to create a PostgreSQL database capable of hosting the data -as described, then load the data.

-
-
-

Loading the data

-

Let’s start the pgloader command with our sqlite.load command file:

-
$ pgloader sqlite.load
-... LOG Starting pgloader, log system is ready.
-... LOG Parsing commands from file "/Users/dim/dev/pgloader/test/sqlite.load"
-... WARNING Postgres warning: table "album" does not exist, skipping
-... WARNING Postgres warning: table "artist" does not exist, skipping
-... WARNING Postgres warning: table "customer" does not exist, skipping
-... WARNING Postgres warning: table "employee" does not exist, skipping
-... WARNING Postgres warning: table "genre" does not exist, skipping
-... WARNING Postgres warning: table "invoice" does not exist, skipping
-... WARNING Postgres warning: table "invoiceline" does not exist, skipping
-... WARNING Postgres warning: table "mediatype" does not exist, skipping
-... WARNING Postgres warning: table "playlist" does not exist, skipping
-... WARNING Postgres warning: table "playlisttrack" does not exist, skipping
-... WARNING Postgres warning: table "track" does not exist, skipping
-            table name       read   imported     errors            time
-----------------------  ---------  ---------  ---------  --------------
-      create, truncate          0          0          0          0.052s
-                 Album        347        347          0          0.070s
-                Artist        275        275          0          0.014s
-              Customer         59         59          0          0.014s
-              Employee          8          8          0          0.012s
-                 Genre         25         25          0          0.018s
-               Invoice        412        412          0          0.032s
-           InvoiceLine       2240       2240          0          0.077s
-             MediaType          5          5          0          0.012s
-              Playlist         18         18          0          0.008s
-         PlaylistTrack       8715       8715          0          0.071s
-                 Track       3503       3503          0          0.105s
-index build completion          0          0          0          0.000s
-----------------------  ---------  ---------  ---------  --------------
-        Create Indexes         20         20          0          0.279s
-       reset sequences          0          0          0          0.043s
-----------------------  ---------  ---------  ---------  --------------
-  Total streaming time      15607      15607          0          0.476s
-
-
-

We can see that pgloader did download the file from -its HTTP URL location then unziped it before loading it.

-

Also, the WARNING messages we see here are expected as the PostgreSQL -database is empty when running the command, and pgloader is using the SQL -commands DROP TABLE IF EXISTS when the given command uses the include -drop option.

-

Note that the output of the command has been edited to facilitate its -browsing online.

-
-
- - -
-
-
- -
-
-
- ©2017, Dimitri Fontaine. - - | - Powered by Sphinx 1.6.5 - & Alabaster 0.7.10 - - | - Page source -
- - - - - - \ No newline at end of file diff --git a/docs/_build/html/tutorial/tutorial.html b/docs/_build/html/tutorial/tutorial.html deleted file mode 100644 index 844d134..0000000 --- a/docs/_build/html/tutorial/tutorial.html +++ /dev/null @@ -1,893 +0,0 @@ - - - - - - - PgLoader Tutorial — pgloader 3.4.1 documentation - - - - - - - - - - - - - - - - - - - - -
-
-
-
- -
-

PgLoader Tutorial

-
-

PgLoader Quick Start

-

In simple cases, pgloader is very easy to use.

-
-

CSV

-

Load data from a CSV file into a pre-existing table in your database:

-
pgloader --type csv                                   \
-         --field id --field field                     \
-         --with truncate                              \
-         --with "fields terminated by ','"            \
-         ./test/data/matching-1.csv                   \
-         postgres:///pgloader?tablename=matching
-
-
-

In that example the whole loading is driven from the command line, bypassing -the need for writing a command in the pgloader command syntax entirely. As -there’s no command though, the extra information needed must be provided on -the command line using the –type and –field and –with switches.

-

For documentation about the available syntaxes for the –field and -–with switches, please refer to the CSV section later in the man page.

-

Note also that the PostgreSQL URI includes the target tablename.

-
-
-

Reading from STDIN

-

File based pgloader sources can be loaded from the standard input, as in the -following example:

-
pgloader --type csv                                         \
-         --field "usps,geoid,aland,awater,aland_sqmi,awater_sqmi,intptlat,intptlong" \
-         --with "skip header = 1"                          \
-         --with "fields terminated by '\t'"                \
-         -                                                 \
-         postgresql:///pgloader?districts_longlat          \
-         < test/data/2013_Gaz_113CDs_national.txt
-
-
-

The dash (-) character as a source is used to mean standard input, as -usual in Unix command lines. It’s possible to stream compressed content to -pgloader with this technique, using the Unix pipe:

-
gunzip -c source.gz | pgloader --type csv ... - pgsql:///target?foo
-
-
-
-
-

Loading from CSV available through HTTP

-

The same command as just above can also be run if the CSV file happens to be -found on a remote HTTP location:

-
pgloader --type csv                                                     \
-         --field "usps,geoid,aland,awater,aland_sqmi,awater_sqmi,intptlat,intptlong" \
-         --with "skip header = 1"                                       \
-         --with "fields terminated by '\t'"                             \
-         http://pgsql.tapoueh.org/temp/2013_Gaz_113CDs_national.txt     \
-         postgresql:///pgloader?districts_longlat
-
-
-

Some more options have to be used in that case, as the file contains a -one-line header (most commonly that’s column names, could be a copyright -notice). Also, in that case, we specify all the fields right into a single -–field option argument.

-

Again, the PostgreSQL target connection string must contain the tablename -option and you have to ensure that the target table exists and may fit the -data. Here’s the SQL command used in that example in case you want to try it -yourself:

-
create table districts_longlat
-(
-         usps        text,
-         geoid       text,
-         aland       bigint,
-         awater      bigint,
-         aland_sqmi  double precision,
-         awater_sqmi double precision,
-         intptlat    double precision,
-         intptlong   double precision
-);
-
-
-

Also notice that the same command will work against an archived version of -the same data.

-
-
-

Streaming CSV data from an HTTP compressed file

-

Finally, it’s important to note that pgloader first fetches the content from -the HTTP URL it to a local file, then expand the archive when it’s -recognized to be one, and only then processes the locally expanded file.

-

In some cases, either because pgloader has no direct support for your -archive format or maybe because expanding the archive is not feasible in -your environment, you might want to stream the content straight from its -remote location into PostgreSQL. Here’s how to do that, using the old battle -tested Unix Pipes trick:

-
curl http://pgsql.tapoueh.org/temp/2013_Gaz_113CDs_national.txt.gz \
-| gunzip -c                                                        \
-| pgloader --type csv                                              \
-           --field "usps,geoid,aland,awater,aland_sqmi,awater_sqmi,intptlat,intptlong"
-           --with "skip header = 1"                                \
-           --with "fields terminated by '\t'"                      \
-           -                                                       \
-           postgresql:///pgloader?districts_longlat
-
-
-

Now the OS will take care of the streaming and buffering between the network -and the commands and pgloader will take care of streaming the data down to -PostgreSQL.

-
-
-

Migrating from SQLite

-

The following command will open the SQLite database, discover its tables -definitions including indexes and foreign keys, migrate those definitions -while casting the data type specifications to their PostgreSQL equivalent -and then migrate the data over:

-
createdb newdb
-pgloader ./test/sqlite/sqlite.db postgresql:///newdb
-
-
-
-
-

Migrating from MySQL

-

Just create a database where to host the MySQL data and definitions and have -pgloader do the migration for you in a single command line:

-
createdb pagila
-pgloader mysql://user@localhost/sakila postgresql:///pagila
-
-
-
-
-

Fetching an archived DBF file from a HTTP remote location

-

It’s possible for pgloader to download a file from HTTP, unarchive it, and -only then open it to discover the schema then load the data:

-
createdb foo
-pgloader --type dbf http://www.insee.fr/fr/methodes/nomenclatures/cog/telechargement/2013/dbf/historiq2013.zip postgresql:///foo
-
-
-

Here it’s not possible for pgloader to guess the kind of data source it’s -being given, so it’s necessary to use the –type command line switch.

-
-
-
-

Loading CSV Data with pgloader

-

CSV means comma separated values and is often found with quite varying -specifications. pgloader allows you to describe those specs in its command.

-
-

The Command

-

To load data with [pgloader](http://pgloader.io/) you need to define in a -command the operations in some details. Here’s our example for loading CSV -data:

-
LOAD CSV
-     FROM 'path/to/file.csv' (x, y, a, b, c, d)
-     INTO postgresql:///pgloader?csv (a, b, d, c)
-
-     WITH truncate,
-          skip header = 1,
-          fields optionally enclosed by '"',
-          fields escaped by double-quote,
-          fields terminated by ','
-
-      SET client_encoding to 'latin1',
-          work_mem to '12MB',
-          standard_conforming_strings to 'on'
-
-   BEFORE LOAD DO
-    $$ drop table if exists csv; $$,
-    $$ create table csv (
-        a bigint,
-        b bigint,
-        c char(2),
-        d text
-       );
-  $$;
-
-
-
-
-

The Data

-

This command allows loading the following CSV file content:

-
Header, with a © sign
-"2.6.190.56","2.6.190.63","33996344","33996351","GB","United Kingdom"
-"3.0.0.0","4.17.135.31","50331648","68257567","US","United States"
-"4.17.135.32","4.17.135.63","68257568","68257599","CA","Canada"
-"4.17.135.64","4.17.142.255","68257600","68259583","US","United States"
-"4.17.143.0","4.17.143.15","68259584","68259599","CA","Canada"
-"4.17.143.16","4.18.32.71","68259600","68296775","US","United States"
-
-
-
-
-

Loading the data

-

Here’s how to start loading the data. Note that the ouput here has been -edited so as to facilitate its browsing online:

-
$ pgloader csv.load
-... LOG Starting pgloader, log system is ready.
-... LOG Parsing commands from file "/Users/dim/dev/pgloader/test/csv.load"
-
-       table name       read   imported     errors            time
------------------  ---------  ---------  ---------  --------------
-      before load          2          2          0          0.039s
------------------  ---------  ---------  ---------  --------------
-              csv          6          6          0          0.019s
------------------  ---------  ---------  ---------  --------------
-Total import time          6          6          0          0.058s
-
-
-
-
-

The result

-

As you can see, the command described above is filtering the input and only -importing some of the columns from the example data file. Here’s what gets -loaded in the PostgreSQL database:

-
pgloader# table csv;
-    a     |    b     | c  |       d
-----------+----------+----+----------------
- 33996344 | 33996351 | GB | United Kingdom
- 50331648 | 68257567 | US | United States
- 68257568 | 68257599 | CA | Canada
- 68257600 | 68259583 | US | United States
- 68259584 | 68259599 | CA | Canada
- 68259600 | 68296775 | US | United States
-(6 rows)
-
-
-
-
-
-

Loading Fixed Width Data File with pgloader

-

Some data providers still use a format where each column is specified with a -starting index position and a given length. Usually the columns are -blank-padded when the data is shorter than the full reserved range.

-
-

The Command

-

To load data with [pgloader](http://pgloader.io/) you need to define in a -command the operations in some details. Here’s our example for loading -Fixed Width Data, using a file provided by the US census.

-

You can find more files from them at the -[Census 2000 Gazetteer Files](http://www.census.gov/geo/maps-data/data/gazetteer2000.html).

-

Here’s our command:

-
LOAD ARCHIVE
-   FROM http://www.census.gov/geo/maps-data/data/docs/gazetteer/places2k.zip
-   INTO postgresql:///pgloader
-
-   BEFORE LOAD DO
-     $$ drop table if exists places; $$,
-     $$ create table places
-       (
-          usps      char(2)  not null,
-          fips      char(2)  not null,
-          fips_code char(5),
-          loc_name  varchar(64)
-       );
-     $$
-
-   LOAD FIXED
-        FROM FILENAME MATCHING ~/places2k.txt/
-             WITH ENCODING latin1
-             (
-                usps           from   0 for  2,
-                fips           from   2 for  2,
-                fips_code      from   4 for  5,
-                "LocationName" from   9 for 64 [trim right whitespace],
-                p              from  73 for  9,
-                h              from  82 for  9,
-                land           from  91 for 14,
-                water          from 105 for 14,
-                ldm            from 119 for 14,
-                wtm            from 131 for 14,
-                lat            from 143 for 10,
-                long           from 153 for 11
-             )
-        INTO postgresql:///pgloader?places
-             (
-            usps, fips, fips_code, "LocationName"
-             );
-
-
-
-
-

The Data

-

This command allows loading the following file content, where we are only -showing the first couple of lines:

-
AL0100124Abbeville city                                                       2987     1353      40301945        120383   15.560669    0.046480 31.566367 -85.251300
-AL0100460Adamsville city                                                      4965     2042      50779330         14126   19.606010    0.005454 33.590411 -86.949166
-AL0100484Addison town                                                          723      339       9101325             0    3.514041    0.000000 34.200042 -87.177851
-AL0100676Akron town                                                            521      239       1436797             0    0.554750    0.000000 32.876425 -87.740978
-AL0100820Alabaster city                                                      22619     8594      53023800        141711   20.472605    0.054715 33.231162 -86.823829
-AL0100988Albertville city                                                    17247     7090      67212867        258738   25.951034    0.099899 34.265362 -86.211261
-AL0101132Alexander City city                                                 15008     6855     100534344        433413   38.816529    0.167342 32.933157 -85.936008
-
-
-
-
-

Loading the data

-

Let’s start the pgloader command with our census-places.load command file:

-
$ pgloader census-places.load
-... LOG Starting pgloader, log system is ready.
-... LOG Parsing commands from file "/Users/dim/dev/pgloader/test/census-places.load"
-... LOG Fetching 'http://www.census.gov/geo/maps-data/data/docs/gazetteer/places2k.zip'
-... LOG Extracting files from archive '//private/var/folders/w7/9n8v8pw54t1gngfff0lj16040000gn/T/pgloader//places2k.zip'
-
-       table name       read   imported     errors            time
------------------  ---------  ---------  ---------  --------------
-         download          0          0          0          1.494s
-          extract          0          0          0          1.013s
-      before load          2          2          0          0.013s
------------------  ---------  ---------  ---------  --------------
-           places      25375      25375          0          0.499s
------------------  ---------  ---------  ---------  --------------
-Total import time      25375      25375          0          3.019s
-
-
-

We can see that pgloader did download the file from its HTTP URL location -then unziped it before the loading itself.

-

Note that the output of the command has been edited to facilitate its -browsing online.

-
-
-
-

Loading MaxMind Geolite Data with pgloader

-

MaxMind provides a free dataset for -geolocation, which is quite popular. Using pgloader you can download the -lastest version of it, extract the CSV files from the archive and load their -content into your database directly.

-
-

The Command

-

To load data with pgloader you need to define in a command the operations -in some details. Here’s our example for loading the Geolite data:

-
/*
- * Loading from a ZIP archive containing CSV files. The full test can be
- * done with using the archive found at
- * http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.zip
- *
- * And a very light version of this data set is found at
- * http://pgsql.tapoueh.org/temp/foo.zip for quick testing.
- */
-
-LOAD ARCHIVE
-   FROM http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.zip
-   INTO postgresql:///ip4r
-
-   BEFORE LOAD DO
-     $$ create extension if not exists ip4r; $$,
-     $$ create schema if not exists geolite; $$,
-     $$ create table if not exists geolite.location
-       (
-          locid      integer primary key,
-          country    text,
-          region     text,
-          city       text,
-          postalcode text,
-          location   point,
-          metrocode  text,
-          areacode   text
-       );
-     $$,
-     $$ create table if not exists geolite.blocks
-       (
-          iprange    ip4r,
-          locid      integer
-       );
-     $$,
-     $$ drop index if exists geolite.blocks_ip4r_idx; $$,
-     $$ truncate table geolite.blocks, geolite.location cascade; $$
-
-   LOAD CSV
-        FROM FILENAME MATCHING ~/GeoLiteCity-Location.csv/
-             WITH ENCODING iso-8859-1
-             (
-                locId,
-                country,
-                region     null if blanks,
-                city       null if blanks,
-                postalCode null if blanks,
-                latitude,
-                longitude,
-                metroCode  null if blanks,
-                areaCode   null if blanks
-             )
-        INTO postgresql:///ip4r?geolite.location
-             (
-                locid,country,region,city,postalCode,
-                location point using (format nil "(~a,~a)" longitude latitude),
-                metroCode,areaCode
-             )
-        WITH skip header = 2,
-             fields optionally enclosed by '"',
-             fields escaped by double-quote,
-             fields terminated by ','
-
-  AND LOAD CSV
-        FROM FILENAME MATCHING ~/GeoLiteCity-Blocks.csv/
-             WITH ENCODING iso-8859-1
-             (
-                startIpNum, endIpNum, locId
-             )
-        INTO postgresql:///ip4r?geolite.blocks
-             (
-                iprange ip4r using (ip-range startIpNum endIpNum),
-                locId
-             )
-        WITH skip header = 2,
-             fields optionally enclosed by '"',
-             fields escaped by double-quote,
-             fields terminated by ','
-
-   FINALLY DO
-     $$ create index blocks_ip4r_idx on geolite.blocks using gist(iprange); $$;
-
-
-

Note that while the Geolite data is using a pair of integers (start, -end) to represent ipv4 data, we use the very poweful ip4r PostgreSQL Extension instead.

-

The transformation from a pair of integers into an IP is done dynamically by -the pgloader process.

-

Also, the location is given as a pair of float columns for the longitude -and the latitude where PostgreSQL offers the -point -datatype, so the pgloader command here will actually transform the data on -the fly to use the appropriate data type and its input representation.

-
-
-

Loading the data

-

Here’s how to start loading the data. Note that the ouput here has been -edited so as to facilitate its browsing online:

-
$ pgloader archive.load
-... LOG Starting pgloader, log system is ready.
-... LOG Parsing commands from file "/Users/dim/dev/pgloader/test/archive.load"
-... LOG Fetching 'http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.zip'
-... LOG Extracting files from archive '//private/var/folders/w7/9n8v8pw54t1gngfff0lj16040000gn/T/pgloader//GeoLiteCity-latest.zip'
-
-       table name       read   imported     errors            time
------------------  ---------  ---------  ---------  --------------
-         download          0          0          0         11.592s
-          extract          0          0          0          1.012s
-      before load          6          6          0          0.019s
------------------  ---------  ---------  ---------  --------------
- geolite.location     470387     470387          0          7.743s
-   geolite.blocks    1903155    1903155          0         16.332s
------------------  ---------  ---------  ---------  --------------
-          finally          1          1          0         31.692s
------------------  ---------  ---------  ---------  --------------
-Total import time    2373542    2373542          0        1m8.390s
-
-
-

The timing of course includes the transformation of the 1.9 million pairs -of integer into a single ipv4 range each. The finally step consists of -creating the GiST specialized index as given in the main command:

-
CREATE INDEX blocks_ip4r_idx ON geolite.blocks USING gist(iprange);
-
-
-

That index will then be used to speed up queries wanting to find which -recorded geolocation contains a specific IP address:

-
ip4r> select *
-        from      geolite.location l
-             join geolite.blocks b using(locid)
-       where iprange >>= '8.8.8.8';
-
--[ RECORD 1 ]------------------
-locid      | 223
-country    | US
-region     |
-city       |
-postalcode |
-location   | (-97,38)
-metrocode  |
-areacode   |
-iprange    | 8.8.8.8-8.8.37.255
-
-Time: 0.747 ms
-
-
-
-
-
-

Loading dBase files with pgloader

-

The dBase format is still in use in some places as modern tools such as -Filemaker and Excel offer some level of support for it. Speaking of -support in modern tools, pgloader is right there on the list too!

-
-

The Command

-

To load data with [pgloader](http://pgloader.io/) you need to define in a -command the operations in some details. Here’s our example for loading a -dBase file, using a file provided by the french administration.

-

You can find more files from them at the Insee -website.

-

Here’s our command:

-
LOAD DBF
-    FROM http://www.insee.fr/fr/methodes/nomenclatures/cog/telechargement/2013/dbf/historiq2013.zip
-    INTO postgresql:///pgloader
-    WITH truncate, create table
-     SET client_encoding TO 'latin1';
-
-
-

Note that here pgloader will benefit from the meta-data information found in -the dBase file to create a PostgreSQL table capable of hosting the data as -described, then load the data.

-
-
-

Loading the data

-

Let’s start the pgloader command with our dbf-zip.load command file:

-
$ pgloader dbf-zip.load
-... LOG Starting pgloader, log system is ready.
-... LOG Parsing commands from file "/Users/dim/dev/pgloader/test/dbf-zip.load"
-... LOG Fetching 'http://www.insee.fr/fr/methodes/nomenclatures/cog/telechargement/2013/dbf/historiq2013.zip'
-... LOG Extracting files from archive '//private/var/folders/w7/9n8v8pw54t1gngfff0lj16040000gn/T/pgloader//historiq2013.zip'
-
-       table name       read   imported     errors            time
------------------  ---------  ---------  ---------  --------------
-         download          0          0          0          0.167s
-          extract          0          0          0          1.010s
- create, truncate          0          0          0          0.071s
------------------  ---------  ---------  ---------  --------------
-     historiq2013       9181       9181          0          0.658s
------------------  ---------  ---------  ---------  --------------
-Total import time       9181       9181          0          1.906s
-
-
-

We can see that pgloader did download the file from -its HTTP URL location then unziped it before the loading itself.

-

Note that the output of the command has been edited to facilitate its -browsing online.

-
-
-
-

Loading SQLite files with pgloader

-

The SQLite database is a respected solution to manage your data with. Its -embeded nature makes it a source of migrations when a projects now needs to -handle more concurrency, which [PostgreSQL](http://www.postgresql.org/) is -very good at. pgloader can help you there.

-
-

In a Single Command Line

-

You can

-
$ createdb chinook
-$ pgloader https://github.com/lerocha/chinook-database/raw/master/ChinookDatabase/DataSources/Chinook_Sqlite_AutoIncrementPKs.sqlite pgsql:///chinook
-
-
-

Done! All with the schema, data, constraints, primary keys and foreign keys, -etc. We also see an error with the Chinook schema that contains several -primary key definitions against the same table, which is not accepted by -PostgreSQL:

-
2017-06-20T16:18:59.019000+02:00 LOG Data errors in '/private/tmp/pgloader/'
-2017-06-20T16:18:59.236000+02:00 LOG Fetching 'https://github.com/lerocha/chinook-database/raw/master/ChinookDatabase/DataSources/Chinook_Sqlite_AutoIncrementPKs.sqlite'
-2017-06-20T16:19:00.664000+02:00 ERROR Database error 42P16: multiple primary keys for table "playlisttrack" are not allowed
-QUERY: ALTER TABLE playlisttrack ADD PRIMARY KEY USING INDEX idx_66873_sqlite_autoindex_playlisttrack_1;
-2017-06-20T16:19:00.665000+02:00 LOG report summary reset
-             table name       read   imported     errors      total time
------------------------  ---------  ---------  ---------  --------------
-                  fetch          0          0          0          0.877s
-        fetch meta data         33         33          0          0.033s
-         Create Schemas          0          0          0          0.003s
-       Create SQL Types          0          0          0          0.006s
-          Create tables         22         22          0          0.043s
-         Set Table OIDs         11         11          0          0.012s
------------------------  ---------  ---------  ---------  --------------
-                  album        347        347          0          0.023s
-                 artist        275        275          0          0.023s
-               customer         59         59          0          0.021s
-               employee          8          8          0          0.018s
-                invoice        412        412          0          0.031s
-                  genre         25         25          0          0.021s
-            invoiceline       2240       2240          0          0.034s
-              mediatype          5          5          0          0.025s
-          playlisttrack       8715       8715          0          0.040s
-               playlist         18         18          0          0.016s
-                  track       3503       3503          0          0.111s
------------------------  ---------  ---------  ---------  --------------
-COPY Threads Completion         33         33          0          0.313s
-         Create Indexes         22         22          0          0.160s
- Index Build Completion         22         22          0          0.027s
-        Reset Sequences          0          0          0          0.017s
-           Primary Keys         12          0          1          0.013s
-    Create Foreign Keys         11         11          0          0.040s
-        Create Triggers          0          0          0          0.000s
-       Install Comments          0          0          0          0.000s
------------------------  ---------  ---------  ---------  --------------
-      Total import time      15607      15607          0          1.669s
-
-
-

You may need to have special cases to take care of tho. In advanced case you -can use the pgloader command.

-
-
-

The Command

-

To load data with [pgloader](http://pgloader.io/) you need to define in a -command the operations in some details. Here’s our command:

-
load database
-     from 'sqlite/Chinook_Sqlite_AutoIncrementPKs.sqlite'
-     into postgresql:///pgloader
-
- with include drop, create tables, create indexes, reset sequences
-
-  set work_mem to '16MB', maintenance_work_mem to '512 MB';
-
-
-

Note that here pgloader will benefit from the meta-data information found in -the SQLite file to create a PostgreSQL database capable of hosting the data -as described, then load the data.

-
-
-

Loading the data

-

Let’s start the pgloader command with our sqlite.load command file:

-
$ pgloader sqlite.load
-... LOG Starting pgloader, log system is ready.
-... LOG Parsing commands from file "/Users/dim/dev/pgloader/test/sqlite.load"
-... WARNING Postgres warning: table "album" does not exist, skipping
-... WARNING Postgres warning: table "artist" does not exist, skipping
-... WARNING Postgres warning: table "customer" does not exist, skipping
-... WARNING Postgres warning: table "employee" does not exist, skipping
-... WARNING Postgres warning: table "genre" does not exist, skipping
-... WARNING Postgres warning: table "invoice" does not exist, skipping
-... WARNING Postgres warning: table "invoiceline" does not exist, skipping
-... WARNING Postgres warning: table "mediatype" does not exist, skipping
-... WARNING Postgres warning: table "playlist" does not exist, skipping
-... WARNING Postgres warning: table "playlisttrack" does not exist, skipping
-... WARNING Postgres warning: table "track" does not exist, skipping
-            table name       read   imported     errors            time
-----------------------  ---------  ---------  ---------  --------------
-      create, truncate          0          0          0          0.052s
-                 Album        347        347          0          0.070s
-                Artist        275        275          0          0.014s
-              Customer         59         59          0          0.014s
-              Employee          8          8          0          0.012s
-                 Genre         25         25          0          0.018s
-               Invoice        412        412          0          0.032s
-           InvoiceLine       2240       2240          0          0.077s
-             MediaType          5          5          0          0.012s
-              Playlist         18         18          0          0.008s
-         PlaylistTrack       8715       8715          0          0.071s
-                 Track       3503       3503          0          0.105s
-index build completion          0          0          0          0.000s
-----------------------  ---------  ---------  ---------  --------------
-        Create Indexes         20         20          0          0.279s
-       reset sequences          0          0          0          0.043s
-----------------------  ---------  ---------  ---------  --------------
-  Total streaming time      15607      15607          0          0.476s
-
-
-

We can see that pgloader did download the file from -its HTTP URL location then unziped it before loading it.

-

Also, the WARNING messages we see here are expected as the PostgreSQL -database is empty when running the command, and pgloader is using the SQL -commands DROP TABLE IF EXISTS when the given command uses the include -drop option.

-

Note that the output of the command has been edited to facilitate its -browsing online.

-
-
-
-

Migrating from MySQL to PostgreSQL

-

If you want to migrate your data over to PostgreSQL from MySQL then pgloader is the tool of -choice!

-

Most tools around are skipping the main problem with migrating from MySQL, -which is to do with the type casting and data sanitizing that needs to be -done. pgloader will not leave you alone on those topics.

-
-

In a Single Command Line

-

As an example, we will use the f1db database from <http://ergast.com/mrd/> -which which provides a historical record of motor racing data for -non-commercial purposes. You can either use their API or download the whole -database at http://ergast.com/downloads/f1db.sql.gz. Once you’ve done that load the -database in MySQL:

-
$ mysql -u root
-> create database f1db;
-> source f1db.sql
-
-
-

Now let’s migrate this database into PostgreSQL in a single command line:

-
$ createdb f1db
-$ pgloader mysql://root@localhost/f1db pgsql:///f1db
-
-
-

Done! All with schema, table definitions, constraints, indexes, primary -keys, auto_increment columns turned into bigserial , foreign keys, -comments, and if you had some MySQL default values such as ON UPDATE -CURRENT_TIMESTAMP they would have been translated to a PostgreSQL before -update trigger -automatically.

-
$ pgloader mysql://root@localhost/f1db pgsql:///f1db
-2017-06-16T08:56:14.064000+02:00 LOG Main logs in '/private/tmp/pgloader/pgloader.log'
-2017-06-16T08:56:14.068000+02:00 LOG Data errors in '/private/tmp/pgloader/'
-2017-06-16T08:56:19.542000+02:00 LOG report summary reset
-               table name       read   imported     errors      total time
--------------------------  ---------  ---------  ---------  --------------
-          fetch meta data         33         33          0          0.365s
-           Create Schemas          0          0          0          0.007s
-         Create SQL Types          0          0          0          0.006s
-            Create tables         26         26          0          0.068s
-           Set Table OIDs         13         13          0          0.012s
--------------------------  ---------  ---------  ---------  --------------
-  f1db.constructorresults      11011      11011          0          0.205s
-            f1db.circuits         73         73          0          0.150s
-        f1db.constructors        208        208          0          0.059s
-f1db.constructorstandings      11766      11766          0          0.365s
-             f1db.drivers        841        841          0          0.268s
-            f1db.laptimes     413578     413578          0          2.892s
-     f1db.driverstandings      31420      31420          0          0.583s
-            f1db.pitstops       5796       5796          0          2.154s
-               f1db.races        976        976          0          0.227s
-          f1db.qualifying       7257       7257          0          0.228s
-             f1db.seasons         68         68          0          0.527s
-             f1db.results      23514      23514          0          0.658s
-              f1db.status        133        133          0          0.130s
--------------------------  ---------  ---------  ---------  --------------
-  COPY Threads Completion         39         39          0          4.303s
-           Create Indexes         20         20          0          1.497s
-   Index Build Completion         20         20          0          0.214s
-          Reset Sequences          0         10          0          0.058s
-             Primary Keys         13         13          0          0.012s
-      Create Foreign Keys          0          0          0          0.000s
-          Create Triggers          0          0          0          0.001s
-         Install Comments          0          0          0          0.000s
--------------------------  ---------  ---------  ---------  --------------
-        Total import time     506641     506641          0          5.547s
-
-
-

You may need to have special cases to take care of tho, or views that you -want to materialize while doing the migration. In advanced case you can use -the pgloader command.

-
-
-

The Command

-

To load data with pgloader you need to define in a command the operations -in some details. Here’s our example for loading the MySQL Sakila Sample -Database.

-

Here’s our command:

-
load database
-     from      mysql://root@localhost/sakila
-     into postgresql:///sakila
-
- WITH include drop, create tables, no truncate,
-      create indexes, reset sequences, foreign keys
-
-  SET maintenance_work_mem to '128MB', work_mem to '12MB', search_path to 'sakila'
-
- CAST type datetime to timestamptz
-                drop default drop not null using zero-dates-to-null,
-      type date drop not null drop default using zero-dates-to-null
-
- MATERIALIZE VIEWS film_list, staff_list
-
- -- INCLUDING ONLY TABLE NAMES MATCHING ~/film/, 'actor'
- -- EXCLUDING TABLE NAMES MATCHING ~<ory>
-
- BEFORE LOAD DO
- $$ create schema if not exists sakila; $$;
-
-
-

Note that here pgloader will benefit from the meta-data information found in -the MySQL database to create a PostgreSQL database capable of hosting the -data as described, then load the data.

-

In particular, some specific casting rules are given here, to cope with -date values such as 0000-00-00 that MySQL allows and PostgreSQL rejects -for not existing in our calendar. It’s possible to add per-column casting -rules too, which is useful is some of your tinyint are in fact smallint -while some others are in fact boolean values.

-

Finaly note that we are using the MATERIALIZE VIEWS clause of pgloader: -the selected views here will be migrated over to PostgreSQL with their -contents.

-

It’s possible to use the MATERIALIZE VIEWS clause and give both the name -and the SQL (in MySQL dialect) definition of view, then pgloader creates the -view before loading the data, then drops it again at the end.

-

## Loading the data

-

Let’s start the pgloader command with our sakila.load command file:

-
$ pgloader sakila.load
-... LOG Starting pgloader, log system is ready.
-... LOG Parsing commands from file "/Users/dim/dev/pgloader/test/sakila.load"
-   <WARNING: table "xxx" does not exists have been edited away>
-
-            table name       read   imported     errors            time
-----------------------  ---------  ---------  ---------  --------------
-           before load          1          1          0          0.007s
-       fetch meta data         45         45          0          0.402s
-          create, drop          0         36          0          0.208s
-----------------------  ---------  ---------  ---------  --------------
-                 actor        200        200          0          0.071s
-               address        603        603          0          0.035s
-              category         16         16          0          0.018s
-                  city        600        600          0          0.037s
-               country        109        109          0          0.023s
-              customer        599        599          0          0.073s
-                  film       1000       1000          0          0.135s
-            film_actor       5462       5462          0          0.236s
-         film_category       1000       1000          0          0.070s
-             film_text       1000       1000          0          0.080s
-             inventory       4581       4581          0          0.136s
-              language          6          6          0          0.036s
-               payment      16049      16049          0          0.539s
-                rental      16044      16044          0          0.648s
-                 staff          2          2          0          0.041s
-                 store          2          2          0          0.036s
-             film_list        997        997          0          0.247s
-            staff_list          2          2          0          0.135s
-Index Build Completion          0          0          0          0.000s
-----------------------  ---------  ---------  ---------  --------------
-        Create Indexes         41         41          0          0.964s
-       Reset Sequences          0          1          0          0.035s
-          Foreign Keys         22         22          0          0.254s
-----------------------  ---------  ---------  ---------  --------------
-     Total import time      48272      48272          0          3.502s
-
-
-

The WARNING messages we see here are expected as the PostgreSQL database -is empty when running the command, and pgloader is using the SQL commands -DROP TABLE IF EXISTS when the given command uses the include drop -option.

-

Note that the output of the command has been edited to facilitate its -browsing online.

-
-
-
- - -
-
-
- -
-
-
- ©2017, Dimitri Fontaine. - - | - Powered by Sphinx 1.6.5 - & Alabaster 0.7.10 - - | - Page source -
- - - - - - \ No newline at end of file diff --git a/pgloader.1 b/pgloader.1 deleted file mode 100644 index ca894cb..0000000 --- a/pgloader.1 +++ /dev/null @@ -1,3382 +0,0 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "PGLOADER" "1" "November 2017" "ff" "" -. -.SH "NAME" -\fBpgloader\fR \- PostgreSQL data loader -. -.SH "SYNOPSIS" -. -.nf - -pgloader [] []\.\.\. -pgloader [] SOURCE TARGET -. -.fi -. -.SH "DESCRIPTION" -pgloader loads data from various sources into PostgreSQL\. It can transform the data it reads on the fly and submit raw SQL before and after the loading\. It uses the \fBCOPY\fR PostgreSQL protocol to stream the data into the server, and manages errors by filling a pair of \fIreject\.dat\fR and \fIreject\.log\fR files\. -. -.P -pgloader operates either using commands which are read from files: -. -.IP "" 4 -. -.nf - -pgloader commands\.load -. -.fi -. -.IP "" 0 -. -.P -or by using arguments and options all provided on the command line: -. -.IP "" 4 -. -.nf - -pgloader SOURCE TARGET -. -.fi -. -.IP "" 0 -. -.SH "ARGUMENTS" -The pgloader arguments can be as many load files as needed, or a couple of connection strings to a specific input file\. -. -.SS "SOURCE CONNECTION STRING" -The source connection string format is as follows: -. -.IP "" 4 -. -.nf - -format:///absolute/path/to/file\.ext -format://\./relative/path/to/file\.ext -. -.fi -. -.IP "" 0 -. -.P -Where format might be one of \fBcsv\fR, \fBfixed\fR, \fBcopy\fR, \fBdbf\fR, \fBdb3\fR or \fBixf\fR\. -. -.IP "" 4 -. -.nf - -db://user:pass@host:port/dbname -. -.fi -. -.IP "" 0 -. -.P -Where db might be of \fBsqlite\fR, \fBmysql\fR or \fBmssql\fR\. -. -.P -When using a file based source format, pgloader also support natively fetching the file from an http location and decompressing an archive if needed\. In that case it\'s necessary to use the \fB\-\-type\fR option to specify the expected format of the file\. See the examples below\. -. -.P -Also note that some file formats require describing some implementation details such as columns to be read and delimiters and quoting when loading from csv\. -. -.P -For more complex loading scenarios, you will need to write a full fledge load command in the syntax described later in this document\. -. -.SS "TARGET CONNECTION STRING" -The target connection string format is described in details later in this document, see Section Connection String\. -. -.SH "OPTIONS" -. -.SS "INQUIRY OPTIONS" -Use these options when you want to know more about how to use \fBpgloader\fR, as those options will cause \fBpgloader\fR not to load any data\. -. -.TP -\fB\-h\fR, \fB\-\-help\fR -Show command usage summary and exit\. -. -.TP -\fB\-V\fR, \fB\-\-version\fR -Show pgloader version string and exit\. -. -.TP -\fB\-E\fR, \fB\-\-list\-encodings\fR -List known encodings in this version of pgloader\. -. -.TP -\fB\-U\fR, \fB\-\-upgrade\-config\fR -Parse given files in the command line as \fBpgloader\.conf\fR files with the \fBINI\fR syntax that was in use in pgloader versions 2\.x, and output the new command syntax for pgloader on standard output\. -. -.SS "GENERAL OPTIONS" -Those options are meant to tweak \fBpgloader\fR behavior when loading data\. -. -.IP "\(bu" 4 -\fB\-v\fR, \fB\-\-verbose\fR: Be verbose\. -. -.IP "\(bu" 4 -\fB\-q\fR, \fB\-\-quiet\fR: Be quiet\. -. -.IP "\(bu" 4 -\fB\-d\fR, \fB\-\-debug\fR: Show debug level information messages\. -. -.IP "\(bu" 4 -\fB\-D\fR, \fB\-\-root\-dir\fR: Set the root working directory (default to "/tmp/pgloader")\. -. -.IP "\(bu" 4 -\fB\-L\fR, \fB\-\-logfile\fR: Set the pgloader log file (default to "/tmp/pgloader\.log")\. -. -.IP "\(bu" 4 -\fB\-\-log\-min\-messages\fR: Minimum level of verbosity needed for log message to make it to the logfile\. One of critical, log, error, warning, notice, info or debug\. -. -.IP "\(bu" 4 -\fB\-\-client\-min\-messages\fR: Minimum level of verbosity needed for log message to make it to the console\. One of critical, log, error, warning, notice, info or debug\. -. -.IP "\(bu" 4 -\fB\-S\fR, \fB\-\-summary\fR: A filename where to copy the summary output\. When relative, the filename is expanded into \fB*root\-dir*\fR\. -. -.IP -The format of the filename defaults to being \fIhuman readable\fR\. It is possible to have the output in machine friendly formats such as \fICSV\fR, \fICOPY\fR (PostgreSQL\'s own COPY format) or \fIJSON\fR by specifying a filename with the extension resp\. \fB\.csv\fR, \fB\.copy\fR or \fB\.json\fR\. -. -.IP "\(bu" 4 -\fB\-l \fR, \fB\-\-load\-lisp\-file \fR: Specify a lisp \fIfile\fR to compile and load into the pgloader image before reading the commands, allowing to define extra transformation function\. Those functions should be defined in the \fBpgloader\.transforms\fR package\. This option can appear more than once in the command line\. -. -.IP "\(bu" 4 -\fB\-\-dry\-run\fR: -. -.IP -Allow testing a \fB\.load\fR file without actually trying to load any data\. It\'s useful to debug it until it\'s ok, in particular to fix connection strings\. -. -.IP "\(bu" 4 -\fB\-\-on\-error\-stop\fR -. -.IP -Alter pgloader behavior: rather than trying to be smart about error handling and continue loading good data, separating away the bad one, just stop as soon as PostgreSQL refuses anything sent to it\. Useful to debug data processing, transformation function and specific type casting\. -. -.IP "\(bu" 4 -\fB\-\-self\-upgrade \fR: -. -.IP -Specify a \fIdirectory\fR where to find pgloader sources so that one of the very first things it does is dynamically loading\-in (and compiling to machine code) another version of itself, usually a newer one like a very recent git checkout\. -. -.IP "" 0 -. -.SS "COMMAND LINE ONLY OPERATIONS" -Those options are meant to be used when using \fBpgloader\fR from the command line only, rather than using a command file and the rich command clauses and parser\. In simple cases, it can be much easier to use the \fISOURCE\fR and \fITARGET\fR directly on the command line, then tweak the loading with those options: -. -.IP "\(bu" 4 -\fB\-\-with "option"\fR: -. -.IP -Allows setting options from the command line\. You can use that option as many times as you want\. The option arguments must follow the \fIWITH\fR clause for the source type of the \fBSOURCE\fR specification, as described later in this document\. -. -.IP "\(bu" 4 -\fB\-\-set "guc_name=\'value\'"\fR -. -.IP -Allows setting PostgreSQL configuration from the command line\. Note that the option parsing is the same as when used from the \fISET\fR command clause, in particular you must enclose the guc value with single\-quotes\. -. -.IP "\(bu" 4 -\fB\-\-field "\.\.\."\fR -. -.IP -Allows setting a source field definition\. Fields are accumulated in the order given on the command line\. It\'s possible to either use a \fB\-\-field\fR option per field in the source file, or to separate field definitions by a comma, as you would do in the \fIHAVING FIELDS\fR clause\. -. -.IP "\(bu" 4 -\fB\-\-cast "\.\.\."\fR -. -.IP -Allows setting a specific casting rule for loading the data\. -. -.IP "\(bu" 4 -\fB\-\-type csv|fixed|db3|ixf|sqlite|mysql|mssql\fR -. -.IP -Allows forcing the source type, in case when the \fISOURCE\fR parsing isn\'t satisfying\. -. -.IP "\(bu" 4 -\fB\-\-encoding \fR -. -.IP -Set the encoding of the source file to load data from\. -. -.IP "\(bu" 4 -\fB\-\-before \fR -. -.IP -Parse given filename for SQL queries and run them against the target database before loading the data from the source\. The queries are parsed by pgloader itself: they need to be terminated by a semi\-colon (;) and the file may include \fB\ei\fR or \fB\eir\fR commands to \fIinclude\fR another file\. -. -.IP "\(bu" 4 -\fB\-\-after \fR -. -.IP -Parse given filename for SQL queries and run them against the target database after having loaded the data from the source\. The queries are parsed in the same way as with the \fB\-\-before\fR option, see above\. -. -.IP "" 0 -. -.SS "MORE DEBUG INFORMATION" -To get the maximum amount of debug information, you can use both the \fB\-\-verbose\fR and the \fB\-\-debug\fR switches at the same time, which is equivalent to saying \fB\-\-client\-min\-messages data\fR\. Then the log messages will show the data being processed, in the cases where the code has explicit support for it\. -. -.SH "USAGE EXAMPLES" -Review the command line options and pgloader\'s version: -. -.IP "" 4 -. -.nf - -pgloader \-\-help -pgloader \-\-version -. -.fi -. -.IP "" 0 -. -.SS "Loading from a complex command" -Use the command file as the pgloader command argument, pgloader will parse that file and execute the commands found in it: -. -.IP "" 4 -. -.nf - -pgloader \-\-verbose \./test/csv\-districts\.load -. -.fi -. -.IP "" 0 -. -.SS "CSV" -Load data from a CSV file into a pre\-existing table in your database, having pgloader guess the CSV properties (separator, quote and escape character): -. -.IP "" 4 -. -.nf - -pgloader \./test/data/matching\-1\.csv pgsql:///pgloader?tablename=matching -. -.fi -. -.IP "" 0 -. -.P -Load data from a CSV file into a pre\-existing table in your database, with expanded options: -. -.IP "" 4 -. -.nf - -pgloader \-\-type csv \e - \-\-field id \-\-field field \e - \-\-with truncate \e - \-\-with "fields terminated by \',\'" \e - \./test/data/matching\-1\.csv \e - postgres:///pgloader?tablename=matching -. -.fi -. -.IP "" 0 -. -.P -In that example the whole loading is driven from the command line, bypassing the need for writing a command in the pgloader command syntax entirely\. As there\'s no command though, the extra inforamtion needed must be provided on the command line using the \fB\-\-type\fR and \fB\-\-field\fR and \fB\-\-with\fR switches\. -. -.P -For documentation about the available syntaxes for the \fB\-\-field\fR and \fB\-\-with\fR switches, please refer to the CSV section later in the man page\. -. -.P -Note also that the PostgreSQL URI includes the target \fItablename\fR\. -. -.SS "Reading from STDIN" -File based pgloader sources can be loaded from the standard input, as in the following example: -. -.IP "" 4 -. -.nf - -pgloader \-\-type csv \e - \-\-field "usps,geoid,aland,awater,aland_sqmi,awater_sqmi,intptlat,intptlong" \e - \-\-with "skip header = 1" \e - \-\-with "fields terminated by \'\et\'" \e - \- \e - postgresql:///pgloader?districts_longlat \e - < test/data/2013_Gaz_113CDs_national\.txt -. -.fi -. -.IP "" 0 -. -.P -The dash (\fB\-\fR) character as a source is used to mean \fIstandard input\fR, as usual in Unix command lines\. It\'s possible to stream compressed content to pgloader with this technique, using the Unix pipe: -. -.IP "" 4 -. -.nf - -gunzip \-c source\.gz | pgloader \-\-type csv \.\.\. \- pgsql:///target?foo -. -.fi -. -.IP "" 0 -. -.SS "Loading from CSV available through HTTP" -The same command as just above can also be run if the CSV file happens to be found on a remote HTTP location: -. -.IP "" 4 -. -.nf - -pgloader \-\-type csv \e - \-\-field "usps,geoid,aland,awater,aland_sqmi,awater_sqmi,intptlat,intptlong" \e - \-\-with "skip header = 1" \e - \-\-with "fields terminated by \'\et\'" \e - http://pgsql\.tapoueh\.org/temp/2013_Gaz_113CDs_national\.txt \e - postgresql:///pgloader?districts_longlat -. -.fi -. -.IP "" 0 -. -.P -Some more options have to be used in that case, as the file contains a one\-line header (most commonly that\'s column names, could be a copyright notice)\. Also, in that case, we specify all the fields right into a single \fB\-\-field\fR option argument\. -. -.P -Again, the PostgreSQL target connection string must contain the \fItablename\fR option and you have to ensure that the target table exists and may fit the data\. Here\'s the SQL command used in that example in case you want to try it yourself: -. -.IP "" 4 -. -.nf - -create table districts_longlat -( - usps text, - geoid text, - aland bigint, - awater bigint, - aland_sqmi double precision, - awater_sqmi double precision, - intptlat double precision, - intptlong double precision -); -. -.fi -. -.IP "" 0 -. -.P -Also notice that the same command will work against an archived version of the same data, e\.g\. http://pgsql\.tapoueh\.org/temp/2013_Gaz_113CDs_national\.txt\.gz\. -. -.P -Finally, it\'s important to note that pgloader first fetches the content from the HTTP URL it to a local file, then expand the archive when it\'s recognized to be one, and only then processes the locally expanded file\. -. -.P -In some cases, either because pgloader has no direct support for your archive format or maybe because expanding the archive is not feasible in your environment, you might want to \fIstream\fR the content straight from its remote location into PostgreSQL\. Here\'s how to do that, using the old battle tested Unix Pipes trick: -. -.IP "" 4 -. -.nf - -curl http://pgsql\.tapoueh\.org/temp/2013_Gaz_113CDs_national\.txt\.gz \e -| gunzip \-c \e -| pgloader \-\-type csv \e - \-\-field "usps,geoid,aland,awater,aland_sqmi,awater_sqmi,intptlat,intptlong" - \-\-with "skip header = 1" \e - \-\-with "fields terminated by \'\et\'" \e - \- \e - postgresql:///pgloader?districts_longlat -. -.fi -. -.IP "" 0 -. -.P -Now the OS will take care of the streaming and buffering between the network and the commands and pgloader will take care of streaming the data down to PostgreSQL\. -. -.SS "Migrating from SQLite" -The following command will open the SQLite database, discover its tables definitions including indexes and foreign keys, migrate those definitions while \fIcasting\fR the data type specifications to their PostgreSQL equivalent and then migrate the data over: -. -.IP "" 4 -. -.nf - -createdb newdb -pgloader \./test/sqlite/sqlite\.db postgresql:///newdb -. -.fi -. -.IP "" 0 -. -.SS "Migrating from MySQL" -Just create a database where to host the MySQL data and definitions and have pgloader do the migration for you in a single command line: -. -.IP "" 4 -. -.nf - -createdb pagila -pgloader mysql://user@localhost/sakila postgresql:///pagila -. -.fi -. -.IP "" 0 -. -.SS "Fetching an archived DBF file from a HTTP remote location" -It\'s possible for pgloader to download a file from HTTP, unarchive it, and only then open it to discover the schema then load the data: -. -.IP "" 4 -. -.nf - -createdb foo -pgloader \-\-type dbf http://www\.insee\.fr/fr/methodes/nomenclatures/cog/telechargement/2013/dbf/historiq2013\.zip postgresql:///foo -. -.fi -. -.IP "" 0 -. -.P -Here it\'s not possible for pgloader to guess the kind of data source it\'s being given, so it\'s necessary to use the \fB\-\-type\fR command line switch\. -. -.SH "BATCHES AND RETRY BEHAVIOUR" -To load data to PostgreSQL, pgloader uses the \fBCOPY\fR streaming protocol\. While this is the faster way to load data, \fBCOPY\fR has an important drawback: as soon as PostgreSQL emits an error with any bit of data sent to it, whatever the problem is, the whole data set is rejected by PostgreSQL\. -. -.P -To work around that, pgloader cuts the data into \fIbatches\fR of 25000 rows each, so that when a problem occurs it\'s only impacting that many rows of data\. Each batch is kept in memory while the \fBCOPY\fR streaming happens, in order to be able to handle errors should some happen\. -. -.P -When PostgreSQL rejects the whole batch, pgloader logs the error message then isolates the bad row(s) from the accepted ones by retrying the batched rows in smaller batches\. To do that, pgloader parses the \fICONTEXT\fR error message from the failed COPY, as the message contains the line number where the error was found in the batch, as in the following example: -. -.IP "" 4 -. -.nf - -CONTEXT: COPY errors, line 3, column b: "2006\-13\-11" -. -.fi -. -.IP "" 0 -. -.P -Using that information, pgloader will reload all rows in the batch before the erroneous one, log the erroneous one as rejected, then try loading the remaining of the batch in a single attempt, which may or may not contain other erroneous data\. -. -.P -At the end of a load containing rejected rows, you will find two files in the \fIroot\-dir\fR location, under a directory named the same as the target database of your setup\. The filenames are the target table, and their extensions are \fB\.dat\fR for the rejected data and \fB\.log\fR for the file containing the full PostgreSQL client side logs about the rejected data\. -. -.P -The \fB\.dat\fR file is formatted in PostgreSQL the text COPY format as documented in http://www\.postgresql\.org/docs/9\.2/static/sql\-copy\.html#AEN66609 \fI\fR\. -. -.SH "A NOTE ABOUT PERFORMANCE" -pgloader has been developed with performance in mind, to be able to cope with ever growing needs in loading large amounts of data into PostgreSQL\. -. -.P -The basic architecture it uses is the old Unix pipe model, where a thread is responsible for loading the data (reading a CSV file, querying MySQL, etc) and fills pre\-processed data into a queue\. Another threads feeds from the queue, apply some more \fItransformations\fR to the input data and stream the end result to PostgreSQL using the COPY protocol\. -. -.P -When given a file that the PostgreSQL \fBCOPY\fR command knows how to parse, and if the file contains no erroneous data, then pgloader will never be as fast as just using the PostgreSQL \fBCOPY\fR command\. -. -.P -Note that while the \fBCOPY\fR command is restricted to read either from its standard input or from a local file on the server\'s file system, the command line tool \fBpsql\fR implements a \fB\ecopy\fR command that knows how to stream a file local to the client over the network and into the PostgreSQL server, using the same protocol as pgloader uses\. -. -.SH "A NOTE ABOUT PARALLELISM" -pgloader uses several concurrent tasks to process the data being loaded: -. -.IP "\(bu" 4 -a reader task reads the data in and pushes it to a queue, -. -.IP "\(bu" 4 -at last one write task feeds from the queue and formats the raw into the PostgreSQL COPY format in batches (so that it\'s possible to then retry a failed batch without reading the data from source again), and then sends the data to PostgreSQL using the COPY protocol\. -. -.IP "" 0 -. -.P -The parameter \fIworkers\fR allows to control how many worker threads are allowed to be active at any time (that\'s the parallelism level); and the parameter \fIconcurrency\fR allows to control how many tasks are started to handle the data (they may not all run at the same time, depending on the \fIworkers\fR setting)\. -. -.P -We allow \fIworkers\fR simultaneous workers to be active at the same time in the context of a single table\. A single unit of work consist of several kinds of workers: -. -.IP "\(bu" 4 -a reader getting raw data from the source, -. -.IP "\(bu" 4 -N writers preparing and sending the data down to PostgreSQL\. -. -.IP "" 0 -. -.P -The N here is setup to the \fIconcurrency\fR parameter: with a \fICONCURRENCY\fR of 2, we start (+ 1 2) = 3 concurrent tasks, with a \fIconcurrency\fR of 4 we start (+ 1 4) = 9 concurrent tasks, of which only \fIworkers\fR may be active simultaneously\. -. -.P -The defaults are \fBworkers = 4, concurrency = 1\fR when loading from a database source, and \fBworkers = 8, concurrency = 2\fR when loading from something else (currently, a file)\. Those defaults are arbitrary and waiting for feedback from users, so please consider providing feedback if you play with the settings\. -. -.P -As the \fBCREATE INDEX\fR threads started by pgloader are only waiting until PostgreSQL is done with the real work, those threads are \fINOT\fR counted into the concurrency levels as detailed here\. -. -.P -By default, as many \fBCREATE INDEX\fR threads as the maximum number of indexes per table are found in your source schema\. It is possible to set the \fBmax parallel create index\fR \fIWITH\fR option to another number in case there\'s just too many of them to create\. -. -.SH "SOURCE FORMATS" -pgloader supports the following input formats: -. -.IP "\(bu" 4 -csv, which includes also tsv and other common variants where you can change the \fIseparator\fR and the \fIquoting\fR rules and how to \fIescape\fR the \fIquotes\fR themselves; -. -.IP "\(bu" 4 -fixed columns file, where pgloader is flexible enough to accomodate with source files missing columns (\fIragged fixed length column files\fR do exist); -. -.IP "\(bu" 4 -PostgreSLQ COPY formatted files, following the COPY TEXT documentation of PostgreSQL, such as the reject files prepared by pgloader; -. -.IP "\(bu" 4 -dbase files known as db3 or dbf file; -. -.IP "\(bu" 4 -ixf formated files, ixf being a binary storage format from IBM; -. -.IP "\(bu" 4 -sqlite databases with fully automated discovery of the schema and advanced cast rules; -. -.IP "\(bu" 4 -mysql databases with fully automated discovery of the schema and advanced cast rules; -. -.IP "\(bu" 4 -MS SQL databases with fully automated discovery of the schema and advanced cast rules\. -. -.IP "" 0 -. -.SH "PGLOADER COMMANDS SYNTAX" -pgloader implements a Domain Specific Language allowing to setup complex data loading scripts handling computed columns and on\-the\-fly sanitization of the input data\. For more complex data loading scenarios, you will be required to learn that DSL\'s syntax\. It\'s meant to look familiar to DBA by being inspired by SQL where it makes sense, which is not that much after all\. -. -.P -The pgloader commands follow the same global grammar rules\. Each of them might support only a subset of the general options and provide specific options\. -. -.IP "" 4 -. -.nf - -LOAD - FROM - [ HAVING FIELDS ] - INTO - [ TARGET TABLE [ "" ]\."" ] - [ TARGET COLUMNS ] - -[ WITH ] - -[ SET ] - -[ BEFORE LOAD [ DO | EXECUTE ] \.\.\. ] -[ AFTER LOAD [ DO | EXECUTE ] \.\.\. ] -; -. -.fi -. -.IP "" 0 -. -.P -The main clauses are the \fBLOAD\fR, \fBFROM\fR, \fBINTO\fR and \fBWITH\fR clauses that each command implements\. Some command then implement the \fBSET\fR command, or some specific clauses such as the \fBCAST\fR clause\. -. -.SH "TEMPLATING WITH MUSTACHE" -pgloader implements the https://mustache\.github\.io/ templating system so that you may have dynamic parts of your commands\. See the documentation for this template system online\. -. -.P -A specific feature of pgloader is the ability to fetch a variable from the OS environment of the pgloader process, making it possible to run pgloader as in the following example: -. -.IP "" 4 -. -.nf - -$ DBPATH=sqlite/sqlite\.db pgloader \./test/sqlite\-env\.load -. -.fi -. -.IP "" 0 -. -.P -or in several steps: -. -.IP "" 4 -. -.nf - -$ export DBPATH=sqlite/sqlite\.db -$ pgloader \./test/sqlite\-env\.load -. -.fi -. -.IP "" 0 -. -.P -The variable can then be used in a typical mustache fashion: -. -.IP "" 4 -. -.nf - -load database - from \'{{DBPATH}}\' - into postgresql:///pgloader; -. -.fi -. -.IP "" 0 -. -.P -It\'s also possible to prepare a INI file such as the following: -. -.IP "" 4 -. -.nf - -[pgloader] - -DBPATH = sqlite/sqlite\.db -. -.fi -. -.IP "" 0 -. -.P -And run the following command, feeding the INI values as a \fIcontext\fR for pgloader templating system: -. -.IP "" 4 -. -.nf - -$ pgloader \-\-context \./test/sqlite\.ini \./test/sqlite\-ini\.load -. -.fi -. -.IP "" 0 -. -.P -The mustache templates implementation with OS environment support replaces former \fBGETENV\fR implementation, which didn\'t work anyway\. -. -.SH "COMMON CLAUSES" -Some clauses are common to all commands: -. -.IP "\(bu" 4 -\fIFROM\fR -. -.IP -The \fIFROM\fR clause specifies where to read the data from, and each command introduces its own variant of sources\. For instance, the \fICSV\fR source supports \fBinline\fR, \fBstdin\fR, a filename, a quoted filename, and a \fIFILENAME MATCHING\fR clause (see above); whereas the \fIMySQL\fR source only supports a MySQL database URI specification\. -. -.IP "\(bu" 4 -\fIINTO\fR -. -.IP -The PostgreSQL connection URI must contains the name of the target table where to load the data into\. That table must have already been created in PostgreSQL, and the name might be schema qualified\. -. -.IP -Then \fIINTO\fR option also supports an optional comma separated list of target columns, which are either the name of an input \fIfield\fR or the white space separated list of the target column name, its PostgreSQL data type and a \fIUSING\fR expression\. -. -.IP -The \fIUSING\fR expression can be any valid Common Lisp form and will be read with the current package set to \fBpgloader\.transforms\fR, so that you can use functions defined in that package, such as functions loaded dynamically with the \fB\-\-load\fR command line parameter\. -. -.IP -Each \fIUSING\fR expression is compiled at runtime to native code\. -. -.IP -This feature allows pgloader to load any number of fields in a CSV file into a possibly different number of columns in the database, using custom code for that projection\. -. -.IP "\(bu" 4 -\fIWITH\fR -. -.IP -Set of options to apply to the command, using a global syntax of either: -. -.IP "\(bu" 4 -\fIkey = value\fR -. -.IP "\(bu" 4 -\fIuse option\fR -. -.IP "\(bu" 4 -\fIdo not use option\fR -. -.IP "" 0 -. -.IP -See each specific command for details\. -. -.IP -All data sources specific commands support the following options: -. -.IP "\(bu" 4 -\fIon error stop\fR -. -.IP "\(bu" 4 -\fIbatch rows = R\fR -. -.IP "\(bu" 4 -\fIbatch size = \.\.\. MB\fR -. -.IP "\(bu" 4 -\fIprefetch rows = \.\.\.\fR -. -.IP "" 0 -. -.IP -See the section BATCH BEHAVIOUR OPTIONS for more details\. -. -.IP -In addition, the following settings are available: -. -.IP "\(bu" 4 -\fIworkers = W\fR -. -.IP "\(bu" 4 -\fIconcurrency = C\fR -. -.IP "\(bu" 4 -\fImax parallel create index = I\fR -. -.IP "" 0 -. -.IP -See section A NOTE ABOUT PARALLELISM for more details\. -. -.IP "\(bu" 4 -\fISET\fR -. -.IP -This clause allows to specify session parameters to be set for all the sessions opened by pgloader\. It expects a list of parameter name, the equal sign, then the single\-quoted value as a comma separated list\. -. -.IP -The names and values of the parameters are not validated by pgloader, they are given as\-is to PostgreSQL\. -. -.IP "\(bu" 4 -\fIBEFORE LOAD DO\fR -. -.IP -You can run SQL queries against the database before loading the data from the \fBCSV\fR file\. Most common SQL queries are \fBCREATE TABLE IF NOT EXISTS\fR so that the data can be loaded\. -. -.IP -Each command must be \fIdollar\-quoted\fR: it must begin and end with a double dollar sign, \fB$$\fR\. Dollar\-quoted queries are then comma separated\. No extra punctuation is expected after the last SQL query\. -. -.IP "\(bu" 4 -\fIBEFORE LOAD EXECUTE\fR -. -.IP -Same behaviour as in the \fIBEFORE LOAD DO\fR clause\. Allows you to read the SQL queries from a SQL file\. Implements support for PostgreSQL dollar\-quoting and the \fB\ei\fR and \fB\eir\fR include facilities as in \fBpsql\fR batch mode (where they are the same thing)\. -. -.IP "\(bu" 4 -\fIAFTER LOAD DO\fR -. -.IP -Same format as \fIBEFORE LOAD DO\fR, the dollar\-quoted queries found in that section are executed once the load is done\. That\'s the right time to create indexes and constraints, or re\-enable triggers\. -. -.IP "\(bu" 4 -\fIAFTER LOAD EXECUTE\fR -. -.IP -Same behaviour as in the \fIAFTER LOAD DO\fR clause\. Allows you to read the SQL queries from a SQL file\. Implements support for PostgreSQL dollar\-quoting and the \fB\ei\fR and \fB\eir\fR include facilities as in \fBpsql\fR batch mode (where they are the same thing)\. -. -.IP "" 0 -. -.SS "Connection String" -The \fB\fR parameter is expected to be given as a \fIConnection URI\fR as documented in the PostgreSQL documentation at http://www\.postgresql\.org/docs/9\.3/static/libpq\-connect\.html#LIBPQ\-CONNSTRING\. -. -.IP "" 4 -. -.nf - -postgresql://[user[:password]@][netloc][:port][/dbname][?option=value&\.\.\.] -. -.fi -. -.IP "" 0 -. -.P -Where: -. -.IP "\(bu" 4 -\fIuser\fR -. -.IP -Can contain any character, including colon (\fB:\fR) which must then be doubled (\fB::\fR) and at\-sign (\fB@\fR) which must then be doubled (\fB@@\fR)\. -. -.IP -When omitted, the \fIuser\fR name defaults to the value of the \fBPGUSER\fR environment variable, and if it is unset, the value of the \fBUSER\fR environment variable\. -. -.IP "\(bu" 4 -\fIpassword\fR -. -.IP -Can contain any character, including the at sign (\fB@\fR) which must then be doubled (\fB@@\fR)\. To leave the password empty, when the \fIuser\fR name ends with at at sign, you then have to use the syntax user:@\. -. -.IP -When omitted, the \fIpassword\fR defaults to the value of the \fBPGPASSWORD\fR environment variable if it is set, otherwise the password is left unset\. -. -.IP -When no \fIpassword\fR is found either in the connection URI nor in the environment, then pgloader looks for a \fB\.pgpass\fR file as documented at https://www\.postgresql\.org/docs/current/static/libpq\-pgpass\.html\. The implementation is not that of \fBlibpq\fR though\. As with \fBlibpq\fR you can set the environment variable \fBPGPASSFILE\fR to point to a \fB\.pgpass\fR file, and pgloader defaults to \fB~/\.pgpass\fR on unix like systems and \fB%APPDATA%\epostgresql\epgpass\.conf\fR on windows\. Matching rules and syntax are the same as with \fBlibpq\fR, refer to its documentation\. -. -.IP "\(bu" 4 -\fInetloc\fR -. -.IP -Can be either a hostname in dotted notation, or an ipv4, or an Unix domain socket path\. Empty is the default network location, under a system providing \fIunix domain socket\fR that method is preferred, otherwise the \fInetloc\fR default to \fBlocalhost\fR\. -. -.IP -It\'s possible to force the \fIunix domain socket\fR path by using the syntax \fBunix:/path/to/where/the/socket/file/is\fR, so to force a non default socket path and a non default port, you would have: -. -.IP "" 4 -. -.nf - -postgresql://unix:/tmp:54321/dbname -. -.fi -. -.IP "" 0 -. -.IP -The \fInetloc\fR defaults to the value of the \fBPGHOST\fR environment variable, and if it is unset, to either the default \fBunix\fR socket path when running on a Unix system, and \fBlocalhost\fR otherwise\. -. -.IP -Socket path containing colons are supported by doubling the colons within the path, as in the following example: -. -.IP "" 4 -. -.nf - -postgresql://unix:/tmp/project::region::instance:5432/dbname -. -.fi -. -.IP "" 0 - -. -.IP "\(bu" 4 -\fIdbname\fR -. -.IP -Should be a proper identifier (letter followed by a mix of letters, digits and the punctuation signs comma (\fB,\fR), dash (\fB\-\fR) and underscore (\fB_\fR)\. -. -.IP -When omitted, the \fIdbname\fR defaults to the value of the environment variable \fBPGDATABASE\fR, and if that is unset, to the \fIuser\fR value as determined above\. -. -.IP "\(bu" 4 -\fIoptions\fR -. -.IP -The optional parameters must be supplied with the form \fBname=value\fR, and you may use several parameters by separating them away using an ampersand (\fB&\fR) character\. -. -.IP -Only some options are supported here, \fItablename\fR (which might be qualified with a schema name) \fIsslmode\fR, \fIhost\fR, \fIport\fR, \fIdbname\fR, \fIuser\fR and \fIpassword\fR\. -. -.IP -The \fIsslmode\fR parameter values can be one of \fBdisable\fR, \fBallow\fR, \fBprefer\fR or \fBrequire\fR\. -. -.IP -For backward compatibility reasons, it\'s possible to specify the \fItablename\fR option directly, without spelling out the \fBtablename=\fR parts\. -. -.IP -The options override the main URI components when both are given, and using the percent\-encoded option parameters allow using passwords starting with a colon and bypassing other URI components parsing limitations\. -. -.IP "" 0 -. -.SS "Regular Expressions" -Several clauses listed in the following accept \fIregular expressions\fR with the following input rules: -. -.IP "\(bu" 4 -A regular expression begins with a tilde sign (\fB~\fR), -. -.IP "\(bu" 4 -is then followed with an opening sign, -. -.IP "\(bu" 4 -then any character is allowed and considered part of the regular expression, except for the closing sign, -. -.IP "\(bu" 4 -then a closing sign is expected\. -. -.IP "" 0 -. -.P -The opening and closing sign are allowed by pair, here\'s the complete list of allowed delimiters: -. -.IP "" 4 -. -.nf - -~// -~[] -~{} -~() -~<> -~"" -~\'\' -~|| -~## -. -.fi -. -.IP "" 0 -. -.P -Pick the set of delimiters that don\'t collide with the \fIregular expression\fR you\'re trying to input\. If your expression is such that none of the solutions allow you to enter it, the places where such expressions are allowed should allow for a list of expressions\. -. -.SS "Comments" -Any command may contain comments, following those input rules: -. -.IP "\(bu" 4 -the \fB\-\-\fR delimiter begins a comment that ends with the end of the current line, -. -.IP "\(bu" 4 -the delimiters \fB/*\fR and \fB*/\fR respectively start and end a comment, which can be found in the middle of a command or span several lines\. -. -.IP "" 0 -. -.P -Any place where you could enter a \fIwhitespace\fR will accept a comment too\. -. -.SS "Batch behaviour options" -All pgloader commands have support for a \fIWITH\fR clause that allows for specifying options\. Some options are generic and accepted by all commands, such as the \fIbatch behaviour options\fR, and some options are specific to a data source kind, such as the CSV \fIskip header\fR option\. -. -.P -The global batch behaviour options are: -. -.IP "\(bu" 4 -\fIbatch rows\fR -. -.IP -Takes a numeric value as argument, used as the maximum number of rows allowed in a batch\. The default is \fB25 000\fR and can be changed to try having better performance characteristics or to control pgloader memory usage; -. -.IP "\(bu" 4 -\fIbatch size\fR -. -.IP -Takes a memory unit as argument, such as \fI20 MB\fR, its default value\. Accepted multipliers are \fIkB\fR, \fIMB\fR, \fIGB\fR, \fITB\fR and \fIPB\fR\. The case is important so as not to be confused about bits versus bytes, we\'re only talking bytes here\. -. -.IP "\(bu" 4 -\fIprefetch rows\fR -. -.IP -Takes a numeric value as argument, defaults to \fB100000\fR\. That\'s the number of rows that pgloader is allowed to read in memory in each reader thread\. See the \fIworkers\fR setting for how many reader threads are allowed to run at the same time\. -. -.IP "" 0 -. -.P -Other options are specific to each input source, please refer to specific parts of the documentation for their listing and covering\. -. -.P -A batch is then closed as soon as either the \fIbatch rows\fR or the \fIbatch size\fR threshold is crossed, whichever comes first\. In cases when a batch has to be closed because of the \fIbatch size\fR setting, a \fIdebug\fR level log message is printed with how many rows did fit in the \fIoversized\fR batch\. -. -.SH "LOAD CSV" -This command instructs pgloader to load data from a \fBCSV\fR file\. Here\'s an example: -. -.IP "" 4 -. -.nf - -LOAD CSV - FROM \'GeoLiteCity\-Blocks\.csv\' WITH ENCODING iso\-646\-us - HAVING FIELDS - ( - startIpNum, endIpNum, locId - ) - INTO postgresql://user@localhost:54393/dbname - TARGET TABLE geolite\.blocks - TARGET COLUMNS - ( - iprange ip4r using (ip\-range startIpNum endIpNum), - locId - ) - WITH truncate, - skip header = 2, - fields optionally enclosed by \'"\', - fields escaped by backslash\-quote, - fields terminated by \'\et\' - - SET work_mem to \'32 MB\', maintenance_work_mem to \'64 MB\'; -. -.fi -. -.IP "" 0 -. -.P -The \fBcsv\fR format command accepts the following clauses and options: -. -.IP "\(bu" 4 -\fIFROM\fR -. -.IP -Filename where to load the data from\. Accepts an \fIENCODING\fR option\. Use the \fB\-\-list\-encodings\fR option to know which encoding names are supported\. -. -.IP -The filename may be enclosed by single quotes, and could be one of the following special values: -. -.IP "\(bu" 4 -\fIinline\fR -. -.IP -The data is found after the end of the parsed commands\. Any number of empty lines between the end of the commands and the beginning of the data is accepted\. -. -.IP "\(bu" 4 -\fIstdin\fR -. -.IP -Reads the data from the standard input stream\. -. -.IP "\(bu" 4 -\fIFILENAMES MATCHING\fR -. -.IP -The whole \fImatching\fR clause must follow the following rule: -. -.IP "" 4 -. -.nf - -[ ALL FILENAMES | [ FIRST ] FILENAME ] -MATCHING regexp -[ IN DIRECTORY \'\.\.\.\' ] -. -.fi -. -.IP "" 0 -. -.IP -The \fImatching\fR clause applies given \fIregular expression\fR (see above for exact syntax, several options can be used here) to filenames\. It\'s then possible to load data from only the first match of all of them\. -. -.IP -The optional \fIIN DIRECTORY\fR clause allows specifying which directory to walk for finding the data files, and can be either relative to where the command file is read from, or absolute\. The given directory must exists\. -. -.IP "" 0 -. -.IP -The \fIFROM\fR option also supports an optional comma separated list of \fIfield\fR names describing what is expected in the \fBCSV\fR data file, optionally introduced by the clause \fBHAVING FIELDS\fR\. -. -.IP -Each field name can be either only one name or a name following with specific reader options for that field, enclosed in square brackets and comma\-separated\. Supported per\-field reader options are: -. -.IP "\(bu" 4 -\fIterminated by\fR -. -.IP -See the description of \fIfield terminated by\fR below\. -. -.IP -The processing of this option is not currently implemented\. -. -.IP "\(bu" 4 -\fIdate format\fR -. -.IP -When the field is expected of the date type, then this option allows to specify the date format used in the file\. -. -.IP -Date format string are template strings modeled against the PostgreSQL \fBto_char\fR template strings support, limited to the following patterns: -. -.IP "\(bu" 4 -YYYY, YYY, YY for the year part -. -.IP "\(bu" 4 -MM for the numeric month part -. -.IP "\(bu" 4 -DD for the numeric day part -. -.IP "\(bu" 4 -HH, HH12, HH24 for the hour part -. -.IP "\(bu" 4 -am, AM, a\.m\., A\.M\. -. -.IP "\(bu" 4 -pm, PM, p\.m\., P\.M\. -. -.IP "\(bu" 4 -MI for the minutes part -. -.IP "\(bu" 4 -SS for the seconds part -. -.IP "\(bu" 4 -MS for the milliseconds part (4 digits) -. -.IP "\(bu" 4 -US for the microseconds part (6 digits) -. -.IP "\(bu" 4 -unparsed punctuation signs: \- \. * # @ T / \e and space -. -.IP "" 0 -. -.IP -Here\'s an example of a \fIdate format\fR specification: -. -.IP "" 4 -. -.nf - -column\-name [date format \'YYYY\-MM\-DD HH24\-MI\-SS\.US\'] -. -.fi -. -.IP "" 0 - -. -.IP "\(bu" 4 -\fInull if\fR -. -.IP -This option takes an argument which is either the keyword \fIblanks\fR or a double\-quoted string\. -. -.IP -When \fIblanks\fR is used and the field value that is read contains only space characters, then it\'s automatically converted to an SQL \fBNULL\fR value\. -. -.IP -When a double\-quoted string is used and that string is read as the field value, then the field value is automatically converted to an SQL \fBNULL\fR value\. -. -.IP "\(bu" 4 -\fItrim both whitespace\fR, \fItrim left whitespace\fR, \fItrim right whitespace\fR -. -.IP -This option allows to trim whitespaces in the read data, either from both sides of the data, or only the whitespace characters found on the left of the streaing, or only those on the right of the string\. -. -.IP "" 0 - -. -.IP "\(bu" 4 -\fIWITH\fR -. -.IP -When loading from a \fBCSV\fR file, the following options are supported: -. -.IP "\(bu" 4 -\fItruncate\fR -. -.IP -When this option is listed, pgloader issues a \fBTRUNCATE\fR command against the PostgreSQL target table before reading the data file\. -. -.IP "\(bu" 4 -\fIdrop indexes\fR -. -.IP -When this option is listed, pgloader issues \fBDROP INDEX\fR commands against all the indexes defined on the target table before copying the data, then \fBCREATE INDEX\fR commands once the \fBCOPY\fR is done\. -. -.IP -In order to get the best performance possible, all the indexes are created in parallel and when done the primary keys are built again from the unique indexes just created\. This two step process allows creating the primary key index in parallel with the other indexes, as only the \fBALTER TABLE\fR command needs an \fIaccess exclusive lock\fR on the target table\. -. -.IP "\(bu" 4 -\fIdisable triggers\fR -. -.IP -When this option is listed, pgloader issues an \fBALTER TABLE \.\.\. DISABLE TRIGGER ALL\fR command against the PostgreSQL target table before copying the data, then the command \fBALTER TABLE \.\.\. ENABLE TRIGGER ALL\fR once the \fBCOPY\fR is done\. -. -.IP -This option allows loading data into a pre\-existing table ignoring the \fIforeign key constraints\fR and user defined triggers and may result in invalid \fIforeign key constraints\fR once the data is loaded\. Use with care\. -. -.IP "\(bu" 4 -\fIskip header\fR -. -.IP -Takes a numeric value as argument\. Instruct pgloader to skip that many lines at the beginning of the input file\. -. -.IP "\(bu" 4 -\fIcsv header\fR -. -.IP -Use the first line read after \fIskip header\fR as the list of csv field names to be found in the CSV file, using the same CSV parameters as for the CSV data\. -. -.IP "\(bu" 4 -\fItrim unquoted blanks\fR -. -.IP -When reading unquoted values in the \fBCSV\fR file, remove the blanks found in between the separator and the value\. That behaviour is the default\. -. -.IP "\(bu" 4 -\fIkeep unquoted blanks\fR -. -.IP -When reading unquoted values in the \fBCSV\fR file, keep blanks found in between the separator and the value\. -. -.IP "\(bu" 4 -\fIfields optionally enclosed by\fR -. -.IP -Takes a single character as argument, which must be found inside single quotes, and might be given as the printable character itself, the special value \et to denote a tabulation character, or \fB0x\fR then an hexadecimal value read as the ASCII code for the character\. -. -.IP -This character is used as the quoting character in the \fBCSV\fR file, and defaults to double\-quote\. -. -.IP "\(bu" 4 -\fIfields not enclosed\fR -. -.IP -By default, pgloader will use the double\-quote character as the enclosing character\. If you have a CSV file where fields are not enclosed and are using double\-quote as an expected ordinary character, then use the option \fIfields not enclosed\fR for the CSV parser to accept those values\. -. -.IP "\(bu" 4 -\fIfields escaped by\fR -. -.IP -Takes either the special value \fIbackslash\-quote\fR or \fIdouble\-quote\fR, or any value supported by the \fIfields terminated by\fR option (see below)\. This value is used to recognize escaped field separators when they are to be found within the data fields themselves\. Defaults to \fIdouble\-quote\fR\. -. -.IP "\(bu" 4 -\fIcsv escape mode\fR -. -.IP -Takes either the special value \fIquote\fR (the default) or \fIfollowing\fR and allows the CSV parser to parse either only escaped field separator or any character (including CSV data) when using the \fIfollowing\fR value\. -. -.IP "\(bu" 4 -\fIfields terminated by\fR -. -.IP -Takes a single character as argument, which must be found inside single quotes, and might be given as the printable character itself, the special value \et to denote a tabulation character, or \fB0x\fR then an hexadecimal value read as the ASCII code for the character\. -. -.IP -This character is used as the \fIfield separator\fR when reading the \fBCSV\fR data\. -. -.IP "\(bu" 4 -\fIlines terminated by\fR -. -.IP -Takes a single character as argument, which must be found inside single quotes, and might be given as the printable character itself, the special value \et to denote a tabulation character, or \fB0x\fR then an hexadecimal value read as the ASCII code for the character\. -. -.IP -This character is used to recognize \fIend\-of\-line\fR condition when reading the \fBCSV\fR data\. -. -.IP "" 0 - -. -.IP "" 0 -. -.SH "LOAD FIXED COLS" -This command instructs pgloader to load data from a text file containing columns arranged in a \fIfixed size\fR manner\. Here\'s an example: -. -.IP "" 4 -. -.nf - -LOAD FIXED - FROM inline - ( - a from 0 for 10, - b from 10 for 8, - c from 18 for 8, - d from 26 for 17 [null if blanks, trim right whitespace] - ) - INTO postgresql:///pgloader - TARGET TABLE fixed - ( - a, b, - c time using (time\-with\-no\-separator c), - d - ) - - WITH truncate - - SET work_mem to \'14MB\', - standard_conforming_strings to \'on\' - -BEFORE LOAD DO - $$ drop table if exists fixed; $$, - $$ create table fixed ( - a integer, - b date, - c time, - d text - ); - $$; - - 01234567892008052011431250firstline - 01234562008052115182300left blank\-padded - 12345678902008052208231560another line - 2345609872014092914371500 - 2345678902014092914371520 -. -.fi -. -.IP "" 0 -. -.P -The \fBfixed\fR format command accepts the following clauses and options: -. -.IP "\(bu" 4 -\fIFROM\fR -. -.IP -Filename where to load the data from\. Accepts an \fIENCODING\fR option\. Use the \fB\-\-list\-encodings\fR option to know which encoding names are supported\. -. -.IP -The filename may be enclosed by single quotes, and could be one of the following special values: -. -.IP "\(bu" 4 -\fIinline\fR -. -.IP -The data is found after the end of the parsed commands\. Any number of empty lines between the end of the commands and the beginning of the data is accepted\. -. -.IP "\(bu" 4 -\fIstdin\fR -. -.IP -Reads the data from the standard input stream\. -. -.IP "\(bu" 4 -\fIFILENAMES MATCHING\fR -. -.IP -The whole \fImatching\fR clause must follow the following rule: -. -.IP "" 4 -. -.nf - -[ ALL FILENAMES | [ FIRST ] FILENAME ] -MATCHING regexp -[ IN DIRECTORY \'\.\.\.\' ] -. -.fi -. -.IP "" 0 -. -.IP -The \fImatching\fR clause applies given \fIregular expression\fR (see above for exact syntax, several options can be used here) to filenames\. It\'s then possible to load data from only the first match of all of them\. -. -.IP -The optional \fIIN DIRECTORY\fR clause allows specifying which directory to walk for finding the data files, and can be either relative to where the command file is read from, or absolute\. The given directory must exists\. -. -.IP "" 0 -. -.IP -The \fIFROM\fR option also supports an optional comma separated list of \fIfield\fR names describing what is expected in the \fBFIXED\fR data file\. -. -.IP -Each field name is composed of the field name followed with specific reader options for that field\. Supported per\-field reader options are the following, where only \fIstart\fR and \fIlength\fR are required\. -. -.IP "\(bu" 4 -\fIstart\fR -. -.IP -Position in the line where to start reading that field\'s value\. Can be entered with decimal digits or \fB0x\fR then hexadecimal digits\. -. -.IP "\(bu" 4 -\fIlength\fR -. -.IP -How many bytes to read from the \fIstart\fR position to read that field\'s value\. Same format as \fIstart\fR\. -. -.IP "" 0 -. -.IP -Those optional parameters must be enclosed in square brackets and comma\-separated: -. -.IP "\(bu" 4 -\fIterminated by\fR -. -.IP -See the description of \fIfield terminated by\fR below\. -. -.IP -The processing of this option is not currently implemented\. -. -.IP "\(bu" 4 -\fIdate format\fR -. -.IP -When the field is expected of the date type, then this option allows to specify the date format used in the file\. -. -.IP -Date format string are template strings modeled against the PostgreSQL \fBto_char\fR template strings support, limited to the following patterns: -. -.IP "\(bu" 4 -YYYY, YYY, YY for the year part -. -.IP "\(bu" 4 -MM for the numeric month part -. -.IP "\(bu" 4 -DD for the numeric day part -. -.IP "\(bu" 4 -HH, HH12, HH24 for the hour part -. -.IP "\(bu" 4 -am, AM, a\.m\., A\.M\. -. -.IP "\(bu" 4 -pm, PM, p\.m\., P\.M\. -. -.IP "\(bu" 4 -MI for the minutes part -. -.IP "\(bu" 4 -SS for the seconds part -. -.IP "\(bu" 4 -MS for the milliseconds part (4 digits) -. -.IP "\(bu" 4 -US for the microseconds part (6 digits) -. -.IP "\(bu" 4 -unparsed punctuation signs: \- \. * # @ T / \e and space -. -.IP "" 0 -. -.IP -Here\'s an example of a \fIdate format\fR specification: -. -.IP "" 4 -. -.nf - -column\-name [date format \'YYYY\-MM\-DD HH24\-MI\-SS\.US\'] -. -.fi -. -.IP "" 0 - -. -.IP "\(bu" 4 -\fInull if\fR -. -.IP -This option takes an argument which is either the keyword \fIblanks\fR or a double\-quoted string\. -. -.IP -When \fIblanks\fR is used and the field value that is read contains only space characters, then it\'s automatically converted to an SQL \fBNULL\fR value\. -. -.IP -When a double\-quoted string is used and that string is read as the field value, then the field value is automatically converted to an SQL \fBNULL\fR value\. -. -.IP "\(bu" 4 -\fItrim both whitespace\fR, \fItrim left whitespace\fR, \fItrim right whitespace\fR -. -.IP -This option allows to trim whitespaces in the read data, either from both sides of the data, or only the whitespace characters found on the left of the streaing, or only those on the right of the string\. -. -.IP "" 0 - -. -.IP "\(bu" 4 -\fIWITH\fR -. -.IP -When loading from a \fBFIXED\fR file, the following options are supported: -. -.IP "\(bu" 4 -\fItruncate\fR -. -.IP -When this option is listed, pgloader issues a \fBTRUNCATE\fR command against the PostgreSQL target table before reading the data file\. -. -.IP "\(bu" 4 -\fIdisable triggers\fR -. -.IP -When this option is listed, pgloader issues an \fBALTER TABLE \.\.\. DISABLE TRIGGER ALL\fR command against the PostgreSQL target table before copying the data, then the command \fBALTER TABLE \.\.\. ENABLE TRIGGER ALL\fR once the \fBCOPY\fR is done\. -. -.IP -This option allows loading data into a pre\-existing table ignoring the \fIforeign key constraints\fR and user defined triggers and may result in invalid \fIforeign key constraints\fR once the data is loaded\. Use with care\. -. -.IP "\(bu" 4 -\fIskip header\fR -. -.IP -Takes a numeric value as argument\. Instruct pgloader to skip that many lines at the beginning of the input file\. -. -.IP "" 0 - -. -.IP "" 0 -. -.SH "LOAD COPY FORMATTED FILES" -This commands instructs pgloader to load from a file containing COPY TEXT data as described in the PostgreSQL documentation\. Here\'s an example: -. -.IP "" 4 -. -.nf - -LOAD COPY - FROM copy://\./data/track\.copy - ( - trackid, track, album, media, genre, composer, - milliseconds, bytes, unitprice - ) - INTO postgresql:///pgloader - TARGET TABLE track_full - - WITH truncate - - SET work_mem to \'14MB\', - standard_conforming_strings to \'on\' - -BEFORE LOAD DO - $$ drop table if exists track_full; $$, - $$ create table track_full ( - trackid bigserial, - track text, - album text, - media text, - genre text, - composer text, - milliseconds bigint, - bytes bigint, - unitprice numeric - ); - $$; -. -.fi -. -.IP "" 0 -. -.P -The \fBCOPY\fR format command accepts the following clauses and options: -. -.IP "\(bu" 4 -\fIFROM\fR -. -.IP -Filename where to load the data from\. This support local files, HTTP URLs and zip files containing a single dbf file of the same name\. Fetch such a zip file from an HTTP address is of course supported\. -. -.IP "\(bu" 4 -\fIinline\fR -. -.IP -The data is found after the end of the parsed commands\. Any number of empty lines between the end of the commands and the beginning of the data is accepted\. -. -.IP "\(bu" 4 -\fIstdin\fR -. -.IP -Reads the data from the standard input stream\. -. -.IP "\(bu" 4 -\fIFILENAMES MATCHING\fR -. -.IP -The whole \fImatching\fR clause must follow the following rule: -. -.IP "" 4 -. -.nf - -[ ALL FILENAMES | [ FIRST ] FILENAME ] -MATCHING regexp -[ IN DIRECTORY \'\.\.\.\' ] -. -.fi -. -.IP "" 0 -. -.IP -The \fImatching\fR clause applies given \fIregular expression\fR (see above for exact syntax, several options can be used here) to filenames\. It\'s then possible to load data from only the first match of all of them\. -. -.IP -The optional \fIIN DIRECTORY\fR clause allows specifying which directory to walk for finding the data files, and can be either relative to where the command file is read from, or absolute\. The given directory must exists\. -. -.IP "" 0 - -. -.IP "\(bu" 4 -\fIWITH\fR -. -.IP -When loading from a \fBCOPY\fR file, the following options are supported: -. -.IP "\(bu" 4 -\fIdelimiter\fR -. -.IP -Takes a single character as argument, which must be found inside single quotes, and might be given as the printable character itself, the special value \et to denote a tabulation character, or \fB0x\fR then an hexadecimal value read as the ASCII code for the character\. -. -.IP -This character is used as the \fIdelimiter\fR when reading the data, in a similar way to the PostgreSQL \fBCOPY\fR option\. -. -.IP "\(bu" 4 -\fInull\fR -. -.IP -Takes a quoted string as an argument (quotes can be either double quotes or single quotes) and uses that string as the \fBNULL\fR representation in the data\. -. -.IP -This is similar to the \fInull\fR \fBCOPY\fR option in PostgreSQL\. -. -.IP "\(bu" 4 -\fItruncate\fR -. -.IP -When this option is listed, pgloader issues a \fBTRUNCATE\fR command against the PostgreSQL target table before reading the data file\. -. -.IP "\(bu" 4 -\fIdisable triggers\fR -. -.IP -When this option is listed, pgloader issues an \fBALTER TABLE \.\.\. DISABLE TRIGGER ALL\fR command against the PostgreSQL target table before copying the data, then the command \fBALTER TABLE \.\.\. ENABLE TRIGGER ALL\fR once the \fBCOPY\fR is done\. -. -.IP -This option allows loading data into a pre\-existing table ignoring the \fIforeign key constraints\fR and user defined triggers and may result in invalid \fIforeign key constraints\fR once the data is loaded\. Use with care\. -. -.IP "\(bu" 4 -\fIskip header\fR -. -.IP -Takes a numeric value as argument\. Instruct pgloader to skip that many lines at the beginning of the input file\. -. -.IP "" 0 - -. -.IP "" 0 -. -.SH "LOAD DBF" -This command instructs pgloader to load data from a \fBDBF\fR file\. Here\'s an example: -. -.IP "" 4 -. -.nf - -LOAD DBF - FROM http://www\.insee\.fr/fr/methodes/nomenclatures/cog/telechargement/2013/dbf/reg2013\.dbf - INTO postgresql://user@localhost/dbname - WITH truncate, create table; -. -.fi -. -.IP "" 0 -. -.P -The \fBdbf\fR format command accepts the following clauses and options: -. -.IP "\(bu" 4 -\fIFROM\fR -. -.IP -Filename where to load the data from\. This support local files, HTTP URLs and zip files containing a single dbf file of the same name\. Fetch such a zip file from an HTTP address is of course supported\. -. -.IP "\(bu" 4 -\fIWITH\fR -. -.IP -When loading from a \fBDBF\fR file, the following options are supported: -. -.IP "\(bu" 4 -\fItruncate\fR -. -.IP -When this option is listed, pgloader issues a \fBTRUNCATE\fR command against the PostgreSQL target table before reading the data file\. -. -.IP "\(bu" 4 -\fIdisable triggers\fR -. -.IP -When this option is listed, pgloader issues an \fBALTER TABLE \.\.\. DISABLE TRIGGER ALL\fR command against the PostgreSQL target table before copying the data, then the command \fBALTER TABLE \.\.\. ENABLE TRIGGER ALL\fR once the \fBCOPY\fR is done\. -. -.IP -This option allows loading data into a pre\-existing table ignoring the \fIforeign key constraints\fR and user defined triggers and may result in invalid \fIforeign key constraints\fR once the data is loaded\. Use with care\. -. -.IP "\(bu" 4 -\fIcreate table\fR -. -.IP -When this option is listed, pgloader creates the table using the meta data found in the \fBDBF\fR file, which must contain a list of fields with their data type\. A standard data type conversion from DBF to PostgreSQL is done\. -. -.IP "\(bu" 4 -\fItable name\fR -. -.IP -This options expects as its value the possibly qualified name of the table to create\. -. -.IP "" 0 - -. -.IP "" 0 -. -.SH "LOAD IXF" -This command instructs pgloader to load data from an IBM \fBIXF\fR file\. Here\'s an example: -. -.IP "" 4 -. -.nf - -LOAD IXF - FROM data/nsitra\.test1\.ixf - INTO postgresql:///pgloader - TARGET TABLE nsitra\.test1 - WITH truncate, create table, timezone UTC - - BEFORE LOAD DO - $$ create schema if not exists nsitra; $$, - $$ drop table if exists nsitra\.test1; $$; -. -.fi -. -.IP "" 0 -. -.P -The \fBixf\fR format command accepts the following clauses and options: -. -.IP "\(bu" 4 -\fIFROM\fR -. -.IP -Filename where to load the data from\. This support local files, HTTP URLs and zip files containing a single ixf file of the same name\. Fetch such a zip file from an HTTP address is of course supported\. -. -.IP "\(bu" 4 -\fIWITH\fR -. -.IP -When loading from a \fBIXF\fR file, the following options are supported: -. -.IP "\(bu" 4 -\fItruncate\fR -. -.IP -When this option is listed, pgloader issues a \fBTRUNCATE\fR command against the PostgreSQL target table before reading the data file\. -. -.IP "\(bu" 4 -\fIdisable triggers\fR -. -.IP -When this option is listed, pgloader issues an \fBALTER TABLE \.\.\. DISABLE TRIGGER ALL\fR command against the PostgreSQL target table before copying the data, then the command \fBALTER TABLE \.\.\. ENABLE TRIGGER ALL\fR once the \fBCOPY\fR is done\. -. -.IP -This option allows loading data into a pre\-existing table ignoring the \fIforeign key constraints\fR and user defined triggers and may result in invalid \fIforeign key constraints\fR once the data is loaded\. Use with care\. -. -.IP "\(bu" 4 -\fIcreate table\fR -. -.IP -When this option is listed, pgloader creates the table using the meta data found in the \fBDBF\fR file, which must contain a list of fields with their data type\. A standard data type conversion from DBF to PostgreSQL is done\. -. -.IP "\(bu" 4 -\fItable name\fR -. -.IP -This options expects as its value the possibly qualified name of the table to create\. -. -.IP "\(bu" 4 -\fItimezone\fR -. -.IP -This options allows to specify which timezone is used when parsing timestamps from an IXF file, and defaults to \fIUTC\fR\. Expected values are either \fBUTC\fR, \fBGMT\fR or a single quoted location name such as \fB\'Universal\'\fR or \fB\'Europe/Paris\'\fR\. -. -.IP "" 0 - -. -.IP "" 0 -. -.SH "LOAD ARCHIVE" -This command instructs pgloader to load data from one or more files contained in an archive\. Currently the only supported archive format is \fIZIP\fR, and the archive might be downloaded from an \fIHTTP\fR URL\. -. -.P -Here\'s an example: -. -.IP "" 4 -. -.nf - -LOAD ARCHIVE - FROM /Users/dim/Downloads/GeoLiteCity\-latest\.zip - INTO postgresql:///ip4r - - BEFORE LOAD - DO $$ create extension if not exists ip4r; $$, - $$ create schema if not exists geolite; $$, - - EXECUTE \'geolite\.sql\' - - LOAD CSV - FROM FILENAME MATCHING ~/GeoLiteCity\-Location\.csv/ - WITH ENCODING iso\-8859\-1 - ( - locId, - country, - region null if blanks, - city null if blanks, - postalCode null if blanks, - latitude, - longitude, - metroCode null if blanks, - areaCode null if blanks - ) - INTO postgresql:///ip4r?geolite\.location - ( - locid,country,region,city,postalCode, - location point using (format nil "(~a,~a)" longitude latitude), - metroCode,areaCode - ) - WITH skip header = 2, - fields optionally enclosed by \'"\', - fields escaped by double\-quote, - fields terminated by \',\' - - AND LOAD CSV - FROM FILENAME MATCHING ~/GeoLiteCity\-Blocks\.csv/ - WITH ENCODING iso\-8859\-1 - ( - startIpNum, endIpNum, locId - ) - INTO postgresql:///ip4r?geolite\.blocks - ( - iprange ip4r using (ip\-range startIpNum endIpNum), - locId - ) - WITH skip header = 2, - fields optionally enclosed by \'"\', - fields escaped by double\-quote, - fields terminated by \',\' - - FINALLY DO - $$ create index blocks_ip4r_idx on geolite\.blocks using gist(iprange); $$; -. -.fi -. -.IP "" 0 -. -.P -The \fBarchive\fR command accepts the following clauses and options: -. -.IP "\(bu" 4 -\fIFROM\fR -. -.IP -Filename or HTTP URI where to load the data from\. When given an HTTP URL the linked file will get downloaded locally before processing\. -. -.IP -If the file is a \fBzip\fR file, the command line utility \fBunzip\fR is used to expand the archive into files in \fB$TMPDIR\fR, or \fB/tmp\fR if \fB$TMPDIR\fR is unset or set to a non\-existing directory\. -. -.IP -Then the following commands are used from the top level directory where the archive has been expanded\. -. -.IP "\(bu" 4 -command [ \fIAND\fR command \.\.\. ] -. -.IP -A series of commands against the contents of the archive, at the moment only \fBCSV\fR,\fB\'FIXED\fR and \fBDBF\fR commands are supported\. -. -.IP -Note that commands are supporting the clause \fIFROM FILENAME MATCHING\fR which allows the pgloader command not to depend on the exact names of the archive directories\. -. -.IP -The same clause can also be applied to several files with using the spelling \fIFROM ALL FILENAMES MATCHING\fR and a regular expression\. -. -.IP -The whole \fImatching\fR clause must follow the following rule: -. -.IP "" 4 -. -.nf - - FROM [ ALL FILENAMES | [ FIRST ] FILENAME ] MATCHING -. -.fi -. -.IP "" 0 - -. -.IP "\(bu" 4 -\fIFINALLY DO\fR -. -.IP -SQL Queries to run once the data is loaded, such as \fBCREATE INDEX\fR\. -. -.IP "" 0 -. -.SH "LOAD MYSQL DATABASE" -This command instructs pgloader to load data from a database connection\. The only supported database source is currently \fIMySQL\fR, and pgloader supports dynamically converting the schema of the source database and the indexes building\. -. -.P -A default set of casting rules are provided and might be overloaded and appended to by the command\. -. -.P -Here\'s an example using as many options as possible, some of them even being defaults\. Chances are you don\'t need that complex a setup, don\'t copy and paste it, use it only as a reference! -. -.IP "" 4 -. -.nf - -LOAD DATABASE - FROM mysql://root@localhost/sakila - INTO postgresql://localhost:54393/sakila - - WITH include drop, create tables, create indexes, reset sequences, - workers = 8, concurrency = 1, - multiple readers per thread, rows per range = 50000 - - SET PostgreSQL PARAMETERS - maintenance_work_mem to \'128MB\', - work_mem to \'12MB\', - search_path to \'sakila, public, "$user"\' - - SET MySQL PARAMETERS - net_read_timeout = \'120\', - net_write_timeout = \'120\' - - CAST type bigint when (= precision 20) to bigserial drop typemod, - type date drop not null drop default using zero\-dates\-to\-null, - \-\- type tinyint to boolean using tinyint\-to\-boolean, - type year to integer - - MATERIALIZE VIEWS film_list, staff_list - - \-\- INCLUDING ONLY TABLE NAMES MATCHING ~/film/, \'actor\' - \-\- EXCLUDING TABLE NAMES MATCHING ~ - \-\- DECODING TABLE NAMES MATCHING ~/messed/, ~/encoding/ AS utf8 - \-\- ALTER TABLE NAMES MATCHING \'film\' RENAME TO \'films\' - \-\- ALTER TABLE NAMES MATCHING ~/_list$/ SET SCHEMA \'mv\' - - ALTER TABLE NAMES MATCHING ~/_list$/, \'sales_by_store\', ~/sales_by/ - SET SCHEMA \'mv\' - - ALTER TABLE NAMES MATCHING \'film\' RENAME TO \'films\' - ALTER TABLE NAMES MATCHING ~/\./ SET (fillfactor=\'40\') - - ALTER SCHEMA \'sakila\' RENAME TO \'pagila\' - - BEFORE LOAD DO - $$ create schema if not exists pagila; $$, - $$ create schema if not exists mv; $$, - $$ alter database sakila set search_path to pagila, mv, public; $$; -. -.fi -. -.IP "" 0 -. -.P -The \fBdatabase\fR command accepts the following clauses and options: -. -.IP "\(bu" 4 -\fIFROM\fR -. -.IP -Must be a connection URL pointing to a MySQL database\. -. -.IP -If the connection URI contains a table name, then only this table is migrated from MySQL to PostgreSQL\. -. -.IP -See the \fBSOURCE CONNECTION STRING\fR section above for details on how to write the connection string\. The MySQL connection string accepts the same parameter \fIsslmode\fR as the PostgreSQL connection string, but the \fIverify\fR mode is not implemented (yet)\. -. -.IP -Environment variables described in \fIhttp://dev\.mysql\.com/doc/refman/5\.0/en/environment\-variables\.html\fR can be used as default values too\. If the user is not provided, then it defaults to \fBUSER\fR environment variable value\. The password can be provided with the environment variable \fBMYSQL_PWD\fR\. The host can be provided with the environment variable \fBMYSQL_HOST\fR and otherwise defaults to \fBlocalhost\fR\. The port can be provided with the environment variable \fBMYSQL_TCP_PORT\fR and otherwise defaults to \fB3306\fR\. -. -.IP "\(bu" 4 -\fIWITH\fR -. -.IP -When loading from a \fBMySQL\fR database, the following options are supported, and the default \fIWITH\fR clause is: \fIno truncate\fR, \fIcreate schema\fR, \fIcreate tables\fR, \fIinclude drop\fR, \fIcreate indexes\fR, \fIreset sequences\fR, \fIforeign keys\fR, \fIdowncase identifiers\fR, \fIuniquify index names\fR\. -. -.IP -\fIWITH\fR options: -. -.IP "\(bu" 4 -\fIinclude drop\fR -. -.IP -When this option is listed, pgloader drops all the tables in the target PostgreSQL database whose names appear in the MySQL database\. This option allows for using the same command several times in a row until you figure out all the options, starting automatically from a clean environment\. Please note that \fBCASCADE\fR is used to ensure that tables are dropped even if there are foreign keys pointing to them\. This is precisely what \fBinclude drop\fR is intended to do: drop all target tables and recreate them\. -. -.IP -Great care needs to be taken when using \fBinclude drop\fR, as it will cascade to \fIall\fR objects referencing the target tables, possibly including other tables that are not being loaded from the source DB\. -. -.IP "\(bu" 4 -\fIinclude no drop\fR -. -.IP -When this option is listed, pgloader will not include any \fBDROP\fR statement when loading the data\. -. -.IP "\(bu" 4 -\fItruncate\fR -. -.IP -When this option is listed, pgloader issue the \fBTRUNCATE\fR command against each PostgreSQL table just before loading data into it\. -. -.IP "\(bu" 4 -\fIno truncate\fR -. -.IP -When this option is listed, pgloader issues no \fBTRUNCATE\fR command\. -. -.IP "\(bu" 4 -\fIdisable triggers\fR -. -.IP -When this option is listed, pgloader issues an \fBALTER TABLE \.\.\. DISABLE TRIGGER ALL\fR command against the PostgreSQL target table before copying the data, then the command \fBALTER TABLE \.\.\. ENABLE TRIGGER ALL\fR once the \fBCOPY\fR is done\. -. -.IP -This option allows loading data into a pre\-existing table ignoring the \fIforeign key constraints\fR and user defined triggers and may result in invalid \fIforeign key constraints\fR once the data is loaded\. Use with care\. -. -.IP "\(bu" 4 -\fIcreate tables\fR -. -.IP -When this option is listed, pgloader creates the table using the meta data found in the \fBMySQL\fR file, which must contain a list of fields with their data type\. A standard data type conversion from DBF to PostgreSQL is done\. -. -.IP "\(bu" 4 -\fIcreate no tables\fR -. -.IP -When this option is listed, pgloader skips the creation of table before loading data, target tables must then already exist\. -. -.IP -Also, when using \fIcreate no tables\fR pgloader fetches the metadata from the current target database and checks type casting, then will remove constraints and indexes prior to loading the data and install them back again once the loading is done\. -. -.IP "\(bu" 4 -\fIcreate indexes\fR -. -.IP -When this option is listed, pgloader gets the definitions of all the indexes found in the MySQL database and create the same set of index definitions against the PostgreSQL database\. -. -.IP "\(bu" 4 -\fIcreate no indexes\fR -. -.IP -When this option is listed, pgloader skips the creating indexes\. -. -.IP "\(bu" 4 -\fIdrop indexes\fR -. -.IP -When this option is listed, pgloader drops the indexes in the target database before loading the data, and creates them again at the end of the data copy\. -. -.IP "\(bu" 4 -\fIuniquify index names\fR, \fIpreserve index names\fR -. -.IP -MySQL index names are unique per\-table whereas in PostgreSQL index names have to be unique per\-schema\. The default for pgloader is to change the index name by prefixing it with \fBidx_OID\fR where \fBOID\fR is the internal numeric identifier of the table the index is built against\. -. -.IP -In somes cases like when the DDL are entirely left to a framework it might be sensible for pgloader to refrain from handling index unique names, that is achieved by using the \fIpreserve index names\fR option\. -. -.IP -The default is to \fIuniquify index names\fR\. -. -.IP -Even when using the option \fIpreserve index names\fR, MySQL primary key indexes named "PRIMARY" will get their names uniquified\. Failing to do so would prevent the primary keys to be created again in PostgreSQL where the index names must be unique per schema\. -. -.IP "\(bu" 4 -\fIdrop schema\fR -. -.IP -When this option is listed, pgloader drops the target schema in the target PostgreSQL database before creating it again and all the objects it contains\. The default behavior doesn\'t drop the target schemas\. -. -.IP "\(bu" 4 -\fIforeign keys\fR -. -.IP -When this option is listed, pgloader gets the definitions of all the foreign keys found in the MySQL database and create the same set of foreign key definitions against the PostgreSQL database\. -. -.IP "\(bu" 4 -\fIno foreign keys\fR -. -.IP -When this option is listed, pgloader skips creating foreign keys\. -. -.IP "\(bu" 4 -\fIreset sequences\fR -. -.IP -When this option is listed, at the end of the data loading and after the indexes have all been created, pgloader resets all the PostgreSQL sequences created to the current maximum value of the column they are attached to\. -. -.IP -The options \fIschema only\fR and \fIdata only\fR have no effects on this option\. -. -.IP "\(bu" 4 -\fIreset no sequences\fR -. -.IP -When this option is listed, pgloader skips resetting sequences after the load\. -. -.IP -The options \fIschema only\fR and \fIdata only\fR have no effects on this option\. -. -.IP "\(bu" 4 -\fIdowncase identifiers\fR -. -.IP -When this option is listed, pgloader converts all MySQL identifiers (table names, index names, column names) to \fIdowncase\fR, except for PostgreSQL \fIreserved\fR keywords\. -. -.IP -The PostgreSQL \fIreserved\fR keywords are determined dynamically by using the system function \fBpg_get_keywords()\fR\. -. -.IP "\(bu" 4 -\fIquote identifiers\fR -. -.IP -When this option is listed, pgloader quotes all MySQL identifiers so that their case is respected\. Note that you will then have to do the same thing in your application code queries\. -. -.IP "\(bu" 4 -\fIschema only\fR -. -.IP -When this option is listed pgloader refrains from migrating the data over\. Note that the schema in this context includes the indexes when the option \fIcreate indexes\fR has been listed\. -. -.IP "\(bu" 4 -\fIdata only\fR -. -.IP -When this option is listed pgloader only issues the \fBCOPY\fR statements, without doing any other processing\. -. -.IP "\(bu" 4 -\fIsingle reader per thread\fR, \fImultiple readers per thread\fR -. -.IP -The default is \fIsingle reader per thread\fR and it means that each MySQL table is read by a single thread as a whole, with a single \fBSELECT\fR statement using no \fBWHERE\fR clause\. -. -.IP -When using \fImultiple readers per thread\fR pgloader may be able to divide the reading work into several threads, as many as the \fIconcurrency\fR setting, which needs to be greater than 1 for this option to kick be activated\. -. -.IP -For each source table, pgloader searches for a primary key over a single numeric column, or a multiple\-column primary key index for which the first column is of a numeric data type (one of \fBinteger\fR or \fBbigint\fR)\. When such an index exists, pgloader runs a query to find the \fImin\fR and \fImax\fR values on this column, and then split that range into many ranges containing a maximum of \fIrows per range\fR\. -. -.IP -When the range list we then obtain contains at least as many ranges than our concurrency setting, then we distribute those ranges to each reader thread\. -. -.IP -So when all the conditions are met, pgloader then starts as many reader thread as the \fIconcurrency\fR setting, and each reader thread issues several queries with a \fBWHERE id >= x AND id < y\fR, where \fBy \- x = rows per range\fR or less (for the last range, depending on the max value just obtained\. -. -.IP "\(bu" 4 -\fIrows per range\fR -. -.IP -How many rows are fetched per \fBSELECT\fR query when using \fImultiple readers per thread\fR, see above for details\. -. -.IP "" 0 - -. -.IP "\(bu" 4 -\fISET MySQL PARAMETERS\fR -. -.IP -The \fISET MySQL PARAMETERS\fR allows setting MySQL parameters using the MySQL \fBSET\fR command each time pgloader connects to it\. -. -.IP "\(bu" 4 -\fICAST\fR -. -.IP -The cast clause allows to specify custom casting rules, either to overload the default casting rules or to amend them with special cases\. -. -.IP -A casting rule is expected to follow one of the forms: -. -.IP "" 4 -. -.nf - -type [ \.\.\. ] to [
" ] - [ TARGET COLUMNS ] - - [ WITH ] - - [ SET ] - - [ BEFORE LOAD [ DO | EXECUTE ] ... ] - [ AFTER LOAD [ DO | EXECUTE ] ... ] - ; - -The main clauses are the `LOAD`, `FROM`, `INTO` and `WITH` clauses that each -command implements. Some command then implement the `SET` command, or some -specific clauses such as the `CAST` clause. - -## TEMPLATING WITH MUSTACHE - -pgloader implements the https://mustache.github.io/ templating system so -that you may have dynamic parts of your commands. See the documentation for -this template system online. - -A specific feature of pgloader is the ability to fetch a variable from the -OS environment of the pgloader process, making it possible to run pgloader -as in the following example: - - $ DBPATH=sqlite/sqlite.db pgloader ./test/sqlite-env.load - -or in several steps: - - $ export DBPATH=sqlite/sqlite.db - $ pgloader ./test/sqlite-env.load - -The variable can then be used in a typical mustache fashion: - - load database - from '{{DBPATH}}' - into postgresql:///pgloader; - -It's also possible to prepare a INI file such as the following: - - [pgloader] - - DBPATH = sqlite/sqlite.db - -And run the following command, feeding the INI values as a *context* for -pgloader templating system: - - $ pgloader --context ./test/sqlite.ini ./test/sqlite-ini.load - -The mustache templates implementation with OS environment support replaces -former `GETENV` implementation, which didn't work anyway. - -## COMMON CLAUSES - -Some clauses are common to all commands: - - - *FROM* - - The *FROM* clause specifies where to read the data from, and each - command introduces its own variant of sources. For instance, the *CSV* - source supports `inline`, `stdin`, a filename, a quoted filename, and a - *FILENAME MATCHING* clause (see above); whereas the *MySQL* source only - supports a MySQL database URI specification. - - - *INTO* - - The PostgreSQL connection URI must contains the name of the target table - where to load the data into. That table must have already been created - in PostgreSQL, and the name might be schema qualified. - - Then *INTO* option also supports an optional comma separated list of - target columns, which are either the name of an input *field* or the - white space separated list of the target column name, its PostgreSQL data - type and a *USING* expression. - - The *USING* expression can be any valid Common Lisp form and will be - read with the current package set to `pgloader.transforms`, so that you - can use functions defined in that package, such as functions loaded - dynamically with the `--load` command line parameter. - - Each *USING* expression is compiled at runtime to native code. - - This feature allows pgloader to load any number of fields in a CSV file - into a possibly different number of columns in the database, using - custom code for that projection. - - - *WITH* - - Set of options to apply to the command, using a global syntax of either: - - - *key = value* - - *use option* - - *do not use option* - - See each specific command for details. - - All data sources specific commands support the following options: - - - *on error stop* - - *batch rows = R* - - *batch size = ... MB* - - *prefetch rows = ...* - - See the section BATCH BEHAVIOUR OPTIONS for more details. - - In addition, the following settings are available: - - - *workers = W* - - *concurrency = C* - - *max parallel create index = I* - - See section A NOTE ABOUT PARALLELISM for more details. - - - *SET* - - This clause allows to specify session parameters to be set for all the - sessions opened by pgloader. It expects a list of parameter name, the - equal sign, then the single-quoted value as a comma separated list. - - The names and values of the parameters are not validated by pgloader, - they are given as-is to PostgreSQL. - - - *BEFORE LOAD DO* - - You can run SQL queries against the database before loading the data - from the `CSV` file. Most common SQL queries are `CREATE TABLE IF NOT - EXISTS` so that the data can be loaded. - - Each command must be *dollar-quoted*: it must begin and end with a - double dollar sign, `$$`. Dollar-quoted queries are then comma - separated. No extra punctuation is expected after the last SQL query. - - - *BEFORE LOAD EXECUTE* - - Same behaviour as in the *BEFORE LOAD DO* clause. Allows you to read - the SQL queries from a SQL file. Implements support for PostgreSQL - dollar-quoting and the `\i` and `\ir` include facilities as in `psql` - batch mode (where they are the same thing). - - - *AFTER LOAD DO* - - Same format as *BEFORE LOAD DO*, the dollar-quoted queries found in that - section are executed once the load is done. That's the right time to - create indexes and constraints, or re-enable triggers. - - - *AFTER LOAD EXECUTE* - - Same behaviour as in the *AFTER LOAD DO* clause. Allows you to read the - SQL queries from a SQL file. Implements support for PostgreSQL - dollar-quoting and the `\i` and `\ir` include facilities as in `psql` - batch mode (where they are the same thing). - -### Connection String - -The `` parameter is expected to be given as a *Connection URI* -as documented in the PostgreSQL documentation at -http://www.postgresql.org/docs/9.3/static/libpq-connect.html#LIBPQ-CONNSTRING. - - postgresql://[user[:password]@][netloc][:port][/dbname][?option=value&...] - -Where: - - - *user* - - Can contain any character, including colon (`:`) which must then be - doubled (`::`) and at-sign (`@`) which must then be doubled (`@@`). - - When omitted, the *user* name defaults to the value of the `PGUSER` - environment variable, and if it is unset, the value of the `USER` - environment variable. - - - *password* - - Can contain any character, including the at sign (`@`) which must then - be doubled (`@@`). To leave the password empty, when the *user* name - ends with at at sign, you then have to use the syntax user:@. - - When omitted, the *password* defaults to the value of the `PGPASSWORD` - environment variable if it is set, otherwise the password is left - unset. - - When no *password* is found either in the connection URI nor in the - environment, then pgloader looks for a `.pgpass` file as documented at - https://www.postgresql.org/docs/current/static/libpq-pgpass.html. The - implementation is not that of `libpq` though. As with `libpq` you can - set the environment variable `PGPASSFILE` to point to a `.pgpass` file, - and pgloader defaults to `~/.pgpass` on unix like systems and - `%APPDATA%\postgresql\pgpass.conf` on windows. Matching rules and syntax - are the same as with `libpq`, refer to its documentation. - - - *netloc* - - Can be either a hostname in dotted notation, or an ipv4, or an Unix - domain socket path. Empty is the default network location, under a - system providing *unix domain socket* that method is preferred, otherwise - the *netloc* default to `localhost`. - - It's possible to force the *unix domain socket* path by using the syntax - `unix:/path/to/where/the/socket/file/is`, so to force a non default - socket path and a non default port, you would have: - - postgresql://unix:/tmp:54321/dbname - - The *netloc* defaults to the value of the `PGHOST` environment - variable, and if it is unset, to either the default `unix` socket path - when running on a Unix system, and `localhost` otherwise. - - Socket path containing colons are supported by doubling the colons - within the path, as in the following example: - - postgresql://unix:/tmp/project::region::instance:5432/dbname - - - *dbname* - - Should be a proper identifier (letter followed by a mix of letters, - digits and the punctuation signs comma (`,`), dash (`-`) and underscore - (`_`). - - When omitted, the *dbname* defaults to the value of the environment - variable `PGDATABASE`, and if that is unset, to the *user* value as - determined above. - - - *options* - - The optional parameters must be supplied with the form `name=value`, and - you may use several parameters by separating them away using an - ampersand (`&`) character. - - Only some options are supported here, *tablename* (which might be - qualified with a schema name) *sslmode*, *host*, *port*, *dbname*, - *user* and *password*. - - The *sslmode* parameter values can be one of `disable`, `allow`, - `prefer` or `require`. - - For backward compatibility reasons, it's possible to specify the - *tablename* option directly, without spelling out the `tablename=` - parts. - - The options override the main URI components when both are given, and - using the percent-encoded option parameters allow using passwords - starting with a colon and bypassing other URI components parsing - limitations. - -### Regular Expressions - -Several clauses listed in the following accept *regular expressions* with -the following input rules: - - - A regular expression begins with a tilde sign (`~`), - - - is then followed with an opening sign, - - - then any character is allowed and considered part of the regular - expression, except for the closing sign, - - - then a closing sign is expected. - -The opening and closing sign are allowed by pair, here's the complete list -of allowed delimiters: - - ~// - ~[] - ~{} - ~() - ~<> - ~"" - ~'' - ~|| - ~## - -Pick the set of delimiters that don't collide with the *regular expression* -you're trying to input. If your expression is such that none of the -solutions allow you to enter it, the places where such expressions are -allowed should allow for a list of expressions. - -### Comments - -Any command may contain comments, following those input rules: - - - the `--` delimiter begins a comment that ends with the end of the - current line, - - - the delimiters `/*` and `*/` respectively start and end a comment, which - can be found in the middle of a command or span several lines. - -Any place where you could enter a *whitespace* will accept a comment too. - -### Batch behaviour options - -All pgloader commands have support for a *WITH* clause that allows for -specifying options. Some options are generic and accepted by all commands, -such as the *batch behaviour options*, and some options are specific to a -data source kind, such as the CSV *skip header* option. - -The global batch behaviour options are: - - - *batch rows* - - Takes a numeric value as argument, used as the maximum number of rows - allowed in a batch. The default is `25 000` and can be changed to try - having better performance characteristics or to control pgloader memory - usage; - - - *batch size* - - Takes a memory unit as argument, such as *20 MB*, its default value. - Accepted multipliers are *kB*, *MB*, *GB*, *TB* and *PB*. The case is - important so as not to be confused about bits versus bytes, we're only - talking bytes here. - - - *prefetch rows* - - Takes a numeric value as argument, defaults to `100000`. That's the - number of rows that pgloader is allowed to read in memory in each reader - thread. See the *workers* setting for how many reader threads are - allowed to run at the same time. - -Other options are specific to each input source, please refer to specific -parts of the documentation for their listing and covering. - -A batch is then closed as soon as either the *batch rows* or the *batch -size* threshold is crossed, whichever comes first. In cases when a batch has -to be closed because of the *batch size* setting, a *debug* level log -message is printed with how many rows did fit in the *oversized* batch. - -## LOAD CSV - -This command instructs pgloader to load data from a `CSV` file. Here's an -example: - - LOAD CSV - FROM 'GeoLiteCity-Blocks.csv' WITH ENCODING iso-646-us - HAVING FIELDS - ( - startIpNum, endIpNum, locId - ) - INTO postgresql://user@localhost:54393/dbname - TARGET TABLE geolite.blocks - TARGET COLUMNS - ( - iprange ip4r using (ip-range startIpNum endIpNum), - locId - ) - WITH truncate, - skip header = 2, - fields optionally enclosed by '"', - fields escaped by backslash-quote, - fields terminated by '\t' - - SET work_mem to '32 MB', maintenance_work_mem to '64 MB'; - -The `csv` format command accepts the following clauses and options: - - - *FROM* - - Filename where to load the data from. Accepts an *ENCODING* option. Use - the `--list-encodings` option to know which encoding names are - supported. - - The filename may be enclosed by single quotes, and could be one of the - following special values: - - - *inline* - - The data is found after the end of the parsed commands. Any number - of empty lines between the end of the commands and the beginning of - the data is accepted. - - - *stdin* - - Reads the data from the standard input stream. - - - *FILENAMES MATCHING* - - The whole *matching* clause must follow the following rule: - - [ ALL FILENAMES | [ FIRST ] FILENAME ] - MATCHING regexp - [ IN DIRECTORY '...' ] - - The *matching* clause applies given *regular expression* (see above - for exact syntax, several options can be used here) to filenames. - It's then possible to load data from only the first match of all of - them. - - The optional *IN DIRECTORY* clause allows specifying which directory - to walk for finding the data files, and can be either relative to - where the command file is read from, or absolute. The given - directory must exists. - - The *FROM* option also supports an optional comma separated list of - *field* names describing what is expected in the `CSV` data file, - optionally introduced by the clause `HAVING FIELDS`. - - Each field name can be either only one name or a name following with - specific reader options for that field, enclosed in square brackets and - comma-separated. Supported per-field reader options are: - - - *terminated by* - - See the description of *field terminated by* below. - - The processing of this option is not currently implemented. - - - *date format* - - When the field is expected of the date type, then this option allows - to specify the date format used in the file. - - Date format string are template strings modeled against the - PostgreSQL `to_char` template strings support, limited to the - following patterns: - - - YYYY, YYY, YY for the year part - - MM for the numeric month part - - DD for the numeric day part - - HH, HH12, HH24 for the hour part - - am, AM, a.m., A.M. - - pm, PM, p.m., P.M. - - MI for the minutes part - - SS for the seconds part - - MS for the milliseconds part (4 digits) - - US for the microseconds part (6 digits) - - unparsed punctuation signs: - . * # @ T / \ and space - - Here's an example of a *date format* specification: - - column-name [date format 'YYYY-MM-DD HH24-MI-SS.US'] - - - *null if* - - This option takes an argument which is either the keyword *blanks* - or a double-quoted string. - - When *blanks* is used and the field value that is read contains only - space characters, then it's automatically converted to an SQL `NULL` - value. - - When a double-quoted string is used and that string is read as the - field value, then the field value is automatically converted to an - SQL `NULL` value. - - - *trim both whitespace*, *trim left whitespace*, *trim right whitespace* - - This option allows to trim whitespaces in the read data, either from - both sides of the data, or only the whitespace characters found on - the left of the streaing, or only those on the right of the string. - - - *WITH* - - When loading from a `CSV` file, the following options are supported: - - - *truncate* - - When this option is listed, pgloader issues a `TRUNCATE` command - against the PostgreSQL target table before reading the data file. - - - *drop indexes* - - When this option is listed, pgloader issues `DROP INDEX` commands - against all the indexes defined on the target table before copying - the data, then `CREATE INDEX` commands once the `COPY` is done. - - In order to get the best performance possible, all the indexes are - created in parallel and when done the primary keys are built again - from the unique indexes just created. This two step process allows - creating the primary key index in parallel with the other indexes, - as only the `ALTER TABLE` command needs an *access exclusive lock* - on the target table. - - - *disable triggers* - - When this option is listed, pgloader issues an `ALTER TABLE ... - DISABLE TRIGGER ALL` command against the PostgreSQL target table - before copying the data, then the command `ALTER TABLE ... ENABLE - TRIGGER ALL` once the `COPY` is done. - - This option allows loading data into a pre-existing table ignoring - the *foreign key constraints* and user defined triggers and may - result in invalid *foreign key constraints* once the data is loaded. - Use with care. - - - *skip header* - - Takes a numeric value as argument. Instruct pgloader to skip that - many lines at the beginning of the input file. - - - *csv header* - - Use the first line read after *skip header* as the list of csv field - names to be found in the CSV file, using the same CSV parameters as - for the CSV data. - - - *trim unquoted blanks* - - When reading unquoted values in the `CSV` file, remove the blanks - found in between the separator and the value. That behaviour is the - default. - - - *keep unquoted blanks* - - When reading unquoted values in the `CSV` file, keep blanks found in - between the separator and the value. - - - *fields optionally enclosed by* - - Takes a single character as argument, which must be found inside - single quotes, and might be given as the printable character itself, - the special value \t to denote a tabulation character, or `0x` then - an hexadecimal value read as the ASCII code for the character. - - This character is used as the quoting character in the `CSV` file, - and defaults to double-quote. - - - *fields not enclosed* - - By default, pgloader will use the double-quote character as the - enclosing character. If you have a CSV file where fields are not - enclosed and are using double-quote as an expected ordinary - character, then use the option *fields not enclosed* for the CSV - parser to accept those values. - - - *fields escaped by* - - Takes either the special value *backslash-quote* or *double-quote*, - or any value supported by the *fields terminated by* option (see - below). This value is used to recognize escaped field separators - when they are to be found within the data fields themselves. - Defaults to *double-quote*. - - - *csv escape mode* - - Takes either the special value *quote* (the default) or *following* - and allows the CSV parser to parse either only escaped field - separator or any character (including CSV data) when using the - *following* value. - - - *fields terminated by* - - Takes a single character as argument, which must be found inside - single quotes, and might be given as the printable character itself, - the special value \t to denote a tabulation character, or `0x` then - an hexadecimal value read as the ASCII code for the character. - - This character is used as the *field separator* when reading the - `CSV` data. - - - *lines terminated by* - - Takes a single character as argument, which must be found inside - single quotes, and might be given as the printable character itself, - the special value \t to denote a tabulation character, or `0x` then - an hexadecimal value read as the ASCII code for the character. - - This character is used to recognize *end-of-line* condition when - reading the `CSV` data. - -## LOAD FIXED COLS - -This command instructs pgloader to load data from a text file containing -columns arranged in a *fixed size* manner. Here's an example: - - LOAD FIXED - FROM inline - ( - a from 0 for 10, - b from 10 for 8, - c from 18 for 8, - d from 26 for 17 [null if blanks, trim right whitespace] - ) - INTO postgresql:///pgloader - TARGET TABLE fixed - ( - a, b, - c time using (time-with-no-separator c), - d - ) - - WITH truncate - - SET work_mem to '14MB', - standard_conforming_strings to 'on' - - BEFORE LOAD DO - $$ drop table if exists fixed; $$, - $$ create table fixed ( - a integer, - b date, - c time, - d text - ); - $$; - - 01234567892008052011431250firstline - 01234562008052115182300left blank-padded - 12345678902008052208231560another line - 2345609872014092914371500 - 2345678902014092914371520 - -The `fixed` format command accepts the following clauses and options: - - - *FROM* - - Filename where to load the data from. Accepts an *ENCODING* option. Use - the `--list-encodings` option to know which encoding names are - supported. - - The filename may be enclosed by single quotes, and could be one of the - following special values: - - - *inline* - - The data is found after the end of the parsed commands. Any number - of empty lines between the end of the commands and the beginning of - the data is accepted. - - - *stdin* - - Reads the data from the standard input stream. - - - *FILENAMES MATCHING* - - The whole *matching* clause must follow the following rule: - - [ ALL FILENAMES | [ FIRST ] FILENAME ] - MATCHING regexp - [ IN DIRECTORY '...' ] - - The *matching* clause applies given *regular expression* (see above - for exact syntax, several options can be used here) to filenames. - It's then possible to load data from only the first match of all of - them. - - The optional *IN DIRECTORY* clause allows specifying which directory - to walk for finding the data files, and can be either relative to - where the command file is read from, or absolute. The given - directory must exists. - - The *FROM* option also supports an optional comma separated list of - *field* names describing what is expected in the `FIXED` data file. - - Each field name is composed of the field name followed with specific - reader options for that field. Supported per-field reader options are - the following, where only *start* and *length* are required. - - - *start* - - Position in the line where to start reading that field's value. Can - be entered with decimal digits or `0x` then hexadecimal digits. - - - *length* - - How many bytes to read from the *start* position to read that - field's value. Same format as *start*. - - Those optional parameters must be enclosed in square brackets and - comma-separated: - - - *terminated by* - - See the description of *field terminated by* below. - - The processing of this option is not currently implemented. - - - *date format* - - When the field is expected of the date type, then this option allows - to specify the date format used in the file. - - Date format string are template strings modeled against the - PostgreSQL `to_char` template strings support, limited to the - following patterns: - - - YYYY, YYY, YY for the year part - - MM for the numeric month part - - DD for the numeric day part - - HH, HH12, HH24 for the hour part - - am, AM, a.m., A.M. - - pm, PM, p.m., P.M. - - MI for the minutes part - - SS for the seconds part - - MS for the milliseconds part (4 digits) - - US for the microseconds part (6 digits) - - unparsed punctuation signs: - . * # @ T / \ and space - - Here's an example of a *date format* specification: - - column-name [date format 'YYYY-MM-DD HH24-MI-SS.US'] - - - *null if* - - This option takes an argument which is either the keyword *blanks* - or a double-quoted string. - - When *blanks* is used and the field value that is read contains only - space characters, then it's automatically converted to an SQL `NULL` - value. - - When a double-quoted string is used and that string is read as the - field value, then the field value is automatically converted to an - SQL `NULL` value. - - - *trim both whitespace*, *trim left whitespace*, *trim right whitespace* - - This option allows to trim whitespaces in the read data, either from - both sides of the data, or only the whitespace characters found on - the left of the streaing, or only those on the right of the string. - - - *WITH* - - When loading from a `FIXED` file, the following options are supported: - - - *truncate* - - When this option is listed, pgloader issues a `TRUNCATE` command - against the PostgreSQL target table before reading the data file. - - - *disable triggers* - - When this option is listed, pgloader issues an `ALTER TABLE ... - DISABLE TRIGGER ALL` command against the PostgreSQL target table - before copying the data, then the command `ALTER TABLE ... ENABLE - TRIGGER ALL` once the `COPY` is done. - - This option allows loading data into a pre-existing table ignoring - the *foreign key constraints* and user defined triggers and may - result in invalid *foreign key constraints* once the data is loaded. - Use with care. - - - *skip header* - - Takes a numeric value as argument. Instruct pgloader to skip that - many lines at the beginning of the input file. - -## LOAD COPY FORMATTED FILES - -This commands instructs pgloader to load from a file containing COPY TEXT -data as described in the PostgreSQL documentation. Here's an example: - - LOAD COPY - FROM copy://./data/track.copy - ( - trackid, track, album, media, genre, composer, - milliseconds, bytes, unitprice - ) - INTO postgresql:///pgloader - TARGET TABLE track_full - - WITH truncate - - SET work_mem to '14MB', - standard_conforming_strings to 'on' - - BEFORE LOAD DO - $$ drop table if exists track_full; $$, - $$ create table track_full ( - trackid bigserial, - track text, - album text, - media text, - genre text, - composer text, - milliseconds bigint, - bytes bigint, - unitprice numeric - ); - $$; - -The `COPY` format command accepts the following clauses and options: - - - *FROM* - - Filename where to load the data from. This support local files, HTTP - URLs and zip files containing a single dbf file of the same name. Fetch - such a zip file from an HTTP address is of course supported. - - - *inline* - - The data is found after the end of the parsed commands. Any number - of empty lines between the end of the commands and the beginning of - the data is accepted. - - - *stdin* - - Reads the data from the standard input stream. - - - *FILENAMES MATCHING* - - The whole *matching* clause must follow the following rule: - - [ ALL FILENAMES | [ FIRST ] FILENAME ] - MATCHING regexp - [ IN DIRECTORY '...' ] - - The *matching* clause applies given *regular expression* (see above - for exact syntax, several options can be used here) to filenames. - It's then possible to load data from only the first match of all of - them. - - The optional *IN DIRECTORY* clause allows specifying which directory - to walk for finding the data files, and can be either relative to - where the command file is read from, or absolute. The given - directory must exists. - - - *WITH* - - When loading from a `COPY` file, the following options are supported: - - - *delimiter* - - Takes a single character as argument, which must be found inside - single quotes, and might be given as the printable character itself, - the special value \t to denote a tabulation character, or `0x` then - an hexadecimal value read as the ASCII code for the character. - - This character is used as the *delimiter* when reading the data, in - a similar way to the PostgreSQL `COPY` option. - - - *null* - - Takes a quoted string as an argument (quotes can be either double - quotes or single quotes) and uses that string as the `NULL` - representation in the data. - - This is similar to the *null* `COPY` option in PostgreSQL. - - - *truncate* - - When this option is listed, pgloader issues a `TRUNCATE` command - against the PostgreSQL target table before reading the data file. - - - *disable triggers* - - When this option is listed, pgloader issues an `ALTER TABLE ... - DISABLE TRIGGER ALL` command against the PostgreSQL target table - before copying the data, then the command `ALTER TABLE ... ENABLE - TRIGGER ALL` once the `COPY` is done. - - This option allows loading data into a pre-existing table ignoring - the *foreign key constraints* and user defined triggers and may - result in invalid *foreign key constraints* once the data is loaded. - Use with care. - - - *skip header* - - Takes a numeric value as argument. Instruct pgloader to skip that - many lines at the beginning of the input file. - -## LOAD DBF - -This command instructs pgloader to load data from a `DBF` file. Here's an -example: - - LOAD DBF - FROM http://www.insee.fr/fr/methodes/nomenclatures/cog/telechargement/2013/dbf/reg2013.dbf - INTO postgresql://user@localhost/dbname - WITH truncate, create table; - -The `dbf` format command accepts the following clauses and options: - - - *FROM* - - Filename where to load the data from. This support local files, HTTP - URLs and zip files containing a single dbf file of the same name. Fetch - such a zip file from an HTTP address is of course supported. - - - *WITH* - - When loading from a `DBF` file, the following options are supported: - - - *truncate* - - When this option is listed, pgloader issues a `TRUNCATE` command - against the PostgreSQL target table before reading the data file. - - - *disable triggers* - - When this option is listed, pgloader issues an `ALTER TABLE ... - DISABLE TRIGGER ALL` command against the PostgreSQL target table - before copying the data, then the command `ALTER TABLE ... ENABLE - TRIGGER ALL` once the `COPY` is done. - - This option allows loading data into a pre-existing table ignoring - the *foreign key constraints* and user defined triggers and may - result in invalid *foreign key constraints* once the data is loaded. - Use with care. - - - *create table* - - When this option is listed, pgloader creates the table using the - meta data found in the `DBF` file, which must contain a list of - fields with their data type. A standard data type conversion from - DBF to PostgreSQL is done. - - - *table name* - - This options expects as its value the possibly qualified name of the - table to create. - -## LOAD IXF - -This command instructs pgloader to load data from an IBM `IXF` file. Here's -an example: - - LOAD IXF - FROM data/nsitra.test1.ixf - INTO postgresql:///pgloader - TARGET TABLE nsitra.test1 - WITH truncate, create table, timezone UTC - - BEFORE LOAD DO - $$ create schema if not exists nsitra; $$, - $$ drop table if exists nsitra.test1; $$; - -The `ixf` format command accepts the following clauses and options: - - - *FROM* - - Filename where to load the data from. This support local files, HTTP - URLs and zip files containing a single ixf file of the same name. Fetch - such a zip file from an HTTP address is of course supported. - - - *WITH* - - When loading from a `IXF` file, the following options are supported: - - - *truncate* - - When this option is listed, pgloader issues a `TRUNCATE` command - against the PostgreSQL target table before reading the data file. - - - *disable triggers* - - When this option is listed, pgloader issues an `ALTER TABLE ... - DISABLE TRIGGER ALL` command against the PostgreSQL target table - before copying the data, then the command `ALTER TABLE ... ENABLE - TRIGGER ALL` once the `COPY` is done. - - This option allows loading data into a pre-existing table ignoring - the *foreign key constraints* and user defined triggers and may - result in invalid *foreign key constraints* once the data is loaded. - Use with care. - - - *create table* - - When this option is listed, pgloader creates the table using the - meta data found in the `DBF` file, which must contain a list of - fields with their data type. A standard data type conversion from - DBF to PostgreSQL is done. - - - *table name* - - This options expects as its value the possibly qualified name of the - table to create. - - - *timezone* - - This options allows to specify which timezone is used when parsing - timestamps from an IXF file, and defaults to *UTC*. Expected values - are either `UTC`, `GMT` or a single quoted location name such as - `'Universal'` or `'Europe/Paris'`. - -## LOAD ARCHIVE - -This command instructs pgloader to load data from one or more files contained -in an archive. Currently the only supported archive format is *ZIP*, and the -archive might be downloaded from an *HTTP* URL. - -Here's an example: - - LOAD ARCHIVE - FROM /Users/dim/Downloads/GeoLiteCity-latest.zip - INTO postgresql:///ip4r - - BEFORE LOAD - DO $$ create extension if not exists ip4r; $$, - $$ create schema if not exists geolite; $$, - - EXECUTE 'geolite.sql' - - LOAD CSV - FROM FILENAME MATCHING ~/GeoLiteCity-Location.csv/ - WITH ENCODING iso-8859-1 - ( - locId, - country, - region null if blanks, - city null if blanks, - postalCode null if blanks, - latitude, - longitude, - metroCode null if blanks, - areaCode null if blanks - ) - INTO postgresql:///ip4r?geolite.location - ( - locid,country,region,city,postalCode, - location point using (format nil "(~a,~a)" longitude latitude), - metroCode,areaCode - ) - WITH skip header = 2, - fields optionally enclosed by '"', - fields escaped by double-quote, - fields terminated by ',' - - AND LOAD CSV - FROM FILENAME MATCHING ~/GeoLiteCity-Blocks.csv/ - WITH ENCODING iso-8859-1 - ( - startIpNum, endIpNum, locId - ) - INTO postgresql:///ip4r?geolite.blocks - ( - iprange ip4r using (ip-range startIpNum endIpNum), - locId - ) - WITH skip header = 2, - fields optionally enclosed by '"', - fields escaped by double-quote, - fields terminated by ',' - - FINALLY DO - $$ create index blocks_ip4r_idx on geolite.blocks using gist(iprange); $$; - -The `archive` command accepts the following clauses and options: - - - *FROM* - - Filename or HTTP URI where to load the data from. When given an HTTP - URL the linked file will get downloaded locally before processing. - - If the file is a `zip` file, the command line utility `unzip` is used - to expand the archive into files in `$TMPDIR`, or `/tmp` if `$TMPDIR` - is unset or set to a non-existing directory. - - Then the following commands are used from the top level directory where - the archive has been expanded. - - - command [ *AND* command ... ] - - A series of commands against the contents of the archive, at the moment - only `CSV`,`'FIXED` and `DBF` commands are supported. - - Note that commands are supporting the clause *FROM FILENAME MATCHING* - which allows the pgloader command not to depend on the exact names of - the archive directories. - - The same clause can also be applied to several files with using the - spelling *FROM ALL FILENAMES MATCHING* and a regular expression. - - The whole *matching* clause must follow the following rule: - - FROM [ ALL FILENAMES | [ FIRST ] FILENAME ] MATCHING - - - *FINALLY DO* - - SQL Queries to run once the data is loaded, such as `CREATE INDEX`. - -## LOAD MYSQL DATABASE - -This command instructs pgloader to load data from a database connection. The -only supported database source is currently *MySQL*, and pgloader supports -dynamically converting the schema of the source database and the indexes -building. - -A default set of casting rules are provided and might be overloaded and -appended to by the command. - -Here's an example using as many options as possible, some of them even being -defaults. Chances are you don't need that complex a setup, don't copy and -paste it, use it only as a reference! - - LOAD DATABASE - FROM mysql://root@localhost/sakila - INTO postgresql://localhost:54393/sakila - - WITH include drop, create tables, create indexes, reset sequences, - workers = 8, concurrency = 1, - multiple readers per thread, rows per range = 50000 - - SET PostgreSQL PARAMETERS - maintenance_work_mem to '128MB', - work_mem to '12MB', - search_path to 'sakila, public, "$user"' - - SET MySQL PARAMETERS - net_read_timeout = '120', - net_write_timeout = '120' - - CAST type bigint when (= precision 20) to bigserial drop typemod, - type date drop not null drop default using zero-dates-to-null, - -- type tinyint to boolean using tinyint-to-boolean, - type year to integer - - MATERIALIZE VIEWS film_list, staff_list - - -- INCLUDING ONLY TABLE NAMES MATCHING ~/film/, 'actor' - -- EXCLUDING TABLE NAMES MATCHING ~ - -- DECODING TABLE NAMES MATCHING ~/messed/, ~/encoding/ AS utf8 - -- ALTER TABLE NAMES MATCHING 'film' RENAME TO 'films' - -- ALTER TABLE NAMES MATCHING ~/_list$/ SET SCHEMA 'mv' - - ALTER TABLE NAMES MATCHING ~/_list$/, 'sales_by_store', ~/sales_by/ - SET SCHEMA 'mv' - - ALTER TABLE NAMES MATCHING 'film' RENAME TO 'films' - ALTER TABLE NAMES MATCHING ~/./ SET (fillfactor='40') - - ALTER SCHEMA 'sakila' RENAME TO 'pagila' - - BEFORE LOAD DO - $$ create schema if not exists pagila; $$, - $$ create schema if not exists mv; $$, - $$ alter database sakila set search_path to pagila, mv, public; $$; - - -The `database` command accepts the following clauses and options: - - - *FROM* - - Must be a connection URL pointing to a MySQL database. - - If the connection URI contains a table name, then only this table is - migrated from MySQL to PostgreSQL. - - See the `SOURCE CONNECTION STRING` section above for details on how to - write the connection string. The MySQL connection string accepts the - same parameter *sslmode* as the PostgreSQL connection string, but the - *verify* mode is not implemented (yet). - - Environment variables described - in - can be used as default values too. If the user is not provided, then it - defaults to `USER` environment variable value. The password can be - provided with the environment variable `MYSQL_PWD`. The host can be - provided with the environment variable `MYSQL_HOST` and otherwise - defaults to `localhost`. The port can be provided with the environment - variable `MYSQL_TCP_PORT` and otherwise defaults to `3306`. - - - *WITH* - - When loading from a `MySQL` database, the following options are - supported, and the default *WITH* clause is: *no truncate*, *create - schema*, *create tables*, *include drop*, *create indexes*, *reset - sequences*, *foreign keys*, *downcase identifiers*, *uniquify index - names*. - - *WITH* options: - - - *include drop* - - When this option is listed, pgloader drops all the tables in the - target PostgreSQL database whose names appear in the MySQL - database. This option allows for using the same command several - times in a row until you figure out all the options, starting - automatically from a clean environment. Please note that `CASCADE` - is used to ensure that tables are dropped even if there are foreign - keys pointing to them. This is precisely what `include drop` is - intended to do: drop all target tables and recreate them. - - Great care needs to be taken when using `include drop`, as - it will cascade to *all* objects referencing the target tables, - possibly including other tables that are not being loaded from - the source DB. - - - *include no drop* - - When this option is listed, pgloader will not include any `DROP` - statement when loading the data. - - - *truncate* - - When this option is listed, pgloader issue the `TRUNCATE` command - against each PostgreSQL table just before loading data into it. - - - *no truncate* - - When this option is listed, pgloader issues no `TRUNCATE` command. - - - *disable triggers* - - When this option is listed, pgloader issues an `ALTER TABLE ... - DISABLE TRIGGER ALL` command against the PostgreSQL target table - before copying the data, then the command `ALTER TABLE ... ENABLE - TRIGGER ALL` once the `COPY` is done. - - This option allows loading data into a pre-existing table ignoring - the *foreign key constraints* and user defined triggers and may - result in invalid *foreign key constraints* once the data is loaded. - Use with care. - - - *create tables* - - When this option is listed, pgloader creates the table using the - meta data found in the `MySQL` file, which must contain a list of - fields with their data type. A standard data type conversion from - DBF to PostgreSQL is done. - - - *create no tables* - - When this option is listed, pgloader skips the creation of table - before loading data, target tables must then already exist. - - Also, when using *create no tables* pgloader fetches the metadata - from the current target database and checks type casting, then will - remove constraints and indexes prior to loading the data and install - them back again once the loading is done. - - - *create indexes* - - When this option is listed, pgloader gets the definitions of all the - indexes found in the MySQL database and create the same set of index - definitions against the PostgreSQL database. - - - *create no indexes* - - When this option is listed, pgloader skips the creating indexes. - - - *drop indexes* - - When this option is listed, pgloader drops the indexes in the target - database before loading the data, and creates them again at the end - of the data copy. - - - *uniquify index names*, *preserve index names* - - MySQL index names are unique per-table whereas in PostgreSQL index - names have to be unique per-schema. The default for pgloader is to - change the index name by prefixing it with `idx_OID` where `OID` is - the internal numeric identifier of the table the index is built - against. - - In somes cases like when the DDL are entirely left to a framework it - might be sensible for pgloader to refrain from handling index unique - names, that is achieved by using the *preserve index names* option. - - The default is to *uniquify index names*. - - Even when using the option *preserve index names*, MySQL primary key - indexes named "PRIMARY" will get their names uniquified. Failing to - do so would prevent the primary keys to be created again in - PostgreSQL where the index names must be unique per schema. - - - *drop schema* - - When this option is listed, pgloader drops the target schema in the - target PostgreSQL database before creating it again and all the - objects it contains. The default behavior doesn't drop the target - schemas. - - - *foreign keys* - - When this option is listed, pgloader gets the definitions of all the - foreign keys found in the MySQL database and create the same set of - foreign key definitions against the PostgreSQL database. - - - *no foreign keys* - - When this option is listed, pgloader skips creating foreign keys. - - - *reset sequences* - - When this option is listed, at the end of the data loading and after - the indexes have all been created, pgloader resets all the - PostgreSQL sequences created to the current maximum value of the - column they are attached to. - - The options *schema only* and *data only* have no effects on this - option. - - - *reset no sequences* - - When this option is listed, pgloader skips resetting sequences after - the load. - - The options *schema only* and *data only* have no effects on this - option. - - - *downcase identifiers* - - When this option is listed, pgloader converts all MySQL identifiers - (table names, index names, column names) to *downcase*, except for - PostgreSQL *reserved* keywords. - - The PostgreSQL *reserved* keywords are determined dynamically by - using the system function `pg_get_keywords()`. - - - *quote identifiers* - - When this option is listed, pgloader quotes all MySQL identifiers so - that their case is respected. Note that you will then have to do the - same thing in your application code queries. - - - *schema only* - - When this option is listed pgloader refrains from migrating the data - over. Note that the schema in this context includes the indexes when - the option *create indexes* has been listed. - - - *data only* - - When this option is listed pgloader only issues the `COPY` - statements, without doing any other processing. - - - *single reader per thread*, *multiple readers per thread* - - The default is *single reader per thread* and it means that each - MySQL table is read by a single thread as a whole, with a single - `SELECT` statement using no `WHERE` clause. - - When using *multiple readers per thread* pgloader may be able to - divide the reading work into several threads, as many as the - *concurrency* setting, which needs to be greater than 1 for this - option to kick be activated. - - For each source table, pgloader searches for a primary key over a - single numeric column, or a multiple-column primary key index for - which the first column is of a numeric data type (one of `integer` - or `bigint`). When such an index exists, pgloader runs a query to - find the *min* and *max* values on this column, and then split that - range into many ranges containing a maximum of *rows per range*. - - When the range list we then obtain contains at least as many ranges - than our concurrency setting, then we distribute those ranges to - each reader thread. - - So when all the conditions are met, pgloader then starts as many - reader thread as the *concurrency* setting, and each reader thread - issues several queries with a `WHERE id >= x AND id < y`, where `y - - x = rows per range` or less (for the last range, depending on the - max value just obtained. - - - *rows per range* - - How many rows are fetched per `SELECT` query when using *multiple - readers per thread*, see above for details. - - - *SET MySQL PARAMETERS* - - The *SET MySQL PARAMETERS* allows setting MySQL parameters using the - MySQL `SET` command each time pgloader connects to it. - - - *CAST* - - The cast clause allows to specify custom casting rules, either to - overload the default casting rules or to amend them with special cases. - - A casting rule is expected to follow one of the forms: - - type [ ... ] to [