diff --git a/BUGS.txt b/BUGS.txt index c2d7434..fedad78 100644 --- a/BUGS.txt +++ b/BUGS.txt @@ -1,6 +1,20 @@ = PGLOADER BUGS LIST = +== Error reporting on last line when alone in its buffer + +When last line is alone on a +COPY+ command and its parsing ends in +error (not enough columns read for example), no information is given +back by +pgloader+. + == Multi-line without quoting +Status:: + Appeared while preparing +2.3.0+, fixed in +~dev+ cycle (release + candidate), not exposed to stable releases. + +Details:: + The new Reader._getopt() facility forgot to consider option types, + which was a pretty bad idea for TextReader.field_count option. + The +cluttered+ example seems to be broken, I just realized this is the case, don't know yet how long it's been the case. diff --git a/Makefile b/Makefile index 86aea74..2fecf0f 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# $Id: Makefile,v 1.16 2008-02-25 17:26:09 dim Exp $ +# $Id: Makefile,v 1.17 2008-03-10 14:39:38 dim Exp $ # # Makefile for debian packaging purpose, make install not intended to work. @@ -53,6 +53,8 @@ pgloader.1.xml: $(DOCS) man: ${DOCS:.txt=.xml} xmlto man $< +doc: man html todo bugs + deb: # working copy from where to make the .orig archive rm -rf $(DEBDIR) diff --git a/debian/changelog b/debian/changelog index 4342000..6b3660c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +pgloader (2.3.0-1) unstable; urgency=low + + * FIX the cluttered test case, see BUGS.txt + * Better release it as stable now and fix bugs when found than wait for + more testing of the ~dev release candidates. + + -- Dimitri Fontaine Mon, 10 Mar 2008 15:36:04 +0100 + pgloader (2.3.0~dev3-1) experimental; urgency=low * Add options for forcing psycopg version to use (-1, -2, --psycopg-version) diff --git a/debian/copyright b/debian/copyright index def199e..0411693 100644 --- a/debian/copyright +++ b/debian/copyright @@ -3,10 +3,10 @@ Mon, 13 Nov 2006 22:56:15 +0100. It was downloaded from http://pgloader.dalibo.org/ -Upstream Author: Dimitri Fontaine +Upstream Author: Dimitri Fontaine Copyright: 2005, Jan Wieck - 2006, Dimitri Fontaine + 2006, 2007, 2008 Dimitri Fontaine License: Redistribution and use in source and binary forms, with or without @@ -28,5 +28,5 @@ On Debian systems, the complete text of the BSD License can be found in `/usr/share/common-licenses/BSD'. -The Debian packaging is (C) 2006, Dimitri Fontaine and +The Debian packaging is (C) 2006, Dimitri Fontaine and is licensed under the GPL, see `/usr/share/common-licenses/GPL'. diff --git a/pgloader.1.txt b/pgloader.1.txt index fa292ff..eb2e98b 100644 --- a/pgloader.1.txt +++ b/pgloader.1.txt @@ -730,11 +730,11 @@ against data from several servers running different RDBMS softwares. == BUGS == -Please report bugs to Dimitri Fontaine . - -When last line is alone on a +COPY+ command and its parsing ends in -error (not enough columns read for example), no information is given -back by +pgloader+. +Please report bugs to Dimitri Fontaine , and see +current list of known bugs in the BUGS.txt distributed file (debian +package includes it at +/usr/share/doc/pgloader/BUGS.txt+ or online at +following url: +http://pgloader.projects.postgresql.org/dev/BUGS.html[]. == AUTHORS == diff --git a/pgloader/csvreader.py b/pgloader/csvreader.py index 89703aa..7c7bffd 100644 --- a/pgloader/csvreader.py +++ b/pgloader/csvreader.py @@ -1,4 +1,4 @@ -# Author: Dimitri Fontaine +# Author: Dimitri Fontaine # # pgloader text format reader # diff --git a/pgloader/db.py b/pgloader/db.py index 5916b92..20398c0 100644 --- a/pgloader/db.py +++ b/pgloader/db.py @@ -1,4 +1,4 @@ -# Author: Dimitri Fontaine +# Author: Dimitri Fontaine # # pgloader database connection handling # COPY dichotomy on error @@ -390,14 +390,17 @@ ORDER BY attnum return ok - def save_copy_buffer(self, tablename): + def save_copy_buffer(self, tablename, debug = False): """ save copy buffer to a temporary file for further inspection """ import tempfile (f, n) = tempfile.mkstemp(prefix='%s.' % tablename, suffix='.pgloader', dir='/tmp') os.write(f, self.buffer.getvalue()) os.close(f) - self.log.warning("COPY data buffer saved in %s" % n) + if debug: + self.log.debug("COPY data buffer saved in %s" % n) + else: + self.log.warning("COPY data buffer saved in %s" % n) return n def copy_from(self, table, columnlist, columns, input_line, @@ -426,7 +429,7 @@ ORDER BY attnum return False if CLIENT_MIN_MESSAGES <= logging.DEBUG: - self.save_copy_buffer(tablename) + self.save_copy_buffer(tablename, debug = True) self.buffer.seek(0) now = time.time() diff --git a/pgloader/lo.py b/pgloader/lo.py index f61f51d..6807c05 100644 --- a/pgloader/lo.py +++ b/pgloader/lo.py @@ -1,4 +1,4 @@ -# Author: Dimitri Fontaine +# Author: Dimitri Fontaine # # pgloader Large Object support diff --git a/pgloader/options.py b/pgloader/options.py index b263de1..41aef3c 100644 --- a/pgloader/options.py +++ b/pgloader/options.py @@ -1,4 +1,4 @@ -# Author: Dimitri Fontaine +# Author: Dimitri Fontaine # # Some common options, for each module to get them diff --git a/pgloader/pgloader.py b/pgloader/pgloader.py index b3a1946..14149d1 100644 --- a/pgloader/pgloader.py +++ b/pgloader/pgloader.py @@ -519,6 +519,8 @@ class PGLoader(threading.Thread): self.newline_escapes = [(a, NEWLINE_ESCAPES) for (a, x) in self.columns] + self.log.debug("self.newline_escapes = '%s'" % self.newline_escapes) + ## # Parallelism knobs if config.has_option(name, 'section_threads'): diff --git a/pgloader/reader.py b/pgloader/reader.py index 32ec01d..bd84278 100644 --- a/pgloader/reader.py +++ b/pgloader/reader.py @@ -78,7 +78,7 @@ class DataReader: self.log.debug("reader.readconfig field_sep: '%s'", self.field_sep) - def _getopt(self, option, config, section, template, default = None): + def _getopt(self, option, config, section, template, default = None, opt_type = "char"): """ Init given configuration option """ if config.has_option(section, option): @@ -95,6 +95,14 @@ class DataReader: % (option, default)) self.__dict__[option] = default + if opt_type == 'int': + try: + self.__dict__[option] = int(self.__dict__[option]) + except ValueError, e: + self.log.error('Configuration option %s.%s is not an int: %s' \ + % (section, option, self.__dict__[option])) + raise PGLoader_Error, e + return self.__dict__[option] def readlines(self): diff --git a/pgloader/textreader.py b/pgloader/textreader.py index 462cf49..bdb8150 100644 --- a/pgloader/textreader.py +++ b/pgloader/textreader.py @@ -1,4 +1,4 @@ -# Author: Dimitri Fontaine +# Author: Dimitri Fontaine # # pgloader text format reader # @@ -39,6 +39,9 @@ class TextReader(DataReader): if 'newline_escapes' not in self.__dict__: self.newline_escapes = newline_escapes + self.log.debug('reader.__init__: newline_escapes %s' \ + % self.newline_escapes) + def readconfig(self, config, name, template): """ get this reader module configuration from config file """ DataReader.readconfig(self, config, name, template) @@ -47,7 +50,7 @@ class TextReader(DataReader): # have to protect ourselves against removing already read # configurations while in second run. - self._getopt('field_count', config, name, template, None) + self._getopt('field_count', config, name, template, None, 'int') self._getopt('trailing_sep', config, name, template, False) if self.trailing_sep is not False: self.trailing_sep = self.trailing_sep == 'True' diff --git a/pgloader/tools.py b/pgloader/tools.py index 87d9c5e..26a76eb 100644 --- a/pgloader/tools.py +++ b/pgloader/tools.py @@ -1,4 +1,4 @@ -# Author: Dimitri Fontaine +# Author: Dimitri Fontaine # # pgloader librairies