FIX the cluttered example (see BUGS.txt), prepare 2.3.0 packaging

BUGS.txt

@@ -1,6 +1,20 @@
 = PGLOADER BUGS LIST =
 
+== Error reporting on last line when alone in its buffer
+
+When last line is alone on a +COPY+ command and its parsing ends in
+error (not enough columns read for example), no information is given
+back by +pgloader+.
+
 == Multi-line without quoting
 
+Status::
+    Appeared while preparing +2.3.0+, fixed in +~dev+ cycle (release
+    candidate), not exposed to stable releases.
+
+Details::
+    The new Reader._getopt() facility forgot to consider option types,
+    which was a pretty bad idea for TextReader.field_count option.
+
 The +cluttered+ example seems to be broken, I just realized this is
 the case, don't know yet how long it's been the case.

Makefile

@@ -1,4 +1,4 @@
-# $Id: Makefile,v 1.16 2008-02-25 17:26:09 dim Exp $
+# $Id: Makefile,v 1.17 2008-03-10 14:39:38 dim Exp $
 #
 # Makefile for debian packaging purpose, make install not intended to work.
 
@@ -53,6 +53,8 @@ pgloader.1.xml: $(DOCS)
 man: ${DOCS:.txt=.xml}
 	xmlto man $<
 
+doc: man html todo bugs
+
 deb:
 	# working copy from where to make the .orig archive
 	rm -rf $(DEBDIR)	

debian/changelog

@@ -1,3 +1,11 @@
+pgloader (2.3.0-1) unstable; urgency=low
+
+  * FIX the cluttered test case, see BUGS.txt
+  * Better release it as stable now and fix bugs when found than wait for
+    more testing of the ~dev release candidates.
+
+ -- Dimitri Fontaine <dim@tapoueh.org>  Mon, 10 Mar 2008 15:36:04 +0100
+
 pgloader (2.3.0~dev3-1) experimental; urgency=low
 
   * Add options for forcing psycopg version to use (-1, -2, --psycopg-version)

debian/copyright

@@ -3,10 +3,10 @@ Mon, 13 Nov 2006 22:56:15 +0100.
 
 It was downloaded from http://pgloader.dalibo.org/
 
-Upstream Author: Dimitri Fontaine <dim@dalibo.com>
+Upstream Author: Dimitri Fontaine <dim@tapoueh.org>
 
 Copyright: 2005, Jan Wieck
-           2006, Dimitri Fontaine
+           2006, 2007, 2008 Dimitri Fontaine
 
 License:
     Redistribution and use in source and binary forms, with or without
@@ -28,5 +28,5 @@ On Debian systems, the complete text of the BSD License can be
 found in `/usr/share/common-licenses/BSD'.
 
 
-The Debian packaging is (C) 2006, Dimitri Fontaine <dim@dalibo.com> and
+The Debian packaging is (C) 2006, Dimitri Fontaine <dim@tapoueh.org> and
 is licensed under the GPL, see `/usr/share/common-licenses/GPL'.

pgloader.1.txt

@@ -730,11 +730,11 @@ against data from several servers running different RDBMS softwares.
 
 == BUGS ==
 
-Please report bugs to Dimitri Fontaine <dim@tapoueh.org>.
-
-When last line is alone on a +COPY+ command and its parsing ends in
-error (not enough columns read for example), no information is given
-back by +pgloader+.
+Please report bugs to Dimitri Fontaine <dim@tapoueh.org>, and see
+current list of known bugs in the BUGS.txt distributed file (debian
+package includes it at +/usr/share/doc/pgloader/BUGS.txt+ or online at
+following url:
+http://pgloader.projects.postgresql.org/dev/BUGS.html[].
 
 == AUTHORS ==
 

pgloader/csvreader.py

@@ -1,4 +1,4 @@
-# Author: Dimitri Fontaine <dimitri@dalibo.com>
+# Author: Dimitri Fontaine <dim@tapoueh.org>
 #
 # pgloader text format reader
 #

pgloader/db.py

@@ -1,4 +1,4 @@
-# Author: Dimitri Fontaine <dimitri@dalibo.com>
+# Author: Dimitri Fontaine <dim@tapoueh.org>
 #
 # pgloader database connection handling
 # COPY dichotomy on error
@@ -390,14 +390,17 @@ ORDER BY attnum
 
         return ok
 
-    def save_copy_buffer(self, tablename):
+    def save_copy_buffer(self, tablename, debug = False):
         """ save copy buffer to a temporary file for further inspection """
         import tempfile
         (f, n) = tempfile.mkstemp(prefix='%s.' % tablename,
                                   suffix='.pgloader', dir='/tmp')
         os.write(f, self.buffer.getvalue())
         os.close(f)
-        self.log.warning("COPY data buffer saved in %s" % n)
+        if debug:
+            self.log.debug("COPY data buffer saved in %s" % n)
+        else:
+            self.log.warning("COPY data buffer saved in %s" % n)
         return n
 
     def copy_from(self, table, columnlist, columns, input_line,
@@ -426,7 +429,7 @@ ORDER BY attnum
                 return False
             
             if CLIENT_MIN_MESSAGES <= logging.DEBUG:
-                self.save_copy_buffer(tablename)
+                self.save_copy_buffer(tablename, debug = True)
 
             self.buffer.seek(0)
             now = time.time()

pgloader/lo.py

@@ -1,4 +1,4 @@
-# Author: Dimitri Fontaine <dimitri@dalibo.com>
+# Author: Dimitri Fontaine <dim@tapoueh.org>
 #
 # pgloader Large Object support
 

pgloader/options.py

@@ -1,4 +1,4 @@
-# Author: Dimitri Fontaine <dimitri@dalibo.com>
+# Author: Dimitri Fontaine <dim@tapoueh.org>
 #
 # Some common options, for each module to get them
 

pgloader/pgloader.py

@@ -519,6 +519,8 @@ class PGLoader(threading.Thread):
             self.newline_escapes = [(a, NEWLINE_ESCAPES)
                                     for (a, x) in self.columns]
 
+        self.log.debug("self.newline_escapes = '%s'" % self.newline_escapes)
+
         ##
         # Parallelism knobs
         if config.has_option(name, 'section_threads'):

pgloader/reader.py

@@ -78,7 +78,7 @@ class DataReader:
             
         self.log.debug("reader.readconfig field_sep: '%s'", self.field_sep)
 
-    def _getopt(self, option, config, section, template, default = None):
+    def _getopt(self, option, config, section, template, default = None, opt_type = "char"):
         """ Init given configuration option """
 
         if config.has_option(section, option):
@@ -95,6 +95,14 @@ class DataReader:
                            % (option, default))
             self.__dict__[option] = default
 
+        if opt_type == 'int':
+            try:
+                self.__dict__[option] = int(self.__dict__[option])
+            except ValueError, e:
+                self.log.error('Configuration option %s.%s is not an int: %s' \
+                               % (section, option, self.__dict__[option]))
+                raise PGLoader_Error, e
+
         return self.__dict__[option]
 
     def readlines(self):

pgloader/textreader.py

@@ -1,4 +1,4 @@
-# Author: Dimitri Fontaine <dimitri@dalibo.com>
+# Author: Dimitri Fontaine <dim@tapoueh.org>
 #
 # pgloader text format reader
 #
@@ -39,6 +39,9 @@ class TextReader(DataReader):
         if 'newline_escapes' not in self.__dict__:
             self.newline_escapes = newline_escapes
 
+        self.log.debug('reader.__init__: newline_escapes %s' \
+                       % self.newline_escapes)
+
     def readconfig(self, config, name, template):
         """ get this reader module configuration from config file """
         DataReader.readconfig(self, config, name, template)
@@ -47,7 +50,7 @@ class TextReader(DataReader):
         # have to protect ourselves against removing already read
         # configurations while in second run.
 
-        self._getopt('field_count', config, name, template, None)
+        self._getopt('field_count', config, name, template, None, 'int')
         self._getopt('trailing_sep', config, name, template, False)
         if self.trailing_sep is not False:
             self.trailing_sep = self.trailing_sep == 'True'

pgloader/tools.py

@@ -1,4 +1,4 @@
-# Author: Dimitri Fontaine <dimitri@dalibo.com>
+# Author: Dimitri Fontaine <dim@tapoueh.org>
 #
 # pgloader librairies