Implement --boundaries to load a part of a file only (given in bytes).

debian/changelog

@@ -9,6 +9,7 @@ pgloader (2.3.3~dev2-1) unstable; urgency=low
   * Fix a bug where pgloader would freeze on early error (no such file)
   * Implement an option to set csv field size limit
   * Implement --load-from-stdin
+  * Implement --boundaries
 
  -- Dimitri Fontaine <dim@tapoueh.org>  Sun,  4 Apr 2010 19:34:39 +0200
 

examples/pgloader.conf

@@ -12,7 +12,7 @@ client_min_messages = WARNING
 ;client_encoding = 'utf-8'
 client_encoding = 'latin1'
 lc_messages         = C
-pg_option_client_encoding = 'utf-8'
+;pg_option_client_encoding = 'utf-8'
 pg_option_standard_conforming_strings = on
 ; This setting has no effect other than allowing to check option precedence
 pg_option_work_mem = 12MB

pgloader.1.txt

@@ -234,6 +234,11 @@ for them.
     loading the data to, it's useful when you want to load from +stdin+ and
     avoid editing a full configuration section.
 
+--boundaries::
+
+    Allow for limiting the range of bytes to read and process, must be given
+    in the X..Y format, with X and Y integers.
+
 == GLOBAL CONFIGURATION SECTION ==
 
 The configuration file has a +.ini+ file syntax, its first section has

pgloader/options.py

@@ -56,3 +56,5 @@ REJECT_DATA_FILE = '%s.rej'
 
 LOAD_FROM_STDIN = None
 LOAD_TO_TABLE   = None
+
+FILE_BOUNDARIES = None  # (start, end) --- file positions in bytes

pgloader/pgloader.py

@@ -289,9 +289,14 @@ class PGLoader(threading.Thread):
                 self.log.debug('%s.%s: %s', name, opt, config.get(name, opt))
                 self.__dict__[opt] = config.get(name, opt)
             else:
-                if not self.template and not self.__dict__[opt]:
+                if not self.template and opt not in self.__dict__:
+                    msg = "Error: Please configure %s.%s" % (name, opt)
+                    raise PGLoader_Error, msg
+                
+                elif not self.template and not self.__dict__[opt]:
                     self.log.error('Error: please configure %s.%s', name, opt)
                     self.config_errors += 1
+                    
                 else:
                     # Reading Configuration Template section
                     # we want the attribute to exists for further usage

pgloader/reader.py

@@ -198,7 +198,6 @@ class UnbufferedFileReader:
                 else:
                     f = open(self.filename, self.mode, self.bufsize)
 
-                self.log.warning('PHOQUE "%s"', f)
                 self.fd = codecs.getreader(self.encoding)(f)
                 self.log.info("Opened '%s' with encoding '%s'" \
                               % (self.filename, self.encoding))
@@ -222,7 +221,7 @@ class UnbufferedFileReader:
             self.fd.seek(self.start)
             self.position = self.fd.tell()
 
-        self.log.debug("Opened '%s' in %s (fileno %s), ftell %d" \
+        self.log.info("Opened '%s' in %s (fileno %s), ftell %d" \
                       % (self.filename, self.fd,
                          self.fd.fileno(), self.position))
         return
@@ -285,6 +284,8 @@ class UnbufferedFileReader:
             if self.end is not None and self.fd.tell() >= self.end:
                 # we want to process current line and stop at next
                 # iteration
+                self.log.info("Reached position %d, reading last line" \
+                              % self.fd.tell())
                 last_line_read = True
 
             yield line