diff --git a/debian/changelog b/debian/changelog index 9c8349b..3acab7a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -9,6 +9,7 @@ pgloader (2.3.3~dev2-1) unstable; urgency=low * Fix a bug where pgloader would freeze on early error (no such file) * Implement an option to set csv field size limit * Implement --load-from-stdin + * Implement --boundaries -- Dimitri Fontaine Sun, 4 Apr 2010 19:34:39 +0200 diff --git a/examples/pgloader.conf b/examples/pgloader.conf index d1f5b82..e0a4f49 100644 --- a/examples/pgloader.conf +++ b/examples/pgloader.conf @@ -12,7 +12,7 @@ client_min_messages = WARNING ;client_encoding = 'utf-8' client_encoding = 'latin1' lc_messages = C -pg_option_client_encoding = 'utf-8' +;pg_option_client_encoding = 'utf-8' pg_option_standard_conforming_strings = on ; This setting has no effect other than allowing to check option precedence pg_option_work_mem = 12MB diff --git a/pgloader.1.txt b/pgloader.1.txt index 8794e71..10dd4b9 100644 --- a/pgloader.1.txt +++ b/pgloader.1.txt @@ -234,6 +234,11 @@ for them. loading the data to, it's useful when you want to load from +stdin+ and avoid editing a full configuration section. +--boundaries:: + + Allow for limiting the range of bytes to read and process, must be given + in the X..Y format, with X and Y integers. + == GLOBAL CONFIGURATION SECTION == The configuration file has a +.ini+ file syntax, its first section has diff --git a/pgloader/options.py b/pgloader/options.py index be46767..b7b6155 100644 --- a/pgloader/options.py +++ b/pgloader/options.py @@ -56,3 +56,5 @@ REJECT_DATA_FILE = '%s.rej' LOAD_FROM_STDIN = None LOAD_TO_TABLE = None + +FILE_BOUNDARIES = None # (start, end) --- file positions in bytes diff --git a/pgloader/pgloader.py b/pgloader/pgloader.py index 5f4da44..5bb2421 100644 --- a/pgloader/pgloader.py +++ b/pgloader/pgloader.py @@ -289,9 +289,14 @@ class PGLoader(threading.Thread): self.log.debug('%s.%s: %s', name, opt, config.get(name, opt)) self.__dict__[opt] = config.get(name, opt) else: - if not self.template and not self.__dict__[opt]: + if not self.template and opt not in self.__dict__: + msg = "Error: Please configure %s.%s" % (name, opt) + raise PGLoader_Error, msg + + elif not self.template and not self.__dict__[opt]: self.log.error('Error: please configure %s.%s', name, opt) self.config_errors += 1 + else: # Reading Configuration Template section # we want the attribute to exists for further usage diff --git a/pgloader/reader.py b/pgloader/reader.py index 6b7cd30..0c7a8b0 100644 --- a/pgloader/reader.py +++ b/pgloader/reader.py @@ -198,7 +198,6 @@ class UnbufferedFileReader: else: f = open(self.filename, self.mode, self.bufsize) - self.log.warning('PHOQUE "%s"', f) self.fd = codecs.getreader(self.encoding)(f) self.log.info("Opened '%s' with encoding '%s'" \ % (self.filename, self.encoding)) @@ -222,7 +221,7 @@ class UnbufferedFileReader: self.fd.seek(self.start) self.position = self.fd.tell() - self.log.debug("Opened '%s' in %s (fileno %s), ftell %d" \ + self.log.info("Opened '%s' in %s (fileno %s), ftell %d" \ % (self.filename, self.fd, self.fd.fileno(), self.position)) return @@ -285,6 +284,8 @@ class UnbufferedFileReader: if self.end is not None and self.fd.tell() >= self.end: # we want to process current line and stop at next # iteration + self.log.info("Reached position %d, reading last line" \ + % self.fd.tell()) last_line_read = True yield line