From d32472bb3e4a0c7ab81f25289fd9af19c241fbf9 Mon Sep 17 00:00:00 2001 From: dim Date: Sat, 20 Oct 2007 14:28:50 +0000 Subject: [PATCH] Add support for --quiet and --summary options, in order to provide some way to only see errors on output. --- debian/changelog | 6 ++ debian/files | 2 +- debian/rules | 3 +- pgloader.1.txt | 8 +++ pgloader.py | 126 ++++++++++++++++++++++++----------------- pgloader/db.py | 27 +++++---- pgloader/options.py | 2 + pgloader/pgloader.py | 21 ++++--- pgloader/textreader.py | 9 +-- pgloader/tools.py | 13 +++-- 10 files changed, 132 insertions(+), 85 deletions(-) diff --git a/debian/changelog b/debian/changelog index ba241f6..9778e05 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +pgloader (2.2.2) unstable; urgency=low + + * New command line options --quiet and --summary (-qs for short) + + -- Dimitri Fontaine Sat, 20 Oct 2007 16:20:18 +0200 + pgloader (2.2.1) unstable; urgency=low * Support for datestyle setting diff --git a/debian/files b/debian/files index 2dc1060..d647361 100644 --- a/debian/files +++ b/debian/files @@ -1 +1 @@ -pgloader_2.2.0_all.deb misc extra +pgloader_2.2.2_all.deb misc extra diff --git a/debian/rules b/debian/rules index 781b540..97a2035 100644 --- a/debian/rules +++ b/debian/rules @@ -35,7 +35,8 @@ build-stamp: configure-stamp # Add here commands to compile the package. #$(MAKE) - docbook-to-man pgloader.1.sgml > pgloader.1 + #docbook-to-man pgloader.1.sgml > pgloader.1 + $(MAKE) man touch $@ diff --git a/pgloader.1.txt b/pgloader.1.txt index b035393..456bc68 100644 --- a/pgloader.1.txt +++ b/pgloader.1.txt @@ -63,6 +63,14 @@ refers to a PostgreSQL table into which some data is to be loaded. makes pgloader very verbose about what it does. +-q, --quiet: + + makes pgloader very quiet about what it does: only output errors. + +-s, --summary:: + + makes pgloader print a 'nice' summary at the end of operations. + -n, --dry-run:: makes pgloader simulate operations, that implies no database connection and diff --git a/pgloader.py b/pgloader.py index c76cd35..3895c73 100644 --- a/pgloader.py +++ b/pgloader.py @@ -55,6 +55,16 @@ def parse_options(): default = False, help = "be verbose and about processing progress") + parser.add_option("-q", "--quiet", action = "store_true", + dest = "quiet", + default = False, + help = "be quiet, only print out errors") + + parser.add_option("-s", "--summary", action = "store_true", + dest = "summary", + default = False, + help = "print a summary") + parser.add_option("-n", "--dry-run", action = "store_true", dest = "dryrun", default = False, @@ -106,10 +116,16 @@ def parse_options(): print "Error: Can't set both options fromcount (-F) AND fromid (-I)" sys.exit(1) + if opts.quiet and (opts.verbose or opts.debug): + print "Error: Can't be verbose and quiet at the same time!" + sys.exit(1) + pgloader.options.DRY_RUN = opts.dryrun pgloader.options.DEBUG = opts.debug # if debug, then verbose pgloader.options.VERBOSE = opts.verbose or opts.debug + pgloader.options.QUIET = opts.quiet + pgloader.options.SUMMARY = opts.summary pgloader.options.PEDANTIC = opts.pedantic pgloader.options.TRUNCATE = opts.truncate @@ -241,7 +257,8 @@ def load_data(): config, dbconn = parse_config(conffile) # load some pgloader package modules - from pgloader.options import DRY_RUN, VERBOSE, DEBUG, PEDANTIC, VACUUM + from pgloader.options import VERBOSE, DEBUG, QUIET, SUMMARY + from pgloader.options import DRY_RUN, PEDANTIC, VACUUM from pgloader.pgloader import PGLoader from pgloader.tools import PGLoader_Error @@ -291,65 +308,68 @@ def load_data(): td = time.time() - begin retcode = 0 - - # print a pretty summary - t= 'Table name | duration | size | updates | errors ' + + t= 'Table name | duration | size | copy rows | errors ' _= '====================================================================' - tu = te = ts = 0 # total updates, errors, size - if not DRY_RUN: - dbconn.reset() - cursor = dbconn.dbconn.cursor() + if SUMMARY: + # print a pretty summary + tu = te = ts = 0 # total updates, errors, size + if not DRY_RUN: + dbconn.reset() + cursor = dbconn.dbconn.cursor() - s_ok = 0 - for s in sections: - if s not in summary: - continue + s_ok = 0 + for s in sections: + if s not in summary: + continue + + s_ok += 1 + if s_ok == 1: + # print pretty sumary header now + print + print t + print _ + + t, d, u, e = summary[s] + d = duration_pprint(d) + + if not DRY_RUN: + sql = "select pg_total_relation_size(%s), " + \ + "pg_size_pretty(pg_total_relation_size(%s));" + cursor.execute(sql, [t, t]) + octets, s = cursor.fetchone() + ts += octets + + if s[5:] == 'bytes': s = s[:-5] + ' B' + else: + s = '-' + + print '%-18s| %ss | %7s | %10d | %10d' % (t, d, s, u, e) + + tu += u + te += e + + if e > 0: + retcode += 1 + + if s_ok > 1: + td = duration_pprint(td) + + # pretty size + cursor.execute("select pg_size_pretty(%s);", [ts]) + [ts] = cursor.fetchone() + if ts[5:] == 'bytes': ts = ts[:-5] + ' B' - s_ok += 1 - if s_ok == 1: - # print pretty sumary header now - print - print t print _ - - t, d, u, e = summary[s] - d = duration_pprint(d) + print 'Total | %ss | %7s | %10d | %10d' \ + % (td, ts, tu, te) - if not DRY_RUN: - sql = "select pg_total_relation_size(%s), " + \ - "pg_size_pretty(pg_total_relation_size(%s));" - cursor.execute(sql, [t, t]) - octets, s = cursor.fetchone() - ts += octets - - if s[5:] == 'bytes': s = s[:-5] + ' B' - else: - s = '-' - - print '%-18s| %ss | %7s | %10d | %10d' % (t, d, s, u, e) + if not DRY_RUN: + cursor.close() + + print - tu += u - te += e - - if e > 0: - retcode += 1 - - if s_ok > 1: - td = duration_pprint(td) - - # pretty size - cursor.execute("select pg_size_pretty(%s);", [ts]) - [ts] = cursor.fetchone() - if ts[5:] == 'bytes': ts = ts[:-5] + ' B' - - print _ - print 'Total | %ss | %7s | %10d | %10d' % (td, ts, tu, te) - - if not DRY_RUN: - cursor.close() - - print if VACUUM and not DRY_RUN: print 'vacuumdb... ' try: diff --git a/pgloader/db.py b/pgloader/db.py index 35e4266..6e03a9f 100644 --- a/pgloader/db.py +++ b/pgloader/db.py @@ -6,7 +6,7 @@ import os, sys, os.path, time, codecs from cStringIO import StringIO -from options import DRY_RUN, VERBOSE, DEBUG, PEDANTIC +from options import DRY_RUN, VERBOSE, DEBUG, QUIET, PEDANTIC from options import TRUNCATE, VACUUM from options import INPUT_ENCODING, PG_CLIENT_ENCODING, DATESTYLE from options import COPY_SEP, FIELD_SEP, CLOB_SEP, NULL, EMPTY_STRING @@ -117,15 +117,15 @@ class db: d = time.time() - self.first_commit_time u = self.commited_rows c = self.commits - print "## %d updates in %d commits took %5.3f seconds" % (u, c, d) + print " %d updates in %d commits took %5.3f seconds" % (u, c, d) if self.errors > 0: - print "## %d database errors occured" % self.errors + print " %d database errors occured" % self.errors if self.copy and not VACUUM: - print "## Please do VACUUM your database to recover space" + print " Please do VACUUM your database to recover space" else: if u > 0: - print "## No database error occured" + print " No database error occured" return def is_null(self, value): @@ -236,9 +236,10 @@ class db: self.commits += 1 duration = now - self.last_commit_time self.last_commit_time = now - - print "-- commit %d: %d updates in %5.3fs --" \ - % (self.commits, self.running_commands, duration) + + if not QUIET: + print "-- commit %d: %d updates in %5.3fs --" \ + % (self.commits, self.running_commands, duration) self.commited_rows += self.running_commands self.running_commands = 1 @@ -271,7 +272,8 @@ class db: os.close(f) # systematicaly write about this - print "--- COPY data buffer saved in %s ---" % n + if not QUIET: + print " -- COPY data buffer saved in %s ---" % n return n def copy_from(self, table, table_colspec, columns, input_line, @@ -311,8 +313,9 @@ class db: duration = now - self.last_commit_time self.last_commit_time = now - print "-- COPY %d: %d rows copied in %5.3fs --" \ - % (self.commits, self.running_commands, duration) + if not QUIET: + print " - COPY %d: %d rows copied in %5.3fs --" \ + % (self.commits, self.running_commands, duration) # prepare next run self.buffer.close() @@ -416,7 +419,7 @@ class db: x.close() if DEBUG: - print "--- COPY ERROR processing progress: %d rows copied"\ + print " -- COPY ERROR handling progress: %d rows copied"\ % (xcount) x.close() diff --git a/pgloader/options.py b/pgloader/options.py index 6f5f979..35ba728 100644 --- a/pgloader/options.py +++ b/pgloader/options.py @@ -16,6 +16,8 @@ NEWLINE_ESCAPES = None DEBUG = False VERBOSE = False +QUIET = False +SUMMARY = False DRY_RUN = False PEDANTIC = False diff --git a/pgloader/pgloader.py b/pgloader/pgloader.py index 626b3cf..00cd46a 100644 --- a/pgloader/pgloader.py +++ b/pgloader/pgloader.py @@ -12,7 +12,7 @@ from tools import PGLoader_Error, Reject, parse_config_string from db import db from lo import ifx_clob, ifx_blob -from options import DRY_RUN, VERBOSE, DEBUG, PEDANTIC +from options import DRY_RUN, VERBOSE, DEBUG, QUIET, PEDANTIC from options import TRUNCATE, VACUUM from options import COUNT, FROM_COUNT, FROM_ID from options import INPUT_ENCODING, PG_CLIENT_ENCODING @@ -304,29 +304,34 @@ class PGLoader: if self.reject is not None: self.errors = self.reject.errors - self.reject.print_stats() + self.reject.print_stats(self.name, QUIET) - if self.db is not None: - self.updates = self.db.commited_rows - self.db.print_stats() + if not QUIET: + if self.db is not None: + self.updates = self.db.commited_rows + self.db.print_stats() return def run(self): """ depending on configuration, do given job """ # Announce the beginning of the work - print "[%s] data import" % self.name + if not QUIET: + print + print "[%s]" % self.name if TRUNCATE and not DRY_RUN: self.db.truncate(self.table) if self.columns is not None: - print "Notice: COPY csv data" + if not QUIET: + print "Notice: COPY csv data" self.data_import() elif self.blob_cols is not None: # elif: COPY process also blob data - print "Notice: UPDATE blob data" + if not QUIET: + print "Notice: UPDATE blob data" # then show up some stats self.print_stats() diff --git a/pgloader/textreader.py b/pgloader/textreader.py index 2bfda61..84e3277 100644 --- a/pgloader/textreader.py +++ b/pgloader/textreader.py @@ -13,7 +13,7 @@ from db import db from lo import ifx_clob, ifx_blob from reader import DataReader -from options import DRY_RUN, VERBOSE, DEBUG, PEDANTIC +from options import DRY_RUN, VERBOSE, DEBUG, QUIET, PEDANTIC from options import TRUNCATE, VACUUM from options import COUNT, FROM_COUNT, FROM_ID from options import INPUT_ENCODING, PG_CLIENT_ENCODING @@ -50,9 +50,10 @@ class TextReader(DataReader): if NEWLINE_ESCAPES is not None: # this parameter is globally set, will ignore local # definition - print "Warning: ignoring %s newline_escapes option" % name - print " option is set to '%s' globally" \ - % NEWLINE_ESCAPES + if not QUIET: + print "Warning: ignoring %s newline_escapes option" % name + print " option is set to '%s' globally" \ + % NEWLINE_ESCAPES else: self._parse_fields('newline_escapes', config.get(name, 'newline_escapes'), diff --git a/pgloader/tools.py b/pgloader/tools.py index bc6e0fe..c1c2457 100644 --- a/pgloader/tools.py +++ b/pgloader/tools.py @@ -5,7 +5,7 @@ import os, sys, os.path, time, codecs from cStringIO import StringIO -from options import DRY_RUN, VERBOSE, DEBUG, PEDANTIC +from options import DRY_RUN, VERBOSE, DEBUG, QUIET, PEDANTIC class PGLoader_Error(Exception): """ Internal pgloader processing error """ @@ -26,17 +26,18 @@ class Reject: # we will open files on first error self.errors = 0 - def print_stats(self): + def print_stats(self, name, quiet): """ give a summary """ if DRY_RUN: return if self.errors == 0: - print "## No data were rejected" + if not quiet: + print " No data were rejected" else: - print "## %d errors found into data" % self.errors - print " please read %s for errors log" % self.reject_log - print " and %s for data still to process" % self.reject_data + print " %d errors found into [%s] data" % (self.errors, name) + print " please read %s for errors log" % self.reject_log + print " and %s for data still to process" % self.reject_data def log(self, messages, data = None): """ log the messages into reject_log, and the data into reject_data