mirrors/pgloader
1
0
Fork 0
mirror of https://github.com/dimitri/pgloader.git synced 2026-05-04 18:36:12 +02:00
Code Issues Projects Releases Wiki Activity

Add support for --quiet and --summary options, in order to provide some

Browse Source 
way to only see errors on output.
This commit is contained in:
dim 2007-10-20 14:28:50 +00:00
parent 921db51d65
commit d32472bb3e
10 changed files with 132 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
debian/changelog vendored
View File

@ -1,3 +1,9 @@
pgloader (2.2.2) unstable; urgency=low
* New command line options --quiet and --summary (-qs for short)
-- Dimitri Fontaine <dim@tapoueh.org> Sat, 20 Oct 2007 16:20:18 +0200
pgloader (2.2.1) unstable; urgency=low
* Support for datestyle setting

2
debian/files vendored
View File

@ -1 +1 @@
pgloader_2.2.0_all.deb misc extra
pgloader_2.2.2_all.deb misc extra

3
debian/rules vendored
View File

@ -35,7 +35,8 @@ build-stamp: configure-stamp
# Add here commands to compile the package.
#$(MAKE)
docbook-to-man pgloader.1.sgml > pgloader.1
#docbook-to-man pgloader.1.sgml > pgloader.1
$(MAKE) man
touch $@

8
pgloader.1.txt
View File

@ -63,6 +63,14 @@ refers to a PostgreSQL table into which some data is to be loaded.
makes pgloader very verbose about what it does.
-q, --quiet:
makes pgloader very quiet about what it does: only output errors.
-s, --summary::
makes pgloader print a 'nice' summary at the end of operations.
-n, --dry-run::
makes pgloader simulate operations, that implies no database connection and

126
pgloader.py
View File

@ -55,6 +55,16 @@ def parse_options():
default = False,
help = "be verbose and about processing progress")
parser.add_option("-q", "--quiet", action = "store_true",
dest = "quiet",
default = False,
help = "be quiet, only print out errors")
parser.add_option("-s", "--summary", action = "store_true",
dest = "summary",
default = False,
help = "print a summary")
parser.add_option("-n", "--dry-run", action = "store_true",
dest = "dryrun",
default = False,
@ -106,10 +116,16 @@ def parse_options():
print "Error: Can't set both options fromcount (-F) AND fromid (-I)"
sys.exit(1)
if opts.quiet and (opts.verbose or opts.debug):
print "Error: Can't be verbose and quiet at the same time!"
sys.exit(1)
pgloader.options.DRY_RUN = opts.dryrun
pgloader.options.DEBUG = opts.debug
# if debug, then verbose
pgloader.options.VERBOSE = opts.verbose or opts.debug
pgloader.options.QUIET = opts.quiet
pgloader.options.SUMMARY = opts.summary
pgloader.options.PEDANTIC = opts.pedantic
pgloader.options.TRUNCATE = opts.truncate
@ -241,7 +257,8 @@ def load_data():
config, dbconn = parse_config(conffile)
# load some pgloader package modules
from pgloader.options import DRY_RUN, VERBOSE, DEBUG, PEDANTIC, VACUUM
from pgloader.options import VERBOSE, DEBUG, QUIET, SUMMARY
from pgloader.options import DRY_RUN, PEDANTIC, VACUUM
from pgloader.pgloader import PGLoader
from pgloader.tools import PGLoader_Error
@ -291,65 +308,68 @@ def load_data():
td = time.time() - begin
retcode = 0
# print a pretty summary
t= 'Table name | duration | size | updates | errors '
t= 'Table name | duration | size | copy rows | errors '
_= '===================================================================='
tu = te = ts = 0 # total updates, errors, size
if not DRY_RUN:
dbconn.reset()
cursor = dbconn.dbconn.cursor()
if SUMMARY:
# print a pretty summary
tu = te = ts = 0 # total updates, errors, size
if not DRY_RUN:
dbconn.reset()
cursor = dbconn.dbconn.cursor()
s_ok = 0
for s in sections:
if s not in summary:
continue
s_ok = 0
for s in sections:
if s not in summary:
continue
s_ok += 1
if s_ok == 1:
# print pretty sumary header now
print
print t
print _
t, d, u, e = summary[s]
d = duration_pprint(d)
if not DRY_RUN:
sql = "select pg_total_relation_size(%s), " + \
"pg_size_pretty(pg_total_relation_size(%s));"
cursor.execute(sql, [t, t])
octets, s = cursor.fetchone()
ts += octets
if s[5:] == 'bytes': s = s[:-5] + ' B'
else:
s = '-'
print '%-18s| %ss | %7s | %10d | %10d' % (t, d, s, u, e)
tu += u
te += e
if e > 0:
retcode += 1
if s_ok > 1:
td = duration_pprint(td)
# pretty size
cursor.execute("select pg_size_pretty(%s);", [ts])
[ts] = cursor.fetchone()
if ts[5:] == 'bytes': ts = ts[:-5] + ' B'
s_ok += 1
if s_ok == 1:
# print pretty sumary header now
print
print t
print _
t, d, u, e = summary[s]
d = duration_pprint(d)
print 'Total | %ss | %7s | %10d | %10d' \
% (td, ts, tu, te)
if not DRY_RUN:
sql = "select pg_total_relation_size(%s), " + \
"pg_size_pretty(pg_total_relation_size(%s));"
cursor.execute(sql, [t, t])
octets, s = cursor.fetchone()
ts += octets
if s[5:] == 'bytes': s = s[:-5] + ' B'
else:
s = '-'
print '%-18s| %ss | %7s | %10d | %10d' % (t, d, s, u, e)
if not DRY_RUN:
cursor.close()
print
tu += u
te += e
if e > 0:
retcode += 1
if s_ok > 1:
td = duration_pprint(td)
# pretty size
cursor.execute("select pg_size_pretty(%s);", [ts])
[ts] = cursor.fetchone()
if ts[5:] == 'bytes': ts = ts[:-5] + ' B'
print _
print 'Total | %ss | %7s | %10d | %10d' % (td, ts, tu, te)
if not DRY_RUN:
cursor.close()
print
if VACUUM and not DRY_RUN:
print 'vacuumdb... '
try:

27
pgloader/db.py
View File

@ -6,7 +6,7 @@
import os, sys, os.path, time, codecs
from cStringIO import StringIO
from options import DRY_RUN, VERBOSE, DEBUG, PEDANTIC
from options import DRY_RUN, VERBOSE, DEBUG, QUIET, PEDANTIC
from options import TRUNCATE, VACUUM
from options import INPUT_ENCODING, PG_CLIENT_ENCODING, DATESTYLE
from options import COPY_SEP, FIELD_SEP, CLOB_SEP, NULL, EMPTY_STRING
@ -117,15 +117,15 @@ class db:
d = time.time() - self.first_commit_time
u = self.commited_rows
c = self.commits
print "## %d updates in %d commits took %5.3f seconds" % (u, c, d)
print " %d updates in %d commits took %5.3f seconds" % (u, c, d)
if self.errors > 0:
print "## %d database errors occured" % self.errors
print " %d database errors occured" % self.errors
if self.copy and not VACUUM:
print "## Please do VACUUM your database to recover space"
print " Please do VACUUM your database to recover space"
else:
if u > 0:
print "## No database error occured"
print " No database error occured"
return
def is_null(self, value):
@ -236,9 +236,10 @@ class db:
self.commits += 1
duration = now - self.last_commit_time
self.last_commit_time = now
print "-- commit %d: %d updates in %5.3fs --" \
% (self.commits, self.running_commands, duration)
if not QUIET:
print "-- commit %d: %d updates in %5.3fs --" \
% (self.commits, self.running_commands, duration)
self.commited_rows += self.running_commands
self.running_commands = 1
@ -271,7 +272,8 @@ class db:
os.close(f)
# systematicaly write about this
print "--- COPY data buffer saved in %s ---" % n
if not QUIET:
print " -- COPY data buffer saved in %s ---" % n
return n
def copy_from(self, table, table_colspec, columns, input_line,
@ -311,8 +313,9 @@ class db:
duration = now - self.last_commit_time
self.last_commit_time = now
print "-- COPY %d: %d rows copied in %5.3fs --" \
% (self.commits, self.running_commands, duration)
if not QUIET:
print " - COPY %d: %d rows copied in %5.3fs --" \
% (self.commits, self.running_commands, duration)
# prepare next run
self.buffer.close()
@ -416,7 +419,7 @@ class db:
x.close()
if DEBUG:
print "--- COPY ERROR processing progress: %d rows copied"\
print " -- COPY ERROR handling progress: %d rows copied"\
% (xcount)
x.close()

2
pgloader/options.py
View File

@ -16,6 +16,8 @@ NEWLINE_ESCAPES = None
DEBUG = False
VERBOSE = False
QUIET = False
SUMMARY = False
DRY_RUN = False
PEDANTIC = False

21
pgloader/pgloader.py
View File

@ -12,7 +12,7 @@ from tools import PGLoader_Error, Reject, parse_config_string
from db import db
from lo import ifx_clob, ifx_blob
from options import DRY_RUN, VERBOSE, DEBUG, PEDANTIC
from options import DRY_RUN, VERBOSE, DEBUG, QUIET, PEDANTIC
from options import TRUNCATE, VACUUM
from options import COUNT, FROM_COUNT, FROM_ID
from options import INPUT_ENCODING, PG_CLIENT_ENCODING
@ -304,29 +304,34 @@ class PGLoader:
if self.reject is not None:
self.errors = self.reject.errors
self.reject.print_stats()
self.reject.print_stats(self.name, QUIET)
if self.db is not None:
self.updates = self.db.commited_rows
self.db.print_stats()
if not QUIET:
if self.db is not None:
self.updates = self.db.commited_rows
self.db.print_stats()
return
def run(self):
""" depending on configuration, do given job """
# Announce the beginning of the work
print "[%s] data import" % self.name
if not QUIET:
print
print "[%s]" % self.name
if TRUNCATE and not DRY_RUN:
self.db.truncate(self.table)
if self.columns is not None:
print "Notice: COPY csv data"
if not QUIET:
print "Notice: COPY csv data"
self.data_import()
elif self.blob_cols is not None:
# elif: COPY process also blob data
print "Notice: UPDATE blob data"
if not QUIET:
print "Notice: UPDATE blob data"
# then show up some stats
self.print_stats()

9
pgloader/textreader.py
View File

@ -13,7 +13,7 @@ from db import db
from lo import ifx_clob, ifx_blob
from reader import DataReader
from options import DRY_RUN, VERBOSE, DEBUG, PEDANTIC
from options import DRY_RUN, VERBOSE, DEBUG, QUIET, PEDANTIC
from options import TRUNCATE, VACUUM
from options import COUNT, FROM_COUNT, FROM_ID
from options import INPUT_ENCODING, PG_CLIENT_ENCODING
@ -50,9 +50,10 @@ class TextReader(DataReader):
if NEWLINE_ESCAPES is not None:
# this parameter is globally set, will ignore local
# definition
print "Warning: ignoring %s newline_escapes option" % name
print " option is set to '%s' globally" \
% NEWLINE_ESCAPES
if not QUIET:
print "Warning: ignoring %s newline_escapes option" % name
print " option is set to '%s' globally" \
% NEWLINE_ESCAPES
else:
self._parse_fields('newline_escapes',
config.get(name, 'newline_escapes'),

13
pgloader/tools.py
View File

@ -5,7 +5,7 @@
import os, sys, os.path, time, codecs
from cStringIO import StringIO
from options import DRY_RUN, VERBOSE, DEBUG, PEDANTIC
from options import DRY_RUN, VERBOSE, DEBUG, QUIET, PEDANTIC
class PGLoader_Error(Exception):
""" Internal pgloader processing error """
@ -26,17 +26,18 @@ class Reject:
# we will open files on first error
self.errors = 0
def print_stats(self):
def print_stats(self, name, quiet):
""" give a summary """
if DRY_RUN:
return
if self.errors == 0:
print "## No data were rejected"
if not quiet:
print " No data were rejected"
else:
print "## %d errors found into data" % self.errors
print " please read %s for errors log" % self.reject_log
print " and %s for data still to process" % self.reject_data
print " %d errors found into [%s] data" % (self.errors, name)
print " please read %s for errors log" % self.reject_log
print " and %s for data still to process" % self.reject_data
def log(self, messages, data = None):
""" log the messages into reject_log, and the data into reject_data