diff --git a/Makefile b/Makefile index a63219e..2a2e22e 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# $Id: Makefile,v 1.9 2007-12-08 20:19:32 dim Exp $ +# $Id: Makefile,v 1.10 2008-02-01 10:24:38 dim Exp $ # # Makefile for debian packaging purpose, make install not intended to work. @@ -20,6 +20,7 @@ refm = $(wildcard reformat/*.py) DEBDIR = /tmp/pgloader EXPORT = $(DEBDIR)/export/pgloader ORIG = $(DEBDIR)/export/pgloader_$(VERSION).orig.tar.gz +ARCHIVE= $(DEBDIR)/export/pgloader-$(VERSION).tar.gz install: install -m 755 $(pgloader) $(DESTDIR)/usr/bin/pgloader @@ -59,6 +60,9 @@ deb: rm -rf $(DEBDIR)/pgloader/debian (cd $(DEBDIR) && tar czf $(ORIG) pgloader) + # have a copy of the $ORIG file named $ARCHIVE for non-debian packagers + cp $(ORIG) $(ARCHIVE) + # build the debian package and copy them to .. (cd $(EXPORT) && debuild) - cp -a $(DEBDIR)/export/pgloader_$(VERSION)* .. + cp -a $(DEBDIR)/export/pgloader[_-]$(VERSION)* .. diff --git a/debian/changelog b/debian/changelog index f9e3983..33daa69 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +pgloader (2.2.6-1) unstable; urgency=low + + * pgloader -V now VACUUM each table separately, no more vacuumdb issued + * New option -D to DISABLE Triggers while loading (ENABLE them one done) + + -- Dimitri Fontaine Fri, 01 Feb 2008 11:01:34 +0100 + pgloader (2.2.5-1) unstable; urgency=low * Now using proper python logging module diff --git a/pgloader.py b/pgloader.py index 246e317..7b28146 100644 --- a/pgloader.py +++ b/pgloader.py @@ -69,14 +69,19 @@ def parse_options(): help = "simulate operations, don't connect to the db") parser.add_option("-T", "--truncate", action = "store_true", - dest = "truncate", + dest = "truncate", default = False, help = "truncate tables before importing data") + parser.add_option("-D", "--disable-triggers", action = "store_true", + dest = "triggers", + default = False, + help = "Disable triggers before loading, Enable them again after") + parser.add_option("-V", "--vacuum", action = "store_true", dest = "vacuum", default = False, - help = "vacuum database after having imported data") + help = "vacuum tables after data loading") parser.add_option("-C", "--count", dest = "count", default = None, type = "int", @@ -140,6 +145,7 @@ def parse_options(): pgloader.options.TRUNCATE = opts.truncate pgloader.options.VACUUM = opts.vacuum + pgloader.options.TRIGGERS = opts.triggers pgloader.options.COUNT = opts.count pgloader.options.FROM_COUNT = opts.fromcount diff --git a/pgloader/db.py b/pgloader/db.py index ae9ff61..ff214f1 100644 --- a/pgloader/db.py +++ b/pgloader/db.py @@ -170,25 +170,62 @@ class db: self.dbconn.commit() except Exception, error: self.log.error(error) - raise PGLoader_Error, "Couldn't truncate table %s" % table + raise PGLoader_Error, "Couldn't TRUNCATE table %s" % table - def vacuum(self): - """ issue an vacuumdb -fvz database """ + def vacuum(self, table): + """ issue VACUUM ANALYZE table """ if DRY_RUN: self.log.info('no vacuum in dry-run mode') return -1 - command = "/usr/bin/vacuumdb %s -fvz %s 2>&1" \ - % (self.connect, self.base) + sql = "VACUUM ANALYZE %s;" % table - self.log.info(command) - - out = os.popen(command) - for line in out.readlines(): - # don't print \n - self.log.debug(line[:-1]) + self.log.info('%s' % sql) - return out.close() + try: + cursor = self.dbconn.cursor() + cursor.execute(sql) + self.dbconn.commit() + except Exception, error: + self.log.error(error) + raise PGLoader_Error, "Couldn't VACUUM table %s" % table + + def disable_triggers(self, table): + """ issue ALTER TABLE table DISABLE TRIGGER ALL """ + if DRY_RUN: + self.log.info("Won't disable triggers on dry-run mode") + return + + sql = "ALTER TABLE %s DISABLE TRIGGER ALL;" % table + + self.log.info('%s' % sql) + + try: + cursor = self.dbconn.cursor() + cursor.execute(sql) + self.dbconn.commit() + except Exception, error: + self.log.error(error) + raise PGLoader_Error, "Couldn't DISABLE TRIGGERS on table %s" % table + + def enable_triggers(self, table): + """ issue ALTER TABLE table ENABLE TRIGGER ALL """ + if DRY_RUN: + self.log.info("Won't enable triggers on dry-run mode") + return + + sql = "ALTER TABLE %s ENABLE TRIGGER ALL;" % table + + self.log.info('%s' % sql) + + try: + cursor = self.dbconn.cursor() + cursor.execute(sql) + self.dbconn.commit() + except Exception, error: + self.log.error(error) + raise PGLoader_Error, "Couldn't ENABLE TRIGGERS on table %s" % table + def insert_blob(self, table, index, rowids, blob_cname, data, btype, diff --git a/pgloader/options.py b/pgloader/options.py index 578610a..96b41ca 100644 --- a/pgloader/options.py +++ b/pgloader/options.py @@ -2,7 +2,7 @@ # # Some common options, for each module to get them -PGLOADER_VERSION = '2.2.5' +PGLOADER_VERSION = '2.2.6~dev' INPUT_ENCODING = None PG_CLIENT_ENCODING = 'latin9' @@ -25,6 +25,7 @@ PEDANTIC = False TRUNCATE = False VACUUM = False +TRIGGERS = False COUNT = None FROM_COUNT = None diff --git a/pgloader/pgloader.py b/pgloader/pgloader.py index b2be7b2..da4802f 100644 --- a/pgloader/pgloader.py +++ b/pgloader/pgloader.py @@ -14,7 +14,7 @@ from db import db from lo import ifx_clob, ifx_blob from options import DRY_RUN, PEDANTIC -from options import TRUNCATE, VACUUM +from options import TRUNCATE, VACUUM, TRIGGERS from options import COUNT, FROM_COUNT, FROM_ID from options import INPUT_ENCODING, PG_CLIENT_ENCODING from options import COPY_SEP, FIELD_SEP, CLOB_SEP, NULL, EMPTY_STRING @@ -608,8 +608,12 @@ class PGLoader: # Announce the beginning of the work self.log.info("[%s]" % self.name) - if TRUNCATE and not DRY_RUN: - self.db.truncate(self.table) + if not DRY_RUN: + if TRUNCATE: + self.db.truncate(self.table) + + if TRIGGERS: + self.db.disable_triggers(self.table) if self.columns is not None: self.log.info("COPY csv data") @@ -619,6 +623,9 @@ class PGLoader: # elif: COPY process also blob data self.log.info("UPDATE blob data") + if TRIGGERS and not DRY_RUN: + self.db.enable_triggers(self.table) + # then show up some stats self.print_stats()