mirrors/pgloader
1
0
Fork 0
mirror of https://github.com/dimitri/pgloader.git synced 2026-05-04 18:36:12 +02:00
Code Issues Projects Releases Wiki Activity

* pgloader -V now VACUUM each table separately, no more vacuumdb issued

Browse Source 
* New option -D to DISABLE Triggers while loading (ENABLE them one done)
This commit is contained in:
dim 2008-02-01 10:24:39 +00:00
parent 3cf6722636
commit 8605b14084
6 changed files with 82 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Makefile
View File

@ -1,4 +1,4 @@
# $Id: Makefile,v 1.9 2007-12-08 20:19:32 dim Exp $
# $Id: Makefile,v 1.10 2008-02-01 10:24:38 dim Exp $
#
# Makefile for debian packaging purpose, make install not intended to work.
@ -20,6 +20,7 @@ refm = $(wildcard reformat/*.py)
DEBDIR = /tmp/pgloader
EXPORT = $(DEBDIR)/export/pgloader
ORIG = $(DEBDIR)/export/pgloader_$(VERSION).orig.tar.gz
ARCHIVE= $(DEBDIR)/export/pgloader-$(VERSION).tar.gz
install:
install -m 755 $(pgloader) $(DESTDIR)/usr/bin/pgloader
@ -59,6 +60,9 @@ deb:
rm -rf $(DEBDIR)/pgloader/debian
(cd $(DEBDIR) && tar czf $(ORIG) pgloader)
# have a copy of the $ORIG file named $ARCHIVE for non-debian packagers
cp $(ORIG) $(ARCHIVE)
# build the debian package and copy them to ..
(cd $(EXPORT) && debuild)
cp -a $(DEBDIR)/export/pgloader_$(VERSION)* ..
cp -a $(DEBDIR)/export/pgloader[_-]$(VERSION)* ..

7
debian/changelog vendored
View File

@ -1,3 +1,10 @@
pgloader (2.2.6-1) unstable; urgency=low
* pgloader -V now VACUUM each table separately, no more vacuumdb issued
* New option -D to DISABLE Triggers while loading (ENABLE them one done)
-- Dimitri Fontaine <dim@tapoueh.org> Fri, 01 Feb 2008 11:01:34 +0100
pgloader (2.2.5-1) unstable; urgency=low
* Now using proper python logging module

10
pgloader.py
View File

@ -69,14 +69,19 @@ def parse_options():
help = "simulate operations, don't connect to the db")
parser.add_option("-T", "--truncate", action = "store_true",
dest = "truncate",
dest = "truncate",
default = False,
help = "truncate tables before importing data")
parser.add_option("-D", "--disable-triggers", action = "store_true",
dest = "triggers",
default = False,
help = "Disable triggers before loading, Enable them again after")
parser.add_option("-V", "--vacuum", action = "store_true",
dest = "vacuum",
default = False,
help = "vacuum database after having imported data")
help = "vacuum tables after data loading")
parser.add_option("-C", "--count", dest = "count",
default = None, type = "int",
@ -140,6 +145,7 @@ def parse_options():
pgloader.options.TRUNCATE = opts.truncate
pgloader.options.VACUUM = opts.vacuum
pgloader.options.TRIGGERS = opts.triggers
pgloader.options.COUNT = opts.count
pgloader.options.FROM_COUNT = opts.fromcount

61
pgloader/db.py
View File

@ -170,25 +170,62 @@ class db:
self.dbconn.commit()
except Exception, error:
self.log.error(error)
raise PGLoader_Error, "Couldn't truncate table %s" % table
raise PGLoader_Error, "Couldn't TRUNCATE table %s" % table
def vacuum(self):
""" issue an vacuumdb -fvz database """
def vacuum(self, table):
""" issue VACUUM ANALYZE table """
if DRY_RUN:
self.log.info('no vacuum in dry-run mode')
return -1
command = "/usr/bin/vacuumdb %s -fvz %s 2>&1" \
% (self.connect, self.base)
sql = "VACUUM ANALYZE %s;" % table
self.log.info(command)
out = os.popen(command)
for line in out.readlines():
# don't print \n
self.log.debug(line[:-1])
self.log.info('%s' % sql)
return out.close()
try:
cursor = self.dbconn.cursor()
cursor.execute(sql)
self.dbconn.commit()
except Exception, error:
self.log.error(error)
raise PGLoader_Error, "Couldn't VACUUM table %s" % table
def disable_triggers(self, table):
""" issue ALTER TABLE table DISABLE TRIGGER ALL """
if DRY_RUN:
self.log.info("Won't disable triggers on dry-run mode")
return
sql = "ALTER TABLE %s DISABLE TRIGGER ALL;" % table
self.log.info('%s' % sql)
try:
cursor = self.dbconn.cursor()
cursor.execute(sql)
self.dbconn.commit()
except Exception, error:
self.log.error(error)
raise PGLoader_Error, "Couldn't DISABLE TRIGGERS on table %s" % table
def enable_triggers(self, table):
""" issue ALTER TABLE table ENABLE TRIGGER ALL """
if DRY_RUN:
self.log.info("Won't enable triggers on dry-run mode")
return
sql = "ALTER TABLE %s ENABLE TRIGGER ALL;" % table
self.log.info('%s' % sql)
try:
cursor = self.dbconn.cursor()
cursor.execute(sql)
self.dbconn.commit()
except Exception, error:
self.log.error(error)
raise PGLoader_Error, "Couldn't ENABLE TRIGGERS on table %s" % table
def insert_blob(self, table, index, rowids,
blob_cname, data, btype,

3
pgloader/options.py
View File

@ -2,7 +2,7 @@
#
# Some common options, for each module to get them
PGLOADER_VERSION = '2.2.5'
PGLOADER_VERSION = '2.2.6~dev'
INPUT_ENCODING = None
PG_CLIENT_ENCODING = 'latin9'
@ -25,6 +25,7 @@ PEDANTIC = False
TRUNCATE = False
VACUUM = False
TRIGGERS = False
COUNT = None
FROM_COUNT = None

13
pgloader/pgloader.py
View File

@ -14,7 +14,7 @@ from db import db
from lo import ifx_clob, ifx_blob
from options import DRY_RUN, PEDANTIC
from options import TRUNCATE, VACUUM
from options import TRUNCATE, VACUUM, TRIGGERS
from options import COUNT, FROM_COUNT, FROM_ID
from options import INPUT_ENCODING, PG_CLIENT_ENCODING
from options import COPY_SEP, FIELD_SEP, CLOB_SEP, NULL, EMPTY_STRING
@ -608,8 +608,12 @@ class PGLoader:
# Announce the beginning of the work
self.log.info("[%s]" % self.name)
if TRUNCATE and not DRY_RUN:
self.db.truncate(self.table)
if not DRY_RUN:
if TRUNCATE:
self.db.truncate(self.table)
if TRIGGERS:
self.db.disable_triggers(self.table)
if self.columns is not None:
self.log.info("COPY csv data")
@ -619,6 +623,9 @@ class PGLoader:
# elif: COPY process also blob data
self.log.info("UPDATE blob data")
if TRIGGERS and not DRY_RUN:
self.db.enable_triggers(self.table)
# then show up some stats
self.print_stats()