mirror of
https://github.com/dimitri/pgloader.git
synced 2025-08-11 08:46:59 +02:00
database object is now initialized from PGLoader init
This commit is contained in:
parent
60ffb4ba3f
commit
526836deca
61
pgloader.py
61
pgloader.py
@ -246,49 +246,11 @@ def parse_config(conffile):
|
|||||||
log.error("Default logfile %s has been used instead",
|
log.error("Default logfile %s has been used instead",
|
||||||
pgloader.options.LOG_FILE)
|
pgloader.options.LOG_FILE)
|
||||||
|
|
||||||
if DRY_RUN:
|
|
||||||
log.info("dry run mode, not connecting to database")
|
|
||||||
return config, None
|
|
||||||
|
|
||||||
try:
|
|
||||||
from pgloader.db import db
|
|
||||||
|
|
||||||
dbconn = db(config.get(section, 'host'),
|
|
||||||
config.getint(section, 'port'),
|
|
||||||
config.get(section, 'base'),
|
|
||||||
config.get(section, 'user'),
|
|
||||||
config.get(section, 'pass'),
|
|
||||||
connect = False)
|
|
||||||
|
|
||||||
if config.has_option(section, 'client_encoding'):
|
|
||||||
client_encoding = pgloader.tools.parse_config_string(
|
|
||||||
config.get(section, 'client_encoding'))
|
|
||||||
dbconn.client_encoding = client_encoding
|
|
||||||
|
|
||||||
if config.has_option(section, 'lc_messages'):
|
|
||||||
lc_messages = pgloader.tools.parse_config_string(
|
|
||||||
config.get(section, 'lc_messages'))
|
|
||||||
dbconn.lc_messages = lc_messages
|
|
||||||
|
|
||||||
if config.has_option(section, 'input_encoding'):
|
if config.has_option(section, 'input_encoding'):
|
||||||
input_encoding = pgloader.tools.parse_config_string(
|
input_encoding = pgloader.tools.parse_config_string(
|
||||||
config.get(section, 'input_encoding'))
|
config.get(section, 'input_encoding'))
|
||||||
pgloader.options.INPUT_ENCODING = input_encoding
|
pgloader.options.INPUT_ENCODING = input_encoding
|
||||||
|
|
||||||
if config.has_option(section, 'datestyle'):
|
|
||||||
datestyle = pgloader.tools.parse_config_string(
|
|
||||||
config.get(section, 'datestyle'))
|
|
||||||
dbconn.datestyle = datestyle
|
|
||||||
|
|
||||||
if config.has_option(section, 'copy_every'):
|
|
||||||
dbconn.copy_every = config.getint(section, 'copy_every')
|
|
||||||
|
|
||||||
if config.has_option(section, 'commit_every'):
|
|
||||||
dbconn.commit_every = config.getint(section, 'commit_every')
|
|
||||||
|
|
||||||
if config.has_option(section, 'copy_delimiter'):
|
|
||||||
dbconn.copy_sep = config.get(section, 'copy_delimiter')
|
|
||||||
|
|
||||||
# optionnal global newline_escapes
|
# optionnal global newline_escapes
|
||||||
if config.has_option(section, 'newline_escapes'):
|
if config.has_option(section, 'newline_escapes'):
|
||||||
setting = pgloader.tools.parse_config_string(
|
setting = pgloader.tools.parse_config_string(
|
||||||
@ -311,12 +273,7 @@ def parse_config(conffile):
|
|||||||
rpath = config.get(section, 'reformat_path')
|
rpath = config.get(section, 'reformat_path')
|
||||||
pgloader.options.REFORMAT_PATH = rpath
|
pgloader.options.REFORMAT_PATH = rpath
|
||||||
|
|
||||||
except Exception, error:
|
return config
|
||||||
log.error("Could not initialize PostgreSQL connection:")
|
|
||||||
print error
|
|
||||||
sys.exit(6)
|
|
||||||
|
|
||||||
return config, dbconn
|
|
||||||
|
|
||||||
def myprint(l, line_prefix = " ", cols = 78):
|
def myprint(l, line_prefix = " ", cols = 78):
|
||||||
""" pretty print list l elements """
|
""" pretty print list l elements """
|
||||||
@ -364,7 +321,8 @@ def print_summary(dbconn, sections, summary, td):
|
|||||||
_= '===================================================================='
|
_= '===================================================================='
|
||||||
|
|
||||||
tu = te = ts = 0 # total updates, errors, size
|
tu = te = ts = 0 # total updates, errors, size
|
||||||
if not DRY_RUN:
|
|
||||||
|
if False and not DRY_RUN:
|
||||||
dbconn.reset()
|
dbconn.reset()
|
||||||
cursor = dbconn.dbconn.cursor()
|
cursor = dbconn.dbconn.cursor()
|
||||||
|
|
||||||
@ -383,7 +341,7 @@ def print_summary(dbconn, sections, summary, td):
|
|||||||
t, d, u, e = summary[s]
|
t, d, u, e = summary[s]
|
||||||
d = duration_pprint(d)
|
d = duration_pprint(d)
|
||||||
|
|
||||||
if not DRY_RUN:
|
if False and not DRY_RUN:
|
||||||
sql = "select pg_total_relation_size(%s), " + \
|
sql = "select pg_total_relation_size(%s), " + \
|
||||||
"pg_size_pretty(pg_total_relation_size(%s));"
|
"pg_size_pretty(pg_total_relation_size(%s));"
|
||||||
cursor.execute(sql, [t, t])
|
cursor.execute(sql, [t, t])
|
||||||
@ -410,15 +368,18 @@ def print_summary(dbconn, sections, summary, td):
|
|||||||
td = duration_pprint(td)
|
td = duration_pprint(td)
|
||||||
|
|
||||||
# pretty size
|
# pretty size
|
||||||
|
if False and not DRY_RUN:
|
||||||
cursor.execute("select pg_size_pretty(%s);", [ts])
|
cursor.execute("select pg_size_pretty(%s);", [ts])
|
||||||
[ts] = cursor.fetchone()
|
[ts] = cursor.fetchone()
|
||||||
if ts[5:] == 'bytes': ts = ts[:-5] + ' B'
|
if ts[5:] == 'bytes': ts = ts[:-5] + ' B'
|
||||||
|
else:
|
||||||
|
ts = '-'
|
||||||
|
|
||||||
print _
|
print _
|
||||||
print 'Total | %ss | %7s | %10d | %10d' \
|
print 'Total | %ss | %7s | %10d | %10d' \
|
||||||
% (td, ts, tu, te)
|
% (td, ts, tu, te)
|
||||||
|
|
||||||
if not DRY_RUN:
|
if False and not DRY_RUN:
|
||||||
cursor.close()
|
cursor.close()
|
||||||
|
|
||||||
return retcode
|
return retcode
|
||||||
@ -433,7 +394,7 @@ def load_data():
|
|||||||
conffile, args = parse_options()
|
conffile, args = parse_options()
|
||||||
|
|
||||||
# now init db connection
|
# now init db connection
|
||||||
config, dbconn = parse_config(conffile)
|
config = parse_config(conffile)
|
||||||
|
|
||||||
from pgloader.logger import log
|
from pgloader.logger import log
|
||||||
from pgloader.tools import read_path, check_path
|
from pgloader.tools import read_path, check_path
|
||||||
@ -492,7 +453,7 @@ def load_data():
|
|||||||
sections.sort()
|
sections.sort()
|
||||||
for s in sections:
|
for s in sections:
|
||||||
try:
|
try:
|
||||||
loader = PGLoader(s, config, dbconn)
|
loader = PGLoader(s, config)
|
||||||
|
|
||||||
if not loader.template:
|
if not loader.template:
|
||||||
loader.run()
|
loader.run()
|
||||||
@ -523,7 +484,7 @@ def load_data():
|
|||||||
|
|
||||||
if SUMMARY:
|
if SUMMARY:
|
||||||
try:
|
try:
|
||||||
retcode = print_summary(dbconn, sections, summary, td)
|
retcode = print_summary(None, sections, summary, td)
|
||||||
print
|
print
|
||||||
except PGLoader_Error, e:
|
except PGLoader_Error, e:
|
||||||
log.error("Can't print summary: %s" % e)
|
log.error("Can't print summary: %s" % e)
|
||||||
|
@ -28,13 +28,14 @@ class PGLoader:
|
|||||||
import data with COPY or update blob data with UPDATE.
|
import data with COPY or update blob data with UPDATE.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, name, config, db):
|
def __init__(self, name, config):
|
||||||
""" Init with a configuration section """
|
""" Init with a configuration section """
|
||||||
# Some settings
|
# Some settings
|
||||||
self.name = name
|
self.name = name
|
||||||
self.db = db
|
|
||||||
self.log = getLogger(name)
|
self.log = getLogger(name)
|
||||||
|
|
||||||
|
self.__dbconnect__(config)
|
||||||
|
|
||||||
self.template = None
|
self.template = None
|
||||||
self.use_template = None
|
self.use_template = None
|
||||||
|
|
||||||
@ -108,6 +109,48 @@ class PGLoader:
|
|||||||
|
|
||||||
self.log.debug('%s init done' % name)
|
self.log.debug('%s init done' % name)
|
||||||
|
|
||||||
|
def __dbconnect__(self, config):
|
||||||
|
""" connects to database """
|
||||||
|
section = 'pgsql'
|
||||||
|
|
||||||
|
if DRY_RUN:
|
||||||
|
log.info("dry run mode, not connecting to database")
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
self.db = db(config.get(section, 'host'),
|
||||||
|
config.getint(section, 'port'),
|
||||||
|
config.get(section, 'base'),
|
||||||
|
config.get(section, 'user'),
|
||||||
|
config.get(section, 'pass'),
|
||||||
|
connect = False)
|
||||||
|
|
||||||
|
if config.has_option(section, 'client_encoding'):
|
||||||
|
self.db.client_encoding = parse_config_string(
|
||||||
|
config.get(section, 'client_encoding'))
|
||||||
|
|
||||||
|
if config.has_option(section, 'lc_messages'):
|
||||||
|
self.db.lc_messages = parse_config_string(
|
||||||
|
config.get(section, 'lc_messages'))
|
||||||
|
|
||||||
|
if config.has_option(section, 'datestyle'):
|
||||||
|
self.db.datestyle = parse_config_string(
|
||||||
|
config.get(section, 'datestyle'))
|
||||||
|
|
||||||
|
if config.has_option(section, 'copy_every'):
|
||||||
|
self.db.copy_every = config.getint(section, 'copy_every')
|
||||||
|
|
||||||
|
if config.has_option(section, 'commit_every'):
|
||||||
|
self.db.commit_every = config.getint(
|
||||||
|
section, 'commit_every')
|
||||||
|
|
||||||
|
if config.has_option(section, 'copy_delimiter'):
|
||||||
|
self.db.copy_sep = config.get(section, 'copy_delimiter')
|
||||||
|
|
||||||
|
except Exception, error:
|
||||||
|
log.error("Could not initialize PostgreSQL connection")
|
||||||
|
raise PGLoader_Error, error
|
||||||
|
|
||||||
def __read_conf__(self, name, config, db, want_template = False):
|
def __read_conf__(self, name, config, db, want_template = False):
|
||||||
""" init self from config section name """
|
""" init self from config section name """
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user