From ea03303e272326c270f3aaee05296e6fa905cb80 Mon Sep 17 00:00:00 2001 From: dim Date: Fri, 5 Sep 2008 12:52:18 +0000 Subject: [PATCH] Use psycopg cursor.copy_expert() when avaiable (> 2.0.6), which should fix problems tied to too small copy buffer, per Marko's comment on Skytools project --- pgloader/db.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pgloader/db.py b/pgloader/db.py index 194357e..8e9fb09 100644 --- a/pgloader/db.py +++ b/pgloader/db.py @@ -445,7 +445,7 @@ ORDER BY attnum try: cursor = self.dbconn.cursor() - r = cursor.copy_from(self.buffer, table, self.copy_sep) + r = self.cursor_copy_from(cursor, self.buffer, table, self.copy_sep) self.dbconn.commit() self.commits += 1 @@ -547,7 +547,7 @@ ORDER BY attnum for (x, xcount) in [(a, m), (b, n-m)]: try: x.seek(0) - cursor.copy_from(x, table, self.copy_sep) + self.cursor_copy_from(cursor, x, table, self.copy_sep) self.dbconn.commit() x.close() @@ -590,6 +590,16 @@ ORDER BY attnum return commits, ok, ko + def cursor_copy_from(self, cursor, buffer, table, delimiter): + """ call psycopg copy command, in expert mode if available """ + + if hasattr(cursor, 'copy_expert'): + self.log.debug("using copy_expert") + sql = "COPY %s FROM STDOUT WITH DELIMITER '%s'" % (table, delimiter) + return cursor.copy_expert(sql, buffer) + else: + return cursor.copy_from(buffer, table, delimiter) + def prepare_copy_data(self, columns, input_line, reject): """ add a data line to copy buffer """