From 1fb69b2039e6e24cff44dfc176bbdad7fa5de5a5 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Tue, 20 Oct 2015 23:15:05 +0200 Subject: [PATCH] Retry connecting to PostgreSQL in some cases. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that we can setup many concurrent threads working against the PostgreSQL database, and before we open the number of workers to our users, install an heuristic to manage the PostgreSQL error classes “too many connection” and “configuration limit exceeded” so that pgloader waits for some time (*retry-connect-delay*) then tries connecting again. It's quite simplistic but should cover lots of border-line cases way more nicely than just throwing the interactive debugger at the end user. --- src/pgsql/queries.lisp | 51 +++++++++++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/src/pgsql/queries.lisp b/src/pgsql/queries.lisp index a5e7407..07b8e3d 100644 --- a/src/pgsql/queries.lisp +++ b/src/pgsql/queries.lisp @@ -34,16 +34,51 @@ :use-ssl (pgconn-use-ssl pgconn) :table-name (pgconn-table-name pgconn))) +;;; +;;; We need to distinguish some special cases of PostgreSQL errors within +;;; Class 53 — Insufficient Resources: in case of "too many connections" we +;;; typically want to leave room for another worker to finish and free one +;;; connection, then try again. +;;; +;;; http://www.postgresql.org/docs/9.4/interactive/errcodes-appendix.html +;;; +;;; The "leave room to finish and try again" heuristic is currently quite +;;; simplistic, but at least it work in my test cases. +;;; +(cl-postgres-error::deferror "53300" + too-many-connections cl-postgres-error:insufficient-resources) +(cl-postgres-error::deferror "53400" + configuration-limit-exceeded cl-postgres-error:insufficient-resources) + +(defvar *retry-connect-times* 5 + "How many times to we try to connect again.") + +(defvar *retry-connect-delay* 0.5 + "How many seconds to wait before trying to connect again.") + (defmethod open-connection ((pgconn pgsql-connection) &key username) "Open a PostgreSQL connection." - (setf (conn-handle pgconn) - (pomo:connect (db-name pgconn) - (or username (db-user pgconn)) - (db-pass pgconn) - (let ((host (db-host pgconn))) - (if (and (consp host) (eq :unix (car host))) :unix host)) - :port (db-port pgconn) - :use-ssl (or (pgconn-use-ssl pgconn) :no))) + (flet ((connect (pgconn username) + (handler-case + (pomo:connect (db-name pgconn) + (or username (db-user pgconn)) + (db-pass pgconn) + (let ((host (db-host pgconn))) + (if (and (consp host) (eq :unix (car host))) + :unix + host)) + :port (db-port pgconn) + :use-ssl (or (pgconn-use-ssl pgconn) :no)) + ((or too-many-connections configuration-limit-exceeded) (e) + (log-message :error + "Failed to connect to ~a: ~a; will try again in ~fs" + pgconn e *retry-connect-delay*) + (sleep *retry-connect-delay*))))) + (loop :while (null (conn-handle pgconn)) + :repeat *retry-connect-times* + :do (setf (conn-handle pgconn) (connect pgconn username)))) + (unless (conn-handle pgconn) + (error "Failed ~d times to connect to ~a" *retry-connect-times* pgconn)) pgconn) (defmethod close-connection ((pgconn pgsql-connection))