From 3a925c155d888aed83608e4ad6474702f28aac02 Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Wed, 4 Sep 2013 17:54:01 +0200 Subject: [PATCH] MEDIUM: stick-tables: flush old entries upon soft-stop When a process with large stick tables is replaced by a new one and remains present until the last connection finishes, it keeps these data in memory for nothing since they will never be used anymore by incoming connections, except during syncing with the new process. This is especially problematic when dealing with long session protocols such as WebSocket as it becomes possible to stack many processes and eat a lot of memory. So the idea here is to know if a table still needs to be synced or not, and to purge all unused entries once the sync is complete. This means that after a few hundred milliseconds when everything has been synchronized with the new process, only a few entries will remain allocated (only the ones held by sessions during the restart) and all the remaining memory will be freed. Note that we carefully do that only after the grace period is expired so as not to impact a possible proxy that needs to accept a few more connections before leaving. Doing this required to add a sync counter to the stick tables, to know how many peer sync sessions are still in progress in order not to flush the entries until all synchronizations are completed. --- include/proto/stick_table.h | 1 + include/types/stick_table.h | 1 + src/peers.c | 3 +++ src/proxy.c | 18 ++++++++++++++++++ src/stick_table.c | 2 +- 5 files changed, 24 insertions(+), 1 deletion(-) diff --git a/include/proto/stick_table.h b/include/proto/stick_table.h index d9a25d0c7..0c26fbea3 100644 --- a/include/proto/stick_table.h +++ b/include/proto/stick_table.h @@ -52,6 +52,7 @@ struct stktable_key *stktable_fetch_key(struct stktable *t, struct proxy *px, int stktable_compatible_sample(struct sample_expr *expr, unsigned long table_type); int stktable_get_data_type(char *name); struct proxy *find_stktable(const char *name); +int stktable_trash_oldest(struct stktable *t, int to_batch); /* return allocation size for standard data type */ static inline int stktable_type_size(int type) diff --git a/include/types/stick_table.h b/include/types/stick_table.h index dcdc405cc..e28492cd5 100644 --- a/include/types/stick_table.h +++ b/include/types/stick_table.h @@ -150,6 +150,7 @@ struct stktable { struct task *sync_task; /* sync task */ unsigned int update; unsigned int localupdate; + unsigned int syncing; /* number of sync tasks watching this table now */ union { struct peers *p; /* sync peers */ char *name; diff --git a/src/peers.c b/src/peers.c index 83781ba3c..7247d7624 100644 --- a/src/peers.c +++ b/src/peers.c @@ -1401,6 +1401,7 @@ static struct task *process_peer_sync(struct task * task) /* add DO NOT STOP flag if not present */ jobs++; st->flags |= SHTABLE_F_DONOTSTOP; + st->table->syncing++; } /* disconnect all connected peers */ @@ -1418,6 +1419,7 @@ static struct task *process_peer_sync(struct task * task) /* resync of new process was complete, current process can die now */ jobs--; st->flags &= ~SHTABLE_F_DONOTSTOP; + st->table->syncing--; } } else if (!ps->session) { @@ -1440,6 +1442,7 @@ static struct task *process_peer_sync(struct task * task) /* unable to resync new process, current process can die now */ jobs--; st->flags &= ~SHTABLE_F_DONOTSTOP; + st->table->syncing--; } } } diff --git a/src/proxy.c b/src/proxy.c index b67f02426..37bda485a 100644 --- a/src/proxy.c +++ b/src/proxy.c @@ -560,6 +560,24 @@ struct task *manage_proxy(struct task *t) } } + /* If the proxy holds a stick table, we need to purge all unused + * entries. These are all the ones in the table with ref_cnt == 0 + * and all the ones in the pool used to allocate new entries. Any + * entry attached to an existing session waiting for a store will + * be in neither list. Any entry being dumped will have ref_cnt > 0. + * However we protect tables that are being synced to peers. + */ + if (unlikely(stopping && p->state == PR_STSTOPPED && p->table.current)) { + if (!p->table.syncing) { + stktable_trash_oldest(&p->table, p->table.current); + pool_gc2(); + } + if (p->table.current) { + /* some entries still remain, let's recheck in one second */ + next = tick_first(next, tick_add(now_ms, 1000)); + } + } + /* the rest below is just for frontends */ if (!(p->cap & PR_CAP_FE)) goto out; diff --git a/src/stick_table.c b/src/stick_table.c index c357b320b..b1d88276d 100644 --- a/src/stick_table.c +++ b/src/stick_table.c @@ -90,7 +90,7 @@ static struct stksess *stksess_init(struct stktable *t, struct stksess * ts) * Trash oldest sticky sessions from table * Returns number of trashed sticky sessions. */ -static int stktable_trash_oldest(struct stktable *t, int to_batch) +int stktable_trash_oldest(struct stktable *t, int to_batch) { struct stksess *ts; struct eb32_node *eb;