diff --git a/doc/configuration.txt b/doc/configuration.txt index 05f07019f..e6ea2cfc8 100644 --- a/doc/configuration.txt +++ b/doc/configuration.txt @@ -587,6 +587,7 @@ The following keywords are supported in the "global" section : - nosplice - nogetaddrinfo - noreuseport + - no-unused-socket - spread-checks - server-state-base - server-state-file @@ -1250,6 +1251,12 @@ noreuseport Disables the use of SO_REUSEPORT - see socket(7). It is equivalent to the command line argument "-dR". +no-unused-socket + By default, each haproxy process keeps all sockets opened, event those that + are only used by another processes, so that any process can provide all the + sockets, to make reloads seamless. This option disables this, and close all + unused sockets, to save some file descriptors. + spread-checks <0..50, in percent> Sometimes it is desirable to avoid sending agent and health checks to servers at exact intervals, for instance when many logical servers are diff --git a/include/proto/fd.h b/include/proto/fd.h index 87309bf0a..1efe32399 100644 --- a/include/proto/fd.h +++ b/include/proto/fd.h @@ -41,6 +41,11 @@ extern int fd_nbupdt; // number of updates in the list */ void fd_delete(int fd); +/* Deletes an FD from the fdsets, and recomputes the maxfd limit. + * The file descriptor is kept open. + */ +void fd_remove(int fd); + /* disable the specified poller */ void disable_poller(const char *poller_name); diff --git a/include/proto/listener.h b/include/proto/listener.h index 552d69595..079d976a3 100644 --- a/include/proto/listener.h +++ b/include/proto/listener.h @@ -88,6 +88,11 @@ void dequeue_all_listeners(struct list *list); */ int unbind_listener(struct listener *listener); +/* This function pretends the listener is dead, but keeps the FD opened, so + * that we can provide it, for conf reloading. + */ +int unbind_listener_no_close(struct listener *listener); + /* This function closes all listening sockets bound to the protocol , * and the listeners end in LI_ASSIGNED state if they were higher. It does not * detach them from the protocol. It always returns ERR_NONE. diff --git a/include/proto/proxy.h b/include/proto/proxy.h index 72f1e1d33..e5d20c4f5 100644 --- a/include/proto/proxy.h +++ b/include/proto/proxy.h @@ -42,6 +42,7 @@ void soft_stop(void); int pause_proxy(struct proxy *p); int resume_proxy(struct proxy *p); void stop_proxy(struct proxy *p); +void zombify_proxy(struct proxy *p); void pause_proxies(void); void resume_proxies(void); int stream_set_backend(struct stream *s, struct proxy *be); diff --git a/include/types/global.h b/include/types/global.h index df8e2c69a..57b969dd1 100644 --- a/include/types/global.h +++ b/include/types/global.h @@ -62,6 +62,8 @@ #define GTUNE_USE_REUSEPORT (1<<6) #define GTUNE_RESOLVE_DONTFAIL (1<<7) +#define GTUNE_SOCKET_TRANSFER (1<<8) + /* Access level for a stats socket */ #define ACCESS_LVL_NONE 0 #define ACCESS_LVL_USER 1 diff --git a/include/types/listener.h b/include/types/listener.h index 227cc284d..2b8f5feb6 100644 --- a/include/types/listener.h +++ b/include/types/listener.h @@ -47,6 +47,7 @@ enum li_state { LI_INIT, /* all parameters filled in, but not assigned yet */ LI_ASSIGNED, /* assigned to the protocol, but not listening yet */ LI_PAUSED, /* listener was paused, it's bound but not listening */ + LI_ZOMBIE, /* The listener doesn't belong to the process, but is kept opened */ LI_LISTEN, /* started, listening but not enabled */ LI_READY, /* started, listening and enabled */ LI_FULL, /* reached its connection limit */ diff --git a/src/cfgparse.c b/src/cfgparse.c index 47d33cf1b..348b9e886 100644 --- a/src/cfgparse.c +++ b/src/cfgparse.c @@ -659,6 +659,11 @@ int cfg_parse_global(const char *file, int linenum, char **args, int kwm) goto out; global.tune.options &= ~GTUNE_USE_REUSEPORT; } + else if (!strcmp(args[0], "no-unused-socket")) { + if (alertif_too_many_args(0, file, linenum, args, &err_code)) + goto out; + global.tune.options &= ~GTUNE_SOCKET_TRANSFER; + } else if (!strcmp(args[0], "quiet")) { if (alertif_too_many_args(0, file, linenum, args, &err_code)) goto out; diff --git a/src/cli.c b/src/cli.c index 54fb43892..20e143b96 100644 --- a/src/cli.c +++ b/src/cli.c @@ -1063,10 +1063,11 @@ static int _getsocks(char **args, struct appctx *appctx, void *private) struct listener *l; list_for_each_entry(l, &px->conf.listeners, by_fe) { - /* Only transfer IPv4/IPv6 sockets */ - if (l->proto->sock_family == AF_INET || + /* Only transfer IPv4/IPv6/UNIX sockets */ + if (l->state >= LI_ZOMBIE && + (l->proto->sock_family == AF_INET || l->proto->sock_family == AF_INET6 || - l->proto->sock_family == AF_UNIX) + l->proto->sock_family == AF_UNIX)) tot_fd_nb++; } px = px->next; @@ -1117,7 +1118,7 @@ static int _getsocks(char **args, struct appctx *appctx, void *private) list_for_each_entry(l, &px->conf.listeners, by_fe) { int ret; /* Only transfer IPv4/IPv6 sockets */ - if (l->state >= LI_LISTEN && + if (l->state >= LI_ZOMBIE && (l->proto->sock_family == AF_INET || l->proto->sock_family == AF_INET6 || l->proto->sock_family == AF_UNIX)) { diff --git a/src/fd.c b/src/fd.c index aeee60261..1a62f9a66 100644 --- a/src/fd.c +++ b/src/fd.c @@ -175,7 +175,7 @@ int fd_nbupdt = 0; // number of updates in the list /* Deletes an FD from the fdsets, and recomputes the maxfd limit. * The file descriptor is also closed. */ -void fd_delete(int fd) +static void fd_dodelete(int fd, int do_close) { if (fdtab[fd].linger_risk) { /* this is generally set when connecting to servers */ @@ -190,7 +190,8 @@ void fd_delete(int fd) port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port); fdinfo[fd].port_range = NULL; - close(fd); + if (do_close) + close(fd); fdtab[fd].owner = NULL; fdtab[fd].new = 0; @@ -198,6 +199,22 @@ void fd_delete(int fd) maxfd--; } +/* Deletes an FD from the fdsets, and recomputes the maxfd limit. + * The file descriptor is also closed. + */ +void fd_delete(int fd) +{ + fd_dodelete(fd, 1); +} + +/* Deletes an FD from the fdsets, and recomputes the maxfd limit. + * The file descriptor is kept open. + */ +void fd_remove(int fd) +{ + fd_dodelete(fd, 0); +} + /* Scan and process the cached events. This should be called right after * the poller. The loop may cause new entries to be created, for example * if a listener causes an accept() to initiate a new incoming connection diff --git a/src/haproxy.c b/src/haproxy.c index bf176df78..01969c90d 100644 --- a/src/haproxy.c +++ b/src/haproxy.c @@ -857,6 +857,7 @@ static void init(int argc, char **argv) #if defined(SO_REUSEPORT) global.tune.options |= GTUNE_USE_REUSEPORT; #endif + global.tune.options |= GTUNE_SOCKET_TRANSFER; pid = getpid(); progname = *argv; @@ -1668,6 +1669,15 @@ void deinit(void) }/* end while(s) */ list_for_each_entry_safe(l, l_next, &p->conf.listeners, by_fe) { + /* + * Zombie proxy, the listener just pretend to be up + * because they still hold an opened fd. + * Close it and give the listener its real state. + */ + if (p->state == PR_STSTOPPED && l->state >= LI_ZOMBIE) { + close(l->fd); + l->state = LI_INIT; + } unbind_listener(l); delete_listener(l); LIST_DEL(&l->by_fe); @@ -2148,8 +2158,12 @@ int main(int argc, char **argv) px = proxy; while (px != NULL) { if (px->bind_proc && px->state != PR_STSTOPPED) { - if (!(px->bind_proc & (1UL << proc))) - stop_proxy(px); + if (!(px->bind_proc & (1UL << proc))) { + if (global.tune.options & GTUNE_SOCKET_TRANSFER) + zombify_proxy(px); + else + stop_proxy(px); + } } px = px->next; } diff --git a/src/listener.c b/src/listener.c index 7a1df0e41..a99e4c0d3 100644 --- a/src/listener.c +++ b/src/listener.c @@ -59,7 +59,12 @@ void enable_listener(struct listener *listener) /* we don't want to enable this listener and don't * want any fd event to reach it. */ - unbind_listener(listener); + if (!(global.tune.options & GTUNE_SOCKET_TRANSFER)) + unbind_listener(listener); + else { + unbind_listener_no_close(listener); + listener->state = LI_LISTEN; + } } else if (listener->nbconn < listener->maxconn) { fd_want_recv(listener->fd); @@ -95,7 +100,7 @@ void disable_listener(struct listener *listener) */ int pause_listener(struct listener *l) { - if (l->state <= LI_PAUSED) + if (l->state <= LI_ZOMBIE) return 1; if (l->proto->pause) { @@ -149,7 +154,7 @@ int resume_listener(struct listener *l) return 0; } - if (l->state < LI_PAUSED) + if (l->state < LI_PAUSED || l->state == LI_ZOMBIE) return 0; if (l->proto->sock_prot == IPPROTO_TCP && @@ -242,12 +247,7 @@ void dequeue_all_listeners(struct list *list) } } -/* This function closes the listening socket for the specified listener, - * provided that it's already in a listening state. The listener enters the - * LI_ASSIGNED state. It always returns ERR_NONE. This function is intended - * to be used as a generic function for standard protocols. - */ -int unbind_listener(struct listener *listener) +static int do_unbind_listener(struct listener *listener, int do_close) { if (listener->state == LI_READY) fd_stop_recv(listener->fd); @@ -256,13 +256,35 @@ int unbind_listener(struct listener *listener) LIST_DEL(&listener->wait_queue); if (listener->state >= LI_PAUSED) { - fd_delete(listener->fd); - listener->fd = -1; + if (do_close) { + fd_delete(listener->fd); + listener->fd = -1; + } + else + fd_remove(listener->fd); listener->state = LI_ASSIGNED; } return ERR_NONE; } +/* This function closes the listening socket for the specified listener, + * provided that it's already in a listening state. The listener enters the + * LI_ASSIGNED state. It always returns ERR_NONE. This function is intended + * to be used as a generic function for standard protocols. + */ +int unbind_listener(struct listener *listener) +{ + return do_unbind_listener(listener, 1); +} + +/* This function pretends the listener is dead, but keeps the FD opened, so + * that we can provide it, for conf reloading. + */ +int unbind_listener_no_close(struct listener *listener) +{ + return do_unbind_listener(listener, 0); +} + /* This function closes all listening sockets bound to the protocol , * and the listeners end in LI_ASSIGNED state if they were higher. It does not * detach them from the protocol. It always returns ERR_NONE. diff --git a/src/proxy.c b/src/proxy.c index d158fac03..dc702139a 100644 --- a/src/proxy.c +++ b/src/proxy.c @@ -991,6 +991,19 @@ void soft_stop(void) p = proxy; tv_update_date(0,1); /* else, the old time before select will be used */ while (p) { + /* Zombie proxy, let's close the file descriptors */ + if (p->state == PR_STSTOPPED && + !LIST_ISEMPTY(&p->conf.listeners) && + LIST_ELEM(p->conf.listeners.n, + struct listener *, by_fe)->state >= LI_ZOMBIE) { + struct listener *l; + list_for_each_entry(l, &p->conf.listeners, by_fe) { + if (l->state >= LI_ZOMBIE) + close(l->fd); + l->state = LI_INIT; + } + } + if (p->state != PR_STSTOPPED) { Warning("Stopping %s %s in %d ms.\n", proxy_cap_str(p->cap), p->id, p->grace); send_log(p, LOG_WARNING, "Stopping %s %s in %d ms.\n", proxy_cap_str(p->cap), p->id, p->grace); @@ -1051,6 +1064,45 @@ int pause_proxy(struct proxy *p) return 1; } +/* This function makes the proxy unusable, but keeps the listening sockets + * opened, so that if any process requests them, we are able to serve them. + * This should only be called early, before we started accepting requests. + */ +void zombify_proxy(struct proxy *p) +{ + struct listener *l; + struct listener *first_to_listen = NULL; + + list_for_each_entry(l, &p->conf.listeners, by_fe) { + enum li_state oldstate = l->state; + + unbind_listener_no_close(l); + if (l->state >= LI_ASSIGNED) { + delete_listener(l); + listeners--; + jobs--; + } + /* + * Pretend we're still up and running so that the fd + * will be sent if asked. + */ + l->state = LI_ZOMBIE; + if (!first_to_listen && oldstate >= LI_LISTEN) + first_to_listen = l; + } + /* Quick hack : at stop time, to know we have to close the sockets + * despite the proxy being marked as stopped, make the first listener + * of the listener list an active one, so that we don't have to + * parse the whole list to be sure. + */ + if (first_to_listen && LIST_ELEM(p->conf.listeners.n, + struct listener *, by_fe) != first_to_listen) { + LIST_DEL(&l->by_fe); + LIST_ADD(&p->conf.listeners, &l->by_fe); + } + + p->state = PR_STSTOPPED; +} /* * This function completely stops a proxy and releases its listeners. It has