haproxy/src/extcheck.c
Willy Tarreau b3250a268b BUG/MINOR: extcheck: add missing checks on extchk_setenv()
Issue #910 reports that we fail to check a few extchk_setenv() in the
child process. These are mostly harmless, but instead of counting on
the external check script to fail the dirty way, better fail cleanly
when detecting the failure.

This could probably be backported to all stable branches.
2020-10-24 13:07:39 +02:00

671 lines
20 KiB
C

/*
* External health-checks functions.
*
* Copyright 2000-2009,2020 Willy Tarreau <w@1wt.eu>
* Copyright 2014 Horms Solutions Ltd, Simon Horman <horms@verge.net.au>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version
* 2 of the License, or (at your option) any later version.
*
*/
#include <sys/resource.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <assert.h>
#include <ctype.h>
#include <errno.h>
#include <fcntl.h>
#include <signal.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include <haproxy/api.h>
#include <haproxy/cfgparse.h>
#include <haproxy/check.h>
#include <haproxy/errors.h>
#include <haproxy/global.h>
#include <haproxy/list.h>
#include <haproxy/proxy.h>
#include <haproxy/server.h>
#include <haproxy/signal.h>
#include <haproxy/task.h>
#include <haproxy/thread.h>
#include <haproxy/time.h>
#include <haproxy/tools.h>
static struct list pid_list = LIST_HEAD_INIT(pid_list);
static struct pool_head *pool_head_pid_list;
__decl_spinlock(pid_list_lock);
struct extcheck_env {
char *name; /* environment variable name */
int vmaxlen; /* value maximum length, used to determine the required memory allocation */
};
/* environment variables memory requirement for different types of data */
#define EXTCHK_SIZE_EVAL_INIT 0 /* size determined during the init phase,
* such environment variables are not updatable. */
#define EXTCHK_SIZE_ULONG 20 /* max string length for an unsigned long value */
#define EXTCHK_SIZE_UINT 11 /* max string length for an unsigned int value */
#define EXTCHK_SIZE_ADDR INET6_ADDRSTRLEN+1 /* max string length for an address */
/* external checks environment variables */
enum {
EXTCHK_PATH = 0,
/* Proxy specific environment variables */
EXTCHK_HAPROXY_PROXY_NAME, /* the backend name */
EXTCHK_HAPROXY_PROXY_ID, /* the backend id */
EXTCHK_HAPROXY_PROXY_ADDR, /* the first bind address if available (or empty) */
EXTCHK_HAPROXY_PROXY_PORT, /* the first bind port if available (or empty) */
/* Server specific environment variables */
EXTCHK_HAPROXY_SERVER_NAME, /* the server name */
EXTCHK_HAPROXY_SERVER_ID, /* the server id */
EXTCHK_HAPROXY_SERVER_ADDR, /* the server address */
EXTCHK_HAPROXY_SERVER_PORT, /* the server port if available (or empty) */
EXTCHK_HAPROXY_SERVER_MAXCONN, /* the server max connections */
EXTCHK_HAPROXY_SERVER_CURCONN, /* the current number of connections on the server */
EXTCHK_SIZE
};
const struct extcheck_env extcheck_envs[EXTCHK_SIZE] = {
[EXTCHK_PATH] = { "PATH", EXTCHK_SIZE_EVAL_INIT },
[EXTCHK_HAPROXY_PROXY_NAME] = { "HAPROXY_PROXY_NAME", EXTCHK_SIZE_EVAL_INIT },
[EXTCHK_HAPROXY_PROXY_ID] = { "HAPROXY_PROXY_ID", EXTCHK_SIZE_EVAL_INIT },
[EXTCHK_HAPROXY_PROXY_ADDR] = { "HAPROXY_PROXY_ADDR", EXTCHK_SIZE_EVAL_INIT },
[EXTCHK_HAPROXY_PROXY_PORT] = { "HAPROXY_PROXY_PORT", EXTCHK_SIZE_EVAL_INIT },
[EXTCHK_HAPROXY_SERVER_NAME] = { "HAPROXY_SERVER_NAME", EXTCHK_SIZE_EVAL_INIT },
[EXTCHK_HAPROXY_SERVER_ID] = { "HAPROXY_SERVER_ID", EXTCHK_SIZE_EVAL_INIT },
[EXTCHK_HAPROXY_SERVER_ADDR] = { "HAPROXY_SERVER_ADDR", EXTCHK_SIZE_ADDR },
[EXTCHK_HAPROXY_SERVER_PORT] = { "HAPROXY_SERVER_PORT", EXTCHK_SIZE_UINT },
[EXTCHK_HAPROXY_SERVER_MAXCONN] = { "HAPROXY_SERVER_MAXCONN", EXTCHK_SIZE_EVAL_INIT },
[EXTCHK_HAPROXY_SERVER_CURCONN] = { "HAPROXY_SERVER_CURCONN", EXTCHK_SIZE_ULONG },
};
void block_sigchld(void)
{
sigset_t set;
sigemptyset(&set);
sigaddset(&set, SIGCHLD);
assert(ha_sigmask(SIG_BLOCK, &set, NULL) == 0);
}
void unblock_sigchld(void)
{
sigset_t set;
sigemptyset(&set);
sigaddset(&set, SIGCHLD);
assert(ha_sigmask(SIG_UNBLOCK, &set, NULL) == 0);
}
static struct pid_list *pid_list_add(pid_t pid, struct task *t)
{
struct pid_list *elem;
struct check *check = t->context;
elem = pool_alloc(pool_head_pid_list);
if (!elem)
return NULL;
elem->pid = pid;
elem->t = t;
elem->exited = 0;
check->curpid = elem;
LIST_INIT(&elem->list);
HA_SPIN_LOCK(PID_LIST_LOCK, &pid_list_lock);
LIST_ADD(&pid_list, &elem->list);
HA_SPIN_UNLOCK(PID_LIST_LOCK, &pid_list_lock);
return elem;
}
static void pid_list_del(struct pid_list *elem)
{
struct check *check;
if (!elem)
return;
HA_SPIN_LOCK(PID_LIST_LOCK, &pid_list_lock);
LIST_DEL(&elem->list);
HA_SPIN_UNLOCK(PID_LIST_LOCK, &pid_list_lock);
if (!elem->exited)
kill(elem->pid, SIGTERM);
check = elem->t->context;
check->curpid = NULL;
pool_free(pool_head_pid_list, elem);
}
/* Called from inside SIGCHLD handler, SIGCHLD is blocked */
static void pid_list_expire(pid_t pid, int status)
{
struct pid_list *elem;
HA_SPIN_LOCK(PID_LIST_LOCK, &pid_list_lock);
list_for_each_entry(elem, &pid_list, list) {
if (elem->pid == pid) {
elem->t->expire = now_ms;
elem->status = status;
elem->exited = 1;
task_wakeup(elem->t, TASK_WOKEN_IO);
break;
}
}
HA_SPIN_UNLOCK(PID_LIST_LOCK, &pid_list_lock);
}
static void sigchld_handler(struct sig_handler *sh)
{
pid_t pid;
int status;
while ((pid = waitpid(0, &status, WNOHANG)) > 0)
pid_list_expire(pid, status);
}
int init_pid_list(void)
{
if (pool_head_pid_list != NULL)
/* Nothing to do */
return 0;
if (!signal_register_fct(SIGCHLD, sigchld_handler, SIGCHLD)) {
ha_alert("Failed to set signal handler for external health checks: %s. Aborting.\n",
strerror(errno));
return 1;
}
pool_head_pid_list = create_pool("pid_list", sizeof(struct pid_list), MEM_F_SHARED);
if (pool_head_pid_list == NULL) {
ha_alert("Failed to allocate memory pool for external health checks: %s. Aborting.\n",
strerror(errno));
return 1;
}
return 0;
}
/* helper macro to set an environment variable and jump to a specific label on failure. */
#define EXTCHK_SETENV(check, envidx, value, fail) { if (extchk_setenv(check, envidx, value)) goto fail; }
/*
* helper function to allocate enough memory to store an environment variable.
* It will also check that the environment variable is updatable, and silently
* fail if not.
*/
static int extchk_setenv(struct check *check, int idx, const char *value)
{
int len, ret;
char *envname;
int vmaxlen;
if (idx < 0 || idx >= EXTCHK_SIZE) {
ha_alert("Illegal environment variable index %d. Aborting.\n", idx);
return 1;
}
envname = extcheck_envs[idx].name;
vmaxlen = extcheck_envs[idx].vmaxlen;
/* Check if the environment variable is already set, and silently reject
* the update if this one is not updatable. */
if ((vmaxlen == EXTCHK_SIZE_EVAL_INIT) && (check->envp[idx]))
return 0;
/* Instead of sending NOT_USED, sending an empty value is preferable */
if (strcmp(value, "NOT_USED") == 0) {
value = "";
}
len = strlen(envname) + 1;
if (vmaxlen == EXTCHK_SIZE_EVAL_INIT)
len += strlen(value);
else
len += vmaxlen;
if (!check->envp[idx])
check->envp[idx] = malloc(len + 1);
if (!check->envp[idx]) {
ha_alert("Failed to allocate memory for the environment variable '%s'. Aborting.\n", envname);
return 1;
}
ret = snprintf(check->envp[idx], len + 1, "%s=%s", envname, value);
if (ret < 0) {
ha_alert("Failed to store the environment variable '%s'. Reason : %s. Aborting.\n", envname, strerror(errno));
return 1;
}
else if (ret > len) {
ha_alert("Environment variable '%s' was truncated. Aborting.\n", envname);
return 1;
}
return 0;
}
int prepare_external_check(struct check *check)
{
struct server *s = check->server;
struct proxy *px = s->proxy;
struct listener *listener = NULL, *l;
int i;
const char *path = px->check_path ? px->check_path : DEF_CHECK_PATH;
char buf[256];
list_for_each_entry(l, &px->conf.listeners, by_fe)
/* Use the first INET, INET6 or UNIX listener */
if (l->rx.addr.ss_family == AF_INET ||
l->rx.addr.ss_family == AF_INET6 ||
l->rx.addr.ss_family == AF_UNIX) {
listener = l;
break;
}
check->curpid = NULL;
check->envp = calloc((EXTCHK_SIZE + 1), sizeof(*check->envp));
if (!check->envp) {
ha_alert("Failed to allocate memory for environment variables. Aborting\n");
goto err;
}
check->argv = calloc(6, sizeof(*check->argv));
if (!check->argv) {
ha_alert("Starting [%s:%s] check: out of memory.\n", px->id, s->id);
goto err;
}
check->argv[0] = px->check_command;
if (!listener) {
check->argv[1] = strdup("NOT_USED");
check->argv[2] = strdup("NOT_USED");
}
else if (listener->rx.addr.ss_family == AF_INET ||
listener->rx.addr.ss_family == AF_INET6) {
addr_to_str(&listener->rx.addr, buf, sizeof(buf));
check->argv[1] = strdup(buf);
port_to_str(&listener->rx.addr, buf, sizeof(buf));
check->argv[2] = strdup(buf);
}
else if (listener->rx.addr.ss_family == AF_UNIX) {
const struct sockaddr_un *un;
un = (struct sockaddr_un *)&listener->rx.addr;
check->argv[1] = strdup(un->sun_path);
check->argv[2] = strdup("NOT_USED");
}
else {
ha_alert("Starting [%s:%s] check: unsupported address family.\n", px->id, s->id);
goto err;
}
if (!check->argv[1] || !check->argv[2]) {
ha_alert("Starting [%s:%s] check: out of memory.\n", px->id, s->id);
goto err;
}
check->argv[3] = calloc(EXTCHK_SIZE_ADDR, sizeof(*check->argv[3]));
check->argv[4] = calloc(EXTCHK_SIZE_UINT, sizeof(*check->argv[4]));
if (!check->argv[3] || !check->argv[4]) {
ha_alert("Starting [%s:%s] check: out of memory.\n", px->id, s->id);
goto err;
}
addr_to_str(&s->addr, check->argv[3], EXTCHK_SIZE_ADDR);
if (s->addr.ss_family == AF_INET || s->addr.ss_family == AF_INET6)
snprintf(check->argv[4], EXTCHK_SIZE_UINT, "%u", s->svc_port);
for (i = 0; i < 5; i++) {
if (!check->argv[i]) {
ha_alert("Starting [%s:%s] check: out of memory.\n", px->id, s->id);
goto err;
}
}
EXTCHK_SETENV(check, EXTCHK_PATH, path, err);
/* Add proxy environment variables */
EXTCHK_SETENV(check, EXTCHK_HAPROXY_PROXY_NAME, px->id, err);
EXTCHK_SETENV(check, EXTCHK_HAPROXY_PROXY_ID, ultoa_r(px->uuid, buf, sizeof(buf)), err);
EXTCHK_SETENV(check, EXTCHK_HAPROXY_PROXY_ADDR, check->argv[1], err);
EXTCHK_SETENV(check, EXTCHK_HAPROXY_PROXY_PORT, check->argv[2], err);
/* Add server environment variables */
EXTCHK_SETENV(check, EXTCHK_HAPROXY_SERVER_NAME, s->id, err);
EXTCHK_SETENV(check, EXTCHK_HAPROXY_SERVER_ID, ultoa_r(s->puid, buf, sizeof(buf)), err);
EXTCHK_SETENV(check, EXTCHK_HAPROXY_SERVER_ADDR, check->argv[3], err);
EXTCHK_SETENV(check, EXTCHK_HAPROXY_SERVER_PORT, check->argv[4], err);
EXTCHK_SETENV(check, EXTCHK_HAPROXY_SERVER_MAXCONN, ultoa_r(s->maxconn, buf, sizeof(buf)), err);
EXTCHK_SETENV(check, EXTCHK_HAPROXY_SERVER_CURCONN, ultoa_r(s->cur_sess, buf, sizeof(buf)), err);
/* Ensure that we don't leave any hole in check->envp */
for (i = 0; i < EXTCHK_SIZE; i++)
if (!check->envp[i])
EXTCHK_SETENV(check, i, "", err);
return 1;
err:
if (check->envp) {
for (i = 0; i < EXTCHK_SIZE; i++)
free(check->envp[i]);
free(check->envp);
check->envp = NULL;
}
if (check->argv) {
for (i = 1; i < 5; i++)
free(check->argv[i]);
free(check->argv);
check->argv = NULL;
}
return 0;
}
/*
* establish a server health-check that makes use of a process.
*
* It can return one of :
* - SF_ERR_NONE if everything's OK
* - SF_ERR_RESOURCE if a system resource is lacking (eg: fd limits, ports, ...)
* Additionally, in the case of SF_ERR_RESOURCE, an emergency log will be emitted.
*
* Blocks and then unblocks SIGCHLD
*/
static int connect_proc_chk(struct task *t)
{
char buf[256];
struct check *check = t->context;
struct server *s = check->server;
struct proxy *px = s->proxy;
int status;
pid_t pid;
status = SF_ERR_RESOURCE;
block_sigchld();
pid = fork();
if (pid < 0) {
ha_alert("Failed to fork process for external health check%s: %s. Aborting.\n",
(global.tune.options & GTUNE_INSECURE_FORK) ?
"" : " (likely caused by missing 'insecure-fork-wanted')",
strerror(errno));
set_server_check_status(check, HCHK_STATUS_SOCKERR, strerror(errno));
goto out;
}
if (pid == 0) {
/* Child */
extern char **environ;
struct rlimit limit;
int fd;
/* close all FDs. Keep stdin/stdout/stderr in verbose mode */
fd = (global.mode & (MODE_QUIET|MODE_VERBOSE)) == MODE_QUIET ? 0 : 3;
my_closefrom(fd);
/* restore the initial FD limits */
limit.rlim_cur = rlim_fd_cur_at_boot;
limit.rlim_max = rlim_fd_max_at_boot;
if (setrlimit(RLIMIT_NOFILE, &limit) == -1) {
getrlimit(RLIMIT_NOFILE, &limit);
ha_warning("External check: failed to restore initial FD limits (cur=%u max=%u), using cur=%u max=%u\n",
rlim_fd_cur_at_boot, rlim_fd_max_at_boot,
(unsigned int)limit.rlim_cur, (unsigned int)limit.rlim_max);
}
environ = check->envp;
/* Update some environment variables and command args: curconn, server addr and server port */
EXTCHK_SETENV(check, EXTCHK_HAPROXY_SERVER_CURCONN, ultoa_r(s->cur_sess, buf, sizeof(buf)), fail);
addr_to_str(&s->addr, check->argv[3], EXTCHK_SIZE_ADDR);
EXTCHK_SETENV(check, EXTCHK_HAPROXY_SERVER_ADDR, check->argv[3], fail);
*check->argv[4] = 0;
if (s->addr.ss_family == AF_INET || s->addr.ss_family == AF_INET6)
snprintf(check->argv[4], EXTCHK_SIZE_UINT, "%u", s->svc_port);
EXTCHK_SETENV(check, EXTCHK_HAPROXY_SERVER_PORT, check->argv[4], fail);
haproxy_unblock_signals();
execvp(px->check_command, check->argv);
ha_alert("Failed to exec process for external health check: %s. Aborting.\n",
strerror(errno));
fail:
exit(-1);
}
/* Parent */
if (check->result == CHK_RES_UNKNOWN) {
if (pid_list_add(pid, t) != NULL) {
t->expire = tick_add(now_ms, MS_TO_TICKS(check->inter));
if (px->timeout.check && px->timeout.connect) {
int t_con = tick_add(now_ms, px->timeout.connect);
t->expire = tick_first(t->expire, t_con);
}
status = SF_ERR_NONE;
goto out;
}
else {
set_server_check_status(check, HCHK_STATUS_SOCKERR, strerror(errno));
}
kill(pid, SIGTERM); /* process creation error */
}
else
set_server_check_status(check, HCHK_STATUS_SOCKERR, strerror(errno));
out:
unblock_sigchld();
return status;
}
/*
* manages a server health-check that uses an external process. Returns
* the time the task accepts to wait, or TIME_ETERNITY for infinity.
*
* Please do NOT place any return statement in this function and only leave
* via the out_unlock label.
*/
struct task *process_chk_proc(struct task *t, void *context, unsigned short state)
{
struct check *check = context;
struct server *s = check->server;
int rv;
int ret;
int expired = tick_is_expired(t->expire, now_ms);
HA_SPIN_LOCK(SERVER_LOCK, &check->server->lock);
if (!(check->state & CHK_ST_INPROGRESS)) {
/* no check currently running */
if (!expired) /* woke up too early */
goto out_unlock;
/* we don't send any health-checks when the proxy is
* stopped, the server should not be checked or the check
* is disabled.
*/
if (((check->state & (CHK_ST_ENABLED | CHK_ST_PAUSED)) != CHK_ST_ENABLED) ||
s->proxy->disabled)
goto reschedule;
/* we'll initiate a new check */
set_server_check_status(check, HCHK_STATUS_START, NULL);
check->state |= CHK_ST_INPROGRESS;
ret = connect_proc_chk(t);
if (ret == SF_ERR_NONE) {
/* the process was forked, we allow up to min(inter,
* timeout.connect) for it to report its status, but
* only when timeout.check is set as it may be to short
* for a full check otherwise.
*/
t->expire = tick_add(now_ms, MS_TO_TICKS(check->inter));
if (s->proxy->timeout.check && s->proxy->timeout.connect) {
int t_con = tick_add(now_ms, s->proxy->timeout.connect);
t->expire = tick_first(t->expire, t_con);
}
task_set_affinity(t, tid_bit);
goto reschedule;
}
/* here, we failed to start the check */
check->state &= ~CHK_ST_INPROGRESS;
check_notify_failure(check);
/* we allow up to min(inter, timeout.connect) for a connection
* to establish but only when timeout.check is set
* as it may be to short for a full check otherwise
*/
while (tick_is_expired(t->expire, now_ms)) {
int t_con;
t_con = tick_add(t->expire, s->proxy->timeout.connect);
t->expire = tick_add(t->expire, MS_TO_TICKS(check->inter));
if (s->proxy->timeout.check)
t->expire = tick_first(t->expire, t_con);
}
}
else {
/* there was a test running.
* First, let's check whether there was an uncaught error,
* which can happen on connect timeout or error.
*/
if (check->result == CHK_RES_UNKNOWN) {
/* good connection is enough for pure TCP check */
struct pid_list *elem = check->curpid;
int status = HCHK_STATUS_UNKNOWN;
if (elem->exited) {
status = elem->status; /* Save in case the process exits between use below */
if (!WIFEXITED(status))
check->code = -1;
else
check->code = WEXITSTATUS(status);
if (!WIFEXITED(status) || WEXITSTATUS(status))
status = HCHK_STATUS_PROCERR;
else
status = HCHK_STATUS_PROCOK;
} else if (expired) {
status = HCHK_STATUS_PROCTOUT;
ha_warning("kill %d\n", (int)elem->pid);
kill(elem->pid, SIGTERM);
}
set_server_check_status(check, status, NULL);
}
if (check->result == CHK_RES_FAILED) {
/* a failure or timeout detected */
check_notify_failure(check);
}
else if (check->result == CHK_RES_CONDPASS) {
/* check is OK but asks for stopping mode */
check_notify_stopping(check);
}
else if (check->result == CHK_RES_PASSED) {
/* a success was detected */
check_notify_success(check);
}
task_set_affinity(t, 1);
check->state &= ~CHK_ST_INPROGRESS;
pid_list_del(check->curpid);
rv = 0;
if (global.spread_checks > 0) {
rv = srv_getinter(check) * global.spread_checks / 100;
rv -= (int) (2 * rv * (ha_random32() / 4294967295.0));
}
t->expire = tick_add(now_ms, MS_TO_TICKS(srv_getinter(check) + rv));
}
reschedule:
while (tick_is_expired(t->expire, now_ms))
t->expire = tick_add(t->expire, MS_TO_TICKS(check->inter));
out_unlock:
HA_SPIN_UNLOCK(SERVER_LOCK, &check->server->lock);
return t;
}
/* Parses the "external-check" proxy keyword */
int proxy_parse_extcheck(char **args, int section, struct proxy *curpx,
struct proxy *defpx, const char *file, int line,
char **errmsg)
{
int cur_arg, ret = 0;
cur_arg = 1;
if (!*(args[cur_arg])) {
memprintf(errmsg, "missing argument after '%s'.\n", args[0]);
goto error;
}
if (strcmp(args[cur_arg], "command") == 0) {
if (too_many_args(2, args, errmsg, NULL))
goto error;
if (!*(args[cur_arg+1])) {
memprintf(errmsg, "missing argument after '%s'.", args[cur_arg]);
goto error;
}
free(curpx->check_command);
curpx->check_command = strdup(args[cur_arg+1]);
}
else if (strcmp(args[cur_arg], "path") == 0) {
if (too_many_args(2, args, errmsg, NULL))
goto error;
if (!*(args[cur_arg+1])) {
memprintf(errmsg, "missing argument after '%s'.", args[cur_arg]);
goto error;
}
free(curpx->check_path);
curpx->check_path = strdup(args[cur_arg+1]);
}
else {
memprintf(errmsg, "'%s' only supports 'command' and 'path'. but got '%s'.",
args[0], args[1]);
goto error;
}
ret = (*errmsg != NULL); /* Handle warning */
return ret;
error:
return -1;
}
int proxy_parse_external_check_opt(char **args, int cur_arg, struct proxy *curpx, struct proxy *defpx,
const char *file, int line)
{
int err_code = 0;
curpx->options2 &= ~PR_O2_CHK_ANY;
curpx->options2 |= PR_O2_EXT_CHK;
if (alertif_too_many_args_idx(0, 1, file, line, args, &err_code))
goto out;
out:
return err_code;
}
static struct cfg_kw_list cfg_kws = {ILH, {
{ CFG_LISTEN, "external-check", proxy_parse_extcheck },
{ 0, NULL, NULL },
}};
INITCALL1(STG_REGISTER, cfg_register_keywords, &cfg_kws);