From 10b72f1d7f9570dc4b033ba045b4a7601a57e919 Mon Sep 17 00:00:00 2001 From: Tobias Waldekranz Date: Tue, 13 Sep 2016 20:26:28 +0200 Subject: [PATCH] service: kill stubborn services, wait before restart Add support for registering callbacks to be called after a given timeout. Use this to implement forceful termination of services that refuse to terminate upon receiving a SIGTERM. If the service PID is not collected within 3 seconds, send a SIGKILL to it. Also, rework restarting of crashing services. When a service crashes for the first time, try to restart it immediately. If it crashes again, wait a few seconds before trying again. Just as before, if it continues to crash, block the service. --- finit.h | 3 ++ helpers.c | 5 ++ service.c | 141 +++++++++++++++++++++++++++++++++++++++++++++++++----- svc.c | 3 ++ svc.h | 6 +++ 5 files changed, 146 insertions(+), 12 deletions(-) diff --git a/finit.h b/finit.h index f81bcff0..a6fcc344 100644 --- a/finit.h +++ b/finit.h @@ -34,6 +34,7 @@ #include #include #include +#include #define CMD_SIZE 256 #define LINE_SIZE 1024 @@ -90,6 +91,8 @@ extern char *hostname; extern char *username; extern char *runparts; extern char *console; +extern uev_ctx_t *ctx; + extern char *__progname; #endif /* FINIT_H_ */ diff --git a/helpers.c b/helpers.c index a33340fc..455527ef 100644 --- a/helpers.c +++ b/helpers.c @@ -127,6 +127,7 @@ void print(int action, const char *fmt, ...) va_list ap; const char success[] = " \e[1m[ OK ]\e[0m\n"; const char failure[] = " \e[7m[FAIL]\e[0m\n"; + const char warning[] = " \e[7m[WARN]\e[0m\n"; const char pending[] = " \e[1m[ \\/ ]\e[0m\n"; const char dots[] = " ....................................................................."; @@ -158,6 +159,10 @@ void print(int action, const char *fmt, ...) write(STDERR_FILENO, failure, sizeof(failure)); break; + case 2: + write(STDERR_FILENO, warning, sizeof(warning)); + break; + default: write(STDERR_FILENO, pending, sizeof(pending)); break; diff --git a/service.c b/service.c index 5fe6cbe2..729bc970 100755 --- a/service.c +++ b/service.c @@ -67,6 +67,63 @@ int service_enabled(svc_t *svc) return 1; } +/** + * service_timeout_cb - libuev callback wrapper for service timeouts + * + * Calls the callback registered with the call to + * service_timeout_after(). + */ +static void service_timeout_cb(uev_t *w, void *_svc, int events) +{ + svc_t *svc = _svc; + + (void)(w); + (void)(events); + svc->timer_cb(svc); +} + +/** + * service_timeout_after - Call a function after some time has elapsed + * @param svc Service to use as argument to the callback + * @param timeout Timeout, in milliseconds + * @param cb Callback function + * + * After @param timeout milliseconds has elapsed, calls @param cb with + * @param svc as the argument. + * + * @return 0 on success, non-zero on error. + */ +static int service_timeout_after(svc_t *svc, int timeout, + void (*cb)(svc_t *svc)) +{ + if (svc->timer_cb) + return -EBUSY; + + svc->timer_cb = cb; + return uev_timer_init(ctx, &svc->timer, service_timeout_cb, + svc, timeout, 0); +} + +/** + * service_timeout_cancel - Cancel timeout associated with service + * @param svc Service whose timeout to cancel + * + * If a timeout is associated with @param svc, cancel it. + * + * @return 0 on success, non-zero on error. + */ +static int service_timeout_cancel(svc_t *svc) +{ + int err; + + if (!svc->timer_cb) + return 0; + + err = uev_timer_stop(&svc->timer); + svc->timer_cb = NULL; + return err; +} + /** * service_stop_is_done - Have all stopped services been collected? * @@ -293,6 +350,27 @@ static int service_start(svc_t *svc) return 0; } +/** + * service_kill - Forcefully terminate a service + * @param svc Service to kill + * + * Called when a service refuses to terminate gracefully. + */ +static void service_kill(svc_t *svc) +{ + service_timeout_cancel(svc); + + if (runlevel != 1 && !silent) + print_desc("Killing ", svc->desc); + + _d("Sending SIGKILL to pid:%d name:%s", svc->pid, pid_get_name(svc->pid, NULL, 0)); + kill(svc->pid, SIGKILL); + + /* Let SIGKILLs stand out, show result as [WARN] */ + if (runlevel != 1 && !silent) + print(2, NULL); +} + /** * service_stop - Stop service * @svc: Service to stop @@ -697,11 +775,52 @@ void service_monitor(pid_t lost) sm_step(&sm); } +static void service_retry(svc_t *svc) +{ + int *restart_counter = (int *)&svc->restart_counter; + int timeout; + service_timeout_cancel(svc); + + if (svc->state != SVC_HALTED_STATE || + svc->block != SVC_BLOCK_RESTARTING) { + _d("%s not crashing anymore", svc->desc); + *restart_counter = 0; + return; + } + + if (*restart_counter >= RESPAWN_MAX) { + _e("%s keeps crashing, not restarting", svc->desc); + svc->block = SVC_BLOCK_CRASHING; + *restart_counter = 0; + service_step(svc); + return; + } + + (*restart_counter)++; + + _d("%s crashed, trying to start it again, attempt %d", + svc->desc, *restart_counter); + + svc->block = SVC_BLOCK_NONE; + service_step(svc); + + /* Wait 2s for the first 5 respawns, then back off to 5s */ + timeout = ((*restart_counter) <= (RESPAWN_MAX / 2)) ? 2000 : 5000; + service_timeout_after(svc, timeout, service_retry); + return; +} + static void svc_set_state(svc_t *svc, svc_state_t new) { svc_state_t *state = (svc_state_t *)&svc->state; *state = new; + + /* if the PID isn't collected within 3s, kill it! */ + if (*state == SVC_STOPPING_STATE) { + service_timeout_cancel(svc); + service_timeout_after(svc, 3000, service_kill); + } } void service_step(svc_t *svc) @@ -725,7 +844,6 @@ restart: switch(svc->state) { case SVC_HALTED_STATE: - *restart_counter = 0; if (enabled) svc_set_state(svc, SVC_READY_STATE); break; @@ -747,6 +865,9 @@ restart: case SVC_STOPPING_STATE: if (!svc->pid) { + /* PID was collected normally, no need to kill it */ + service_timeout_cancel(svc); + switch (svc->type) { case SVC_TYPE_SERVICE: case SVC_TYPE_INETD: @@ -770,13 +891,6 @@ restart: if (!enabled) { svc_set_state(svc, SVC_HALTED_STATE); } else if (cond_get_agg(svc->cond) == COND_ON) { - if (*restart_counter >= RESPAWN_MAX) { - _e("%s keeps crashing, not restarting", svc->desc); - svc->block = SVC_BLOCK_CRASHING; - svc_set_state(svc, SVC_HALTED_STATE); - break; - } - /* wait until all processes has been stopped before continuing... */ if (sm_is_in_teardown(&sm)) break; @@ -818,10 +932,13 @@ restart: } if (!svc->pid && !svc_is_inetd(svc)) { - (*restart_counter)++; - /* TODO: There should be an async wait here - * before moving back to READY */ - svc_set_state(svc, SVC_READY_STATE); + svc->block = SVC_BLOCK_RESTARTING; + svc_set_state(svc, SVC_HALTED_STATE); + + /* Restart directly after the first crash, + * then retry after 2s. */ + _d("delayed restart of %s", svc->desc); + service_timeout_after(svc, 1, service_retry); break; } diff --git a/svc.c b/svc.c index e10d413f..51fa7729 100644 --- a/svc.c +++ b/svc.c @@ -443,6 +443,9 @@ char *svc_status(svc_t *svc) case SVC_BLOCK_INETD_BUSY: return "busy"; + + case SVC_BLOCK_RESTARTING: + return "restart"; } case SVC_DONE_STATE: diff --git a/svc.h b/svc.h index 078a9add..d28aa859 100755 --- a/svc.h +++ b/svc.h @@ -65,6 +65,7 @@ typedef enum { SVC_BLOCK_CRASHING, SVC_BLOCK_USER, SVC_BLOCK_INETD_BUSY, + SVC_BLOCK_RESTARTING, } svc_block_t; #define FINIT_SHM_ID 0x494E4954 /* "INIT", see ascii(7) */ @@ -123,6 +124,11 @@ typedef struct svc { int dynamic_stop; int private; svc_cmd_t (*cb)(struct svc *svc, int event, void *event_arg); + + /* Used to forcefully kill services that won't shutdown on + * termination and to delay restarts of crashing services. */ + uev_t timer; + void (*timer_cb)(struct svc *svc); } svc_t; typedef struct svc_map svc_map_t;