diff --git a/finit.h b/finit.h index f81bcff0..a6fcc344 100644 --- a/finit.h +++ b/finit.h @@ -34,6 +34,7 @@ #include #include #include +#include #define CMD_SIZE 256 #define LINE_SIZE 1024 @@ -90,6 +91,8 @@ extern char *hostname; extern char *username; extern char *runparts; extern char *console; +extern uev_ctx_t *ctx; + extern char *__progname; #endif /* FINIT_H_ */ diff --git a/helpers.c b/helpers.c index 061b2d76..7deb60ce 100644 --- a/helpers.c +++ b/helpers.c @@ -127,6 +127,7 @@ void print(int action, const char *fmt, ...) va_list ap; const char success[] = " \e[1m[ OK ]\e[0m\n"; const char failure[] = " \e[7m[FAIL]\e[0m\n"; + const char warning[] = " \e[7m[WARN]\e[0m\n"; const char pending[] = " \e[1m[ \\/ ]\e[0m\n"; const char dots[] = " ....................................................................."; @@ -158,6 +159,10 @@ void print(int action, const char *fmt, ...) write(STDERR_FILENO, failure, sizeof(failure)); break; + case 2: + write(STDERR_FILENO, warning, sizeof(warning)); + break; + default: write(STDERR_FILENO, pending, sizeof(pending)); break; diff --git a/service.c b/service.c index 8c5a1b7e..ee597802 100755 --- a/service.c +++ b/service.c @@ -70,6 +70,63 @@ int service_enabled(svc_t *svc) return 1; } +/** + * service_timeout_cb - libuev callback wrapper for service timeouts + * + * Calls the callback registered with the call to + * service_timeout_after(). + */ +static void service_timeout_cb(uev_t *w, void *_svc, int events) +{ + svc_t *svc = _svc; + + (void)(w); + (void)(events); + svc->timer_cb(svc); +} + +/** + * service_timeout_after - Call a function after some time has elapsed + * @param svc Service to use as argument to the callback + * @param timeout Timeout, in milliseconds + * @param cb Callback function + * + * After @param timeout milliseconds has elapsed, calls @param cb with + * @param svc as the argument. + * + * @return 0 on success, non-zero on error. + */ +static int service_timeout_after(svc_t *svc, int timeout, + void (*cb)(svc_t *svc)) +{ + if (svc->timer_cb) + return -EBUSY; + + svc->timer_cb = cb; + return uev_timer_init(ctx, &svc->timer, service_timeout_cb, + svc, timeout, 0); +} + +/** + * service_timeout_cancel - Cancel timeout associated with service + * @param svc Service whose timeout to cancel + * + * If a timeout is associated with @param svc, cancel it. + * + * @return 0 on success, non-zero on error. + */ +static int service_timeout_cancel(svc_t *svc) +{ + int err; + + if (!svc->timer_cb) + return 0; + + err = uev_timer_stop(&svc->timer); + svc->timer_cb = NULL; + return err; +} + /** * service_stop_is_done - Have all stopped services been collected? * @@ -296,6 +353,27 @@ static int service_start(svc_t *svc) return 0; } +/** + * service_kill - Forcefully terminate a service + * @param svc Service to kill + * + * Called when a service refuses to terminate gracefully. + */ +static void service_kill(svc_t *svc) +{ + service_timeout_cancel(svc); + + if (runlevel != 1 && !silent) + print_desc("Killing ", svc->desc); + + _d("Sending SIGKILL to pid:%d name:%s", svc->pid, pid_get_name(svc->pid, NULL, 0)); + kill(svc->pid, SIGKILL); + + /* Let SIGKILLs stand out, show result as [WARN] */ + if (runlevel != 1 && !silent) + print(2, NULL); +} + /** * service_stop - Stop service * @svc: Service to stop @@ -692,11 +770,52 @@ void service_monitor(pid_t lost) sm_step(&sm); } +static void service_retry(svc_t *svc) +{ + int *restart_counter = (int *)&svc->restart_counter; + int timeout; + service_timeout_cancel(svc); + + if (svc->state != SVC_HALTED_STATE || + svc->block != SVC_BLOCK_RESTARTING) { + _d("%s not crashing anymore", svc->desc); + *restart_counter = 0; + return; + } + + if (*restart_counter >= RESPAWN_MAX) { + _e("%s keeps crashing, not restarting", svc->desc); + svc->block = SVC_BLOCK_CRASHING; + *restart_counter = 0; + service_step(svc); + return; + } + + (*restart_counter)++; + + _d("%s crashed, trying to start it again, attempt %d", + svc->desc, *restart_counter); + + svc->block = SVC_BLOCK_NONE; + service_step(svc); + + /* Wait 2s for the first 5 respawns, then back off to 5s */ + timeout = ((*restart_counter) <= (RESPAWN_MAX / 2)) ? 2000 : 5000; + service_timeout_after(svc, timeout, service_retry); + return; +} + static void svc_set_state(svc_t *svc, svc_state_t new) { svc_state_t *state = (svc_state_t *)&svc->state; *state = new; + + /* if the PID isn't collected within 3s, kill it! */ + if (*state == SVC_STOPPING_STATE) { + service_timeout_cancel(svc); + service_timeout_after(svc, 3000, service_kill); + } } void service_step(svc_t *svc) @@ -720,7 +839,6 @@ restart: switch(svc->state) { case SVC_HALTED_STATE: - *restart_counter = 0; if (enabled) svc_set_state(svc, SVC_READY_STATE); break; @@ -742,6 +860,9 @@ restart: case SVC_STOPPING_STATE: if (!svc->pid) { + /* PID was collected normally, no need to kill it */ + service_timeout_cancel(svc); + switch (svc->type) { case SVC_TYPE_SERVICE: case SVC_TYPE_INETD: @@ -765,13 +886,6 @@ restart: if (!enabled) { svc_set_state(svc, SVC_HALTED_STATE); } else if (cond_get_agg(svc->cond) == COND_ON) { - if (*restart_counter >= RESPAWN_MAX) { - _e("%s keeps crashing, not restarting", svc->desc); - svc_crashing(svc); - svc_set_state(svc, SVC_HALTED_STATE); - break; - } - /* wait until all processes has been stopped before continuing... */ if (sm_is_in_teardown(&sm)) break; @@ -813,10 +927,13 @@ restart: } if (!svc->pid && !svc_is_inetd(svc)) { - (*restart_counter)++; - /* TODO: There should be an async wait here - * before moving back to READY */ - svc_set_state(svc, SVC_READY_STATE); + svc->block = SVC_BLOCK_RESTARTING; + svc_set_state(svc, SVC_HALTED_STATE); + + /* Restart directly after the first crash, + * then retry after 2s. */ + _d("delayed restart of %s", svc->desc); + service_timeout_after(svc, 1, service_retry); break; } diff --git a/svc.c b/svc.c index e10d413f..51fa7729 100644 --- a/svc.c +++ b/svc.c @@ -443,6 +443,9 @@ char *svc_status(svc_t *svc) case SVC_BLOCK_INETD_BUSY: return "busy"; + + case SVC_BLOCK_RESTARTING: + return "restart"; } case SVC_DONE_STATE: diff --git a/svc.h b/svc.h index f7500810..9dc5c960 100755 --- a/svc.h +++ b/svc.h @@ -61,6 +61,7 @@ typedef enum { SVC_BLOCK_CRASHING, SVC_BLOCK_USER, SVC_BLOCK_INETD_BUSY, + SVC_BLOCK_RESTARTING, } svc_block_t; #define FINIT_SHM_ID 0x494E4954 /* "INIT", see ascii(7) */ @@ -112,6 +113,11 @@ typedef struct svc { char cmd[MAX_ARG_LEN]; char args[MAX_NUM_SVC_ARGS][MAX_ARG_LEN]; char desc[MAX_STR_LEN]; + + /* Used to forcefully kill services that won't shutdown on + * termination and to delay restarts of crashing services. */ + uev_t timer; + void (*timer_cb)(struct svc *svc); } svc_t; typedef struct svc_map svc_map_t;