diff --git a/src/conf.c b/src/conf.c index ff3eb02a..9268626e 100644 --- a/src/conf.c +++ b/src/conf.c @@ -41,6 +41,7 @@ #include "finit.h" #include "cond.h" #include "iwatch.h" +#include "private.h" #include "service.h" #include "tty.h" #include "helpers.h" @@ -572,6 +573,26 @@ static void parse_static(char *line, int is_rcsd) cfglevel = 2; /* Fallback */ return; } + + /* + * Periodic check and instability index leveler, seconds + */ + if (MATCH_CMD(line, "service-interval ", x)) { + char *token = strip_line(x); + const char *err = NULL; + int val; + + /* 0 min to 1 day, should check at least daily */ + val = strtonum(token, 0, 1440, &err); + if (!err) { + int disabled = !service_interval; + + service_interval = val * 1000; /* to milliseconds */ + if (disabled) + service_init(); + } + return; + } } static void parse_dynamic(char *line, struct rlimit rlimit[], char *file) diff --git a/src/finit.c b/src/finit.c index 5c2ea956..500645e9 100644 --- a/src/finit.c +++ b/src/finit.c @@ -705,6 +705,11 @@ int main(int argc, char *argv[]) _d("Starting bootstrap finalize timer ..."); schedule_work(&bootstrap_work); + /* + * Background service tasks + */ + service_init(); + /* * Enter main loop to monitor /dev/initctl and services */ diff --git a/src/private.h b/src/private.h index 41545a2d..18cd71e8 100644 --- a/src/private.h +++ b/src/private.h @@ -28,6 +28,9 @@ #include "svc.h" #include "plugin.h" +#define SERVICE_INTERVAL_DEFAULT 300000 /* 5 mins */ +extern int service_interval; + int api_init (uev_ctx_t *ctx); int api_exit (void); diff --git a/src/service.c b/src/service.c index 0add643b..5c913fa1 100644 --- a/src/service.c +++ b/src/service.c @@ -56,6 +56,7 @@ static struct wq work = { .cb = service_worker, }; +int service_interval = SERVICE_INTERVAL_DEFAULT; static void svc_set_state(svc_t *svc, svc_state_t new); @@ -1628,10 +1629,12 @@ static void service_retry(svc_t *svc) if (svc->state != SVC_HALTED_STATE || svc->block != SVC_BLOCK_RESTARTING) { - *restart_cnt = 0; + logit(LOG_CONSOLE | LOG_NOTICE, "Successfully restarted crashing service %s.", + svc_ident(svc, NULL, 0)); return; } + /* Peak instability index */ if (*restart_cnt >= svc->restart_max) { logit(LOG_CONSOLE | LOG_WARNING, "Service %s keeps crashing, not restarting.", svc_ident(svc, NULL, 0)); @@ -1968,6 +1971,65 @@ int service_completed(void) return 1; } +/* + * Every five¹ minutes we sweep over all services, skipping crashed or + * otherwise no longer running ones. Decrement non-zero crash counters + * to allow services that have started after an initial crash to slowly + * prove themselves again as stable services. Previously this counter + * was reset as soon as such services had stopped crashing at least once + * per second. This new scheme allows us to catch those that rage-quit + * immediately when we try to start them, but now also those that are + * only slightly buggy -- when they reach their restart_max, they too + * are marked 'crashed'. + * + * This does not affect the restart_tot counter, which you can see in + * the output from 'initctl status foo', along with this instability + * "index" in parethesis: total (cnt/max) + */ +static void service_interval_cb(uev_t *w, void *arg, int events) +{ + svc_t *svc, *iter = NULL; + + (void)arg; + if (UEV_ERROR == events) { + uev_timer_start(w); + return; + } + + for (svc = svc_iterator(&iter, 1); svc; svc = svc_iterator(&iter, 0)) { + if (svc_is_daemon(svc) || svc_is_sysv(svc)) { + char *restart_cnt = (char *)&svc->restart_cnt; + + if (!svc_is_running(svc)) + continue; + + if (*restart_cnt > 0) { + logit(LOG_CONSOLE | LOG_DEBUG, "Aging %s instability index (%d/%d)", + svc_ident(svc, NULL, 0), svc->restart_cnt, svc->restart_max); + (*restart_cnt)--; + } + } + } + + service_init(); +} + +/* + * The service_interval may change (conf) between invocations, so we + * periodically reset the one-shot timer instead of using a periodic. + */ +void service_init(void) +{ + static int initialized = 0; + static uev_t watcher; + + if (!initialized) { + uev_timer_init(ctx, &watcher, service_interval_cb, NULL, service_interval, 0); + initialized = 1; + } else + uev_timer_set(&watcher, service_interval, 0); +} + /** * Local Variables: * indent-tabs-mode: t diff --git a/src/service.h b/src/service.h index e9688868..9e720381 100644 --- a/src/service.h +++ b/src/service.h @@ -41,6 +41,8 @@ void service_worker (void *unused); int service_completed (void); +void service_init (void); + #endif /* FINIT_SERVICE_H_ */ /**