From 449bc7d12bee656811ceb5f7748445970569e311 Mon Sep 17 00:00:00 2001 From: Joachim Wiberg Date: Thu, 13 Jan 2022 22:38:36 +0100 Subject: [PATCH] Follow-up to 10b72f1: periodically age service crash counter Traditionally we've reset the service restart counter on two occasions: 1) when a service reaches its max crash count (default 10), and 2) when a service somehow "stabilizes" In the second case the service is restored to normal running state and we "forget" its bad previous behavior. Meaning we cannot catch daemons that act in an unstable manner outside the "rage quit" scenario. This patch allows for slowing aging (decrementing) the restart counter once every five minutes. Meaning we still catch rage quitters but now are also able catch other types of misbehavior. Signed-off-by: Joachim Wiberg --- src/conf.c | 21 +++++++++++++++++ src/finit.c | 5 ++++ src/private.h | 3 +++ src/service.c | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++- src/service.h | 2 ++ 5 files changed, 94 insertions(+), 1 deletion(-) diff --git a/src/conf.c b/src/conf.c index ff3eb02a..9268626e 100644 --- a/src/conf.c +++ b/src/conf.c @@ -41,6 +41,7 @@ #include "finit.h" #include "cond.h" #include "iwatch.h" +#include "private.h" #include "service.h" #include "tty.h" #include "helpers.h" @@ -572,6 +573,26 @@ static void parse_static(char *line, int is_rcsd) cfglevel = 2; /* Fallback */ return; } + + /* + * Periodic check and instability index leveler, seconds + */ + if (MATCH_CMD(line, "service-interval ", x)) { + char *token = strip_line(x); + const char *err = NULL; + int val; + + /* 0 min to 1 day, should check at least daily */ + val = strtonum(token, 0, 1440, &err); + if (!err) { + int disabled = !service_interval; + + service_interval = val * 1000; /* to milliseconds */ + if (disabled) + service_init(); + } + return; + } } static void parse_dynamic(char *line, struct rlimit rlimit[], char *file) diff --git a/src/finit.c b/src/finit.c index 5c2ea956..500645e9 100644 --- a/src/finit.c +++ b/src/finit.c @@ -705,6 +705,11 @@ int main(int argc, char *argv[]) _d("Starting bootstrap finalize timer ..."); schedule_work(&bootstrap_work); + /* + * Background service tasks + */ + service_init(); + /* * Enter main loop to monitor /dev/initctl and services */ diff --git a/src/private.h b/src/private.h index 41545a2d..18cd71e8 100644 --- a/src/private.h +++ b/src/private.h @@ -28,6 +28,9 @@ #include "svc.h" #include "plugin.h" +#define SERVICE_INTERVAL_DEFAULT 300000 /* 5 mins */ +extern int service_interval; + int api_init (uev_ctx_t *ctx); int api_exit (void); diff --git a/src/service.c b/src/service.c index 0add643b..5c913fa1 100644 --- a/src/service.c +++ b/src/service.c @@ -56,6 +56,7 @@ static struct wq work = { .cb = service_worker, }; +int service_interval = SERVICE_INTERVAL_DEFAULT; static void svc_set_state(svc_t *svc, svc_state_t new); @@ -1628,10 +1629,12 @@ static void service_retry(svc_t *svc) if (svc->state != SVC_HALTED_STATE || svc->block != SVC_BLOCK_RESTARTING) { - *restart_cnt = 0; + logit(LOG_CONSOLE | LOG_NOTICE, "Successfully restarted crashing service %s.", + svc_ident(svc, NULL, 0)); return; } + /* Peak instability index */ if (*restart_cnt >= svc->restart_max) { logit(LOG_CONSOLE | LOG_WARNING, "Service %s keeps crashing, not restarting.", svc_ident(svc, NULL, 0)); @@ -1968,6 +1971,65 @@ int service_completed(void) return 1; } +/* + * Every five¹ minutes we sweep over all services, skipping crashed or + * otherwise no longer running ones. Decrement non-zero crash counters + * to allow services that have started after an initial crash to slowly + * prove themselves again as stable services. Previously this counter + * was reset as soon as such services had stopped crashing at least once + * per second. This new scheme allows us to catch those that rage-quit + * immediately when we try to start them, but now also those that are + * only slightly buggy -- when they reach their restart_max, they too + * are marked 'crashed'. + * + * This does not affect the restart_tot counter, which you can see in + * the output from 'initctl status foo', along with this instability + * "index" in parethesis: total (cnt/max) + */ +static void service_interval_cb(uev_t *w, void *arg, int events) +{ + svc_t *svc, *iter = NULL; + + (void)arg; + if (UEV_ERROR == events) { + uev_timer_start(w); + return; + } + + for (svc = svc_iterator(&iter, 1); svc; svc = svc_iterator(&iter, 0)) { + if (svc_is_daemon(svc) || svc_is_sysv(svc)) { + char *restart_cnt = (char *)&svc->restart_cnt; + + if (!svc_is_running(svc)) + continue; + + if (*restart_cnt > 0) { + logit(LOG_CONSOLE | LOG_DEBUG, "Aging %s instability index (%d/%d)", + svc_ident(svc, NULL, 0), svc->restart_cnt, svc->restart_max); + (*restart_cnt)--; + } + } + } + + service_init(); +} + +/* + * The service_interval may change (conf) between invocations, so we + * periodically reset the one-shot timer instead of using a periodic. + */ +void service_init(void) +{ + static int initialized = 0; + static uev_t watcher; + + if (!initialized) { + uev_timer_init(ctx, &watcher, service_interval_cb, NULL, service_interval, 0); + initialized = 1; + } else + uev_timer_set(&watcher, service_interval, 0); +} + /** * Local Variables: * indent-tabs-mode: t diff --git a/src/service.h b/src/service.h index e9688868..9e720381 100644 --- a/src/service.h +++ b/src/service.h @@ -41,6 +41,8 @@ void service_worker (void *unused); int service_completed (void); +void service_init (void); + #endif /* FINIT_SERVICE_H_ */ /**