diff --git a/doc/ChangeLog.md b/doc/ChangeLog.md index a4eac5c0..ef5b233f 100644 --- a/doc/ChangeLog.md +++ b/doc/ChangeLog.md @@ -11,6 +11,10 @@ All relevant changes are documented in this file. - `shutdown` now takes a time, `now`, `+MIN`, or `HH:MM`, and a message. Finit warns logged-in users as the time approaches, refuses new logins from five minutes before, and `shutdown -c` cancels. Issue #149 +- Add `restart-max-sec`, a limit for the delay between restarts of a + crashing service. When set, the delay doubles for each restart, from + `restart-sec` up to the limit, and comes down again as the service keeps + running. Issue #306 - Add `stop-scope`, which processes a stop applies to: `pgroup`, the process group of the main process (default), `process`, only the main process, `cgroup`, the cgroup of the service, or `mixed` like systemd. Issue #413 diff --git a/doc/config/migration.md b/doc/config/migration.md index d8edb6a2..e91cbb67 100644 --- a/doc/config/migration.md +++ b/doc/config/migration.md @@ -64,6 +64,7 @@ Service options | `remain:yes` | `remain-after-exit = true` | | `restart:always` / `restart:NUM` | `restart = "always"` / `restart-max = NUM` | | `restart_sec:SEC` | `restart-sec = SEC` | +| `restart-max-sec:SEC` | `restart-max-sec = SEC` | | `norestart` | `restart = "never"` | | `respawn` | `respawn = true` | | `oncrash:reboot` | `oncrash = "reboot"` | diff --git a/doc/config/service-opts.md b/doc/config/service-opts.md index 70a27c9c..ef4a16e7 100644 --- a/doc/config/service-opts.md +++ b/doc/config/service-opts.md @@ -179,6 +179,16 @@ following settings are available: the default `restart-max`, and with `restart = "always"`, that is the first five retries. The greater of this configured value and the back-off is used + * `restart-max-sec = SEC` -- replaces that back-off with a delay that + doubles for each restart, from `restart-sec` up to this limit. With + `restart-sec = 10` and `restart-max-sec = 300` Finit waits 10, 20, + 40, 80, 160, and then 300 seconds. Without `restart-sec` the first + restart is at once, then after 2, 4, 8 seconds, and so on. A service + that stays up longer than the current delay is restarted after + `restart-sec` the next time it crashes, and the delay comes down + again the longer the service keeps running. Useful with + `restart = "always"`, for a service that should be retried forever, + only less and less often * `restart = "always"` -- no upper limit on the number of times Finit tries to restart a crashing service * `restart = "never"` -- do not restart on failures. `false` is diff --git a/man/finit.conf.5 b/man/finit.conf.5 index e4edc2bc..b1fa7b57 100644 --- a/man/finit.conf.5 +++ b/man/finit.conf.5 @@ -384,6 +384,15 @@ number of seconds before Finit tries to restart a crashing service, default: 2 seconds for the first five retries, then back-off to 5 seconds. The maximum of this configured value and the above (2 and 5) will be used +.It Cm restart-max-sec:SEC +replaces that back-off with a delay that doubles for each restart, from +.Cm restart_sec +up to this limit, e.g. 10, 20, 40, and so on up to 300 seconds with +.Cm restart_sec:10 restart-max-sec:300 . +Without +.Cm restart_sec +the first restart is at once, then after 2, 4, 8 seconds. The delay +comes down again as the service keeps running .It Cm restart:always no upper limit on the number of times Finit tries to restart a crashing service. Same as diff --git a/src/conf.c b/src/conf.c index 333a9ac1..76de4690 100644 --- a/src/conf.c +++ b/src/conf.c @@ -261,6 +261,7 @@ static cfg_opt_t svc_opts[] = { CFG_STR ("restart", NULL, CFGF_NODEFAULT), CFG_INT ("restart-max", 0, CFGF_NODEFAULT), CFG_INT ("restart-sec", 0, CFGF_NODEFAULT), + CFG_INT ("restart-max-sec", 0, CFGF_NODEFAULT), CFG_STR ("oncrash", NULL, CFGF_NODEFAULT), CFG_STR ("reload-signal", NULL, CFGF_NODEFAULT), CFG_BOOL ("required", cfg_true, CFGF_NODEFAULT), @@ -1668,6 +1669,8 @@ static void svc_translate(cfg_t *sec, int type, struct rlimit rlimit[], char *fi if (sec_getint(sec, "restart-sec", NULL, &num)) addtok(line, sizeof(line), "restart_sec:%ld", num); + if (sec_getint(sec, "restart-max-sec", NULL, &num)) + addtok(line, sizeof(line), "restart-max-sec:%ld", num); if ((str = sec_getstr(sec, "oncrash", NULL))) addtok(line, sizeof(line), "oncrash:%s", str); diff --git a/src/service.c b/src/service.c index 3545a556..d3c742cc 100644 --- a/src/service.c +++ b/src/service.c @@ -2332,6 +2332,7 @@ svc_t *service_register_desc(int type, char *cfg, const char *desc, int forking = 0, manual = 0, remain = 0, nowarn = 0; int restart_max = SVC_RESPAWN_MAX; int restart_tmo = 0; + int restart_tmo_max = 0; unsigned oncrash_action = SVC_ONCRASH_IGNORE; char *line, *args; svc_t *svc; @@ -2399,6 +2400,13 @@ svc_t *service_register_desc(int type, char *cfg, const char *desc, restart_tmo = atoi(arg) * 1000; else if (MATCH_CMD(cmd, "restart_sec:", arg)) restart_tmo = atoi(arg) * 1000; + else if (MATCH_CMD(cmd, "restart-max-sec:", arg)) { + const char *errstr; + + restart_tmo_max = (int)strtonum(arg, 0, 86400, &errstr) * 1000; + if (errstr) + warnx("restart-max-sec %s is %s (0-86400), ignoring", arg, errstr); + } else if (!strcasecmp(cmd, "norestart")) restart_max = 0; else if (!strcasecmp(cmd, "nowarn")) @@ -2744,6 +2752,7 @@ svc_t *service_register_desc(int type, char *cfg, const char *desc, svc->forking = forking; svc->restart_max = restart_max; svc->restart_tmo = restart_tmo; + svc->restart_tmo_max = restart_tmo_max; svc->oncrash_action = oncrash_action; /* Decode any (optional) pid:/optional/path/to/file.pid */ @@ -3265,11 +3274,53 @@ static void service_cleanup_script(svc_t *svc) service_timeout_after(svc, svc->cleanup_tmo, service_kill_script); } -/* Saturates, restart always has no restart_max to stop the count */ +/* Doubles with the instability index, up to restart-max-sec, issue #306 */ +static int restart_backoff(svc_t *svc, int cnt) +{ + int tmo = svc->restart_saved; + + /* No restart-sec, the first restart was immediate */ + if (tmo < 1000) { + tmo = 2000; + cnt--; + } + + while (cnt-- > 0 && tmo < svc->restart_tmo_max) + tmo = tmo > svc->restart_tmo_max / 2 ? svc->restart_tmo_max : tmo * 2; + + return tmo; +} + +/* Delay before the next restart of a crashing service */ +static int restart_delay(svc_t *svc) +{ + int half; + + if (svc->restart_tmo_max) + return restart_backoff(svc, svc->restart_cnt); + + /* Wait 2s for the first half of the attempts, then back off to 5s */ + half = (svc->restart_max == -1 ? SVC_RESPAWN_MAX : svc->restart_max) / 2; + + /* If a longer timeout was specified in the conf, use that instead. */ + return max(svc->restart_tmo, svc->restart_cnt <= half ? 2000 : 5000); +} + +/* + * Saturates, restart always has no restart_max to stop the count. + * It stops where the back-off peaks, so aging soon brings it down. + */ static void restart_cnt_inc(svc_t *svc) { int *restart_cnt = (int *)&svc->restart_cnt; - int limit = svc->restart_max == -1 ? SVC_RESPAWN_MAX : INT_MAX; + int limit = INT_MAX; + + if (svc->restart_max == -1) { + if (!svc->restart_tmo_max) + limit = SVC_RESPAWN_MAX; + else if (*restart_cnt && restart_backoff(svc, *restart_cnt) >= svc->restart_tmo_max) + return; + } if (*restart_cnt < limit) (*restart_cnt)++; @@ -3278,7 +3329,6 @@ static void restart_cnt_inc(svc_t *svc) static void service_retry(svc_t *svc) { int *restart_cnt = (int *)&svc->restart_cnt; - int timeout, half; service_timeout_cancel(svc); @@ -3335,11 +3385,7 @@ static void service_retry(svc_t *svc) restart_cnt_inc(svc); dbg("%s crashed, trying to start it again, attempt %d", svc_ident(svc, NULL, 0), *restart_cnt); - /* Wait 2s for the first half of the attempts, then back off to 5s */ - half = (svc->restart_max == -1 ? SVC_RESPAWN_MAX : svc->restart_max) / 2; - timeout = *restart_cnt <= half ? 2000 : 5000; - /* If a longer timeout was specified in the conf, use that instead. */ - svc->restart_tmo = max(svc->restart_tmo, timeout); + svc->restart_tmo = restart_delay(svc); if (WIFEXITED(svc->status)) logit(LOG_CONSOLE|LOG_WARNING, "Service %s[%d] died (exit status: %d), restarting (retry in %d msec) (attempt: %d/%d)", diff --git a/src/svc.h b/src/svc.h index e2f65528..6f070ee8 100644 --- a/src/svc.h +++ b/src/svc.h @@ -177,6 +177,7 @@ typedef struct svc { int restart_max; /* Maximum number of restarts allowed */ int restart_saved; /* INTERNAL, saved copy of .conf value */ int restart_tmo; /* Time before restarting a crashing service */ + int restart_tmo_max;/* restart-max-sec, 0: fixed back-off, issue #306 */ unsigned char oncrash_action; /* Action to perform in crashed state. */ char respawn; /* ttys, or services with `respawn`, never increment restart_cnt */ const int restart_cnt; /* Incremented for each restart by service monitor. */ diff --git a/test/restart-backoff.sh b/test/restart-backoff.sh index 0d1c363e..a8f34c71 100755 --- a/test/restart-backoff.sh +++ b/test/restart-backoff.sh @@ -54,3 +54,16 @@ crash 'restart = "always"' assert_delay 1 2 0 1 assert_delay 2 3 2 3 assert_delay 3 4 2 3 + +sep +say 'restart-max-sec, the delay doubles from restart-sec up to the limit' +crash 'restart = "always" + restart-sec = 1 + restart-max-sec = 4' +assert_delay 1 2 1 2 +assert_delay 2 3 2 3 +assert_delay 3 4 4 5 +assert_delay 4 5 4 5 + +say 'The restart counter stops where the delay peaks' +assert "Restart counter is 2" "$(texec initctl status crash | awk '/Restarts/{ print $4 }')" = "(2/-1)"