mirror of
https://github.com/troglobit/finit.git
synced 2026-11-20 19:41:51 +00:00
service: add restart-max-sec, a limit for the restart back-off
The delay between restarts of a crashing service is fixed: two
seconds, then five, or restart-sec if that is longer. A service that
should be retried quickly at first and then less and less often, e.g.
a modem dialer, cannot be set up.
Add restart-max-sec, in the block format and as the restart-max-sec:
token:
service dialer {
restart = "always"
restart-sec = 10
restart-max-sec = 300
command = "dialer"
}
When set, the delay doubles for each restart, from restart-sec up to
the limit: 10, 20, 40, 80, 160, 300. Without restart-sec the first
restart is at once, then after 2, 4, 8 seconds. The delay follows the
restart counter, so it comes down again as the counter ages. With
restart always the counter stops where the delay peaks.
Fixes #306
Signed-off-by: Joachim Wiberg <troglobit@gmail.com>
This commit is contained in:
8 files changed
+95
-8
No files matched your search
@@ -11,6 +11,10 @@ All relevant changes are documented in this file.
|
||||
- `shutdown` now takes a time, `now`, `+MIN`, or `HH:MM`, and a message.
|
||||
Finit warns logged-in users as the time approaches, refuses new logins
|
||||
from five minutes before, and `shutdown -c` cancels. Issue #149
|
||||
- Add `restart-max-sec`, a limit for the delay between restarts of a
|
||||
crashing service. When set, the delay doubles for each restart, from
|
||||
`restart-sec` up to the limit, and comes down again as the service keeps
|
||||
running. Issue #306
|
||||
- Add `stop-scope`, which processes a stop applies to: `pgroup`, the process
|
||||
group of the main process (default), `process`, only the main process,
|
||||
`cgroup`, the cgroup of the service, or `mixed` like systemd. Issue #413
|
||||
|
||||
@@ -64,6 +64,7 @@ Service options
|
||||
| `remain:yes` | `remain-after-exit = true` |
|
||||
| `restart:always` / `restart:NUM` | `restart = "always"` / `restart-max = NUM` |
|
||||
| `restart_sec:SEC` | `restart-sec = SEC` |
|
||||
| `restart-max-sec:SEC` | `restart-max-sec = SEC` |
|
||||
| `norestart` | `restart = "never"` |
|
||||
| `respawn` | `respawn = true` |
|
||||
| `oncrash:reboot` | `oncrash = "reboot"` |
|
||||
|
||||
@@ -179,6 +179,16 @@ following settings are available:
|
||||
the default `restart-max`, and with `restart = "always"`, that is the
|
||||
first five retries. The greater of this configured value and the
|
||||
back-off is used
|
||||
* `restart-max-sec = SEC` -- replaces that back-off with a delay that
|
||||
doubles for each restart, from `restart-sec` up to this limit. With
|
||||
`restart-sec = 10` and `restart-max-sec = 300` Finit waits 10, 20,
|
||||
40, 80, 160, and then 300 seconds. Without `restart-sec` the first
|
||||
restart is at once, then after 2, 4, 8 seconds, and so on. A service
|
||||
that stays up longer than the current delay is restarted after
|
||||
`restart-sec` the next time it crashes, and the delay comes down
|
||||
again the longer the service keeps running. Useful with
|
||||
`restart = "always"`, for a service that should be retried forever,
|
||||
only less and less often
|
||||
* `restart = "always"` -- no upper limit on the number of times Finit
|
||||
tries to restart a crashing service
|
||||
* `restart = "never"` -- do not restart on failures. `false` is
|
||||
|
||||
@@ -384,6 +384,15 @@ number of seconds before Finit tries to restart
|
||||
a crashing service, default: 2 seconds for the first five retries,
|
||||
then back-off to 5 seconds. The maximum of this configured value
|
||||
and the above (2 and 5) will be used
|
||||
.It Cm restart-max-sec:SEC
|
||||
replaces that back-off with a delay that doubles for each restart, from
|
||||
.Cm restart_sec
|
||||
up to this limit, e.g. 10, 20, 40, and so on up to 300 seconds with
|
||||
.Cm restart_sec:10 restart-max-sec:300 .
|
||||
Without
|
||||
.Cm restart_sec
|
||||
the first restart is at once, then after 2, 4, 8 seconds. The delay
|
||||
comes down again as the service keeps running
|
||||
.It Cm restart:always
|
||||
no upper limit on the number of times Finit tries to restart a crashing
|
||||
service. Same as
|
||||
|
||||
@@ -261,6 +261,7 @@ static cfg_opt_t svc_opts[] = {
|
||||
CFG_STR ("restart", NULL, CFGF_NODEFAULT),
|
||||
CFG_INT ("restart-max", 0, CFGF_NODEFAULT),
|
||||
CFG_INT ("restart-sec", 0, CFGF_NODEFAULT),
|
||||
CFG_INT ("restart-max-sec", 0, CFGF_NODEFAULT),
|
||||
CFG_STR ("oncrash", NULL, CFGF_NODEFAULT),
|
||||
CFG_STR ("reload-signal", NULL, CFGF_NODEFAULT),
|
||||
CFG_BOOL ("required", cfg_true, CFGF_NODEFAULT),
|
||||
@@ -1668,6 +1669,8 @@ static void svc_translate(cfg_t *sec, int type, struct rlimit rlimit[], char *fi
|
||||
|
||||
if (sec_getint(sec, "restart-sec", NULL, &num))
|
||||
addtok(line, sizeof(line), "restart_sec:%ld", num);
|
||||
if (sec_getint(sec, "restart-max-sec", NULL, &num))
|
||||
addtok(line, sizeof(line), "restart-max-sec:%ld", num);
|
||||
|
||||
if ((str = sec_getstr(sec, "oncrash", NULL)))
|
||||
addtok(line, sizeof(line), "oncrash:%s", str);
|
||||
|
||||
+54
-8
@@ -2332,6 +2332,7 @@ svc_t *service_register_desc(int type, char *cfg, const char *desc,
|
||||
int forking = 0, manual = 0, remain = 0, nowarn = 0;
|
||||
int restart_max = SVC_RESPAWN_MAX;
|
||||
int restart_tmo = 0;
|
||||
int restart_tmo_max = 0;
|
||||
unsigned oncrash_action = SVC_ONCRASH_IGNORE;
|
||||
char *line, *args;
|
||||
svc_t *svc;
|
||||
@@ -2399,6 +2400,13 @@ svc_t *service_register_desc(int type, char *cfg, const char *desc,
|
||||
restart_tmo = atoi(arg) * 1000;
|
||||
else if (MATCH_CMD(cmd, "restart_sec:", arg))
|
||||
restart_tmo = atoi(arg) * 1000;
|
||||
else if (MATCH_CMD(cmd, "restart-max-sec:", arg)) {
|
||||
const char *errstr;
|
||||
|
||||
restart_tmo_max = (int)strtonum(arg, 0, 86400, &errstr) * 1000;
|
||||
if (errstr)
|
||||
warnx("restart-max-sec %s is %s (0-86400), ignoring", arg, errstr);
|
||||
}
|
||||
else if (!strcasecmp(cmd, "norestart"))
|
||||
restart_max = 0;
|
||||
else if (!strcasecmp(cmd, "nowarn"))
|
||||
@@ -2744,6 +2752,7 @@ svc_t *service_register_desc(int type, char *cfg, const char *desc,
|
||||
svc->forking = forking;
|
||||
svc->restart_max = restart_max;
|
||||
svc->restart_tmo = restart_tmo;
|
||||
svc->restart_tmo_max = restart_tmo_max;
|
||||
svc->oncrash_action = oncrash_action;
|
||||
|
||||
/* Decode any (optional) pid:/optional/path/to/file.pid */
|
||||
@@ -3265,11 +3274,53 @@ static void service_cleanup_script(svc_t *svc)
|
||||
service_timeout_after(svc, svc->cleanup_tmo, service_kill_script);
|
||||
}
|
||||
|
||||
/* Saturates, restart always has no restart_max to stop the count */
|
||||
/* Doubles with the instability index, up to restart-max-sec, issue #306 */
|
||||
static int restart_backoff(svc_t *svc, int cnt)
|
||||
{
|
||||
int tmo = svc->restart_saved;
|
||||
|
||||
/* No restart-sec, the first restart was immediate */
|
||||
if (tmo < 1000) {
|
||||
tmo = 2000;
|
||||
cnt--;
|
||||
}
|
||||
|
||||
while (cnt-- > 0 && tmo < svc->restart_tmo_max)
|
||||
tmo = tmo > svc->restart_tmo_max / 2 ? svc->restart_tmo_max : tmo * 2;
|
||||
|
||||
return tmo;
|
||||
}
|
||||
|
||||
/* Delay before the next restart of a crashing service */
|
||||
static int restart_delay(svc_t *svc)
|
||||
{
|
||||
int half;
|
||||
|
||||
if (svc->restart_tmo_max)
|
||||
return restart_backoff(svc, svc->restart_cnt);
|
||||
|
||||
/* Wait 2s for the first half of the attempts, then back off to 5s */
|
||||
half = (svc->restart_max == -1 ? SVC_RESPAWN_MAX : svc->restart_max) / 2;
|
||||
|
||||
/* If a longer timeout was specified in the conf, use that instead. */
|
||||
return max(svc->restart_tmo, svc->restart_cnt <= half ? 2000 : 5000);
|
||||
}
|
||||
|
||||
/*
|
||||
* Saturates, restart always has no restart_max to stop the count.
|
||||
* It stops where the back-off peaks, so aging soon brings it down.
|
||||
*/
|
||||
static void restart_cnt_inc(svc_t *svc)
|
||||
{
|
||||
int *restart_cnt = (int *)&svc->restart_cnt;
|
||||
int limit = svc->restart_max == -1 ? SVC_RESPAWN_MAX : INT_MAX;
|
||||
int limit = INT_MAX;
|
||||
|
||||
if (svc->restart_max == -1) {
|
||||
if (!svc->restart_tmo_max)
|
||||
limit = SVC_RESPAWN_MAX;
|
||||
else if (*restart_cnt && restart_backoff(svc, *restart_cnt) >= svc->restart_tmo_max)
|
||||
return;
|
||||
}
|
||||
|
||||
if (*restart_cnt < limit)
|
||||
(*restart_cnt)++;
|
||||
@@ -3278,7 +3329,6 @@ static void restart_cnt_inc(svc_t *svc)
|
||||
static void service_retry(svc_t *svc)
|
||||
{
|
||||
int *restart_cnt = (int *)&svc->restart_cnt;
|
||||
int timeout, half;
|
||||
|
||||
service_timeout_cancel(svc);
|
||||
|
||||
@@ -3335,11 +3385,7 @@ static void service_retry(svc_t *svc)
|
||||
restart_cnt_inc(svc);
|
||||
|
||||
dbg("%s crashed, trying to start it again, attempt %d", svc_ident(svc, NULL, 0), *restart_cnt);
|
||||
/* Wait 2s for the first half of the attempts, then back off to 5s */
|
||||
half = (svc->restart_max == -1 ? SVC_RESPAWN_MAX : svc->restart_max) / 2;
|
||||
timeout = *restart_cnt <= half ? 2000 : 5000;
|
||||
/* If a longer timeout was specified in the conf, use that instead. */
|
||||
svc->restart_tmo = max(svc->restart_tmo, timeout);
|
||||
svc->restart_tmo = restart_delay(svc);
|
||||
if (WIFEXITED(svc->status))
|
||||
logit(LOG_CONSOLE|LOG_WARNING,
|
||||
"Service %s[%d] died (exit status: %d), restarting (retry in %d msec) (attempt: %d/%d)",
|
||||
|
||||
@@ -177,6 +177,7 @@ typedef struct svc {
|
||||
int restart_max; /* Maximum number of restarts allowed */
|
||||
int restart_saved; /* INTERNAL, saved copy of .conf value */
|
||||
int restart_tmo; /* Time before restarting a crashing service */
|
||||
int restart_tmo_max;/* restart-max-sec, 0: fixed back-off, issue #306 */
|
||||
unsigned char oncrash_action; /* Action to perform in crashed state. */
|
||||
char respawn; /* ttys, or services with `respawn`, never increment restart_cnt */
|
||||
const int restart_cnt; /* Incremented for each restart by service monitor. */
|
||||
|
||||
@@ -54,3 +54,16 @@ crash 'restart = "always"'
|
||||
assert_delay 1 2 0 1
|
||||
assert_delay 2 3 2 3
|
||||
assert_delay 3 4 2 3
|
||||
|
||||
sep
|
||||
say 'restart-max-sec, the delay doubles from restart-sec up to the limit'
|
||||
crash 'restart = "always"
|
||||
restart-sec = 1
|
||||
restart-max-sec = 4'
|
||||
assert_delay 1 2 1 2
|
||||
assert_delay 2 3 2 3
|
||||
assert_delay 3 4 4 5
|
||||
assert_delay 4 5 4 5
|
||||
|
||||
say 'The restart counter stops where the delay peaks'
|
||||
assert "Restart counter is 2" "$(texec initctl status crash | awk '/Restarts/{ print $4 }')" = "(2/-1)"
|
||||
Reference in new issue
Block a user