Merge pull request #489 from finit-project/stale-pidfile

Handle stale pid file for crashing daemons
This commit is contained in:
Joachim Wiberg authored and GitHub committed 2026-05-12 10:40:01 +02:00
commit a3eac3a85e
8 files changed
+139 -11

No files matched your search

+25
View File
@@ -3,6 +3,31 @@ Change Log
All relevant changes are documented in this file. All relevant changes are documented in this file.
[Unreleased]
------------
### Changes
- Restart log now spells out the signal name and flags core dumps,
e.g. `killed by SIGKILL` or `killed by SIGSEGV, core dumped`, in
place of the bare numeric `by signal: N`. Gives operators a much
stronger breadcrumb when a daemon dies unexpectedly
### Fixes
- Remove stale daemon-owned (`pid:!`) pidfiles after unclean exits.
When a daemon dies via SIGKILL/OOM/segfault, or exits early during
startup, its pidfile lingers and can prevent the next instance from
starting -- `dbus-daemon` for example refuses to start when its
pidfile already exists, so Finit's restart loop gives up. Finit now
drops the file when it still names the just-reaped PID and that PID
is no longer alive (the liveness check guards against PID reuse).
This is a controlled exception to the long-standing rule that Finit
does not touch service-owned pidfiles
- Fix misspelled `SIGUNKOWN` returned by `sig_name()` for unknown
signal numbers, now spelled correctly as `SIGUNKNOWN`. Surfaced by
the new restart log above
[4.17][] - 2026-04-28 [4.17][] - 2026-04-28
--------------------- ---------------------
+8
View File
@@ -31,6 +31,14 @@ basename of the binary to guess the PID file to watch for the PID:
`/var/run/ntpd.pid`. If Finit guesses wrong, you have to submit the `/var/run/ntpd.pid`. If Finit guesses wrong, you have to submit the
full `pid:!/path/to/file.pid`. full `pid:!/path/to/file.pid`.
With `pid:!/path`, the file belongs to the service: Finit reads it
but does not create or remove it. The one exception is *stale*
cleanup — if the service dies without removing its own pidfile
(SIGKILL, OOM, segfault), and the file still names the just-reaped
PID, Finit removes it before the next retry. This prevents daemons
that refuse to start on an existing pidfile (e.g. `dbus-daemon`)
from getting stuck in a crash-restart loop.
**Example:** **Example:**
In the case of `ospfd` (below), we omit the `-d` flag (daemonize) to In the case of `ospfd` (below), we omit the `-d` flag (daemonize) to
+4 -1
View File
@@ -201,7 +201,10 @@ int pid_file_set(svc_t *svc, char *file, int not)
* pid:!foo --> !/run/foo.pid * pid:!foo --> !/run/foo.pid
* pid:!/run/foo.pid --> !/run/foo.pid * pid:!/run/foo.pid --> !/run/foo.pid
* *
* Note, nothing is created or removed by Finit in this latter form. * Nothing is created or removed by Finit in this latter form, with one
* exception: a verifiably stale pidfile (still names the just-reaped
* PID, and that PID is no longer alive) is removed so the next instance
* can start. See service_clean_pidfile() in src/service.c.
*/ */
int pid_file_parse(svc_t *svc, char *arg) int pid_file_parse(svc_t *svc, char *arg)
{ {
+45 -6
View File
@@ -1120,6 +1120,35 @@ static void service_notify_stop(svc_t *svc)
} }
} }
/*
* Drop a daemon-owned (pid:!) pidfile if it still names the just-reaped
* PID and that PID is gone. The liveness check guards against reuse.
*/
static void service_clean_pidfile(svc_t *svc, pid_t reaped)
{
pid_t pid;
char *fn;
if (reaped <= 1)
return;
fn = pid_file(svc);
if (!fn)
return;
pid = pid_file_read(fn);
if (pid != reaped || pid_alive(pid))
return;
if (remove(fn) && errno != ENOENT) {
logit(LOG_CRIT, "Failed removing stale service %s pidfile %s",
svc_ident(svc, NULL, 0), fn);
return;
}
dbg("Removed stale service %s pidfile %s", svc_ident(svc, NULL, 0), fn);
}
/* /*
* Clean up any lingering state from dead/killed services * Clean up any lingering state from dead/killed services
*/ */
@@ -1137,6 +1166,8 @@ static void service_cleanup(svc_t *svc)
if (remove(fn) && errno != ENOENT) if (remove(fn) && errno != ENOENT)
logit(LOG_CRIT, "Failed removing service %s pidfile %s", logit(LOG_CRIT, "Failed removing service %s pidfile %s",
svc_ident(svc, NULL, 0), fn); svc_ident(svc, NULL, 0), fn);
} else if (svc->pidfile[0] == '!') {
service_clean_pidfile(svc, svc->pid);
} }
/* /*
@@ -2405,6 +2436,9 @@ void service_monitor(pid_t lost, int status)
if (svc_is_forking(svc)) { if (svc_is_forking(svc)) {
/* Likely start script exiting */ /* Likely start script exiting */
if (svc_is_starting(svc)) { if (svc_is_starting(svc)) {
/* Daemon died before clearing 'starting'; drop any stale pidfile. */
service_clean_pidfile(svc, lost);
svc->oldpid = lost; /* So service_retry() logs the real PID */
svc->pid = 0; /* Expect no more activity from this one */ svc->pid = 0; /* Expect no more activity from this one */
goto cont; goto cont;
} }
@@ -2794,13 +2828,18 @@ static void service_retry(svc_t *svc)
timeout = ((*restart_cnt) <= (svc->restart_max / 2)) ? 2000 : 5000; timeout = ((*restart_cnt) <= (svc->restart_max / 2)) ? 2000 : 5000;
/* If a longer timeout was specified in the conf, use that instead. */ /* If a longer timeout was specified in the conf, use that instead. */
svc->restart_tmo = max(svc->restart_tmo, timeout); svc->restart_tmo = max(svc->restart_tmo, timeout);
logit(LOG_CONSOLE|LOG_WARNING, "Service %s[%d] died (%s%d), restarting (retry in %d msec) (attempt: %d/%d)", if (WIFEXITED(svc->status))
logit(LOG_CONSOLE|LOG_WARNING,
"Service %s[%d] died (exit status: %d), restarting (retry in %d msec) (attempt: %d/%d)",
svc_ident(svc, NULL, 0), svc->oldpid, WEXITSTATUS(svc->status),
svc->restart_tmo, *restart_cnt, svc->restart_max);
else
logit(LOG_CONSOLE|LOG_WARNING,
"Service %s[%d] died (killed by %s%s), restarting (retry in %d msec) (attempt: %d/%d)",
svc_ident(svc, NULL, 0), svc->oldpid, svc_ident(svc, NULL, 0), svc->oldpid,
WIFEXITED(svc->status) ? "with exit status: " : "by signal: ", sig_name(WTERMSIG(svc->status)),
WIFEXITED(svc->status) ? WEXITSTATUS(svc->status) : WTERMSIG(svc->status), WCOREDUMP(svc->status) ? ", core dumped" : "",
svc->restart_tmo, svc->restart_tmo, *restart_cnt, svc->restart_max);
*restart_cnt,
svc->restart_max);
svc_unblock(svc); svc_unblock(svc);
service_step(svc); service_step(svc);
+1 -1
View File
@@ -630,7 +630,7 @@ const char *sig_name(int signo)
return signames[i].name; return signames[i].name;
} }
return "SIGUNKOWN"; return "SIGUNKNOWN";
} }
/* /*
+2
View File
@@ -41,6 +41,7 @@ EXTRA_DIST += global-envs.sh
EXTRA_DIST += initctl-status-subset.sh EXTRA_DIST += initctl-status-subset.sh
EXTRA_DIST += notify.sh EXTRA_DIST += notify.sh
EXTRA_DIST += pidfile.sh EXTRA_DIST += pidfile.sh
EXTRA_DIST += stale-pidfile.sh
EXTRA_DIST += pre-post-serv.sh EXTRA_DIST += pre-post-serv.sh
EXTRA_DIST += pre-fail.sh EXTRA_DIST += pre-fail.sh
EXTRA_DIST += process-depends.sh EXTRA_DIST += process-depends.sh
@@ -84,6 +85,7 @@ TESTS += global-envs.sh
TESTS += initctl-status-subset.sh TESTS += initctl-status-subset.sh
TESTS += notify.sh TESTS += notify.sh
TESTS += pidfile.sh TESTS += pidfile.sh
TESTS += stale-pidfile.sh
TESTS += pre-post-serv.sh TESTS += pre-post-serv.sh
TESTS += pre-fail.sh TESTS += pre-fail.sh
TESTS += process-depends.sh TESTS += process-depends.sh
+16 -3
View File
@@ -34,6 +34,7 @@ volatile sig_atomic_t reloading = 1;
volatile sig_atomic_t running = 1; volatile sig_atomic_t running = 1;
static char *ident = PROGNM; static char *ident = PROGNM;
static char fn[80]; static char fn[80];
static int exclusive = 0; /* -x: refuse to start if pidfile exists (dbus-style) */
static void verify_env(char *arg) static void verify_env(char *arg)
{ {
@@ -122,7 +123,7 @@ static void writefn(char *fn, int val)
static int checkfn(char *fn) static int checkfn(char *fn)
{ {
return !access(fn, R_OK); return !access(fn, F_OK);
} }
static void mine(char *fn) static void mine(char *fn)
@@ -133,12 +134,18 @@ static void mine(char *fn)
static void pidfile(char *pidfn) static void pidfile(char *pidfn)
{ {
static int once = 0;
if (!pidfn) { if (!pidfn) {
if (fn[0] == 0) if (fn[0] == 0)
snprintf(fn, sizeof(fn), "%s%s.pid", _PATH_VARRUN, ident); snprintf(fn, sizeof(fn), "%s%s.pid", _PATH_VARRUN, ident);
pidfn = fn; pidfn = fn;
} }
if (exclusive && !once && checkfn(pidfn))
errx(EX_SOFTWARE, "PID file %s exists, refusing to start", pidfn);
once = 1;
if (!checkfn(pidfn)) { if (!checkfn(pidfn)) {
pid_t pid; pid_t pid;
@@ -148,7 +155,7 @@ static void pidfile(char *pidfn)
atexit(cleanup); atexit(cleanup);
} else { } else {
inf("Touching PID file %s", pidfn); inf("Touching PID file %s", pidfn);
utimensat(0, fn, NULL, 0); utimensat(AT_FDCWD, pidfn, NULL, 0);
} }
} }
@@ -182,6 +189,7 @@ static int usage(int rc)
" -p Create PID file despite running in foreground\n" " -p Create PID file despite running in foreground\n"
" -P FILE Create PID file using FILE\n" " -P FILE Create PID file using FILE\n"
" -r SVC Call initctl to restart service SVC (self)\n" " -r SVC Call initctl to restart service SVC (self)\n"
" -x Refuse to start if PID file already exists (dbus-style)\n"
"\n" "\n"
"By default this program daemonizes itself to the background, and,\n" "By default this program daemonizes itself to the background, and,\n"
"when it's done setting up its signal handler(s), creates a PID file\n" "when it's done setting up its signal handler(s), creates a PID file\n"
@@ -212,7 +220,7 @@ int main(int argc, char *argv[])
char cmd[80]; char cmd[80];
int c; int c;
while ((c = getopt(argc, argv, "cCe:E:f:F:hi:nN:pP:r:")) != EOF) { while ((c = getopt(argc, argv, "cCe:E:f:F:hi:nN:pP:r:x")) != EOF) {
switch (c) { switch (c) {
case 'c': case 'c':
do_crash = 1; do_crash = 1;
@@ -249,12 +257,17 @@ int main(int argc, char *argv[])
break; break;
case 'P': case 'P':
pidfn = optarg; pidfn = optarg;
if ((size_t)snprintf(fn, sizeof(fn), "%s", optarg) >= sizeof(fn))
errx(EX_USAGE, "-P path too long (max %zu)", sizeof(fn) - 1);
do_pidfile++; do_pidfile++;
break; break;
case 'r': case 'r':
snprintf(cmd, sizeof(cmd), "initctl restart %s", optarg); snprintf(cmd, sizeof(cmd), "initctl restart %s", optarg);
do_restart = 1; do_restart = 1;
break; break;
case 'x':
exclusive = 1;
break;
default: default:
return usage(1); return usage(1);
} }
+38
View File
@@ -0,0 +1,38 @@
#!/bin/sh
# Regression: Finit must remove a daemon-owned (pid:!) stale pidfile
# left behind by an unclean exit (SIGKILL), so the next instance can
# start. Simulates the dbus-daemon pattern: 'serv -x' refuses to
# start if its pidfile already exists.
set -eu
TEST_DIR=$(dirname "$0")
PIDFN="/run/serv.pid"
test_teardown()
{
say "Running test teardown."
run "rm -f $FINIT_CONF $PIDFN"
}
# shellcheck source=/dev/null
. "$TEST_DIR/lib/setup.sh"
say "Add service stanza '$FINIT_CONF' with pid:!$PIDFN"
run "echo 'service pid:!$PIDFN serv -np -P $PIDFN -x' > $FINIT_CONF"
run "initctl reload"
retry "assert_num_children 1 serv"
assert_is_pidfile "serv" "$PIDFN"
PID=$(texec cat "$PIDFN")
say "SIGKILL serv ($PID) -- leaves stale pidfile"
run "kill -9 $PID"
# Without the fix, 'serv -x' refuses to start on each retry until
# Finit hits restart_max and marks the service crashed. With the fix
# Finit removes the stale pidfile and the next instance comes up.
retry "assert_pidiff serv $PID"
retry "assert_num_children 1 serv"
sep