Fix #109: Support PID files in subdirectories to /var/run

Services, like dbus and teamd for instance, may create their PID files
in a subdirectory of /var/run (today often /run). E.g.,

   - /var/run/teamd/a1.pid    -- For aggregate A1
   - /var/run/dbus/pid
   - /var/run/lxc/foo.pid     -- For container foo

This patch adds support for dynamically adding inotify watchers to any
new subdirectory created in /var/run (discarding too deep directories).

To match services in this directory the run/task/service/sysv stanza
must contain the pid:!/path/to/pidfile.pid syntax.  This pid file name
is also used to create the condition this service asserts using the
following formula:

   svc/ + <dirname of service> + <subdir and file without .pid>

E.g., the case of teamd (above) gives condition 'svc/usr/bin/teamd/a1'

The special case of dbus is interesting, since it may not be a special
case, but rather the norm for services using a subdirectory.  It is
handled as follows; when a new subdirectory is detected, the directory
is scanned for files matching *.pid.  Matching files follow the teamd
case.  The directory is also scanned for 'pid', which then gives us the
condition 'svc/usr/bin/dbus'

One last example, illustrated by lxc-start, where we want to track the
condition for the LXC container foo.  The service stanza:

   service pid:!/run/lxc/foo.pid lxc-start -n foo -F -p /run/lxc/foo.pid -- Container foo

This command has no leading path so the condition is composed entirely
from the PID file location:

   svc/  + '' + lxc/foo => svc/lxc/foo

Signed-off-by: Joachim Nilsson <troglobit@gmail.com>

plugins/pidfile.c

@@ -1,6 +1,7 @@
 /* Simple pidfile event monitor for the Finit condition engine
  *
  * Copyright (c) 2015-2016  Tobias Waldekranz <tobias@waldekranz.com>
+ * Copyright (c) 2016-2020  Joachim Nilsson <troglobit@gmail.com>
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -21,9 +22,10 @@
  * THE SOFTWARE.
  */
 
+#include <glob.h>
 #include <limits.h>
 #include <paths.h>
-
+#include <lite/queue.h>
 #include <sys/inotify.h>
 
 #include "finit.h"
@@ -33,52 +35,217 @@
 #include "plugin.h"
 #include "service.h"
 
+struct wd_entry {
+	TAILQ_ENTRY(wd_entry) link;
+	char *path;
+	int fd;
+};
+
 struct context {
 	int fd;
-	int wd;
+	TAILQ_HEAD(, wd_entry) wd_list;
 };
 
+static struct context pidfile_ctx;
+
+static int watcher_add(struct context *ctx, char *path)
+{
+	struct wd_entry *wd;
+	uint32_t mask = IN_ONLYDIR | IN_CREATE | IN_ATTRIB | IN_DELETE | IN_MODIFY | IN_MOVED_TO;
+	char *ptr;
+	int fd;
+
+	_d("pidfile: Adding new watcher for path %s", path);
+	ptr = strstr(path, "run/");
+	if (ptr) {
+		char *slash;
+
+		ptr += 4;
+		slash = strchr(ptr, '/');
+		if (slash) {
+			ptr = slash++;
+			slash = strchr(ptr, '/');
+			if (slash) {
+				_d("pidfile: Path too deep, skipping.");
+				return -1;
+			}
+		}
+	}
+
+	fd = inotify_add_watch(ctx->fd, path, mask);
+	if (fd < 0) {
+		_pe("inotify_add_watch()");
+		return -1;
+	}
+
+	wd = malloc(sizeof(struct wd_entry));
+	if (!wd) {
+		_pe("Failed allocating new `struct wd_entry`");
+		inotify_rm_watch(ctx->fd, fd);
+		return -1;
+	}
+
+	wd->path = path;
+	wd->fd = fd;
+	TAILQ_INSERT_HEAD(&ctx->wd_list, wd, link);
+
+	return 0;
+}
+
+static int watcher_del(struct context *ctx, struct wd_entry *wde)
+{
+	_d("pidfile: Removing watcher for removed path %s", wde->path);
+	TAILQ_REMOVE(&ctx->wd_list, wde, link);
+	inotify_rm_watch(ctx->fd, wde->fd);
+	free(wde->path);
+	free(wde);
+
+	return 0;
+}
+
+static void update_conds(char *name, uint32_t mask)
+{
+	svc_t *svc;
+	char cond[MAX_COND_LEN];
+
+	_d("name: %s", name);
+	svc = svc_find_by_pidfile(name);
+	if (!svc) {
+		_d("pidfile: No matching svc for %s", name);
+		return;
+	}
+
+	_d("Event %s for svc %s", name, svc->cmd);
+
+	mkcond(svc, cond, sizeof(cond));
+	if (mask & (IN_CREATE | IN_ATTRIB | IN_MODIFY | IN_MOVED_TO)) {
+		svc_started(svc);
+		if (svc_is_forking(svc)) {
+			pid_t pid;
+
+			pid = pid_file_read(pid_file(svc));
+			_d("Forking service %s changed PID from %d to %d",
+			   svc->cmd, svc->pid, pid);
+			svc->pid = pid;
+		}
+
+		cond_set(cond);
+	} else if (mask & IN_DELETE)
+		cond_clear(cond);
+}
+
+/* synthesize events in case of new run dirs */
+static void scan_for_pidfiles(struct context *ctx, char *dir, int len)
+{
+	glob_t gl;
+	size_t i;
+	char path[len + 6];
+	int rc;
+
+	snprintf(path, sizeof(path), "%s/*.pid", dir);
+	rc = glob(path, GLOB_NOSORT, NULL, &gl);
+	if (rc && rc != GLOB_NOMATCH)
+		return;
+
+	snprintf(path, sizeof(path), "%s/pid", dir);
+	rc = glob(path, GLOB_NOSORT | GLOB_APPEND, NULL, &gl);
+	if (rc && rc != GLOB_NOMATCH)
+		return;
+
+	for (i = 0; i < gl.gl_pathc; i++) {
+		_d("scan found %s", gl.gl_pathv[i]);
+		update_conds(gl.gl_pathv[i], IN_CREATE);
+	}
+	globfree(&gl);
+}
+
 static void pidfile_callback(void *arg, int fd, int events)
 {
-	static char ev_buf[8 *(sizeof(struct inotify_event) + NAME_MAX + 1) + 1];
-	static char cond[MAX_COND_LEN];
-
 	struct inotify_event *ev;
-	ssize_t sz, len;
+	ssize_t sz, off;
-	svc_t *svc;
+	size_t buflen = 8 *(sizeof(struct inotify_event) + NAME_MAX + 1) + 1;
+	char *buf;
 
-	sz = read(fd, ev_buf, sizeof(ev_buf) - 1);
+        buf= malloc(buflen);
-	if (sz <= 0) {
+	if (!buf) {
-		_pe("invalid inotify event");
+		_pe("Failed allocating buffer for inotify events");
 		return;
 	}
-	ev_buf[sz] = 0;
 
-	for (ev = (void *)ev_buf;
+	_d("pidfile: Entering ... reading %zu bytes into ev_buf[]", buflen - 1);
-	     sz > (ssize_t)sizeof(*ev);
+	sz = read(fd, buf, buflen - 1);
-	     len = sizeof(*ev) + ev->len, ev = (void *)ev + len, sz -= len) {
+	if (sz <= 0) {
-		if (!ev->mask || !strstr(ev->name, ".pid"))
+		_pe("invalid inotify event");
+		free(buf);
+		return;
+	}
+	buf[sz] = 0;
+	_d("pidfile: Read %zd bytes, processing ...", sz);
+
+	off = 0;
+	for (off = 0; off < sz; off += sizeof(*ev) + ev->len) {
+		struct wd_entry *wde;
+
+		ev = (struct inotify_event *)&buf[off];
+
+		_d("pidfile: path %s, event: 0x%08x", ev->name, ev->mask);
+		if (!ev->mask)
 			continue;
 
-		svc = svc_find_by_pidfile(ev->name);
+		/* Find base path for this event */
-		if (!svc)
+		TAILQ_FOREACH(wde, &pidfile_ctx.wd_list, link) {
-			continue;
+			if (wde->fd == ev->wd)
+				break;
+		}
 
-		mkcond(cond, sizeof(cond), svc->cmd);
+		if (ev->mask & IN_ISDIR) {
-		if (ev->mask & (IN_CREATE | IN_ATTRIB | IN_MODIFY | IN_MOVED_TO)) {
+			char *path;
-			svc_started(svc);
+			int plen;
-			if (svc_is_forking(svc)) {
+			int exist = 0;
-				pid_t pid;
 
-				pid = pid_file_read(pid_file(svc));
+			plen = snprintf(NULL, 0, "%s/%s", wde->path, ev->name) + 1;
-				_d("Forking service %s changed PID from %d to %d",
+			path = malloc(plen);
-				   svc->cmd, svc->pid, pid);
+			if (!path) {
-				svc->pid = pid;
+				_pe("Failed allocating path buffer for watcher");
+				free(buf);
+				return;
+			}
+			snprintf(path, plen, "%s/%s", wde->path, ev->name);
+			_d("pidfile: path is %s", path);
+
+			TAILQ_FOREACH(wde, &pidfile_ctx.wd_list, link) {
+				if (!strcmp(wde->path, path)) {
+					exist = 1;
+					break;
+				}
 			}
 
-			cond_set(cond);
+			if (ev->mask & IN_CREATE) {
-		} else if (ev->mask & IN_DELETE)
+				if (!exist) {
-			cond_clear(cond);
+					int rc;
+
+					rc = watcher_add(&pidfile_ctx, path);
+					scan_for_pidfiles(&pidfile_ctx, path, plen);
+					if (rc)
+						free(path);
+					continue;
+				}
+			} else if (ev->mask & IN_DELETE) {
+				if (exist)
+					watcher_del(&pidfile_ctx, wde);
+			}
+
+			free(path);
+			continue; /* no more processing for dirs */
+		}
+
+		if (ev->mask & IN_DELETE) {
+			_d("pidfile %s/%s removed ...", wde->path, ev->name);
+			continue;
+		}
+
+		if (ev->mask & (IN_CREATE | IN_ATTRIB | IN_MODIFY | IN_MOVED_TO))
+			update_conds(ev->name, ev->mask);
 	}
 }
 
@@ -101,7 +268,7 @@ static void pidfile_reconf(void *arg)
 		if (svc_is_changed(svc) || svc_is_starting(svc))
 			continue;
 
-		mkcond(cond, sizeof(cond), svc->cmd);
+		mkcond(svc, cond, sizeof(cond));
 		if (cond_get(cond) == COND_ON)
 			continue;
 
@@ -120,8 +287,6 @@ static void pidfile_reconf(void *arg)
 static void pidfile_init(void *arg)
 {
 	char *path;
-	struct context *ctx = arg;
-	uint32_t mask = IN_CREATE | IN_ATTRIB | IN_DELETE | IN_MODIFY | IN_MOVED_TO;
 
 	/*
 	 * The bootmisc plugin is responsible for setting up /var/run.
@@ -134,15 +299,9 @@ static void pidfile_init(void *arg)
 		return;
 	}
 
-	ctx->wd = inotify_add_watch(ctx->fd, path, mask);
+	TAILQ_INIT(&pidfile_ctx.wd_list);
-	free(path);
+	if (watcher_add(&pidfile_ctx, path))
-
+		close(pidfile_ctx.fd);
-	if (ctx->wd < 0) {
-		_pe("inotify_add_watch()");
-		close(ctx->fd);
-		return;
-	}
-
 	_d("pidfile monitor active");
 }
 
@@ -154,7 +313,7 @@ static struct context pidfile_ctx;
  */
 static plugin_t plugin = {
 	.name = __FILE__,
-	.hook[HOOK_BASEFS_UP]  = { .arg = &pidfile_ctx, .cb = pidfile_init },
+	.hook[HOOK_BASEFS_UP]  = { .cb = pidfile_init   },
 	.hook[HOOK_SVC_RECONF] = { .cb = pidfile_reconf },
 	.io = {
 		.cb    = pidfile_callback,
@@ -177,7 +336,13 @@ PLUGIN_INIT(plugin_init)
 
 PLUGIN_EXIT(plugin_exit)
 {
-	inotify_rm_watch(pidfile_ctx.fd, pidfile_ctx.wd);
+	struct wd_entry *wde, *tmp;
+
+	TAILQ_FOREACH_SAFE(wde, &pidfile_ctx.wd_list, link, tmp) {
+		inotify_rm_watch(pidfile_ctx.fd, wde->fd);
+		free(wde->path);
+		free(wde);
+	}
 	close(pidfile_ctx.fd);
 
 	plugin_unregister(&plugin);

src/cond-w.c

@@ -31,6 +31,68 @@
 #include "pid.h"
 #include "service.h"
 
+/*
+ * The service condition name is constructed from the 'svc/' prefix, the
+ * dirname of the svc->cmd, e.g., '/usr/bin/teamd' => 'usr/bin', and the
+ * service's pid:filename, including an optional subdirectory, without
+ * the .pid extension.
+ *
+ * The following example uses the team (aggregate) service:
+ *
+ *    service pid:!/run/teamd/a1.pid /usr/bin/teamd -f /etc/teamd/a1.conf
+ *
+ * => 'svc/' + 'usr/bin' + 'teamd/a1' => svc/usr/bin/teamd/a1
+ *
+ * The next example uses the dbus-daemon:
+ *
+ *    service pid:!/run/dbus/pid /usr/bin/dbus-daemon
+ *
+ * => 'svc/' + 'usr/bin' + 'dbus' => svc/usr/bin/dbus
+ *
+ * The last example uses lxc-start to start container foo:
+ *
+ *    service pid:!/run/lxc/foo.pid lxc-start -n foo -F -p /run/lxc/foo.pid -- Container foo
+ *
+ * => 'svc/' + '' + 'lxc/foo' => svc/lxc/foo
+ */
+char *mkcond(svc_t *svc, char *buf, size_t len)
+{
+	char path[256];
+	char *pidfile;
+	char *ptr;
+
+	strlcpy(path, svc->cmd, sizeof(path));
+	ptr = rindex(path, '/');
+	if (ptr)
+		*ptr = 0;
+	else
+		path[0] = 0;
+
+	pidfile = pid_file(svc);
+	ptr = strstr(pidfile, "run");
+	if (ptr)
+		ptr += 3;
+	else
+		ptr = rindex(pidfile, '/');
+
+	snprintf(buf, len, "svc%s%s%s", path[0] != 0 && path[0] != '/' ? "/" : "", path, ptr);
+	_d("Composed condition from cmd %s (path %s) and pidfile %s => %s", svc->cmd, path, ptr, buf);
+
+	/* Case: /var/run/dbus/pid */
+	ptr = strstr(buf, "/pid");
+	if (ptr && !strcmp(ptr, "/pid"))
+		*ptr = 0;
+
+	/* Case /var/run/teamd/a1.pid */
+	ptr = strstr(buf, ".pid");
+	if (ptr && !strcmp(ptr, ".pid"))
+		*ptr = 0;
+
+	_d("Creating condition => %s", buf);
+
+	return buf;
+}
+
 static int cond_set_gen(const char *file, unsigned int gen)
 {
 	char *ptr, path[256];

src/cond.c

@@ -29,13 +29,6 @@
 #include "pid.h"
 #include "service.h"
 
-
-char *mkcond(char *buf, size_t len, char *nm)
-{
-	snprintf(buf, len, "svc%s%s", nm[0] != '/' ? "/" : "", nm);
-	return buf;
-}
-
 const char *condstr(enum cond_state s)
 {
 	static const char *strs[] = {

src/cond.h

@@ -2,6 +2,7 @@
 #define FINIT_COND_H_
 
 #include <paths.h>
+#include <svc.h>
 
 #define COND_DIR      "finit/cond"
 #define COND_PATH     _PATH_VARRUN COND_DIR
@@ -13,7 +14,7 @@ typedef enum cond_state {
 	COND_ON
 } cond_state_t;
 
-char           *mkcond       (char *buf, size_t len, char *nm);
+char           *mkcond       (svc_t *svc, char *buf, size_t len);
 const char     *condstr      (enum cond_state s);
 const char     *cond_path    (const char *name);
 unsigned int    cond_get_gen (const char *path);

src/service.c

@@ -1353,7 +1353,7 @@ restart:
 			if (!svc_is_changed(svc)) {
 				char name[MAX_COND_LEN];
 
-				mkcond(name, sizeof(name), svc->cmd);
+				mkcond(svc, name, sizeof(name));
 				cond_set_path(cond_path(name), COND_ON);
 			}
 			break;

src/svc.c

@@ -60,7 +60,7 @@ static void svc_gc(void *arg)
 			continue;
 
 		TAILQ_REMOVE(&gc_list, svc, link);
-		cond_clear(mkcond(cond, sizeof(cond), svc->cmd));
+		cond_clear(mkcond(svc, cond, sizeof(cond)));
 		free(svc);
 	}