Fix #86: Ensure that run/tasks are run at least once per runlevel

This patch fixes the problem with run/task jobs with conditions, run at
runlevel S (bootstrap), not being given enough time to actually start
and run even once.

The new svc counter 'once' is now inspected before transitioning away
from runlevel S to the configured runlevel.  A global (currently hard
coded) timeout of 10 sec is given to all run/tasks.

As a spin-off, this patch starts up the event loop much earlier than
before.  Meaning the initial runlevel change at boot is now done in
a finalize() callback when all run/tasks have completed, or when the
global timeout has expired.  Starting the event loop earlier gives us
the possibility to rely on the pidfile plugin to provide our run/tasks
with the conditions necessary to start.

TODO: What remains is to add similar handling to all runlevel
      transitions, not just the initial one at boot.

Signed-off-by: Joachim Nilsson <troglobit@gmail.com>

src/finit.c

@@ -240,11 +240,64 @@ static void emergency_shell(void)
 #endif /* EMERGENCY_SHELL */
 }
 
+/*
+ * Handle bootstrap transition to configured runlevel, start TTYs
+ *
+ * This is the final stage of bootstrap.  It changes to the default
+ * (configured) runlevel, calls all external start scripts and final
+ * bootstrap hooks before bringing up TTYs.
+ *
+ * We must ensure that all declared `task [S]` and `run [S]` jobs in
+ * finit.conf, or *.conf in finit.d/, run to completion before we
+ * finalize the bootstrap process by calling this function.
+ */
+static void finalize(void)
+{
+	/*
+	 * Start all tasks/services in the configured runlevel
+	 */
+	service_runlevel(cfglevel);
+
+	/* Clean up bootstrap-only tasks/services that never started */
+	svc_prune_bootstrap();
+
+	/* All services/tasks/inetd/etc. in configure runlevel have started */
+	_d("Running svc up hooks ...");
+	plugin_run_hooks(HOOK_SVC_UP);
+	service_step_all(SVC_TYPE_ANY);
+
+	/*
+	 * Run startup scripts in the runparts directory, if any.
+	 */
+	if (runparts && fisdir(runparts)) {
+		_d("Running startup scripts in %s ...", runparts);
+		run_parts(runparts, NULL);
+		service_reload_dynamic();
+	}
+
+	/* Convenient SysV compat for when you just don't care ... */
+	if (!access(FINIT_RC_LOCAL, X_OK)) {
+		run_interactive(FINIT_RC_LOCAL, "Calling %s", FINIT_RC_LOCAL);
+		service_reload_dynamic();
+	}
+
+	/* Hooks that should run at the very end */
+	plugin_run_hooks(HOOK_SYSTEM_UP);
+	service_step_all(SVC_TYPE_ANY);
+
+	/* Enable silent mode before starting TTYs */
+	log_silent();
+
+	/* Delayed start of TTYs at bootstrap */
+	tty_runlevel();
+}
+
 int main(int argc, char* argv[])
 {
 	char *path;
 	char cmd[256];
 	int udev = 0;
+	uev_t timer;	       /* Bootstrap timer, on timeout call finalize() */
 	uev_ctx_t loop;
 
 	/*
@@ -449,45 +502,15 @@ int main(int argc, char* argv[])
 	/* Hooks that rely on loopback, or basic networking being up. */
 	plugin_run_hooks(HOOK_NETWORK_UP);
 
-	/*
-	 * Start all tasks/services in the configured runlevel
-	 */
-	service_runlevel(cfglevel);
-
-	/* Clean up bootstrap-only tasks/services that never started */
-	svc_prune_bootstrap();
-
-	/* All services/tasks/inetd/etc. in configure runlevel have started */
-	_d("Running svc up hooks ...");
-	plugin_run_hooks(HOOK_SVC_UP);
-
-	/*
-	 * Run startup scripts in the runparts directory, if any.
-	 */
-	if (runparts && fisdir(runparts)) {
-		_d("Running startup scripts in %s ...", runparts);
-		run_parts(runparts, NULL);
-		service_reload_dynamic();
-	}
-
-	/* Convenient SysV compat for when you just don't care ... */
-	if (!access(FINIT_RC_LOCAL, X_OK)) {
-		run_interactive(FINIT_RC_LOCAL, "Calling %s", FINIT_RC_LOCAL);
-		service_reload_dynamic();
-	}
-
-	/* Hooks that should run at the very end */
-	plugin_run_hooks(HOOK_SYSTEM_UP);
-
-	/* Enable silent mode before starting TTYs */
-	log_silent();
-
-	/* Delayed start of TTYs at bootstrap */
-	tty_runlevel();
-
 	/* Start new initctl API responder */
 	api_init(&loop);
 
+	/*
+	 * Wait for all SVC_TYPE_RUNTASK to have completed their work in
+	 * [S], or timeout, before calling finalize()
+	 */
+	uev_timer_init(&loop, &timer, service_bootstrap_cb, finalize, 1000, 1000);
+
 	/*
 	 * Enter main loop to monior /dev/initctl and services
 	 */

src/service.c

@@ -1030,6 +1030,67 @@ void service_runtask_clean(void)
 	}
 }
 
+/**
+ * service_runtask_completed - Have run/task completed in current runlevel
+ * @skip: Hardcoded list of conditions to skip
+ *
+ * This function checks if all run/task have run once in the current
+ * runlevel.  E.g., at bootstrap we must wait for these scripts or
+ * programs to complete their run before switching to the configured
+ * runlevel.
+ *
+ * Setting @skip means skipping tasks with %HOOK_SVC_UP, %HOOK_SYSTEM_UP
+ * set in their condition mask.  These tasks cannot run until finalize()
+ *
+ * Returns:
+ * %TRUE(1) or %FALSE(0)
+ */
+int service_runtask_completed(int skip)
+{
+	svc_t *svc;
+
+	for (svc = svc_iterator(1); svc; svc = svc_iterator(0)) {
+		if (!svc_is_runtask(svc))
+			continue;
+
+		if (!svc_enabled(svc))
+			continue;
+
+		if (strstr(svc->cond, plugin_hook_str(HOOK_SVC_UP)) ||
+		    strstr(svc->cond, plugin_hook_str(HOOK_SYSTEM_UP))) {
+			_d("Skipping %s(%s), post-strap hook", svc->desc, svc->cmd);
+			continue;
+		}
+
+		if (!svc->once) {
+			_d("%s has not yet completed ...", svc->cmd);
+			return 0;
+		}
+		_d("%s has completed ...", svc->cmd);
+	}
+
+	return 1;
+}
+
+void service_bootstrap_cb(uev_t *w, void *arg, int events)
+{
+	static int cnt = 10;
+	void (*finalize)(void) = arg;
+
+	_d("Checking completion of all run/task ... %d", cnt);
+	if (cnt-- > 0 && !service_runtask_completed(!runlevel))
+		return;
+
+	if (cnt > 0)
+		_d("All run/task have completed, resuming bootstrap.");
+	else
+		_d("Timeout, resuming bootstrap.");
+	cnt = 10;
+	uev_timer_stop(w);
+	finalize();
+}
+
+
 /**
  * Local Variables:
  *  indent-tabs-mode: t

src/service.h

@@ -37,6 +37,8 @@ void      service_reload_dynamic (void);
 void      service_step           (svc_t *svc);
 void      service_step_all       (int types);
 
+void      service_bootstrap_cb   (uev_t *w, void *arg, int events);
+
 #endif	/* FINIT_SERVICE_H_ */
 
 /**