service: kill stubborn services, wait before restart

Add support for registering callbacks to be called after a given
timeout.

Use this to implement forceful termination of services that refuse to
terminate upon receiving a SIGTERM. If the service PID is not collected
within 3 seconds, send a SIGKILL to it.

Also, rework restarting of crashing services. When a service crashes for
the first time, try to restart it immediately. If it crashes again, wait
a few seconds before trying again. Just as before, if it continues to
crash, block the service.

finit.h

@@ -34,6 +34,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 #include <lite/lite.h>
+#include <uev/uev.h>
 
 #define CMD_SIZE                256
 #define LINE_SIZE               1024
@@ -90,6 +91,8 @@ extern char  *hostname;
 extern char  *username;
 extern char  *runparts;
 extern char  *console;
+extern uev_ctx_t *ctx;
+
 extern char  *__progname;
 
 #endif /* FINIT_H_ */

helpers.c

@@ -127,6 +127,7 @@ void print(int action, const char *fmt, ...)
 	va_list ap;
 	const char success[] = " \e[1m[ OK ]\e[0m\n";
 	const char failure[] = " \e[7m[FAIL]\e[0m\n";
+	const char warning[] = " \e[7m[WARN]\e[0m\n";
 	const char pending[] = " \e[1m[ \\/ ]\e[0m\n";
 	const char dots[] = " .....................................................................";
 
@@ -158,6 +159,10 @@ void print(int action, const char *fmt, ...)
 		write(STDERR_FILENO, failure, sizeof(failure));
 		break;
 
+	case 2:
+		write(STDERR_FILENO, warning, sizeof(warning));
+		break;
+
 	default:
 		write(STDERR_FILENO, pending, sizeof(pending));
 		break;

service.c

@@ -67,6 +67,63 @@ int service_enabled(svc_t *svc)
 	return 1;
 }
 
+/**
+ * service_timeout_cb - libuev callback wrapper for service timeouts
+ *
+ * Calls the callback registered with the call to
+ * service_timeout_after().
+ */
+static void service_timeout_cb(uev_t *w, void *_svc, int events)
+{
+	svc_t *svc = _svc;
+
+	(void)(w);
+	(void)(events);
+	svc->timer_cb(svc);
+}
+
+/**
+ * service_timeout_after - Call a function after some time has elapsed
+ * @param svc      Service to use as argument to the callback
+ * @param timeout  Timeout, in milliseconds
+ * @param cb       Callback function
+ *
+ * After @param timeout milliseconds has elapsed, calls @param cb with
+ * @param svc as the argument.
+ *
+ * @return 0 on success, non-zero on error.
+ */
+static int service_timeout_after(svc_t *svc, int timeout,
+				 void (*cb)(svc_t *svc))
+{
+	if (svc->timer_cb)
+		return -EBUSY;
+
+	svc->timer_cb = cb;
+	return uev_timer_init(ctx, &svc->timer, service_timeout_cb,
+			      svc, timeout, 0);
+}
+
+/**
+ * service_timeout_cancel - Cancel timeout associated with service
+ * @param svc      Service whose timeout to cancel
+ *
+ * If a timeout is associated with @param svc, cancel it.
+ *
+ * @return 0 on success, non-zero on error.
+ */
+static int service_timeout_cancel(svc_t *svc)
+{
+	int err;
+
+	if (!svc->timer_cb)
+		return 0;
+
+	err = uev_timer_stop(&svc->timer);
+	svc->timer_cb = NULL;
+	return err;
+}
+
 /**
  * service_stop_is_done - Have all stopped services been collected?
  *
@@ -293,6 +350,27 @@ static int service_start(svc_t *svc)
 	return 0;
 }
 
+/**
+ * service_kill - Forcefully terminate a service
+ * @param svc  Service to kill
+ *
+ * Called when a service refuses to terminate gracefully.
+ */
+static void service_kill(svc_t *svc)
+{
+	service_timeout_cancel(svc);
+
+	if (runlevel != 1 && !silent)
+		print_desc("Killing ", svc->desc);
+
+	_d("Sending SIGKILL to pid:%d name:%s", svc->pid, pid_get_name(svc->pid, NULL, 0));
+	kill(svc->pid, SIGKILL);
+
+	/* Let SIGKILLs stand out, show result as [WARN] */
+	if (runlevel != 1 && !silent)
+		print(2, NULL);
+}
+
 /**
  * service_stop - Stop service
  * @svc: Service to stop
@@ -697,11 +775,52 @@ void service_monitor(pid_t lost)
 	sm_step(&sm);
 }
 
+static void service_retry(svc_t *svc)
+{
+	int *restart_counter = (int *)&svc->restart_counter;
+	int timeout;
+	service_timeout_cancel(svc);
+
+	if (svc->state != SVC_HALTED_STATE ||
+	    svc->block != SVC_BLOCK_RESTARTING) {
+		_d("%s not crashing anymore", svc->desc);
+		*restart_counter = 0;
+		return;
+	}
+
+	if (*restart_counter >= RESPAWN_MAX) {
+		_e("%s keeps crashing, not restarting", svc->desc);
+		svc->block = SVC_BLOCK_CRASHING;
+		*restart_counter = 0;
+		service_step(svc);
+		return;
+	}
+
+	(*restart_counter)++;
+
+	_d("%s crashed, trying to start it again, attempt %d",
+	   svc->desc, *restart_counter);
+
+	svc->block = SVC_BLOCK_NONE;
+	service_step(svc);
+
+	/* Wait 2s for the first 5 respawns, then back off to 5s */
+	timeout = ((*restart_counter) <= (RESPAWN_MAX / 2)) ? 2000 : 5000;
+	service_timeout_after(svc, timeout, service_retry);
+	return;
+}
+
 static void svc_set_state(svc_t *svc, svc_state_t new)
 {
 	svc_state_t *state = (svc_state_t *)&svc->state;
 
 	*state = new;
+
+	/* if the PID isn't collected within 3s, kill it! */
+	if (*state == SVC_STOPPING_STATE) {
+		service_timeout_cancel(svc);
+		service_timeout_after(svc, 3000, service_kill);
+	}
 }
 
 void service_step(svc_t *svc)
@@ -725,7 +844,6 @@ restart:
 
 	switch(svc->state) {
 	case SVC_HALTED_STATE:
-		*restart_counter = 0;
 		if (enabled)
 			svc_set_state(svc, SVC_READY_STATE);
 		break;
@@ -747,6 +865,9 @@ restart:
 
 	case SVC_STOPPING_STATE:
 		if (!svc->pid) {
+			/* PID was collected normally, no need to kill it */
+			service_timeout_cancel(svc);
+
 			switch (svc->type) {
 			case SVC_TYPE_SERVICE:
 			case SVC_TYPE_INETD:
@@ -770,13 +891,6 @@ restart:
 		if (!enabled) {
 			svc_set_state(svc, SVC_HALTED_STATE);
 		} else if (cond_get_agg(svc->cond) == COND_ON) {
-			if (*restart_counter >= RESPAWN_MAX) {
-				_e("%s keeps crashing, not restarting", svc->desc);
-				svc->block = SVC_BLOCK_CRASHING;
-				svc_set_state(svc, SVC_HALTED_STATE);
-				break;
-			}
-
 			/* wait until all processes has been stopped before continuing... */
 			if (sm_is_in_teardown(&sm))
 				break;
@@ -818,10 +932,13 @@ restart:
 		}
 
 		if (!svc->pid && !svc_is_inetd(svc)) {
-			(*restart_counter)++;
-			/* TODO: There should be an async wait here
-			 * before moving back to READY */
-			svc_set_state(svc, SVC_READY_STATE);
+			svc->block = SVC_BLOCK_RESTARTING;
+			svc_set_state(svc, SVC_HALTED_STATE);
+
+			/* Restart directly after the first crash,
+			 * then retry after 2s. */
+			_d("delayed restart of %s", svc->desc);
+			service_timeout_after(svc, 1, service_retry);
 			break;
 		}
 

svc.c

@@ -443,6 +443,9 @@ char *svc_status(svc_t *svc)
 
 		case SVC_BLOCK_INETD_BUSY:
 			return "busy";
+
+		case SVC_BLOCK_RESTARTING:
+			return "restart";
 		}
 
 	case SVC_DONE_STATE:

svc.h

@@ -65,6 +65,7 @@ typedef enum {
 	SVC_BLOCK_CRASHING,
 	SVC_BLOCK_USER,
 	SVC_BLOCK_INETD_BUSY,
+	SVC_BLOCK_RESTARTING,
 } svc_block_t;
 
 #define FINIT_SHM_ID     0x494E4954  /* "INIT", see ascii(7) */
@@ -123,6 +124,11 @@ typedef struct svc {
 	int	       dynamic_stop;
 	int	       private;
 	svc_cmd_t    (*cb)(struct svc *svc, int event, void *event_arg);
+
+	/* Used to forcefully kill services that won't shutdown on
+	 * termination and to delay restarts of crashing services. */
+	uev_t          timer;
+	void           (*timer_cb)(struct svc *svc);
 } svc_t;
 
 typedef struct svc_map svc_map_t;