Wait for post:script and cleanup:script before returning OK

This change introduces two new states for the big Finit state machine:
runlevel-clean and reload-clean.  Here Finit now waits for any post or
cleanup script to finish before returning OK to the initctl command.

Additionally, the service state machine has been updated to ensure a
run/task/sysv/service calls any post or cleanup script before they are
removed.  A new 'dead' state for svc_t is introduced which any removed
svc_t ends up in now instead of becing collected from 'halted' state.

Signed-off-by: Joachim Wiberg <troglobit@gmail.com>

Add cleanup:script support, runs at service removal

Signed-off-by: Joachim Wiberg <troglobit@gmail.com>

doc/svc-machine.svg

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<!-- Do not edit this file with editors other than diagrams.net -->
+<!-- Do not edit this file with editors other than draw.io -->
 <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<svg xmlns="http://www.w3.org/2000/svg" style="background-color: rgb(255, 255, 255);" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" width="732px" height="888px" viewBox="-0.5 -0.5 732 888" content="&lt;mxfile host=&quot;app.diagrams.net&quot; modified=&quot;2022-08-23T13:29:07.417Z&quot; agent=&quot;5.0 (X11)&quot; version=&quot;20.2.4&quot; etag=&quot;2-w-AmAkF4Yj3b3JyTHu&quot; type=&quot;google&quot;&gt;&lt;diagram id=&quot;2QNQ_rVyMpIu31uE8ppT&quot;&gt;7V1Zc6O4Gv01rpl+iEuA2B6dpXu6KpNJJenq248EZJvbGPkCznIf5rePxGaEZAwYMOnBU5M2QggkHc63Sp4pV5u3L4G1Xf+JHeTNZOC8zZTrmSzLQDXIP7TkPSmRgAmTklXgOkkZ2Bc8uv9HWcW0dOc6KEzLkqIIYy9yt2yhjX0f2RFTZgUBfmWrLbHnMAVba4W4gkfb8vjS764TrZNSQ9b35X8gd7XO7ixpZnJmY2WV056Ea8vBr4Ui5WamXAUYR8m3zdsV8ujosePy+cDZ/MEC5Ed1LpCTC14sb5f2LX2u6D3rLHJI39NDHERrvMK+5d3sSy8DvPMdRFsE5Cj8iSKb9lIiB/sLbjHepoX/RVH0ns6qtYswKVpHGy89u8R+9NnauB5FxyKw125E5jAkD0UqkzFFQVrpEe8Cm7axjiIy8bKqLMgf0lX6h1YI5yuMVx6ytm44t/EmPmGHcdXPy+QW5CtzE1W+LN8meVAJkuNkcOiIMOMdZk9CDlMkR1awQukU6PysSPlck7cE4Q2KAvIsIECeFbkvbOtWitZVXm8/oeRLOqfi+VVOmk6pg+kkXQ7e/0OxMVezwx8pVOKD6zfm6D09+vVgIJnd4yC+dBEE1nuhwha7Pu1E3vI9LSAVUgaGMpgb6cOkFCwbGssTpUtkDR65hHxJnmOPy7xDtaAKBVSkeVE6/AyItf/tcHbiIownhsw5kLTt2/4k+bZK/41beS4XpM06VmRdJC0FNtN8AijaNAHTASjFQMphxIJoD6HCQzXvBlhaNnuB+HXgev749Nf9/de7L1mH006Ux4EUc2PDVS2xCMsRr/RpHrfJY74SoV9idNfzrrCHg/haZWnYyLYptUQB/okKZ54NFaqcDElxgYIIvR0UcwdepPQCReMQL5kJfMHrXoJLmVheF6S3Bk6nYbVbMdsJL7+5UULLspoe/iic2rMyPchIuS2Z1yDMTLcrMmaDie6XMRWTw4+mmk0Zs3TJyYypnV1XK+BB/mWke6s3owbCdYFKAM6nGuqTvO1Y3v6xuH26uZ6krUDaQmVAaWuM0KhlJ8VRkeFA0aQY8rOiaQwLjVY8K30ZNAKLRSojSjcNtpXkydILS2BpJlfN8eGnMLM6M7XSkandw0gv4ghU4qg5hbRE0Bk9I9njTPKvM/n3ffH1aYTmpkr/E6FXiz/lt5M+SKFS8ulNMKoaHE4wShI3smczO4tklIu4ruhoT5eMGNxLxRMEocLTGBxKDkoCAAGtlhxsavKSO5nQLHyYu5qaXmy84dWZ6dqVISyNMIyRAV1igK5XI/0QbvVjwG2uX7aEunROka1MIrtjkX2/+PY4OpPVsZCxFBK8ZhvoedmDySqpHLGaij6gZIajIawmBmdbElHOSCIj9MW3UHDyyTJNVsJABVZOGT24R4FLBo56P2tPYw+8f8DWN8iraBxTHbpWd6AhNVJwdF07qX6lCiR4Oi6YIAFosiORTFcXDhBJG68+xRoOck19KneTFC9roE8dl0gN3qHRUOHhEIDjvhzQhSYVq6GK9fDt7q4jr0hezExPl5rXcbuh+2AB59odVvMyzi/+P0QwtFut0OSp8JyWpcjlP9HeKbR3dXuzuPt2PzLTcrlcymLV2tGeNbUPgpMBp7spmcNqCILLU41H4PRtQCKjJsVC2gtRogqq5QWYAyAfUS7joxY2WJ7rPkQMtqkBJch30nW50iZSDKODSwygl16HKktKdE9TZ8eiu0hyNsqTXOkuqfXmaZIqQqmiGQPm2GQegQKytziM4skujSzpZMQOHztMPvZRaUzTIstzVz45tMmgxERMh8y1LW+Rnti4jhPLK9F8VfgfxixcugAHn64qazw4oAAbchfY4OM0f19NsBgjLEyF10R7gwW/wuNigsX5YWGW/O0lhEDeGdMbQkSxmcsJI+fHiIg6jAGpgw9IXNxMsDg7LKBKnbdg/4EMRFQoDQcRUTCDcgewfId2KqCWSGgH7jaakHN25BB9dG6qrP2iGAL7Re4JLvwiAcoogMytDCa1ZAQIUaHO5hCW1BKVA4vWF7WIogOTWjICjIjWhejD6auKaA1B2dfuOwu65QidXc8KQ9eefZzgX8s0aoGzi3Nst4v8FSZVFUxqVnZqmjVgTSJDZyAGtVIO0oHMG65dlc8yVI16+UwtnM8K73wOSeuT9nN+3pKAVJUfDyG/KKQ3DuMduUS4TcbV+UGiQJNjiwENKuVwtn2cfgSyed3HaDy0jPiIDVWrL8j/N7717JGZPJDG1Kbpq7TpK+w7buRi//fwU5ft36ft338lVQHBNoqFi+W85wG44Fh6Vpv7PqT3fSC3cn1Egdjn7RLbOL0nObjIzOQOb5EFh5J7kKPSTY737CghVXBP/FD/DubxrGfkXeLAQUFpMELSP9dfkQK4P3qKczjAvuA2Hqq4ZJvlKDzmVwKO+GdEDY4/3bCeZlR5o2XhPgLGPFsEt2a23dM7YMFpW6oD3Wgdwb/+6+5mbAH88awRVmWpwu2hSLwC0FuMX1F5a3ZKjRVYx8122+gppVaw42BGXmdPDoOSUfb2ymr1joOiS45sUqgaVa9O5X6F/O2VwykUTc3+eEVT6T3O3Jc9mP18rCzY+eSqyAp/Tmbd2c06aHI+yyGTc7K1FROpD7neQRGw82ArKmWdz0A35ZZcJnFOCZh1rgcu46N0G8vfWaT+4h1RxMTB3bIF6fpuZEe0od8D9Glyeo6E+HS5LNEVQbAm8Y32RH6H1/lMKx473Hd48dDVRlD/kiWP2SKGQew6WCds+UtukFhHUgtWJg64+QFvdmhl8VpbVCs8zMouse5ENeSjjWzKFXCo+JzE8LnFsKwRMSwzsNCzuDnLPkKvahdiGLbhm1/Z6rB3wUsuv/ryD/Ww08SB9IlS5pbBos2E9eyFpk4mSWeVSxNUu4ukLKu4bf20H539AAYfcN0bMmRKsCUwdfiSx69f6A8/0Ld5nQQsl+5qF9DwjQzwjnbIXVIcBNiOI4sgXOOdR9oGz7S261Ppt3bpGYJun9ZdIR8FFg2w1iDwHrma8z6M+aXuNUxm6FUJJLpoKaCu8GwugdIuyq3onI+RtQFunh+QZmw/3Cyuf1BUxph9tdwowfAyztG1ZvHveyVh/5lMMwHQfDWPr8bRmtolGcTnE2jHAdrqlF6oKkOCll9rlCR2TIsERqOqKrpZ9hhJQJM5lBhqT3pqx/ugDWYbMyDpYO/4Bj+E4ljhOu9sDbVUuCnaYPa2YkIeYVlKZGODm4buy41J9X6moPkvEgnupVbvQUhXax255nQltuaecjTF7Wh2WV5kbSgR+s/hNn6DZvTlvrw/rFQ0Th2bmLxnJjeVuWKWXw0+y1kVMXnJXGzF5G12eZs8Did6HPram4hnNk1hPQ5lRq+XbdF4j3xY8gjIRxJUsn2/WtbPO9IZW/PxsO9rRB0AIQpeXJv6A0JCbJZHUftbrCD/Rt9YeoKwnhe7CgpJ0SD39iY2XJS0hjZulFfN+Tym6vDFzgtSto7tusSsA+SdIn8Cd7WKLbrMsitmtHxOnzWswfOTmTeEmacStb3CNyEy82Se940OrDy1RqiLpcIjy/U+4Hz3GCzLPD+jWbmnmlUeBq2sStTW7E25Qr6oZikxsCPxUt2Z7K6HFxvK9a8+WZKofPjv7tvtbdFJF6frBFQCEHGCy2fm8zO76T6USv7BF3lACMrmgJbFJItiIQuJMEzRfL9fchhgmqKyBzeZ/fWf2EG0xj8=&lt;/diagram&gt;&lt;/mxfile&gt;"><defs><style type="text/css">@font-face { font-family: "Architects Daughter"; src: url("data:application/x-font-ttf;charset=utf-8;base64,AAEAAAAMAIAAAwBAT1MvMoWwV7IAAGWAAAAAYGNtYXDPBugkAABl4AAAARxnYXNwAAAAEAAAkvgAAAAIZ2x5Zv2v7C0AAADMAABb/GhlYWQC0dksAABfqAAAADZoaGVhCEMEFgAAZVwAAAAkaG10eN2yH1kAAF/gAAAFfGxvY2GKxKMcAABc6AAAAsBtYXhwAa4AzAAAXMgAAAAgbmFtZUarZSoAAGcEAAAlEHBvc3QANIw5AACMFAAABuNwcmVwaAaMhQAAZvwAAAAHAAIAFv/5AHwDhQAKABMAADcyFhUGBwYiJicmEwYQByM0NxI1PyoTCyUGDhUFC1sGD0MIDkMQCSgIAQgKGQNhGv2rLWebAS1pAAIANgHqANcC+wALABcAABM3JzQ3FxUXDgEjIic3JzQ3FxUXDgEjIpIBASQREAMZBSRcAQEkEBEEGAUkAlIcKVMREMciAw1XHSpTERDHIgQNAAACADgAxgKdApgAQwBQAAATNDMyFjM1Ii4BJyY0Nz4BNzYzNTQ2MzIWHQE3NTMXMxQjIiYjFTI2MxUUBw4BBw4BBwYjNSYiDgEeAQYHIyInJicuASQmIgYHBh0BMjc2NTRWKhIlAwLine truncated
+<svg xmlns="http://www.w3.org/2000/svg" style="background: transparent; background-color: transparent; color-scheme: light dark;" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" width="732px" height="1006px" viewBox="-0.5 -0.5 732 1006" content="&lt;mxfile host=&quot;app.diagrams.net&quot; agent=&quot;Mozilla/5.0 (X11; Linux x86_64; rv:135.0) Gecko/20100101 Firefox/135.0&quot; version=&quot;26.0.15&quot; scale=&quot;1&quot; border=&quot;0&quot;&gt;&#xA;  &lt;diagram name=&quot;Page-1&quot; id=&quot;FNw5HM3Bq2xN372-o7V3&quot;&gt;&#xA;    &lt;mxGraphModel dx=&quot;2049&quot; dy=&quot;1094&quot; grid=&quot;0&quot; gridSize=&quot;10&quot; guides=&quot;1&quot; tooltips=&quot;1&quot; connect=&quot;1&quot; arrows=&quot;1&quot; fold=&quot;1&quot; page=&quot;1&quot; pageScale=&quot;1&quot; pageWidth=&quot;827&quot; pageHeight=&quot;1169&quot; math=&quot;0&quot; shadow=&quot;0&quot;&gt;&#xA;      &lt;root&gt;&#xA;        &lt;mxCell id=&quot;0&quot; /&gt;&#xA;        &lt;mxCell id=&quot;1&quot; parent=&quot;0&quot; /&gt;&#xA;        &lt;mxCell id=&quot;2&quot; value=&quot;&quot; style=&quot;edgeStyle=orthogonalEdgeStyle;rounded=0;sketch=1;orthogonalLoop=1;jettySize=auto;html=1;fontFamily=Architects Daughter;fontSource=https%3A%2F%2Ffonts.googleapis.com%2Fcss%3Ffamily%3DArchitects%2BDaughter;fontSize=14;&quot; edge=&quot;1&quot; source=&quot;4&quot; target=&quot;7&quot; parent=&quot;1&quot;&gt;&#xA;          &lt;mxGeometry relative=&quot;1&quot; as=&quot;geometry&quot; /&gt;&#xA;        &lt;/mxCell&gt;&#xA;        &lt;mxCell id=&quot;3&quot; style=&quot;edgeStyle=orthogonalEdgeStyle;rounded=1;sketch=1;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;fontFamily=Architects Daughter;fontSource=https%3A%2F%2Ffonts.googleapis.com%2Fcss%3Ffamily%3DArchitects%2BDaughter;fontSize=14;&quot; edge=&quot;1&quot; source=&quot;4&quot; target=&quot;19&quot; parent=&quot;1&quot;&gt;&#xA;          &lt;mxGeometry relative=&quot;1&quot; as=&quot;geometry&quot;&gt;&#xA;            &lt;Array as=&quot;points&quot;&gt;&#xA;              &lt;mxPoint x=&quot;421&quot; y=&quot;328&quot; /&gt;&#xA;              &lt;mxPoint x=&quot;265&quot; y=&quot;328&quot; /&gt;&#xA;              &lt;mxPoint x=&quot;183&quot; y=&quot;329&quot; /&gt;&#xA;            &lt;/Array&gt;&#xA;          &lt;/mxGeometry&gt;&#xA;        &lt;/mxCell&gt;&#xA;        &lt;mxCell id=&quot;4&quot; value=&quot;&amp;lt;font style=&amp;quot;font-size: 16px&amp;quot;&amp;gt;&amp;lt;b&amp;gt;&amp;lt;font data-font-src=&amp;quot;https://fonts.googleapis.com/css?family=Architects+Daughter&amp;quot; style=&amp;quot;font-size: 16px&amp;quot; face=&amp;quot;Architects Daughter&amp;quot;&amp;gt;STOPPING&amp;lt;/font&amp;gt;&amp;lt;/b&amp;gt;&amp;lt;/font&amp;gt;&quot; style=&quot;rounded=1;whiteSpace=wrap;html=1;fillColor=#f8cecc;strokeColor=#b85450;sketch=1;&quot; vertex=&quot;1&quot; parent=&quot;1&quot;&gt;&#xA;          &lt;mxGeometry x=&quot;360.89&quot; y=&quot;226&quot; width=&quot;120&quot; height=&quot;60&quot; as=&quot;geometry&quot; /&gt;&#xA;        &lt;/mxCell&gt;&#xA;        &lt;mxCell id=&quot;5&quot; value=&quot;&quot; style=&quot;edgeStyle=orthogonalEdgeStyle;rounded=1;sketch=1;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.25;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;&quot; edge=&quot;1&quot; source=&quot;10&quot; target=&quot;21&quot; parent=&quot;1&quot;&gt;&#xA;          &lt;mxGeometry relative=&quot;1&quot; as=&quot;geometry&quot;&gt;&#xA;            &lt;Array as=&quot;points&quot;&gt;&#xA;              &lt;mxPoint x=&quot;390.89&quot; y=&quot;729&quot; /&gt;&#xA;              &lt;mxPoint x=&quot;264.89&quot; y=&quot;729&quot; /&gt;&#xA;            &lt;/Array&gt;&#xA;          &lt;/mxGeometry&gt;&#xA;        &lt;/mxCell&gt;&#xA;        &lt;mxCell id=&quot;6&quot; style=&quot;edgeStyle=orthogonalEdgeStyle;rounded=0;sketch=1;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.25;entryY=0;entryDx=0;entryDy=0;fontFamily=Architects Daughter;fontSource=https%3A%2F%2Ffonts.googleapis.com%2Fcss%3Ffamily%3DArchitects%2BDaughter;fontSize=14;exitX=0.25;exitY=1;exitDx=0;exitDy=0;&quot; edge=&quot;1&quot; source=&quot;7&quot; target=&quot;10&quot; parent=&quot;1&quot;&gt;&#xA;          &lt;mxGeometry relative=&quot;1&quot; as=&quot;geometry&quot; /&gt;&#xA;        &lt;/mxCell&gt;&#xA;        &lt;mxCell id=&quot;7&quot; value=&quot;&amp;lt;font style=&amp;quot;font-size: 16px&amp;quot;&amp;gt;&amp;lt;b&amp;gt;&amp;lt;font data-font-src=&amp;quot;https://fonts.googleapis.com/css?family=Architects+Daughter&amp;quot; style=&amp;quot;font-size: 16px&amp;quot; face=&amp;quot;Architects Daughter&amp;quot;&amp;gt;HALTED&amp;lt;/font&amp;gt;&amp;lt;/b&amp;gt;&amp;lt;/font&amp;gt;&quot; style=&quot;rounded=1;whiteSpace=wrap;html=1;fillColor=#f8cecc;strokeColor=#b85450;sketch=1;&quot; vertex=&quot;1&quot; parent=&quot;1&quot;&gt;&#xA;          &lt;mxGeometry x=&quot;360.89&quot; y=&quot;526&quot; width=&quot;120&quot; height=&quot;Line truncated

src/service.c

@@ -2483,16 +2483,13 @@ restart:
 	   condstr(cond_get_agg(svc->cond)));
 
 	switch (svc->state) {
-	case SVC_CLEANUP_STATE:
-		if (!svc->pid)
-			svc_set_state(svc, SVC_HALTED_STATE);
-		break;
-
 	case SVC_HALTED_STATE:
 		if (enabled)
 			svc_set_state(svc, SVC_WAITING_STATE);
 		else {
-			if (svc_is_conflict(svc)) {
+			if (svc_is_removed(svc)) {
+				svc_set_state(svc, SVC_DEAD_STATE);
+			} else if (svc_is_conflict(svc)) {
 #if 0
 				logit(svc->nowarn ? LOG_DEBUG : LOG_INFO,
 				      "%s in conflict with %s, checking again ...",
@@ -2504,6 +2501,30 @@ restart:
 		}
 		break;
 
+	case SVC_TEARDOWN_STATE:
+		if (!svc->pid) {
+			dbg("%s: post script done.", svc_ident(svc, NULL, 0));
+			service_timeout_cancel(svc);
+
+			if (svc_is_removed(svc) && svc_has_cleanup(svc)) {
+				svc_set_state(svc, SVC_CLEANUP_STATE);
+				service_cleanup_script(svc);
+			} else
+				svc_set_state(svc, SVC_HALTED_STATE);
+		}
+		break;
+
+	case SVC_CLEANUP_STATE:
+		if (!svc->pid) {
+			dbg("%s: cleanup script done.", svc_ident(svc, NULL, 0));
+			svc_set_state(svc, SVC_HALTED_STATE);
+		}
+		break;
+
+	case SVC_DEAD_STATE:
+		/* End of the line ☠ */
+		break;
+
 	case SVC_DONE_STATE:
 		if (svc_is_changed(svc))
 			svc_set_state(svc, SVC_HALTED_STATE);
@@ -2528,6 +2549,9 @@ restart:
 				if (svc_has_post(svc)) {
 					svc_set_state(svc, SVC_TEARDOWN_STATE);
 					service_post_script(svc);
+				} else if (svc_is_removed(svc) && svc_has_cleanup(svc)) {
+					svc_set_state(svc, SVC_CLEANUP_STATE);
+					service_cleanup_script(svc);
 				} else
 					svc_set_state(svc, SVC_HALTED_STATE);
 				break;
@@ -2546,16 +2570,6 @@ restart:
 		}
 		break;
 
-	case SVC_TEARDOWN_STATE:
-		if (!svc->pid) {
-			if (svc_is_removed(svc) && svc_has_cleanup(svc)) {
-				svc_set_state(svc, SVC_CLEANUP_STATE);
-				service_cleanup_script(svc);
-			} else
-				svc_set_state(svc, SVC_HALTED_STATE);
-		}
-		break;
-
 	case SVC_SETUP_STATE:
 		if (!svc->pid) {
 			int rc = WEXITSTATUS(svc->status);

src/sm.c

@@ -107,6 +107,9 @@ static char *sm_status(sm_state_t state)
 	case SM_BOOTSTRAP_STATE:
 		return "bootstrap";
 
+	case SM_BOOTSTRAP_WAIT_STATE:
+		return "bootstrap/wait";
+
 	case SM_RUNNING_STATE:
 		return "running";
 
@@ -116,15 +119,20 @@ static char *sm_status(sm_state_t state)
 	case SM_RUNLEVEL_WAIT_STATE:
 		return "runlevel/wait";
 
+	case SM_RUNLEVEL_CLEAN_STATE:
+		return "runlevel/clean";
+
 	case SM_RELOAD_CHANGE_STATE:
 		return "reload/change";
 
 	case SM_RELOAD_WAIT_STATE:
 		return "reload/wait";
 
-	default:
+	case SM_RELOAD_CLEAN_STATE:
-		return "unknown";
+		return "reload/clean";
 	}
+
+	return "unknown";
 }
 
 static char sm_runlevel(int lvl)
@@ -319,6 +327,22 @@ restart:
 		sm->in_teardown = 0;
 		service_step_all(SVC_TYPE_ANY);
 
+		sm->state = SM_RUNLEVEL_CLEAN_STATE;
+		break;
+
+	case SM_RUNLEVEL_CLEAN_STATE:
+		/*
+		 * Wait for post:script or cleanup:script to be collected,
+		 * which moves the svc to HALTED or DEAD state.  We will
+		 * be called by the service_monitor() on collect.
+		 */
+		svc = svc_clean_completed();
+		if (svc) {
+			dbg("Waiting to collect post/cleanup script for %s, cmd %s(%d) ...",
+			    svc_ident(svc, NULL, 0), svc->cmd, svc->pid);
+			break;
+		}
+
 		/* Cleanup stale services */
 		svc_clean_dynamic(service_unregister);
 
@@ -372,6 +396,22 @@ restart:
 		dbg("Starting services after reconf ...");
 		service_step_all(SVC_TYPE_ANY);
 
+		sm->state = SM_RELOAD_CLEAN_STATE;
+		break;
+
+	case SM_RELOAD_CLEAN_STATE:
+		/*
+		 * Wait for post:script or cleanup:script to be collected,
+		 * which moves the svc to HALTED or DEAD state.  We will
+		 * be called by the service_monitor() on collect.
+		 */
+		svc = svc_clean_completed();
+		if (svc) {
+			dbg("Waiting to collect post/cleanup script for %s, cmd %s(%d) ...",
+			    svc_ident(svc, NULL, 0), svc->cmd, svc->pid);
+			break;
+		}
+
 		/* Cleanup stale services */
 		svc_clean_dynamic(service_unregister);
 

src/sm.h

@@ -29,9 +29,11 @@ typedef enum {
 	SM_BOOTSTRAP_WAIT_STATE,  /* Waiting for bootstrap to complete */
 	SM_RUNNING_STATE,         /* Normal state, services running */
 	SM_RUNLEVEL_CHANGE_STATE, /* A runlevel change has occurred */
-	SM_RUNLEVEL_WAIT_STATE,   /* Waiting for all stopped runlevel processes to be halted */
+	SM_RUNLEVEL_WAIT_STATE,   /* Waiting for all stopped processes to halt */
+	SM_RUNLEVEL_CLEAN_STATE,  /* Wait for post:scripts and cleanup:scripts */
 	SM_RELOAD_CHANGE_STATE,   /* A reload event has occurred */
-	SM_RELOAD_WAIT_STATE,     /* Waiting for all stopped reload processes to be halted */
+	SM_RELOAD_WAIT_STATE,     /* Waiting for all stopped processes to halt */
+	SM_RELOAD_CLEAN_STATE,    /* Wait for post:scripts and cleanup:scripts */
 } sm_state_t;
 
 typedef struct sm {

src/svc.c

@@ -348,6 +348,43 @@ svc_t *svc_stop_completed(void)
 	return NULL;
 }
 
+
+/**
+ * svc_clean_completed - Have post: and cleanup: scripts finished?
+ *
+ * Called late in runlevel and reload transitions to check if all post: and
+ * cleanup:scripts have completed.  A removed service should end up in the
+ * DEAD state, while a service with a post:script ends up in HALTED.
+ *
+ * Returns:
+ * %NULL if post: and cleanup: scripts have run, otherwise a pointer to
+ * the first found svc_t waiting for post: or cleanup:script.
+ */
+svc_t *svc_clean_completed(void)
+{
+	svc_t *svc, *iter = NULL;
+
+	for (svc = svc_iterator(&iter, 1); svc; svc = svc_iterator(&iter, 0)) {
+		if (svc_enabled(svc))
+			continue;
+
+		if (svc_is_runtask(svc) && svc->state == SVC_DONE_STATE)
+			continue;
+
+		if (svc_is_removed(svc)) {
+			if (svc->state == SVC_DEAD_STATE)
+				continue; /* any cleanup:script done */
+		} else {
+			if (svc->state == SVC_HALTED_STATE)
+				continue; /* any post:script done */
+		}
+
+		return svc;
+	}
+
+	return NULL;
+}
+
 /**
  * svc_find - Find a service object by its full path name
  * @name: Full path name, e.g., /sbin/syslogd
@@ -580,7 +617,7 @@ void svc_clean_dynamic(void (*cb)(svc_t *))
 	svc_t *svc, *iter = NULL;
 
 	for (svc = svc_iterator(&iter, 1); svc; svc = svc_iterator(&iter, 0)) {
-		if (svc->removed && cb)
+		if (svc_is_removed(svc) && cb)
 			cb(svc);
 	}
 }

src/svc.h

@@ -58,10 +58,11 @@ typedef enum {
 typedef enum {
 	SVC_HALTED_STATE = 0,	/* Not allowed in runlevel, or not enabled. */
 	SVC_DONE_STATE,		/* Task/Run job has been run */
+	SVC_DEAD_STATE,		/* Process is dead and scheduled for removal */
+	SVC_CLEANUP_STATE,	/* Running cleanup: script */
+	SVC_TEARDOWN_STATE,	/* Running post: script */
 	SVC_STOPPING_STATE,	/* Waiting to collect the child process */
 	SVC_SETUP_STATE,	/* Running pre: script */
-	SVC_TEARDOWN_STATE,	/* Running post: script */
-	SVC_CLEANUP_STATE,	/* Running cleanup: script */
 	SVC_PAUSED_STATE,	/* Condition is in flux, process SIGSTOPed */
 	SVC_WAITING_STATE,	/* Enabled but condition not satisfied */
 	SVC_STARTING_STATE,	/* Conditions OK and pre: script done, start */
@@ -247,6 +248,7 @@ void	    svc_foreach	           (int (*cb)(svc_t *));
 void        svc_foreach_type       (int types, int (*cb)(svc_t *));
 
 svc_t	   *svc_stop_completed	   (void);
+svc_t      *svc_clean_completed    (void);
 
 void	    svc_mark		   (svc_t *svc);
 void	    svc_mark_dynamic       (void);
@@ -405,7 +407,7 @@ static inline char *svc_status(svc_t *svc)
 		return "setup";
 
 	case SVC_CLEANUP_STATE:
-		return "setup";
+		return "cleanup";
 
 	case SVC_PAUSED_STATE:
 		return "paused";
@@ -419,6 +421,9 @@ static inline char *svc_status(svc_t *svc)
 	case SVC_RUNNING_STATE:
 		return "running";
 
+	case SVC_DEAD_STATE:
+		return "dead";
+
 	default:
 		return "UNKNOWN";
 	}