cgroup: refactor for cgroups v2, much cleaner + unified hieararchy

Since the cgroups support in Finit is not yet officially released, we
decided to change the back-end to use cgroups v2 instead of the aging
and rather clumsy cgroups v1.  Even this initial refactor is a lot
easier to read and understand, more can still be done since there's
a lot of concepts, data values and the ilk that can now be shared
between different controllers.

Still ToDo: inotify for cgroup.events to clean up leaf nodes, which
            in cgroups v1 was handled by cgreaper.sh.  This is fixed
	    in the next commit in the series.

Signed-off-by: Joachim Wiberg <troglobit@gmail.com>

src/Makefile.am

@@ -1,4 +1,4 @@
-EXTRA_DIST         = cgreaper.sh rescue.conf sample.conf
+EXTRA_DIST         = rescue.conf sample.conf
 AM_CPPFLAGS        = -D_XOPEN_SOURCE=600 -D_BSD_SOURCE -D_GNU_SOURCE -D_DEFAULT_SOURCE
 if STATIC
 AM_CPPFLAGS       += -DENABLE_STATIC
@@ -8,7 +8,6 @@ AM_LDFLAGS         = -export-dynamic
 endif
 
 sbin_PROGRAMS      = finit initctl
-pkglibexec_SCRIPTS = cgreaper.sh
 
 finit_pkglibdir      = $(pkglibdir)
 finit_pkglib_SCRIPTS = rescue.conf sample.conf

src/cgreaper.sh

@@ -1,11 +0,0 @@
-#!/bin/sh
-# Called by kernel when last child in cgroup exists
-
-BASE=/sys/fs/cgroup/finit
-PROC=$(basename "$1")
-DIR="$BASE$1"
-
-#/libexec/finit/logit -p daemon.info -t finit "$PROC stopped, cleaning up control group $DIR"
-rmdir "$DIR"
-
-exit 0

src/cgroup.c

@@ -32,49 +32,64 @@
 #include "log.h"
 #include "util.h"
 
-static char controllers[256] = "none";
+static char controllers[256];
 
-static void group_init(char *path, char *nproc, int share)
+static void group_init(char *path, int leaf, int weight)
 {
 	char file[strlen(path) + 64];
 
-	paste(file, sizeof(file), path, "cgroup.clone_children");
+	if (mkdir(path, 0755) && EEXIST != errno) {
-	echo(file, 0, "1");
+		_pe("Failed creating cgroup %s", path);
+		return;
+	}
 
-	paste(file, sizeof(file), path, "cpuset.cpus");
+	/* enable detected controllers on domain groups */
-	echo(file, 0, nproc);
+	if (!leaf) {
+		paste(file, sizeof(file), path, "cgroup.subtree_control");
+		echo(file, 0, controllers);
+	}
 
-	paste(file, sizeof(file), path, "cpuset.mems");
+	/* set initial group weight */
-	echo(file, 0, "0");
+	if (weight) {
+		paste(file, sizeof(file), path, "cpu.weight");
+		echo(file, 0, "%d", weight);
+	}
+}
 
-	paste(file, sizeof(file), path, "cpu.shares");
+static int cgroup_leaf_init(char *group, char *name, int pid)
-	echo(file, 0, "%d", share);
+{
+	char path[256];
 
-	paste(file, sizeof(file), path, "memory.use_hierarchy");
+	if (pid <= 1) {
-	echo(file, 0, "1");
+		errno = EINVAL;
+		return 1;
+	}
+
+	snprintf(path, sizeof(path), "/sys/fs/cgroup/%s/%s", group, name);
+	group_init(path, 1, 0);
+
+	strlcat(path, "/cgroup.procs", sizeof(path));
+
+	return echo(path, 0, "%d", pid);
+}
+
+int cgroup_user(char *name, int pid)
+{
+	return cgroup_leaf_init("user", name, pid);
+}
+
+int cgroup_service(char *name, int pid)
+{
+	return cgroup_leaf_init("system", name, pid);
 }
 
 static void append_ctrl(char *ctrl)
 {
-	char *wanted[] = {
+	if (controllers[0])
-		"cpu",
+		strlcat(controllers, " ", sizeof(controllers));
-		"cpuacct",
-		"cpuset",
-		"memory"
-	};
-	size_t i;
 
-	for (i = 0; i < NELEMS(wanted); i++) {
+	strlcat(controllers, "+", sizeof(controllers));
-		if (strcmp(wanted[i], ctrl))
+	strlcat(controllers, ctrl, sizeof(controllers));
-			continue;
-
-		if (strcmp(controllers, "none"))
-			strlcat(controllers, ",", sizeof(controllers));
-		else
-			strlcpy(controllers, "", sizeof(controllers));
-
-		strlcat(controllers, ctrl, sizeof(controllers));
-	}
 }
 
 /*
@@ -86,116 +101,40 @@ void cgroup_init(void)
 	char buf[80];
 	FILE *fp;
 
-	fp = fopen("/proc/cgroups", "r");
+	if (mount("none", FINIT_CGPATH, "cgroup2", opts, NULL)) {
-	if (!fp) {
+		_pe("Failed mounting cgroup v2");
-		_d("No cgroup support");
 		return;
 	}
 
-	if (mount("none", "/sys/fs/cgroup", "tmpfs", opts, NULL)) {
+	/* Find available controllers */
-		_d("Failed mounting cgroups");
+	fp = fopen(FINIT_CGPATH "/cgroup.controllers", "r");
-		goto fail;
+	if (!fp) {
+		_pe("Failed opening %s", FINIT_CGPATH "/cgroup.controllers");
+		return;
 	}
 
-	/* Skip first line, header */
+	if (fgets(buf, sizeof(buf), fp)) {
-	(void)fgets(buf, sizeof(buf), fp);
-
-	/* Create and mount traditional cgroups v1 hier */
-	while (fgets(buf, sizeof(buf), fp)) {
 		char *cgroup;
-		char rc[80];
 
-		cgroup = strtok(buf, "\t ");
+		cgroup = strtok(chomp(buf), "\t ");
-		if (!cgroup)
+		while (cgroup) {
-			continue;
+			append_ctrl(cgroup);
-
+			cgroup = strtok(NULL, "\t ");
-		snprintf(rc, sizeof(rc), "/sys/fs/cgroup/%s", cgroup);
+		}
-#if 0
-		if (mkdir(rc, 0755) && EEXIST != errno)
-			continue;
-
-		if (mount("cgroup", rc, "cgroup", opts, cgroup))
-			_d("Failed mounting %s cgroup on %s", cgroup, rc);
-#else
-		symlink("finit", rc);
-		append_ctrl(cgroup);
-#endif
 	}
 
-	/* Finit default cgroups for process monitoring */
+	fclose(fp);
-	if (mkdir(FINIT_CGPATH, 0755) && EEXIST != errno)
-		goto fail;
 
-#if 0
+	/* Enable all controllers */
-	strlcpy(controllers, "none,name=finit", sizeof(controllers));
+	echo(FINIT_CGPATH "/cgroup.subtree_control", 0, controllers);
-#else
-	strlcat(controllers, ",name=finit", sizeof(controllers));
-#endif
-	if (mount("none", FINIT_CGPATH, "cgroup", opts, controllers)) {
-		_pe("Failed mounting Finit cgroup hierarchy");
-		goto fail;
-	}
-
-	/* Set up reaper and enable callbacks */
-	echo(FINIT_CGPATH "/release_agent", 0, FINIT_LIBPATH_ "/cgreaper.sh");
-	echo(FINIT_CGPATH "/notify_on_release", 0, "1");
 
 	/* Default groups, PID 1, services, and user/login processes */
-	if (mkdir(FINIT_CGPATH "/init", 0755) && EEXIST != errno)
+	group_init(FINIT_CGPATH "/init",   1,  100);
-		goto fail;
+	group_init(FINIT_CGPATH "/user",   0,  100);
-	if (mkdir(FINIT_CGPATH "/system", 0755) && EEXIST != errno)
+	group_init(FINIT_CGPATH "/system", 0, 9800);
-		goto fail;
-	if (mkdir(FINIT_CGPATH "/user", 0755) && EEXIST != errno)
-		goto fail;
-
-	/*
-	 * Set up basic limits for our groups.  Finit optimized defaults
-	 * for embedded and server systems include limiting the groups
-	 * init and user to a single CPU with 10% share, the system
-	 * group gets the rest.  Override this from finit.conf
-	 */
-#if 0
-#else
-	snprintf(buf, sizeof(buf), "0-%d", get_nprocs_conf() - 1);
-
-	group_init(FINIT_CGPATH "/init",   "0", 50);
-	group_init(FINIT_CGPATH "/user",   "0", 75);
-	group_init(FINIT_CGPATH "/system", buf, 900);
-#endif
 
 	/* Move ourselves to init */
 	echo(FINIT_CGPATH "/init/cgroup.procs", 0, "1");
-
-fail:
-	fclose(fp);
-}
-
-static int move_pid(char *group, char *name, int pid)
-{
-	char path[256];
-
-	snprintf(path, sizeof(path), "/sys/fs/cgroup/%s/%s", group, name);
-	if (mkdir(path, 0755) && errno != EEXIST)
-		return 1;
-
-	strlcat(path, "/cgroup.procs", sizeof(path));
-
-	return echo(path, 0, "%d", pid);
-}
-
-int cgroup_user(char *name, int pid)
-{
-	return move_pid("finit/user", name, pid);
-}
-
-int cgroup_service(char *name, int pid)
-{
-	if (pid <= 0) {
-		errno = EINVAL;
-		return 1;
-	}
-
-	return move_pid("finit/system", name, pid);
 }
 
 /**

src/cgutil.c

@@ -118,16 +118,18 @@ static uint64_t cgroup_uint64(char *path, char *file)
 	if (!fp)
 		return val;
 
-	if (fgets(buf, sizeof(buf), fp))
+	if (fgets(buf, sizeof(buf), fp)) {
+		chomp(buf);
 		val = strtoull(buf, NULL, 10);
+	}
 	fclose(fp);
 
 	return val;
 }
 
-static int cgroup_shares(char *path)
+static int cgroup_weight(char *path)
 {
-	return (int)cgroup_uint64(path, "cpu.shares");
+	return (int)cgroup_uint64(path, "cpu.weight");
 }
 
 static uint64_t cgroup_memuse(struct cg *cg)
@@ -135,16 +137,17 @@ static uint64_t cgroup_memuse(struct cg *cg)
 	char buf[42];
 	FILE *fp;
 
-
 	fp = fopenf("r", "%s/memory.stat", cg->cg_path);
 	if (fp) {
 		while (fgets(buf, sizeof(buf), fp)) {
-			if (!strncmp(buf, "total_rss", 9)) {
+			chomp(buf);
-				cg->cg_rss = strtoull(&buf[10], NULL, 10);
+
+			if (!strncmp(buf, "anon", 4)) {
+				cg->cg_rss = strtoull(&buf[5], NULL, 10);
 				continue;
 			}
-			if (!strncmp(buf, "total_mapped_file", 17)) {
+			if (!strncmp(buf, "file", 4)) {
-				cg->cg_vmlib = strtoull(&buf[18], NULL, 10);
+				cg->cg_vmlib = strtoull(&buf[5], NULL, 10);
 				break;	/* done */
 			}
 		}
@@ -153,8 +156,7 @@ static uint64_t cgroup_memuse(struct cg *cg)
 
 	cg->cg_mem = (float)(cg->cg_rss * 100 / total_ram);
 
-	/* "fuzz value for efficient access", sort of like total_cache + total_rss */
+	return cg->cg_vmsize = cgroup_uint64(cg->cg_path, "memory.current");
-	return cg->cg_vmsize = cgroup_uint64(cg->cg_path, "memory.usage_in_bytes");
 }
 
 static char *memsz(uint64_t sz, char *buf, size_t len)
@@ -184,23 +186,28 @@ static float cgroup_cpuload(struct cg *cg)
 	char buf[64];
 	FILE *fp;
 
-	snprintf(fn, sizeof(fn), "%s/cpuacct.usage", cg->cg_path);
+	snprintf(fn, sizeof(fn), "%s/cpu.stat", cg->cg_path);
 	fp = fopen(fn, "r");
 	if (!fp)
 		err(1, "Cannot open %s", fn);
 
-	if (fgets(buf, sizeof(buf), fp)) {
+	while (fgets(buf, sizeof(buf), fp)) {
 		uint64_t curr;
 
-		curr = strtoull(buf, NULL, 10);
+		chomp(buf);
+		if (strncmp(buf, "usage_usec", 10))
+			continue;
+
+		curr = strtoull(&buf[11], NULL, 10);
 		if (cg->cg_prev != 0) {
 			uint64_t diff = curr - cg->cg_prev;
 
 			/* this expects 1 sec poll interval */
 			cg->cg_load = (float)(diff / 1000000);
-			cg->cg_load /= 10.0;
+			cg->cg_load *= 100.0;
 		}
 		cg->cg_prev = curr;
+		break;
 	}
 
 	fclose(fp);
@@ -214,9 +221,9 @@ static struct cg *append(char *path)
 	char fn[256];
 	ENTRY item;
 
-	snprintf(fn, sizeof(fn), "%s/cpuacct.usage", path);
+	snprintf(fn, sizeof(fn), "%s/cpu.stat", path);
 	if (access(fn, F_OK)) {
-		warn("not a cgroup path with cpuacct controller, %s", path);
+		warn("not a cgroup path with cpu controller, %s", path);
 		return NULL;
 	}
 
@@ -357,7 +364,7 @@ static int cgroup_tree(char *path, char *pfx, int top)
 			strlcat(row, nm,   rplen);
 			strlcat(row, "/ ", rplen);
 
-			snprintf(tmp, sizeof(tmp), "[cpu.shares: %d]", cgroup_shares(buf));
+			snprintf(tmp, sizeof(tmp), "[cpu.weight: %d]", cgroup_weight(buf));
 			strlcat(row, tmp, rplen);
 			puts(row);
 

src/finit.c

@@ -518,7 +518,6 @@ int main(int argc, char *argv[])
 	 */
 	cgroup_init();
 
-
 	/* Check and mount filesystems. */
 	fs_mount();
 

src/finit.h

@@ -46,7 +46,7 @@
 #endif
 
 #ifndef FINIT_CGPATH
-#define FINIT_CGPATH            "/sys/fs/cgroup/finit"
+#define FINIT_CGPATH            "/sys/fs/cgroup"
 #endif
 
 #define LOGIT_PATH              FINIT_LIBPATH_ "/logit"