From 6a6f3a82911fd0d6ae5b4c07c2ce4af26dd11ac5 Mon Sep 17 00:00:00 2001 From: Joachim Wiberg Date: Wed, 17 Mar 2021 23:06:49 +0100 Subject: [PATCH] cgroup: refactor for cgroups v2, much cleaner + unified hieararchy Since the cgroups support in Finit is not yet officially released, we decided to change the back-end to use cgroups v2 instead of the aging and rather clumsy cgroups v1. Even this initial refactor is a lot easier to read and understand, more can still be done since there's a lot of concepts, data values and the ilk that can now be shared between different controllers. Still ToDo: inotify for cgroup.events to clean up leaf nodes, which in cgroups v1 was handled by cgreaper.sh. This is fixed in the next commit in the series. Signed-off-by: Joachim Wiberg --- src/Makefile.am | 3 +- src/cgreaper.sh | 11 --- src/cgroup.c | 189 ++++++++++++++++-------------------------------- src/cgutil.c | 41 ++++++----- src/finit.c | 1 - src/finit.h | 2 +- 6 files changed, 90 insertions(+), 157 deletions(-) delete mode 100755 src/cgreaper.sh diff --git a/src/Makefile.am b/src/Makefile.am index fb113fb9..68e804e1 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,4 +1,4 @@ -EXTRA_DIST = cgreaper.sh rescue.conf sample.conf +EXTRA_DIST = rescue.conf sample.conf AM_CPPFLAGS = -D_XOPEN_SOURCE=600 -D_BSD_SOURCE -D_GNU_SOURCE -D_DEFAULT_SOURCE if STATIC AM_CPPFLAGS += -DENABLE_STATIC @@ -8,7 +8,6 @@ AM_LDFLAGS = -export-dynamic endif sbin_PROGRAMS = finit initctl -pkglibexec_SCRIPTS = cgreaper.sh finit_pkglibdir = $(pkglibdir) finit_pkglib_SCRIPTS = rescue.conf sample.conf diff --git a/src/cgreaper.sh b/src/cgreaper.sh deleted file mode 100755 index d6bcf174..00000000 --- a/src/cgreaper.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -# Called by kernel when last child in cgroup exists - -BASE=/sys/fs/cgroup/finit -PROC=$(basename "$1") -DIR="$BASE$1" - -#/libexec/finit/logit -p daemon.info -t finit "$PROC stopped, cleaning up control group $DIR" -rmdir "$DIR" - -exit 0 diff --git a/src/cgroup.c b/src/cgroup.c index 995fe07a..e48125f5 100644 --- a/src/cgroup.c +++ b/src/cgroup.c @@ -32,49 +32,64 @@ #include "log.h" #include "util.h" -static char controllers[256] = "none"; +static char controllers[256]; -static void group_init(char *path, char *nproc, int share) +static void group_init(char *path, int leaf, int weight) { char file[strlen(path) + 64]; - paste(file, sizeof(file), path, "cgroup.clone_children"); - echo(file, 0, "1"); + if (mkdir(path, 0755) && EEXIST != errno) { + _pe("Failed creating cgroup %s", path); + return; + } - paste(file, sizeof(file), path, "cpuset.cpus"); - echo(file, 0, nproc); + /* enable detected controllers on domain groups */ + if (!leaf) { + paste(file, sizeof(file), path, "cgroup.subtree_control"); + echo(file, 0, controllers); + } - paste(file, sizeof(file), path, "cpuset.mems"); - echo(file, 0, "0"); + /* set initial group weight */ + if (weight) { + paste(file, sizeof(file), path, "cpu.weight"); + echo(file, 0, "%d", weight); + } +} - paste(file, sizeof(file), path, "cpu.shares"); - echo(file, 0, "%d", share); +static int cgroup_leaf_init(char *group, char *name, int pid) +{ + char path[256]; - paste(file, sizeof(file), path, "memory.use_hierarchy"); - echo(file, 0, "1"); + if (pid <= 1) { + errno = EINVAL; + return 1; + } + + snprintf(path, sizeof(path), "/sys/fs/cgroup/%s/%s", group, name); + group_init(path, 1, 0); + + strlcat(path, "/cgroup.procs", sizeof(path)); + + return echo(path, 0, "%d", pid); +} + +int cgroup_user(char *name, int pid) +{ + return cgroup_leaf_init("user", name, pid); +} + +int cgroup_service(char *name, int pid) +{ + return cgroup_leaf_init("system", name, pid); } static void append_ctrl(char *ctrl) { - char *wanted[] = { - "cpu", - "cpuacct", - "cpuset", - "memory" - }; - size_t i; + if (controllers[0]) + strlcat(controllers, " ", sizeof(controllers)); - for (i = 0; i < NELEMS(wanted); i++) { - if (strcmp(wanted[i], ctrl)) - continue; - - if (strcmp(controllers, "none")) - strlcat(controllers, ",", sizeof(controllers)); - else - strlcpy(controllers, "", sizeof(controllers)); - - strlcat(controllers, ctrl, sizeof(controllers)); - } + strlcat(controllers, "+", sizeof(controllers)); + strlcat(controllers, ctrl, sizeof(controllers)); } /* @@ -86,116 +101,40 @@ void cgroup_init(void) char buf[80]; FILE *fp; - fp = fopen("/proc/cgroups", "r"); - if (!fp) { - _d("No cgroup support"); + if (mount("none", FINIT_CGPATH, "cgroup2", opts, NULL)) { + _pe("Failed mounting cgroup v2"); return; } - if (mount("none", "/sys/fs/cgroup", "tmpfs", opts, NULL)) { - _d("Failed mounting cgroups"); - goto fail; + /* Find available controllers */ + fp = fopen(FINIT_CGPATH "/cgroup.controllers", "r"); + if (!fp) { + _pe("Failed opening %s", FINIT_CGPATH "/cgroup.controllers"); + return; } - /* Skip first line, header */ - (void)fgets(buf, sizeof(buf), fp); - - /* Create and mount traditional cgroups v1 hier */ - while (fgets(buf, sizeof(buf), fp)) { + if (fgets(buf, sizeof(buf), fp)) { char *cgroup; - char rc[80]; - cgroup = strtok(buf, "\t "); - if (!cgroup) - continue; - - snprintf(rc, sizeof(rc), "/sys/fs/cgroup/%s", cgroup); -#if 0 - if (mkdir(rc, 0755) && EEXIST != errno) - continue; - - if (mount("cgroup", rc, "cgroup", opts, cgroup)) - _d("Failed mounting %s cgroup on %s", cgroup, rc); -#else - symlink("finit", rc); - append_ctrl(cgroup); -#endif + cgroup = strtok(chomp(buf), "\t "); + while (cgroup) { + append_ctrl(cgroup); + cgroup = strtok(NULL, "\t "); + } } - /* Finit default cgroups for process monitoring */ - if (mkdir(FINIT_CGPATH, 0755) && EEXIST != errno) - goto fail; + fclose(fp); -#if 0 - strlcpy(controllers, "none,name=finit", sizeof(controllers)); -#else - strlcat(controllers, ",name=finit", sizeof(controllers)); -#endif - if (mount("none", FINIT_CGPATH, "cgroup", opts, controllers)) { - _pe("Failed mounting Finit cgroup hierarchy"); - goto fail; - } - - /* Set up reaper and enable callbacks */ - echo(FINIT_CGPATH "/release_agent", 0, FINIT_LIBPATH_ "/cgreaper.sh"); - echo(FINIT_CGPATH "/notify_on_release", 0, "1"); + /* Enable all controllers */ + echo(FINIT_CGPATH "/cgroup.subtree_control", 0, controllers); /* Default groups, PID 1, services, and user/login processes */ - if (mkdir(FINIT_CGPATH "/init", 0755) && EEXIST != errno) - goto fail; - if (mkdir(FINIT_CGPATH "/system", 0755) && EEXIST != errno) - goto fail; - if (mkdir(FINIT_CGPATH "/user", 0755) && EEXIST != errno) - goto fail; - - /* - * Set up basic limits for our groups. Finit optimized defaults - * for embedded and server systems include limiting the groups - * init and user to a single CPU with 10% share, the system - * group gets the rest. Override this from finit.conf - */ -#if 0 -#else - snprintf(buf, sizeof(buf), "0-%d", get_nprocs_conf() - 1); - - group_init(FINIT_CGPATH "/init", "0", 50); - group_init(FINIT_CGPATH "/user", "0", 75); - group_init(FINIT_CGPATH "/system", buf, 900); -#endif + group_init(FINIT_CGPATH "/init", 1, 100); + group_init(FINIT_CGPATH "/user", 0, 100); + group_init(FINIT_CGPATH "/system", 0, 9800); /* Move ourselves to init */ echo(FINIT_CGPATH "/init/cgroup.procs", 0, "1"); - -fail: - fclose(fp); -} - -static int move_pid(char *group, char *name, int pid) -{ - char path[256]; - - snprintf(path, sizeof(path), "/sys/fs/cgroup/%s/%s", group, name); - if (mkdir(path, 0755) && errno != EEXIST) - return 1; - - strlcat(path, "/cgroup.procs", sizeof(path)); - - return echo(path, 0, "%d", pid); -} - -int cgroup_user(char *name, int pid) -{ - return move_pid("finit/user", name, pid); -} - -int cgroup_service(char *name, int pid) -{ - if (pid <= 0) { - errno = EINVAL; - return 1; - } - - return move_pid("finit/system", name, pid); } /** diff --git a/src/cgutil.c b/src/cgutil.c index cc8be99b..da46411a 100644 --- a/src/cgutil.c +++ b/src/cgutil.c @@ -118,16 +118,18 @@ static uint64_t cgroup_uint64(char *path, char *file) if (!fp) return val; - if (fgets(buf, sizeof(buf), fp)) + if (fgets(buf, sizeof(buf), fp)) { + chomp(buf); val = strtoull(buf, NULL, 10); + } fclose(fp); return val; } -static int cgroup_shares(char *path) +static int cgroup_weight(char *path) { - return (int)cgroup_uint64(path, "cpu.shares"); + return (int)cgroup_uint64(path, "cpu.weight"); } static uint64_t cgroup_memuse(struct cg *cg) @@ -135,16 +137,17 @@ static uint64_t cgroup_memuse(struct cg *cg) char buf[42]; FILE *fp; - fp = fopenf("r", "%s/memory.stat", cg->cg_path); if (fp) { while (fgets(buf, sizeof(buf), fp)) { - if (!strncmp(buf, "total_rss", 9)) { - cg->cg_rss = strtoull(&buf[10], NULL, 10); + chomp(buf); + + if (!strncmp(buf, "anon", 4)) { + cg->cg_rss = strtoull(&buf[5], NULL, 10); continue; } - if (!strncmp(buf, "total_mapped_file", 17)) { - cg->cg_vmlib = strtoull(&buf[18], NULL, 10); + if (!strncmp(buf, "file", 4)) { + cg->cg_vmlib = strtoull(&buf[5], NULL, 10); break; /* done */ } } @@ -153,8 +156,7 @@ static uint64_t cgroup_memuse(struct cg *cg) cg->cg_mem = (float)(cg->cg_rss * 100 / total_ram); - /* "fuzz value for efficient access", sort of like total_cache + total_rss */ - return cg->cg_vmsize = cgroup_uint64(cg->cg_path, "memory.usage_in_bytes"); + return cg->cg_vmsize = cgroup_uint64(cg->cg_path, "memory.current"); } static char *memsz(uint64_t sz, char *buf, size_t len) @@ -184,23 +186,28 @@ static float cgroup_cpuload(struct cg *cg) char buf[64]; FILE *fp; - snprintf(fn, sizeof(fn), "%s/cpuacct.usage", cg->cg_path); + snprintf(fn, sizeof(fn), "%s/cpu.stat", cg->cg_path); fp = fopen(fn, "r"); if (!fp) err(1, "Cannot open %s", fn); - if (fgets(buf, sizeof(buf), fp)) { + while (fgets(buf, sizeof(buf), fp)) { uint64_t curr; - curr = strtoull(buf, NULL, 10); + chomp(buf); + if (strncmp(buf, "usage_usec", 10)) + continue; + + curr = strtoull(&buf[11], NULL, 10); if (cg->cg_prev != 0) { uint64_t diff = curr - cg->cg_prev; /* this expects 1 sec poll interval */ cg->cg_load = (float)(diff / 1000000); - cg->cg_load /= 10.0; + cg->cg_load *= 100.0; } cg->cg_prev = curr; + break; } fclose(fp); @@ -214,9 +221,9 @@ static struct cg *append(char *path) char fn[256]; ENTRY item; - snprintf(fn, sizeof(fn), "%s/cpuacct.usage", path); + snprintf(fn, sizeof(fn), "%s/cpu.stat", path); if (access(fn, F_OK)) { - warn("not a cgroup path with cpuacct controller, %s", path); + warn("not a cgroup path with cpu controller, %s", path); return NULL; } @@ -357,7 +364,7 @@ static int cgroup_tree(char *path, char *pfx, int top) strlcat(row, nm, rplen); strlcat(row, "/ ", rplen); - snprintf(tmp, sizeof(tmp), "[cpu.shares: %d]", cgroup_shares(buf)); + snprintf(tmp, sizeof(tmp), "[cpu.weight: %d]", cgroup_weight(buf)); strlcat(row, tmp, rplen); puts(row); diff --git a/src/finit.c b/src/finit.c index 260cc3a4..5ce83cd0 100644 --- a/src/finit.c +++ b/src/finit.c @@ -518,7 +518,6 @@ int main(int argc, char *argv[]) */ cgroup_init(); - /* Check and mount filesystems. */ fs_mount(); diff --git a/src/finit.h b/src/finit.h index 997e37cc..0018e3c7 100644 --- a/src/finit.h +++ b/src/finit.h @@ -46,7 +46,7 @@ #endif #ifndef FINIT_CGPATH -#define FINIT_CGPATH "/sys/fs/cgroup/finit" +#define FINIT_CGPATH "/sys/fs/cgroup" #endif #define LOGIT_PATH FINIT_LIBPATH_ "/logit"