From e7cb324188903ce22ddd86a46049058a238134a4 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 05:25:00 +0100 Subject: [PATCH 01/82] service: report time namespace support unconditionally put_namespace() stats /proc/self/ns/ and only advertises namespaces the running kernel actually provides, so guarding the "time" entry with #ifdef CLONE_NEWTIME cannot enable anything the runtime check would not. The guard tests the toolchain headers at build time, and the guarded code does not even use CLONE_NEWTIME, so its only possible effect is to hide time namespace support from the features reply when procd was built against headers predating the flag. Drop the guard together with the include which was added solely to provide it. Fixes: 47a9f0d65267 ("service: add method to query available container features") Signed-off-by: Daniel Golle --- service/service.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/service/service.c b/service/service.c index 73999fe..28e9017 100644 --- a/service/service.c +++ b/service/service.c @@ -19,7 +19,6 @@ #include #include -#include #include #include @@ -416,9 +415,7 @@ container_handle_features(struct ubus_context *ctx, struct ubus_object *obj, put_namespace(&b, "mnt"); put_namespace(&b, "net"); put_namespace(&b, "pid"); -#ifdef CLONE_NEWTIME put_namespace(&b, "time"); -#endif put_namespace(&b, "user"); put_namespace(&b, "uts"); blobmsg_close_array(&b, nsarray); From d9d0b36a2cf546408ab0ca2a42d53808befa37f5 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 17 Jun 2026 05:13:47 +0100 Subject: [PATCH 02/82] jail: tolerate exact-duplicate mount registrations _add_mount() keyed mounts by target alone and silently skipped any second registration for an already-known target. With OCI bundles a target can legitimately be registered twice from independent sources (for instance an implicit mount and an explicit one), and the two may carry conflicting parameters that the old code discarded without notice. Compare the full descriptor instead: an exact duplicate (same source, filesystemtype, optstr, flags, error and inner flags) is accepted as a no-op, whereas a genuine conflict on the same target now returns EEXIST so the caller can fail loudly rather than honour whichever registration happened to win. Fixes: 71e75f401133 ("jail: refactor mount support to cover OCI spec") Signed-off-by: Daniel Golle --- jail/fs.c | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/jail/fs.c b/jail/fs.c index 1969425..3a8d998 100644 --- a/jail/fs.c +++ b/jail/fs.c @@ -380,16 +380,43 @@ static int do_mount(const char *root, const char *orig_source, const char *targe return ret; } +static bool nullable_str_eq(const char *a, const char *b) +{ + if (a == b) + return true; + if (!a || !b) + return false; + return !strcmp(a, b); +} + static int _add_mount(const char *source, const char *target, const char *filesystemtype, unsigned long mountflags, unsigned long propflags, const char *optstr, int error, bool inner) { + struct mount *m; + assert(target != NULL); - if (avl_find(&mounts, target)) - return 1; + m = avl_find_element(&mounts, target, m, avl); + if (m) { + bool source_match; + if (m->source == (void *)(-1) || source == (void *)(-1)) + source_match = (m->source == source); + else + source_match = nullable_str_eq(m->source, source); + + if (source_match && + nullable_str_eq(m->filesystemtype, filesystemtype) && + nullable_str_eq(m->optstr, optstr) && + m->mountflags == mountflags && + m->propflags == propflags && + m->error == error && + m->inner == inner) + return 0; + + return EEXIST; + } - struct mount *m; m = calloc(1, sizeof(struct mount)); if (!m) return ENOMEM; From 4c0f70e9044a2fbae793e3cd96ea143210af30ac Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 17 Jun 2026 05:13:47 +0100 Subject: [PATCH 03/82] jail: do not catch synchronous fault signals in signals_init signals_init() installed the jail_handle_signal forwarding handler for every signal in the mask except a short denylist. That denylist omitted the synchronous fault signals, so a fault inside the jailed process would enter the handler, return, and let the kernel restart the faulting instruction, faulting again and spinning forever instead of dumping core or terminating. Skip the synchronous faults (SIGSEGV, SIGBUS, SIGFPE, SIGILL, SIGSYS, SIGABRT, SIGTRAP) alongside the existing un-catchable and default-ignored signals, so they retain their default disposition. Rewrite the test as a switch for legibility now that the list is long. Fixes: cdc3dab3cd5d ("ujail: fix signal forwarding") Signed-off-by: Daniel Golle --- jail/jail.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/jail/jail.c b/jail/jail.c index d74f62f..5830ff3 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -1842,8 +1842,22 @@ static void signals_init(void) if (!sigismember(&sigmask, i)) continue; - if ((i == SIGCHLD) || (i == SIGPIPE) || (i == SIGSEGV) || (i == SIGSTOP) || (i == SIGKILL)) + switch (i) { + case SIGCHLD: + case SIGPIPE: + case SIGKILL: + case SIGSTOP: + case SIGSEGV: + case SIGBUS: + case SIGFPE: + case SIGILL: + case SIGSYS: + case SIGABRT: + case SIGTRAP: continue; + default: + break; + } s.sa_handler = jail_handle_signal; sigaction(i, &s, NULL); From 56734716fd2dd4adf0e04be647af6b6169739b57 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 17 Jun 2026 05:13:47 +0100 Subject: [PATCH 04/82] jail: fix free_hooklist/free_sysctl iteration over arrays of pointers Both hooklist and opts.sysctl are NULL-terminated arrays of pointers, but the cleanup loops dereferenced the first element once and then advanced a single struct pointer with cur++. That walked element fields as if they were the array and incremented by one struct rather than one pointer, freeing the wrong addresses and reading past the allocation. Iterate over the array itself: hold a pointer-to-pointer, free each pointed-to element, then advance the cursor by one slot until the NULL terminator. Fixes: fc9f614bf701 ("jail: parse and run OCI hooks") Signed-off-by: Daniel Golle --- jail/jail.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/jail/jail.c b/jail/jail.c index 5830ff3..59a3597 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -202,33 +202,33 @@ static void free_oci_envp(char **p) { static void free_hooklist(struct hook_execvpe **hooklist) { - struct hook_execvpe *cur; + struct hook_execvpe **cur; if (!hooklist) return; - cur = *hooklist; - while (cur) { - free_oci_envp(cur->argv); - free_oci_envp(cur->envp); - free(cur->file); - free(cur++); + cur = hooklist; + while (*cur) { + free_oci_envp((*cur)->argv); + free_oci_envp((*cur)->envp); + free((*cur)->file); + free(*(cur++)); } free(hooklist); } static void free_sysctl(void) { - struct sysctl_val *cur; + struct sysctl_val **cur; if (!opts.sysctl) return; - cur = *opts.sysctl; + cur = opts.sysctl; - while (cur) { - free(cur->entry); - free(cur->value); - free(cur++); + while (*cur) { + free((*cur)->entry); + free((*cur)->value); + free(*(cur++)); } free(opts.sysctl); } From 3e956d5e0bede72318fc5ac9b0663212c042af6a Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 17 Jun 2026 05:13:48 +0100 Subject: [PATCH 05/82] jail/capabilities: treat a missing OCI capability set as empty, not all-ones parseOCIcap() returned JAIL_CAP_ALL when a capability set was absent from the OCI config, on the assumption that an omitted set means "leave everything as is". The opposite is correct: the OCI runtime spec defines an absent set as empty, so returning an all-ones mask silently granted the process every capability instead of dropping them. Return 0 for a missing set so an unspecified capability set yields no capabilities, matching the spec and the principle of least privilege. Fixes: ea7a790f210c ("jail: add support for running OCI bundle") Signed-off-by: Daniel Golle --- jail/capabilities.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jail/capabilities.c b/jail/capabilities.c index 434fc2f..bad5af5 100644 --- a/jail/capabilities.c +++ b/jail/capabilities.c @@ -60,9 +60,8 @@ static uint64_t parseOCIcap(struct blob_attr *msg) uint64_t caps = 0; int capnum; - /* each capset is optional, set all-1 mask if absent */ if (!msg) - return JAIL_CAP_ALL; + return 0; blobmsg_for_each_attr(cur, msg, rem) { capnum = find_capabilities(blobmsg_get_string(cur)); From 33f270f91b0ceb7937bf1f1bc1a4b0cd0eb65c6f Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 17 Jun 2026 05:13:48 +0100 Subject: [PATCH 06/82] jail: retry pipe sync reads interrupted by a signal post_jail_fs() reads a one-byte synchronisation token from the parent over a pipe and treats any short read as fatal. A signal delivered while blocked in read() returns -1 with EINTR, which the old check counted as failure, aborting the jail setup spuriously whenever a signal arrived at that moment. Loop on the read while it fails with EINTR so a benign interruption is retried, and only treat a genuine short or failed read as the parent going away. Fixes: 6f3dbd283bbd ("jail: add support for userns and cgroupsns") Signed-off-by: Daniel Golle --- jail/jail.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/jail/jail.c b/jail/jail.c index 59a3597..d8129fd 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -1874,6 +1874,7 @@ static int parent_pidfd = -1; static int exec_jail(void *arg) { char buf[1]; + ssize_t n; exit_from_child = true; prctl(PR_SET_SECUREBITS, 0); @@ -1922,7 +1923,10 @@ static int exec_jail(void *arg) return EXIT_FAILURE; } close(pipes[1]); - if (read(pipes[2], buf, 1) < 1) { + do { + n = read(pipes[2], buf, 1); + } while (n < 0 && errno == EINTR); + if (n < 1) { ERROR("can't read from parent\n"); return EXIT_FAILURE; } @@ -2000,8 +2004,12 @@ static void post_start_hook(void); static void post_jail_fs(void) { char buf[1]; + ssize_t n; - if (read(pipes[2], buf, 1) < 1) { + do { + n = read(pipes[2], buf, 1); + } while (n < 0 && errno == EINTR); + if (n < 1) { ERROR("can't read from parent\n"); free_and_exit(EXIT_FAILURE); } From 014c684c58d2626f765b1c9c83e558933197d05f Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 17 Jun 2026 05:23:20 +0100 Subject: [PATCH 07/82] jail: parse and apply further OCI process and linux fields Parse and apply the remaining process scheduling controls: scheduler (policy, nice, priority, flags and the deadline runtime/deadline/period triplet via sched_setattr) and ioPriority (class and 0..7 priority via ioprio_set), both applied in the post-start hook with range validation. Honour the spec hostname alongside the new domainname, gating both on a UTS namespace whether created or joined via setns. Parse linux.netDevices and move the named host interfaces into the container network namespace over rtnetlink, optionally renaming them. Accept linux.personality only when its domain matches the current one, since cross-personality execution is not supported. Reject fields that cannot be implemented on OpenWrt rather than silently ignoring them: apparmorProfile, selinuxLabel, mountLabel, memoryPolicy and personality flags return ENOTSUP. Define a CLONE_NEWTIME fallback beside the CLONE_NEWCGROUP one and drop the ifdef guards around it, so support for the namespace depends on the running kernel rather than on whether the libc headers happen to expose the macro. Signed-off-by: Daniel Golle --- jail/jail.c | 541 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 513 insertions(+), 28 deletions(-) diff --git a/jail/jail.c b/jail/jail.c index d8129fd..24e5eec 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -21,8 +21,13 @@ #include #include #include +#include +#include #include +#include #include +#include +#include /* musl only defined 15 limit types, make sure all 16 are supported */ #ifndef RLIMIT_RTTIME @@ -70,6 +75,10 @@ #define CLONE_NEWCGROUP 0x02000000 #endif +#ifndef CLONE_NEWTIME +#define CLONE_NEWTIME 0x00000080 +#endif + #define STACK_SIZE (1024 * 1024) #define OPT_ARGS "cC:d:De:EfFG:h:ij:J:ln:NoO:pP:r:R:sS:uU:w:t:T:y" @@ -98,6 +107,7 @@ struct mknod_args { static struct { char *name; char *hostname; + char *domainname; char **jail_argv; char *cwd; char *seccomp; @@ -124,9 +134,7 @@ static struct { int uts; int user; int cgroup; -#ifdef CLONE_NEWTIME int time; -#endif } setns; int procfs; int ronly; @@ -157,7 +165,23 @@ static struct { char **oci_deferred_readonly; bool immediately; struct blob_attr *annotations; + struct blob_attr *netdevices; int term_timeout; + struct { + bool set; + uint32_t policy; + uint64_t flags; + int32_t nice; + uint32_t priority; + uint64_t runtime; + uint64_t deadline; + uint64_t period; + } scheduler; + struct { + bool set; + int class; + int priority; + } ioprio; } opts; static struct blob_buf ocibuf; @@ -182,9 +206,7 @@ return ((opts.setns.pid != -1) || (opts.setns.uts != -1) || (opts.setns.user != -1) || (opts.setns.cgroup != -1) || -#ifdef CLONE_NEWTIME (opts.setns.time != -1) || -#endif opts.namespace); } @@ -276,10 +298,12 @@ static void free_opts(bool parent) { free_sysctl(); free_devices(); free(opts.hostname); + free(opts.domainname); free(opts.cwd); free(opts.uidmap); free(opts.gidmap); free(opts.annotations); + free(opts.netdevices); free(opts.extroot); free(opts.overlaydir); free_hooklist(opts.hooks.createRuntime); @@ -1753,10 +1777,8 @@ static int* get_namespace_fd(const unsigned int nstype) return &opts.setns.user; case CLONE_NEWCGROUP: return &opts.setns.cgroup; -#ifdef CLONE_NEWTIME case CLONE_NEWTIME: return &opts.setns.time; -#endif default: return NULL; } @@ -1864,6 +1886,359 @@ static void signals_init(void) } } +enum { + OCI_PROCESS_SCHEDULER_POLICY, + OCI_PROCESS_SCHEDULER_NICE, + OCI_PROCESS_SCHEDULER_PRIORITY, + OCI_PROCESS_SCHEDULER_FLAGS, + OCI_PROCESS_SCHEDULER_RUNTIME, + OCI_PROCESS_SCHEDULER_DEADLINE, + OCI_PROCESS_SCHEDULER_PERIOD, + __OCI_PROCESS_SCHEDULER_MAX, +}; + +static const struct blobmsg_policy oci_process_scheduler_policy[] = { + [OCI_PROCESS_SCHEDULER_POLICY] = { "policy", BLOBMSG_TYPE_STRING }, + [OCI_PROCESS_SCHEDULER_NICE] = { "nice", BLOBMSG_TYPE_INT32 }, + [OCI_PROCESS_SCHEDULER_PRIORITY] = { "priority", BLOBMSG_TYPE_INT32 }, + [OCI_PROCESS_SCHEDULER_FLAGS] = { "flags", BLOBMSG_TYPE_ARRAY }, + [OCI_PROCESS_SCHEDULER_RUNTIME] = { "runtime", BLOBMSG_CAST_INT64 }, + [OCI_PROCESS_SCHEDULER_DEADLINE] = { "deadline", BLOBMSG_CAST_INT64 }, + [OCI_PROCESS_SCHEDULER_PERIOD] = { "period", BLOBMSG_CAST_INT64 }, +}; + +#ifndef SCHED_DEADLINE +#define SCHED_DEADLINE 6 +#endif + +#ifndef SCHED_FLAG_RESET_ON_FORK +#define SCHED_FLAG_RESET_ON_FORK 0x01 +#endif + +#ifndef SCHED_FLAG_RECLAIM +#define SCHED_FLAG_RECLAIM 0x02 +#endif + +#ifndef SCHED_FLAG_DL_OVERRUN +#define SCHED_FLAG_DL_OVERRUN 0x04 +#endif + +struct procd_sched_attr { + uint32_t size; + uint32_t sched_policy; + uint64_t sched_flags; + int32_t sched_nice; + uint32_t sched_priority; + uint64_t sched_runtime; + uint64_t sched_deadline; + uint64_t sched_period; +}; + +static int parseOCIprocessscheduler(struct blob_attr *msg) +{ + struct blob_attr *tb[__OCI_PROCESS_SCHEDULER_MAX]; + struct blob_attr *cur; + const char *policy; + int rem; + + blobmsg_parse(oci_process_scheduler_policy, __OCI_PROCESS_SCHEDULER_MAX, tb, + blobmsg_data(msg), blobmsg_len(msg)); + + if (!tb[OCI_PROCESS_SCHEDULER_POLICY]) + return ENODATA; + + policy = blobmsg_get_string(tb[OCI_PROCESS_SCHEDULER_POLICY]); + if (!strcmp(policy, "SCHED_OTHER")) + opts.scheduler.policy = SCHED_OTHER; + else if (!strcmp(policy, "SCHED_FIFO")) + opts.scheduler.policy = SCHED_FIFO; + else if (!strcmp(policy, "SCHED_RR")) + opts.scheduler.policy = SCHED_RR; + else if (!strcmp(policy, "SCHED_BATCH")) + opts.scheduler.policy = SCHED_BATCH; + else if (!strcmp(policy, "SCHED_IDLE")) + opts.scheduler.policy = SCHED_IDLE; + else if (!strcmp(policy, "SCHED_DEADLINE")) + opts.scheduler.policy = SCHED_DEADLINE; + else + return EINVAL; + + if (tb[OCI_PROCESS_SCHEDULER_NICE]) + opts.scheduler.nice = blobmsg_get_u32(tb[OCI_PROCESS_SCHEDULER_NICE]); + + if (tb[OCI_PROCESS_SCHEDULER_PRIORITY]) { + int32_t prio = (int32_t)blobmsg_get_u32(tb[OCI_PROCESS_SCHEDULER_PRIORITY]); + + if (prio < 0) { + ERROR("scheduler: priority %d out of range\n", prio); + return EINVAL; + } + if ((opts.scheduler.policy == SCHED_FIFO || opts.scheduler.policy == SCHED_RR) && + (prio < 1 || prio > 99)) { + ERROR("scheduler: priority %d outside 1..99 for FIFO/RR\n", prio); + return EINVAL; + } + opts.scheduler.priority = prio; + } + + if (tb[OCI_PROCESS_SCHEDULER_RUNTIME]) + opts.scheduler.runtime = blobmsg_cast_u64(tb[OCI_PROCESS_SCHEDULER_RUNTIME]); + + if (tb[OCI_PROCESS_SCHEDULER_DEADLINE]) + opts.scheduler.deadline = blobmsg_cast_u64(tb[OCI_PROCESS_SCHEDULER_DEADLINE]); + + if (tb[OCI_PROCESS_SCHEDULER_PERIOD]) + opts.scheduler.period = blobmsg_cast_u64(tb[OCI_PROCESS_SCHEDULER_PERIOD]); + + if (tb[OCI_PROCESS_SCHEDULER_FLAGS]) { + if (blobmsg_check_array(tb[OCI_PROCESS_SCHEDULER_FLAGS], BLOBMSG_TYPE_STRING) < 0) + return EINVAL; + blobmsg_for_each_attr(cur, tb[OCI_PROCESS_SCHEDULER_FLAGS], rem) { + const char *flag = blobmsg_get_string(cur); + if (!strcmp(flag, "SCHED_FLAG_RESET_ON_FORK")) + opts.scheduler.flags |= SCHED_FLAG_RESET_ON_FORK; + else if (!strcmp(flag, "SCHED_FLAG_RECLAIM")) + opts.scheduler.flags |= SCHED_FLAG_RECLAIM; + else if (!strcmp(flag, "SCHED_FLAG_DL_OVERRUN")) + opts.scheduler.flags |= SCHED_FLAG_DL_OVERRUN; + else + return EINVAL; + } + } + + opts.scheduler.set = true; + return 0; +} + +static int applyOCIprocessscheduler(void) +{ + struct procd_sched_attr attr = { + .size = sizeof(attr), + .sched_policy = opts.scheduler.policy, + .sched_flags = opts.scheduler.flags, + .sched_nice = opts.scheduler.nice, + .sched_priority = opts.scheduler.priority, + .sched_runtime = opts.scheduler.runtime, + .sched_deadline = opts.scheduler.deadline, + .sched_period = opts.scheduler.period, + }; + + if (syscall(SYS_sched_setattr, 0, &attr, 0)) { + ERROR("sched_setattr: %m\n"); + return errno; + } + + return 0; +} + +enum { + OCI_PROCESS_IOPRIORITY_CLASS, + OCI_PROCESS_IOPRIORITY_PRIORITY, + __OCI_PROCESS_IOPRIORITY_MAX, +}; + +static const struct blobmsg_policy oci_process_iopriority_policy[] = { + [OCI_PROCESS_IOPRIORITY_CLASS] = { "class", BLOBMSG_TYPE_STRING }, + [OCI_PROCESS_IOPRIORITY_PRIORITY] = { "priority", BLOBMSG_TYPE_INT32 }, +}; + +#ifndef IOPRIO_WHO_PROCESS +#define IOPRIO_WHO_PROCESS 1 +#endif + +#ifndef IOPRIO_CLASS_RT +#define IOPRIO_CLASS_RT 1 +#endif + +#ifndef IOPRIO_CLASS_BE +#define IOPRIO_CLASS_BE 2 +#endif + +#ifndef IOPRIO_CLASS_SHIFT +#define IOPRIO_CLASS_SHIFT 13 +#endif + +#ifndef IOPRIO_CLASS_IDLE +#define IOPRIO_CLASS_IDLE 3 +#endif + +static int parseOCIprocessiopriority(struct blob_attr *msg) +{ + struct blob_attr *tb[__OCI_PROCESS_IOPRIORITY_MAX]; + const char *class; + int priority; + + blobmsg_parse(oci_process_iopriority_policy, __OCI_PROCESS_IOPRIORITY_MAX, tb, + blobmsg_data(msg), blobmsg_len(msg)); + + if (!tb[OCI_PROCESS_IOPRIORITY_CLASS] || !tb[OCI_PROCESS_IOPRIORITY_PRIORITY]) + return ENODATA; + + class = blobmsg_get_string(tb[OCI_PROCESS_IOPRIORITY_CLASS]); + if (!strcmp(class, "IOPRIO_CLASS_RT")) + opts.ioprio.class = IOPRIO_CLASS_RT; + else if (!strcmp(class, "IOPRIO_CLASS_BE")) + opts.ioprio.class = IOPRIO_CLASS_BE; + else if (!strcmp(class, "IOPRIO_CLASS_IDLE")) + opts.ioprio.class = IOPRIO_CLASS_IDLE; + else + return EINVAL; + + priority = blobmsg_get_u32(tb[OCI_PROCESS_IOPRIORITY_PRIORITY]); + if (priority < 0 || priority > 7) + return EINVAL; + + opts.ioprio.priority = priority; + opts.ioprio.set = true; + return 0; +} + +static int applyOCIprocessiopriority(void) +{ + int ioprio = (opts.ioprio.class << IOPRIO_CLASS_SHIFT) | opts.ioprio.priority; + + if (syscall(SYS_ioprio_set, IOPRIO_WHO_PROCESS, 0, ioprio)) { + ERROR("ioprio_set: %m\n"); + return errno; + } + + return 0; +} + +static int move_netdev_to_ns(int netns_fd, const char *host_name, const char *new_name) +{ + struct { + struct nlmsghdr hdr; + struct ifinfomsg ifi; + char attrbuf[256]; + } req = { 0 }; + struct sockaddr_nl sa = { .nl_family = AF_NETLINK }; + struct rtattr *rta; + int sock, ifindex; + char buf[4096]; + ssize_t n; + + int saved_err; + + ifindex = if_nametoindex(host_name); + if (!ifindex) { + ERROR("netDevices: interface %s not found\n", host_name); + return ENODEV; + } + + sock = socket(AF_NETLINK, SOCK_RAW | SOCK_CLOEXEC, NETLINK_ROUTE); + if (sock < 0) { + ERROR("netDevices: socket(AF_NETLINK): %m\n"); + return errno; + } + if (bind(sock, (struct sockaddr *)&sa, sizeof(sa)) < 0) { + saved_err = errno; + ERROR("netDevices: bind: %m\n"); + close(sock); + errno = saved_err; + return saved_err; + } + + req.hdr.nlmsg_len = NLMSG_LENGTH(sizeof(req.ifi)); + req.hdr.nlmsg_type = RTM_NEWLINK; + req.hdr.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK; + req.hdr.nlmsg_seq = 1; + req.ifi.ifi_family = AF_UNSPEC; + req.ifi.ifi_index = ifindex; + + rta = (struct rtattr *)((char *)&req + NLMSG_ALIGN(req.hdr.nlmsg_len)); + rta->rta_type = IFLA_NET_NS_FD; + rta->rta_len = RTA_LENGTH(sizeof(int)); + memcpy(RTA_DATA(rta), &netns_fd, sizeof(int)); + req.hdr.nlmsg_len = NLMSG_ALIGN(req.hdr.nlmsg_len) + RTA_ALIGN(rta->rta_len); + + if (new_name) { + size_t namelen = strlen(new_name) + 1; + rta = (struct rtattr *)((char *)&req + NLMSG_ALIGN(req.hdr.nlmsg_len)); + rta->rta_type = IFLA_IFNAME; + rta->rta_len = RTA_LENGTH(namelen); + memcpy(RTA_DATA(rta), new_name, namelen); + req.hdr.nlmsg_len = NLMSG_ALIGN(req.hdr.nlmsg_len) + RTA_ALIGN(rta->rta_len); + } + + if (send(sock, &req, req.hdr.nlmsg_len, 0) < 0) { + saved_err = errno; + ERROR("netDevices: send: %m\n"); + close(sock); + errno = saved_err; + return saved_err; + } + + n = recv(sock, buf, sizeof(buf), 0); + saved_err = (n < 0) ? errno : 0; + close(sock); + if (n < 0) { + errno = saved_err; + ERROR("netDevices: recv: %m\n"); + return saved_err; + } + + if (n < (ssize_t)NLMSG_HDRLEN || + !NLMSG_OK((struct nlmsghdr *)buf, (size_t)n)) { + ERROR("netDevices: short or malformed nlmsg (%zd bytes)\n", n); + return EIO; + } + + struct nlmsghdr *nh = (struct nlmsghdr *)buf; + if (nh->nlmsg_type == NLMSG_ERROR) { + struct nlmsgerr *err = NLMSG_DATA(nh); + if (err->error) { + ERROR("netDevices: kernel rejected move of %s: %s\n", + host_name, strerror(-err->error)); + return -err->error; + } + } + + return 0; +} + +static int move_netdevs_into_jail(pid_t pid) +{ + enum { + OCI_LINUX_NETDEVICES_NAME, + __OCI_LINUX_NETDEVICES_MAX, + }; + static const struct blobmsg_policy policy[] = { + [OCI_LINUX_NETDEVICES_NAME] = { "name", BLOBMSG_TYPE_STRING }, + }; + struct blob_attr *cur, *tb[__OCI_LINUX_NETDEVICES_MAX]; + char path[64]; + int rem, netns_fd, ret = 0; + + if (!opts.netdevices) + return 0; + + snprintf(path, sizeof(path), "/proc/%d/ns/net", pid); + netns_fd = open(path, O_RDONLY | O_CLOEXEC); + if (netns_fd < 0) { + ERROR("netDevices: open(%s): %m\n", path); + return errno; + } + + blobmsg_for_each_attr(cur, opts.netdevices, rem) { + const char *host_name = blobmsg_name(cur); + const char *new_name = NULL; + + blobmsg_parse(policy, __OCI_LINUX_NETDEVICES_MAX, tb, + blobmsg_data(cur), blobmsg_len(cur)); + if (tb[OCI_LINUX_NETDEVICES_NAME]) + new_name = blobmsg_get_string(tb[OCI_LINUX_NETDEVICES_NAME]); + + ret = move_netdev_to_ns(netns_fd, host_name, new_name); + if (ret) + break; + } + + close(netns_fd); + return ret; +} + + static void pre_exec_jail(struct uloop_timeout *t); static struct uloop_timeout pre_exec_timeout = { .cb = pre_exec_jail, @@ -1973,12 +2348,20 @@ static int exec_jail(void *arg) } } - if (opts.namespace && opts.hostname && strlen(opts.hostname) > 0 + if (((opts.namespace & CLONE_NEWUTS) || opts.setns.uts != -1) + && opts.hostname && strlen(opts.hostname) > 0 && sethostname(opts.hostname, strlen(opts.hostname))) { ERROR("sethostname(%s) failed: %m\n", opts.hostname); free_and_exit(EXIT_FAILURE); } + if (((opts.namespace & CLONE_NEWUTS) || opts.setns.uts != -1) + && opts.domainname && strlen(opts.domainname) > 0 + && setdomainname(opts.domainname, strlen(opts.domainname))) { + ERROR("setdomainname(%s) failed: %m\n", opts.domainname); + free_and_exit(EXIT_FAILURE); + } + uloop_timeout_add(&pre_exec_timeout); uloop_run(); @@ -2026,6 +2409,12 @@ static void post_start_hook(void) { int pw_uid, pw_gid, gr_gid; + if (opts.scheduler.set && applyOCIprocessscheduler()) + free_and_exit(EXIT_FAILURE); + + if (opts.ioprio.set && applyOCIprocessiopriority()) + free_and_exit(EXIT_FAILURE); + /* * make sure setuid/setgid won't drop capabilities in case capabilities * have been specified explicitely. @@ -2549,26 +2938,36 @@ static int parseOCIrlimit(struct blob_attr *msg) }; enum { + OCI_PROCESS_APPARMORPROFILE, OCI_PROCESS_ARGS, OCI_PROCESS_CAPABILITIES, OCI_PROCESS_CWD, OCI_PROCESS_ENV, + OCI_PROCESS_EXECCPUAFFINITY, + OCI_PROCESS_IOPRIORITY, OCI_PROCESS_OOMSCOREADJ, OCI_PROCESS_NONEWPRIVILEGES, OCI_PROCESS_RLIMITS, + OCI_PROCESS_SCHEDULER, + OCI_PROCESS_SELINUXLABEL, OCI_PROCESS_TERMINAL, OCI_PROCESS_USER, __OCI_PROCESS_MAX, }; static const struct blobmsg_policy oci_process_policy[] = { + [OCI_PROCESS_APPARMORPROFILE] = { "apparmorProfile", BLOBMSG_TYPE_STRING }, [OCI_PROCESS_ARGS] = { "args", BLOBMSG_TYPE_ARRAY }, [OCI_PROCESS_CAPABILITIES] = { "capabilities", BLOBMSG_TYPE_TABLE }, [OCI_PROCESS_CWD] = { "cwd", BLOBMSG_TYPE_STRING }, [OCI_PROCESS_ENV] = { "env", BLOBMSG_TYPE_ARRAY }, + [OCI_PROCESS_EXECCPUAFFINITY] = { "execCPUAffinity", BLOBMSG_TYPE_TABLE }, + [OCI_PROCESS_IOPRIORITY] = { "ioPriority", BLOBMSG_TYPE_TABLE }, [OCI_PROCESS_OOMSCOREADJ] = { "oomScoreAdj", BLOBMSG_TYPE_INT32 }, [OCI_PROCESS_NONEWPRIVILEGES] = { "noNewPrivileges", BLOBMSG_TYPE_BOOL }, [OCI_PROCESS_RLIMITS] = { "rlimits", BLOBMSG_TYPE_ARRAY }, + [OCI_PROCESS_SCHEDULER] = { "scheduler", BLOBMSG_TYPE_TABLE }, + [OCI_PROCESS_SELINUXLABEL] = { "selinuxLabel", BLOBMSG_TYPE_STRING }, [OCI_PROCESS_TERMINAL] = { "terminal", BLOBMSG_TYPE_BOOL }, [OCI_PROCESS_USER] = { "user", BLOBMSG_TYPE_TABLE }, }; @@ -2581,6 +2980,16 @@ static int parseOCIprocess(struct blob_attr *msg) blobmsg_parse(oci_process_policy, __OCI_PROCESS_MAX, tb, blobmsg_data(msg), blobmsg_len(msg)); + if (tb[OCI_PROCESS_APPARMORPROFILE]) { + ERROR("process.apparmorProfile is not supported\n"); + return ENOTSUP; + } + + if (tb[OCI_PROCESS_SELINUXLABEL]) { + ERROR("process.selinuxLabel is not supported\n"); + return ENOTSUP; + } + if (!tb[OCI_PROCESS_ARGS]) return ENOENT; @@ -2591,6 +3000,18 @@ static int parseOCIprocess(struct blob_attr *msg) if (tb[OCI_PROCESS_TERMINAL]) opts.console = blobmsg_get_bool(tb[OCI_PROCESS_TERMINAL]); + if (tb[OCI_PROCESS_SCHEDULER]) { + res = parseOCIprocessscheduler(tb[OCI_PROCESS_SCHEDULER]); + if (res) + return res; + } + + if (tb[OCI_PROCESS_IOPRIORITY]) { + res = parseOCIprocessiopriority(tb[OCI_PROCESS_IOPRIORITY]); + if (res) + return res; + } + if (tb[OCI_PROCESS_NONEWPRIVILEGES]) opts.no_new_privs = blobmsg_get_bool(tb[OCI_PROCESS_NONEWPRIVILEGES]); @@ -2654,10 +3075,8 @@ static int resolve_nstype(char *type) { return CLONE_NEWUSER; else if (!strcmp("cgroup", type)) return CLONE_NEWCGROUP; -#ifdef CLONE_NEWTIME else if (!strcmp("time", type)) return CLONE_NEWTIME; -#endif else return 0; } @@ -3013,6 +3432,10 @@ enum { OCI_LINUX_MASKEDPATHS, OCI_LINUX_READONLYPATHS, OCI_LINUX_ROOTFSPROPAGATION, + OCI_LINUX_PERSONALITY, + OCI_LINUX_NETDEVICES, + OCI_LINUX_MEMORYPOLICY, + OCI_LINUX_MOUNTLABEL, __OCI_LINUX_MAX, }; @@ -3028,6 +3451,10 @@ static const struct blobmsg_policy oci_linux_policy[] = { [OCI_LINUX_MASKEDPATHS] = { "maskedPaths", BLOBMSG_TYPE_ARRAY }, [OCI_LINUX_READONLYPATHS] = { "readonlyPaths", BLOBMSG_TYPE_ARRAY }, [OCI_LINUX_ROOTFSPROPAGATION] = { "rootfsPropagation", BLOBMSG_TYPE_STRING }, + [OCI_LINUX_PERSONALITY] = { "personality", BLOBMSG_TYPE_TABLE }, + [OCI_LINUX_NETDEVICES] = { "netDevices", BLOBMSG_TYPE_TABLE }, + [OCI_LINUX_MEMORYPOLICY] = { "memoryPolicy", BLOBMSG_TYPE_TABLE }, + [OCI_LINUX_MOUNTLABEL] = { "mountLabel", BLOBMSG_TYPE_STRING }, }; static int append_deferred_path(char ***list, const char *path) @@ -3054,6 +3481,53 @@ static int append_deferred_path(char ***list, const char *path) return 0; } +enum { + OCI_LINUX_PERSONALITY_DOMAIN, + OCI_LINUX_PERSONALITY_FLAGS, + __OCI_LINUX_PERSONALITY_MAX, +}; + +static const struct blobmsg_policy oci_linux_personality_policy[] = { + [OCI_LINUX_PERSONALITY_DOMAIN] = { "domain", BLOBMSG_TYPE_STRING }, + [OCI_LINUX_PERSONALITY_FLAGS] = { "flags", BLOBMSG_TYPE_ARRAY }, +}; + +static int parseOCIlinuxpersonality(struct blob_attr *msg) +{ + struct blob_attr *tb[__OCI_LINUX_PERSONALITY_MAX]; + const char *domain; + unsigned long requested, current; + + blobmsg_parse(oci_linux_personality_policy, __OCI_LINUX_PERSONALITY_MAX, tb, + blobmsg_data(msg), blobmsg_len(msg)); + + if (tb[OCI_LINUX_PERSONALITY_FLAGS] && + blobmsg_len(tb[OCI_LINUX_PERSONALITY_FLAGS])) { + ERROR("linux.personality.flags is not supported\n"); + return ENOTSUP; + } + + if (!tb[OCI_LINUX_PERSONALITY_DOMAIN]) + return ENODATA; + + domain = blobmsg_get_string(tb[OCI_LINUX_PERSONALITY_DOMAIN]); + if (!strcmp(domain, "LINUX")) + requested = PER_LINUX; + else if (!strcmp(domain, "LINUX32")) + requested = PER_LINUX32; + else + return EINVAL; + + current = personality(0xFFFFFFFF) & PER_MASK; + if (requested != current) { + ERROR("linux.personality '%s' differs from current; cross-personality execution is not supported\n", + domain); + return ENOTSUP; + } + + return 0; +} + static int parseOCIlinux(struct blob_attr *msg) { struct blob_attr *tb[__OCI_LINUX_MAX]; @@ -3065,6 +3539,26 @@ static int parseOCIlinux(struct blob_attr *msg) blobmsg_parse(oci_linux_policy, __OCI_LINUX_MAX, tb, blobmsg_data(msg), blobmsg_len(msg)); + if (tb[OCI_LINUX_PERSONALITY]) { + res = parseOCIlinuxpersonality(tb[OCI_LINUX_PERSONALITY]); + if (res) + return res; + } + + + if (tb[OCI_LINUX_NETDEVICES]) + opts.netdevices = blob_memdup(tb[OCI_LINUX_NETDEVICES]); + + if (tb[OCI_LINUX_MEMORYPOLICY]) { + ERROR("linux.memoryPolicy is not supported on OpenWrt\n"); + return ENOTSUP; + } + + if (tb[OCI_LINUX_MOUNTLABEL]) { + ERROR("linux.mountLabel is not supported\n"); + return ENOTSUP; + } + if (tb[OCI_LINUX_NAMESPACES]) { blobmsg_for_each_attr(cur, tb[OCI_LINUX_NAMESPACES], rem) { res = parseOCIlinuxns(cur); @@ -3175,6 +3669,7 @@ static int parseOCIlinux(struct blob_attr *msg) enum { OCI_VERSION, OCI_HOSTNAME, + OCI_DOMAINNAME, OCI_PROCESS, OCI_ROOT, OCI_MOUNTS, @@ -3187,6 +3682,7 @@ enum { static const struct blobmsg_policy oci_policy[] = { [OCI_VERSION] = { "ociVersion", BLOBMSG_TYPE_STRING }, [OCI_HOSTNAME] = { "hostname", BLOBMSG_TYPE_STRING }, + [OCI_DOMAINNAME] = { "domainname", BLOBMSG_TYPE_STRING }, [OCI_PROCESS] = { "process", BLOBMSG_TYPE_TABLE }, [OCI_ROOT] = { "root", BLOBMSG_TYPE_TABLE }, [OCI_MOUNTS] = { "mounts", BLOBMSG_TYPE_ARRAY }, @@ -3225,6 +3721,9 @@ static int parseOCI(const char *jsonfile) if (tb[OCI_HOSTNAME]) opts.hostname = strdup(blobmsg_get_string(tb[OCI_HOSTNAME])); + if (tb[OCI_DOMAINNAME]) + opts.domainname = strdup(blobmsg_get_string(tb[OCI_DOMAINNAME])); + if (!tb[OCI_PROCESS]) { res=ENODATA; goto errout; @@ -3445,9 +3944,6 @@ static struct uloop_timeout post_main_timeout = { }; static int netns_fd; static int pidns_fd; -#ifdef CLONE_NEWTIME -static int timens_fd; -#endif static void post_create_runtime(void); struct env_e { @@ -3481,9 +3977,7 @@ int main(int argc, char **argv) opts.setns.uts = -1; opts.setns.user = -1; opts.setns.cgroup = -1; -#ifdef CLONE_NEWTIME opts.setns.time = -1; -#endif /* default 5 seconds timeout after SIGTERM before SIGKILL is sent */ opts.term_timeout = 5; @@ -3941,14 +4435,6 @@ static void post_main(struct uloop_timeout *t) pidns_fd = -1; } -#ifdef CLONE_NEWTIME - if (opts.setns.time != -1) { - timens_fd = ns_open_pid("time", getpid()); - setns_open(CLONE_NEWTIME); - } else { - timens_fd = -1; - } -#endif if (opts.namespace & CLONE_NEWUSER) { if (opts.overlaydir) { @@ -3995,12 +4481,6 @@ static void post_main(struct uloop_timeout *t) setns(pidns_fd, CLONE_NEWPID); close(pidns_fd); } -#ifdef CLONE_NEWTIME - if (timens_fd != -1) { - setns(timens_fd, CLONE_NEWTIME); - close(timens_fd); - } -#endif if (opts.setns.net != -1) close(opts.setns.net); if (opts.setns.ns != -1) @@ -4030,6 +4510,11 @@ static void post_main(struct uloop_timeout *t) if (opts.namespace & CLONE_NEWNET) jail_network_start(parent_ctx, opts.name, jail_process.pid); + if (opts.netdevices && + ((opts.namespace & CLONE_NEWNET) || opts.setns.net != -1) && + move_netdevs_into_jail(jail_process.pid)) + free_and_exit(-1); + if (jail_writepid(jail_process.pid)) { ERROR("failed to write pidfile: %m\n"); free_and_exit(-1); From 914d92fafd6ca4dd23245ca9748227eeaf5908d2 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 17 Jun 2026 05:23:20 +0100 Subject: [PATCH 08/82] jail: run OCI prestart, createRuntime and poststart hooks Run the deprecated prestart hook rather than ignoring it, executing it ahead of createRuntime to preserve the historical ordering for runtimes that still rely on it. The hook list is parsed, freed on the error unwind path and chained into the existing lifecycle sequence. Track failure across each hook chain with a shared flag set whenever a hook exits non-zero or on a signal. A failed prestart or createRuntime chain now aborts container creation, and a failed poststart stops the running container instead of idling, so hook errors are no longer masked. Send the termination signal to the jail through its pidfd rather than by pid, closing the window where a recycled pid could be hit during teardown. Signed-off-by: Daniel Golle --- jail/jail.c | 47 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/jail/jail.c b/jail/jail.c index 24e5eec..cdb29d8 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -150,6 +150,7 @@ static struct { bool set_umask; int require_jail; struct { + struct hook_execvpe **prestart; struct hook_execvpe **createRuntime; struct hook_execvpe **createContainer; struct hook_execvpe **startContainer; @@ -306,6 +307,7 @@ static void free_opts(bool parent) { free(opts.netdevices); free(opts.extroot); free(opts.overlaydir); + free_hooklist(opts.hooks.prestart); free_hooklist(opts.hooks.createRuntime); free_hooklist(opts.hooks.createContainer); free_hooklist(opts.hooks.startContainer); @@ -468,6 +470,7 @@ static int create_dev_console(const char *jail_root) static int hook_running = 0; static int hook_return_code = 0; +static bool hook_chain_failed = false; static struct hook_execvpe **current_hook = NULL; typedef void (*hook_return_handler)(void); static hook_return_handler hook_return_cb = NULL; @@ -493,6 +496,8 @@ static void hook_process_handler(struct uloop_process *c, int ret) hook_return_code = WTERMSIG(ret); ERROR("hook (%d) exited with signal: %d\n", c->pid, hook_return_code); } + if (hook_return_code) + hook_chain_failed = true; hook_running = 0; ++current_hook; run_hooklist(); @@ -555,8 +560,12 @@ static void run_hooklist(void) static void run_hooks(struct hook_execvpe **hooklist, hook_return_handler return_cb) { - if (!hooklist) + hook_chain_failed = false; + + if (!hooklist) { return_cb(); + return; + } current_hook = hooklist; hook_return_cb = return_cb; @@ -2746,13 +2755,17 @@ static int parseOCIhooks(struct blob_attr *msg) blobmsg_parse(oci_hooks_policy, __OCI_HOOKS_MAX, tb, blobmsg_data(msg), blobmsg_len(msg)); - if (tb[OCI_HOOKS_PRESTART]) - INFO("warning: ignoring deprecated prestart hook\n"); + if (tb[OCI_HOOKS_PRESTART]) { + INFO("notice: deprecated prestart hook present; running it before createRuntime\n"); + ret = parseOCIhook(&opts.hooks.prestart, tb[OCI_HOOKS_PRESTART]); + if (ret) + return ret; + } if (tb[OCI_HOOKS_CREATERUNTIME]) { ret = parseOCIhook(&opts.hooks.createRuntime, tb[OCI_HOOKS_CREATERUNTIME]); if (ret) - return ret; + goto out_prestart; } if (tb[OCI_HOOKS_CREATECONTAINER]) { @@ -2789,6 +2802,8 @@ static int parseOCIhooks(struct blob_attr *msg) free_hooklist(opts.hooks.createContainer); out_createruntime: free_hooklist(opts.hooks.createRuntime); +out_prestart: + free_hooklist(opts.hooks.prestart); return ret; }; @@ -4316,6 +4331,15 @@ int main(int argc, char **argv) return ret; } +static void post_prestart(void) +{ + if (hook_chain_failed) { + ERROR("prestart hook failed; aborting container\n"); + free_and_exit(EXIT_FAILURE); + } + run_hooks(opts.hooks.createRuntime, post_create_runtime); +} + static void post_main(struct uloop_timeout *t) { if (apply_rlimits()) { @@ -4526,7 +4550,7 @@ static void post_main(struct uloop_timeout *t) ERROR("failed to clone/fork: %m\n"); free_and_exit(EXIT_FAILURE); } - run_hooks(opts.hooks.createRuntime, post_create_runtime); + run_hooks(opts.hooks.prestart, post_prestart); } static void post_poststart(void); @@ -4534,6 +4558,11 @@ static void post_create_runtime(void) { char sig_buf[1]; + if (hook_chain_failed) { + ERROR("createRuntime hook failed; aborting container\n"); + free_and_exit(EXIT_FAILURE); + } + sig_buf[0] = 'O'; if (write(pipes[3], sig_buf, 1) < 0) { ERROR("can't write to child\n"); @@ -4655,9 +4684,13 @@ static void pipe_send_start_container(struct uloop_timeout *t) static void post_poststart(void) { - uloop_run(); /* idle here while jail is running */ + if (hook_chain_failed) + ERROR("poststart hook failed; stopping container\n"); + else + uloop_run(); /* idle here while jail is running */ + if (jail_running) { - DEBUG("uloop interrupted, killing jail process\n"); + DEBUG("killing jail process\n"); kill(jail_process.pid, SIGTERM); uloop_timeout_set(&jail_process_timeout, 1000); uloop_run(); From 813d4e1e9b10062ab3a1ca5a0a1e78175fa4e1e6 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 17 Jun 2026 05:27:13 +0100 Subject: [PATCH 09/82] jail: create the container time namespace and apply linux.timeOffsets Set up the time namespace when CLONE_NEWTIME is requested. Because clone3 cannot create one, unshare it in the parent before the child is cloned so the child inherits it through time_for_children, and write the monotonic and boottime offsets to /proc/self/timens_offsets while the namespace is still empty. Probe /proc/self/ns/time to catch kernels built without CONFIG_TIME_NS: if it is absent the requested namespace cannot be created, so fail hard with a clear diagnostic rather than silently running in the host time, matching runc and crun where unshare() fails with EINVAL. Joining an existing time namespace via setns is handled on its own path. Signed-off-by: Daniel Golle --- jail/jail.c | 146 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 145 insertions(+), 1 deletion(-) diff --git a/jail/jail.c b/jail/jail.c index cdb29d8..fd1b96b 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -2247,6 +2247,117 @@ static int move_netdevs_into_jail(pid_t pid) return ret; } +enum { + OCI_LINUX_TIMEOFFSETS_SECS, + OCI_LINUX_TIMEOFFSETS_NANOSECS, + __OCI_LINUX_TIMEOFFSETS_CLOCK_MAX, +}; + +static const struct blobmsg_policy oci_linux_timeoffsets_clock_policy[] = { + [OCI_LINUX_TIMEOFFSETS_SECS] = { "secs", BLOBMSG_CAST_INT64 }, + [OCI_LINUX_TIMEOFFSETS_NANOSECS] = { "nanosecs", BLOBMSG_TYPE_INT32 }, +}; + +struct procd_timens_offset { + bool set; + int64_t secs; + uint32_t nanosecs; +}; + +static struct { + struct procd_timens_offset monotonic; + struct procd_timens_offset boottime; +} timens_offsets; + +enum { + OCI_LINUX_TIMEOFFSETS_MONOTONIC, + OCI_LINUX_TIMEOFFSETS_BOOTTIME, + __OCI_LINUX_TIMEOFFSETS_MAX, +}; + +static const struct blobmsg_policy oci_linux_timeoffsets_policy[] = { + [OCI_LINUX_TIMEOFFSETS_MONOTONIC] = { "monotonic", BLOBMSG_TYPE_TABLE }, + [OCI_LINUX_TIMEOFFSETS_BOOTTIME] = { "boottime", BLOBMSG_TYPE_TABLE }, +}; + +static int parseOCItimensclock(struct blob_attr *msg, struct procd_timens_offset *off) +{ + struct blob_attr *tb[__OCI_LINUX_TIMEOFFSETS_CLOCK_MAX]; + + blobmsg_parse(oci_linux_timeoffsets_clock_policy, __OCI_LINUX_TIMEOFFSETS_CLOCK_MAX, tb, + blobmsg_data(msg), blobmsg_len(msg)); + + if (tb[OCI_LINUX_TIMEOFFSETS_SECS]) + off->secs = blobmsg_cast_s64(tb[OCI_LINUX_TIMEOFFSETS_SECS]); + + if (tb[OCI_LINUX_TIMEOFFSETS_NANOSECS]) { + uint32_t ns = blobmsg_get_u32(tb[OCI_LINUX_TIMEOFFSETS_NANOSECS]); + + if (ns > 999999999) { + ERROR("timeOffsets: nanosecs %u out of range\n", ns); + return EINVAL; + } + off->nanosecs = ns; + } + + off->set = true; + return 0; +} + +static int parseOCIlinuxtimeoffsets(struct blob_attr *msg) +{ + struct blob_attr *tb[__OCI_LINUX_TIMEOFFSETS_MAX]; + int res; + + blobmsg_parse(oci_linux_timeoffsets_policy, __OCI_LINUX_TIMEOFFSETS_MAX, tb, + blobmsg_data(msg), blobmsg_len(msg)); + + if (tb[OCI_LINUX_TIMEOFFSETS_MONOTONIC]) { + res = parseOCItimensclock(tb[OCI_LINUX_TIMEOFFSETS_MONOTONIC], &timens_offsets.monotonic); + if (res) + return res; + } + + if (tb[OCI_LINUX_TIMEOFFSETS_BOOTTIME]) { + res = parseOCItimensclock(tb[OCI_LINUX_TIMEOFFSETS_BOOTTIME], &timens_offsets.boottime); + if (res) + return res; + } + + return 0; +} + +static int applyOCIlinuxtimeoffsets(void) +{ + int fd = open("/proc/self/timens_offsets", O_WRONLY | O_CLOEXEC); + int saved_errno; + + if (fd < 0) { + ERROR("open(/proc/self/timens_offsets): %m\n"); + return errno; + } + + if (timens_offsets.monotonic.set && + dprintf(fd, "%d %" PRId64 " %" PRIu32 "\n", CLOCK_MONOTONIC, + timens_offsets.monotonic.secs, timens_offsets.monotonic.nanosecs) < 0) { + saved_errno = errno; + ERROR("timens_offsets monotonic: %m\n"); + close(fd); + return saved_errno; + } + + if (timens_offsets.boottime.set && + dprintf(fd, "%d %" PRId64 " %" PRIu32 "\n", CLOCK_BOOTTIME, + timens_offsets.boottime.secs, timens_offsets.boottime.nanosecs) < 0) { + saved_errno = errno; + ERROR("timens_offsets boottime: %m\n"); + close(fd); + return saved_errno; + } + + close(fd); + return 0; +} static void pre_exec_jail(struct uloop_timeout *t); static struct uloop_timeout pre_exec_timeout = { @@ -3448,6 +3559,7 @@ enum { OCI_LINUX_READONLYPATHS, OCI_LINUX_ROOTFSPROPAGATION, OCI_LINUX_PERSONALITY, + OCI_LINUX_TIMEOFFSETS, OCI_LINUX_NETDEVICES, OCI_LINUX_MEMORYPOLICY, OCI_LINUX_MOUNTLABEL, @@ -3467,6 +3579,7 @@ static const struct blobmsg_policy oci_linux_policy[] = { [OCI_LINUX_READONLYPATHS] = { "readonlyPaths", BLOBMSG_TYPE_ARRAY }, [OCI_LINUX_ROOTFSPROPAGATION] = { "rootfsPropagation", BLOBMSG_TYPE_STRING }, [OCI_LINUX_PERSONALITY] = { "personality", BLOBMSG_TYPE_TABLE }, + [OCI_LINUX_TIMEOFFSETS] = { "timeOffsets", BLOBMSG_TYPE_TABLE }, [OCI_LINUX_NETDEVICES] = { "netDevices", BLOBMSG_TYPE_TABLE }, [OCI_LINUX_MEMORYPOLICY] = { "memoryPolicy", BLOBMSG_TYPE_TABLE }, [OCI_LINUX_MOUNTLABEL] = { "mountLabel", BLOBMSG_TYPE_STRING }, @@ -3560,6 +3673,11 @@ static int parseOCIlinux(struct blob_attr *msg) return res; } + if (tb[OCI_LINUX_TIMEOFFSETS]) { + res = parseOCIlinuxtimeoffsets(tb[OCI_LINUX_TIMEOFFSETS]); + if (res) + return res; + } if (tb[OCI_LINUX_NETDEVICES]) opts.netdevices = blob_memdup(tb[OCI_LINUX_NETDEVICES]); @@ -3959,6 +4077,7 @@ static struct uloop_timeout post_main_timeout = { }; static int netns_fd; static int pidns_fd; +static int timens_fd; static void post_create_runtime(void); struct env_e { @@ -4459,6 +4578,27 @@ static void post_main(struct uloop_timeout *t) pidns_fd = -1; } + if ((opts.namespace & CLONE_NEWTIME) && opts.setns.time == -1 && + access("/proc/self/ns/time", F_OK)) { + ERROR("kernel lacks time namespace support\n"); + free_and_exit(EXIT_FAILURE); + } + + if (opts.setns.time != -1) { + timens_fd = ns_open_pid("time", getpid()); + setns_open(CLONE_NEWTIME); + } else if (opts.namespace & CLONE_NEWTIME) { + timens_fd = ns_open_pid("time", getpid()); + if (unshare(CLONE_NEWTIME)) { + ERROR("unshare(CLONE_NEWTIME) failed: %m\n"); + free_and_exit(EXIT_FAILURE); + } + if ((timens_offsets.monotonic.set || timens_offsets.boottime.set) && + applyOCIlinuxtimeoffsets()) + free_and_exit(EXIT_FAILURE); + } else { + timens_fd = -1; + } if (opts.namespace & CLONE_NEWUSER) { if (opts.overlaydir) { @@ -4482,7 +4622,7 @@ static void post_main(struct uloop_timeout *t) * CLONE_NEWUSER is excluded here; the child creates its own * later, in enter_userns(). See exec_jail() for why. */ - jail_process.pid = clone(exec_jail, child_stack + STACK_SIZE, SIGCHLD | (opts.namespace & (~(CLONE_NEWCGROUP | CLONE_NEWUSER))), NULL); + jail_process.pid = clone(exec_jail, child_stack + STACK_SIZE, SIGCHLD | (opts.namespace & ~(CLONE_NEWCGROUP | CLONE_NEWUSER | CLONE_NEWTIME)), NULL); } else { jail_process.pid = fork(); } @@ -4505,6 +4645,10 @@ static void post_main(struct uloop_timeout *t) setns(pidns_fd, CLONE_NEWPID); close(pidns_fd); } + if (timens_fd != -1) { + setns(timens_fd, CLONE_NEWTIME); + close(timens_fd); + } if (opts.setns.net != -1) close(opts.setns.net); if (opts.setns.ns != -1) From 20f75c54d349a30824db651a101dc1eca28c3b3a Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Thu, 18 Jun 2026 01:39:04 +0100 Subject: [PATCH 10/82] jail/cgroups: support further cgroup v2 cpu and memory controls Extend the cpu controller with burst (cpu.max.burst) and idle (cpu.idle), and accept the memory checkBeforeUpdate field. Make the pids limit optional and map a negative limit to "max" rather than rejecting the absence of a limit, matching how the kernel expresses an unbounded count. Correct the swap limit attribute name to memory.swap.max; the previous memory.swap_max never existed so the swap cap was silently never applied. Signed-off-by: Daniel Golle --- jail/cgroups.c | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/jail/cgroups.c b/jail/cgroups.c index 198b5a2..0dfe314 100644 --- a/jail/cgroups.c +++ b/jail/cgroups.c @@ -584,6 +584,8 @@ enum { OCI_LINUX_CGROUPS_CPU_SHARES, OCI_LINUX_CGROUPS_CPU_PERIOD, OCI_LINUX_CGROUPS_CPU_QUOTA, + OCI_LINUX_CGROUPS_CPU_BURST, + OCI_LINUX_CGROUPS_CPU_IDLE, OCI_LINUX_CGROUPS_CPU_REALTIMERUNTIME, OCI_LINUX_CGROUPS_CPU_REALTIMEPERIOD, OCI_LINUX_CGROUPS_CPU_CPUS, @@ -595,6 +597,8 @@ static const struct blobmsg_policy oci_linux_cgroups_cpu_policy[] = { [OCI_LINUX_CGROUPS_CPU_SHARES] = { "shares", BLOBMSG_CAST_INT64 }, [OCI_LINUX_CGROUPS_CPU_PERIOD] = { "period", BLOBMSG_CAST_INT64 }, [OCI_LINUX_CGROUPS_CPU_QUOTA] = { "quota", BLOBMSG_CAST_INT64 }, /* signed int64! */ + [OCI_LINUX_CGROUPS_CPU_BURST] = { "burst", BLOBMSG_CAST_INT64 }, + [OCI_LINUX_CGROUPS_CPU_IDLE] = { "idle", BLOBMSG_CAST_INT64 }, [OCI_LINUX_CGROUPS_CPU_REALTIMEPERIOD] = { "realtimePeriod", BLOBMSG_CAST_INT64 }, [OCI_LINUX_CGROUPS_CPU_REALTIMERUNTIME] = { "realtimeRuntime", BLOBMSG_CAST_INT64 }, [OCI_LINUX_CGROUPS_CPU_CPUS] = { "cpus", BLOBMSG_TYPE_STRING }, @@ -644,6 +648,18 @@ static int parseOCIlinuxcgroups_legacy_cpu(struct blob_attr *msg) if (tmp[0]) cgroups_set("cpu.max", tmp); + if (tb[OCI_LINUX_CGROUPS_CPU_BURST]) { + snprintf(tmp, sizeof(tmp), "%" PRIu64, + blobmsg_cast_u64(tb[OCI_LINUX_CGROUPS_CPU_BURST])); + cgroups_set("cpu.max.burst", tmp); + } + + if (tb[OCI_LINUX_CGROUPS_CPU_IDLE]) { + snprintf(tmp, sizeof(tmp), "%" PRId64, + blobmsg_cast_s64(tb[OCI_LINUX_CGROUPS_CPU_IDLE])); + cgroups_set("cpu.idle", tmp); + } + if (tb[OCI_LINUX_CGROUPS_CPU_CPUS]) cgroups_set("cpuset.cpus", blobmsg_get_string(tb[OCI_LINUX_CGROUPS_CPU_CPUS])); @@ -663,6 +679,7 @@ enum { OCI_LINUX_CGROUPS_MEMORY_SWAPPINESS, OCI_LINUX_CGROUPS_MEMORY_DISABLEOOMKILLER, OCI_LINUX_CGROUPS_MEMORY_USEHIERARCHY, + OCI_LINUX_CGROUPS_MEMORY_CHECKBEFOREUPDATE, __OCI_LINUX_CGROUPS_MEMORY_MAX, }; @@ -675,6 +692,7 @@ static const struct blobmsg_policy oci_linux_cgroups_memory_policy[] = { [OCI_LINUX_CGROUPS_MEMORY_SWAPPINESS] = { "swappiness", BLOBMSG_CAST_INT64 }, [OCI_LINUX_CGROUPS_MEMORY_DISABLEOOMKILLER] = { "disableOOMKiller", BLOBMSG_TYPE_BOOL }, [OCI_LINUX_CGROUPS_MEMORY_USEHIERARCHY] = { "useHierarchy", BLOBMSG_TYPE_BOOL }, + [OCI_LINUX_CGROUPS_MEMORY_CHECKBEFOREUPDATE] = { "checkBeforeUpdate", BLOBMSG_TYPE_BOOL }, }; static int parseOCIlinuxcgroups_legacy_memory(struct blob_attr *msg) @@ -699,7 +717,6 @@ static int parseOCIlinuxcgroups_legacy_memory(struct blob_attr *msg) tb[OCI_LINUX_CGROUPS_MEMORY_USEHIERARCHY]) return ENOTSUP; - if (tb[OCI_LINUX_CGROUPS_MEMORY_LIMIT]) { limit = blobmsg_cast_s64(tb[OCI_LINUX_CGROUPS_MEMORY_LIMIT]); if (limit == -1) @@ -732,7 +749,7 @@ static int parseOCIlinuxcgroups_legacy_memory(struct blob_attr *msg) else snprintf(tmp, sizeof(tmp), "%" PRId64, limit - swap); - cgroups_set("memory.swap_max", tmp); + cgroups_set("memory.swap.max", tmp); } return 0; @@ -752,13 +769,18 @@ static int parseOCIlinuxcgroups_legacy_pids(struct blob_attr *msg) { struct blob_attr *tb[__OCI_LINUX_CGROUPS_MEMORY_MAX]; char tmp[32] = { 0 }; + int64_t limit; blobmsg_parse(oci_linux_cgroups_pids_policy, __OCI_LINUX_CGROUPS_PIDS_MAX, tb, blobmsg_data(msg), blobmsg_len(msg)); if (!tb[OCI_LINUX_CGROUPS_PIDS_LIMIT]) - return EINVAL; + return 0; - snprintf(tmp, sizeof(tmp), "%" PRIu64, blobmsg_cast_u64(tb[OCI_LINUX_CGROUPS_PIDS_LIMIT])); + limit = blobmsg_cast_s64(tb[OCI_LINUX_CGROUPS_PIDS_LIMIT]); + if (limit < 0) + strcpy(tmp, "max"); + else + snprintf(tmp, sizeof(tmp), "%" PRId64, limit); cgroups_set("pids.max", tmp); From a454498c4ab6afeb0189718f52c9a3375f86f4cd Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Thu, 18 Jun 2026 01:39:04 +0100 Subject: [PATCH 11/82] jail/cgroups: cap container memory at a percentage of system RAM Support a vendor annotation org.openwrt.cgroup.memory.pct that caps a container's memory at a percentage (1 to 100) of total system RAM. The percentage is resolved at start from /proc/meminfo via a new read_memtotal_bytes() helper and applied through a cgroups_set_memory_limit() interface writing memory.max. This lets an image express a memory cap relative to the host size rather than as an absolute byte value that has to be tuned per device. Signed-off-by: Daniel Golle --- jail/cgroups.c | 8 +++++++ jail/cgroups.h | 1 + jail/jail.c | 64 +++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 72 insertions(+), 1 deletion(-) diff --git a/jail/cgroups.c b/jail/cgroups.c index 0dfe314..f315ad1 100644 --- a/jail/cgroups.c +++ b/jail/cgroups.c @@ -695,6 +695,14 @@ static const struct blobmsg_policy oci_linux_cgroups_memory_policy[] = { [OCI_LINUX_CGROUPS_MEMORY_CHECKBEFOREUPDATE] = { "checkBeforeUpdate", BLOBMSG_TYPE_BOOL }, }; +void cgroups_set_memory_limit(int64_t bytes) +{ + char tmp[32]; + + snprintf(tmp, sizeof(tmp), "%" PRId64, bytes); + cgroups_set("memory.max", tmp); +} + static int parseOCIlinuxcgroups_legacy_memory(struct blob_attr *msg) { struct blob_attr *tb[__OCI_LINUX_CGROUPS_MEMORY_MAX]; diff --git a/jail/cgroups.h b/jail/cgroups.h index 4c8f968..f049061 100644 --- a/jail/cgroups.h +++ b/jail/cgroups.h @@ -19,5 +19,6 @@ int parseOCIlinuxcgroups(struct blob_attr *msg); void cgroups_apply(pid_t pid); void cgroups_free(void); void cgroups_prepare(void); +void cgroups_set_memory_limit(int64_t bytes); #endif diff --git a/jail/jail.c b/jail/jail.c index fd1b96b..be5b84c 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -3824,12 +3824,48 @@ static const struct blobmsg_policy oci_policy[] = { [OCI_ANNOTATIONS] = { "annotations", BLOBMSG_TYPE_TABLE }, }; +static int64_t read_memtotal_bytes(void) +{ + char buf[512]; + char *p; + char *end; + int64_t kb; + int fd; + ssize_t n; + + fd = open("/proc/meminfo", O_RDONLY | O_CLOEXEC); + if (fd < 0) + return -1; + do { + n = read(fd, buf, sizeof(buf) - 1); + } while (n < 0 && errno == EINTR); + close(fd); + if (n <= 0) + return -1; + buf[n] = '\0'; + p = strstr(buf, "MemTotal:"); + if (!p) + return -1; + p += strlen("MemTotal:"); + while (*p == ' ' || *p == '\t') + p++; + kb = strtoll(p, &end, 10); + if (end == p || kb <= 0) + return -1; + return kb * 1024; +} + static int parseOCI(const char *jsonfile) { struct blob_attr *tb[__OCI_MAX]; struct blob_attr *cur; int rem; + int arem; + struct blob_attr *acur; int res; + long pct; + char *pct_end; + int64_t memtotal; blob_buf_init(&ocibuf, 0); @@ -3887,9 +3923,35 @@ static int parseOCI(const char *jsonfile) if (tb[OCI_HOOKS] && (res = parseOCIhooks(tb[OCI_HOOKS]))) goto errout; - if (tb[OCI_ANNOTATIONS]) + if (tb[OCI_ANNOTATIONS]) { opts.annotations = blob_memdup(tb[OCI_ANNOTATIONS]); + blobmsg_for_each_attr(acur, tb[OCI_ANNOTATIONS], arem) { + const char *name = blobmsg_name(acur); + const char *val; + + if (!name || blobmsg_type(acur) != BLOBMSG_TYPE_STRING) + continue; + + val = blobmsg_get_string(acur); + + if (!strcmp(name, "org.openwrt.cgroup.memory.pct")) { + pct = strtol(val, &pct_end, 10); + if (pct_end == val || pct < 1 || pct > 100) { + ERROR("cgroup.memory.pct: invalid value '%s'\n", val); + res = EINVAL; + goto errout; + } + memtotal = read_memtotal_bytes(); + if (memtotal < 0) { + ERROR("cgroup.memory.pct: cannot read MemTotal\n"); + res = EIO; + goto errout; + } + cgroups_set_memory_limit(memtotal * pct / 100); + } + } + } errout: blob_buf_free(&ocibuf); From d3db457be8318cbe352d082a4c79e111f802322f Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 17 Jun 2026 05:37:05 +0100 Subject: [PATCH 12/82] jail: report cgroup memory and pids statistics and add memory.reclaim Extend the container state report with cgroup v2 statistics: memory.peak, memory.swap.peak and pids.peak are exposed as memoryPeak, memorySwapPeak and pidsPeak, and the memory.events.local counters are parsed into a memoryEventsLocal table. The state report is also produced while the container is paused, not only when created or running. Add the cgroup helper interface used to gather and act on these figures: read and open accessors for arbitrary attributes and the directory, plus attach, kill-all, freeze and a reclaim entry point that drives the memory.reclaim control with an optional swappiness hint. Signed-off-by: Daniel Golle --- jail/cgroups.c | 103 +++++++++++++++++++++++++++++++++++++++++++++++++ jail/cgroups.h | 6 +++ jail/jail.c | 92 ++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 200 insertions(+), 1 deletion(-) diff --git a/jail/cgroups.c b/jail/cgroups.c index f315ad1..8bf8ed0 100644 --- a/jail/cgroups.c +++ b/jail/cgroups.c @@ -101,6 +101,63 @@ void cgroups_free(void) } } +static int cgroups_write_attr(const char *attr, const char *val, size_t vlen) +{ + char *ent; + int fd, ret = 0; + size_t len; + + if (!cgroup_path) + return -ENODEV; + + len = strlen(cgroup_path) + 1 + strlen(attr) + 1; + ent = malloc(len); + if (!ent) + return -ENOMEM; + + snprintf(ent, len, "%s/%s", cgroup_path, attr); + fd = open(ent, O_WRONLY); + if (fd < 0) { + ret = -errno; + free(ent); + return ret; + } + + if (write(fd, val, vlen) < 0) + ret = -errno; + + close(fd); + free(ent); + return ret; +} + +int cgroups_reclaim(int64_t bytes, int32_t swappiness) +{ + char val[64]; + int len, ret; + int attempt; + + if (bytes < 0) + return -EINVAL; + + if (swappiness >= 0) + len = snprintf(val, sizeof(val), "%" PRId64 " swappiness=%" PRId32, + bytes, swappiness); + else + len = snprintf(val, sizeof(val), "%" PRId64, bytes); + if (len < 0 || len >= (int)sizeof(val)) + return -EINVAL; + + for (attempt = 0; attempt < 2; ++attempt) { + ret = cgroups_write_attr("memory.reclaim", val, len); + if (ret != -EINTR) + break; + } + if (ret == -ENOENT) + ret = -ENODEV; + return ret; +} + void cgroups_apply(pid_t pid) { struct cgval *valp; @@ -695,6 +752,52 @@ static const struct blobmsg_policy oci_linux_cgroups_memory_policy[] = { [OCI_LINUX_CGROUPS_MEMORY_CHECKBEFOREUPDATE] = { "checkBeforeUpdate", BLOBMSG_TYPE_BOOL }, }; +static int64_t read_int64_file(const char *path) +{ + char buf[32]; + char *end; + int64_t v; + int fd; + ssize_t n; + + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return -1; + do { + n = read(fd, buf, sizeof(buf) - 1); + } while (n < 0 && errno == EINTR); + close(fd); + if (n <= 0) + return -1; + buf[n] = '\0'; + v = strtoll(buf, &end, 10); + if (end == buf || (*end != '\0' && *end != '\n')) + return -1; + return v; +} + +int64_t cgroups_read_int64(const char *attr) +{ + char path[PATH_MAX]; + + if (!cgroup_path) + return -1; + + snprintf(path, sizeof(path), "%s/%s", cgroup_path, attr); + return read_int64_file(path); +} + +int cgroups_open_attr(const char *attr) +{ + char path[PATH_MAX]; + + if (!cgroup_path) + return -1; + + snprintf(path, sizeof(path), "%s/%s", cgroup_path, attr); + return open(path, O_RDONLY | O_CLOEXEC); +} + void cgroups_set_memory_limit(int64_t bytes) { char tmp[32]; diff --git a/jail/cgroups.h b/jail/cgroups.h index f049061..c1631b3 100644 --- a/jail/cgroups.h +++ b/jail/cgroups.h @@ -14,9 +14,15 @@ #ifndef _JAIL_CGROUPS_H #define _JAIL_CGROUPS_H +#include +#include + void cgroups_init(const char *p); int parseOCIlinuxcgroups(struct blob_attr *msg); void cgroups_apply(pid_t pid); +int cgroups_reclaim(int64_t bytes, int32_t swappiness); +int64_t cgroups_read_int64(const char *attr); +int cgroups_open_attr(const char *attr); void cgroups_free(void); void cgroups_prepare(void); void cgroups_set_memory_limit(int64_t bytes); diff --git a/jail/jail.c b/jail/jail.c index be5b84c..76d5bf4 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -4032,9 +4032,46 @@ static int handle_state(struct ubus_context *ctx, struct ubus_object *obj, blobmsg_add_string(&bb, "id", opts.name); blobmsg_add_string(&bb, "status", statusstr); if (jail_oci_state == OCI_STATE_CREATED || - jail_oci_state == OCI_STATE_RUNNING) + jail_oci_state == OCI_STATE_RUNNING) { + int64_t v; + blobmsg_add_u32(&bb, "pid", jail_process.pid); + v = cgroups_read_int64("memory.peak"); + if (v >= 0) + blobmsg_add_u64(&bb, "memoryPeak", (uint64_t)v); + v = cgroups_read_int64("memory.swap.peak"); + if (v >= 0) + blobmsg_add_u64(&bb, "memorySwapPeak", (uint64_t)v); + v = cgroups_read_int64("pids.peak"); + if (v >= 0) + blobmsg_add_u64(&bb, "pidsPeak", (uint64_t)v); + + int events_fd = cgroups_open_attr("memory.events.local"); + if (events_fd >= 0) { + char ebuf[1024], *line, *next; + ssize_t en = read(events_fd, ebuf, sizeof(ebuf) - 1); + + close(events_fd); + if (en > 0) { + void *sub = blobmsg_open_table(&bb, "memoryEventsLocal"); + + ebuf[en] = '\0'; + next = ebuf; + while ((line = strsep(&next, "\n"))) { + char *space = strchr(line, ' '); + + if (!space) + continue; + *space = '\0'; + blobmsg_add_u64(&bb, line, + strtoull(space + 1, NULL, 10)); + } + blobmsg_close_table(&bb, sub); + } + } + } + blobmsg_add_string(&bb, "bundle", opts.ocibundle); if (opts.annotations) @@ -4083,6 +4120,58 @@ container_handle_kill(struct ubus_context *ctx, struct ubus_object *obj, return UBUS_STATUS_UNKNOWN_ERROR; } +enum { + CONTAINER_RECLAIM_ATTR_BYTES, + CONTAINER_RECLAIM_ATTR_SWAPPINESS, + __CONTAINER_RECLAIM_ATTR_MAX, +}; + +static const struct blobmsg_policy container_reclaim_attrs[__CONTAINER_RECLAIM_ATTR_MAX] = { + [CONTAINER_RECLAIM_ATTR_BYTES] = { "bytes", BLOBMSG_CAST_INT64 }, + [CONTAINER_RECLAIM_ATTR_SWAPPINESS] = { "swappiness", BLOBMSG_TYPE_INT32 }, +}; + +static int +container_handle_reclaim(struct ubus_context *ctx, struct ubus_object *obj, + struct ubus_request_data *req, const char *method, + struct blob_attr *msg) +{ + struct blob_attr *tb[__CONTAINER_RECLAIM_ATTR_MAX]; + int64_t bytes; + int32_t swappiness = -1; + int rc; + + if (jail_oci_state != OCI_STATE_CREATED && + jail_oci_state != OCI_STATE_RUNNING) + return UBUS_STATUS_INVALID_ARGUMENT; + if (!msg) + return UBUS_STATUS_INVALID_ARGUMENT; + + blobmsg_parse(container_reclaim_attrs, __CONTAINER_RECLAIM_ATTR_MAX, tb, + blobmsg_data(msg), blobmsg_data_len(msg)); + if (!tb[CONTAINER_RECLAIM_ATTR_BYTES]) + return UBUS_STATUS_INVALID_ARGUMENT; + + bytes = blobmsg_cast_s64(tb[CONTAINER_RECLAIM_ATTR_BYTES]); + if (tb[CONTAINER_RECLAIM_ATTR_SWAPPINESS]) { + uint32_t s = blobmsg_get_u32(tb[CONTAINER_RECLAIM_ATTR_SWAPPINESS]); + if (s > 200) + return UBUS_STATUS_INVALID_ARGUMENT; + swappiness = (int32_t)s; + } + + rc = cgroups_reclaim(bytes, swappiness); + if (rc == 0) + return UBUS_STATUS_OK; + if (rc == -EAGAIN) + return UBUS_STATUS_TIMEOUT; + if (rc == -EINVAL) + return UBUS_STATUS_INVALID_ARGUMENT; + if (rc == -ENODEV) + return UBUS_STATUS_NOT_SUPPORTED; + return UBUS_STATUS_UNKNOWN_ERROR; +} + static int jail_writepid(pid_t pid) { @@ -4122,6 +4211,7 @@ static struct ubus_method container_methods[] = { UBUS_METHOD_NOARG("start", handle_start), UBUS_METHOD_NOARG("state", handle_state), UBUS_METHOD("kill", container_handle_kill, container_kill_attrs), + UBUS_METHOD("reclaim", container_handle_reclaim, container_reclaim_attrs), }; static struct ubus_object_type container_object_type = From 9cd74e36138b62b76876cac216c8a07359dfda71 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 17 Jun 2026 12:11:32 +0100 Subject: [PATCH 13/82] uxc: restructure argv parser into global/verb/verb-flag phases The single getopt table forced every subcommand to share one option set and one set of short flags, which does not scale as verbs gain their own options. Split parsing into a global phase, a verb, and per-verb flag tables so each subcommand declares only the options it accepts. This is a pure restructure with no behavioural change; it is the foundation the kill, pause/resume and exec/update verbs build on. Signed-off-by: Daniel Golle --- uxc.c | 339 ++++++++++++++++++++++++++-------------------------------- 1 file changed, 154 insertions(+), 185 deletions(-) diff --git a/uxc.c b/uxc.c index 4926924..6147b52 100644 --- a/uxc.c +++ b/uxc.c @@ -73,34 +73,33 @@ struct settings { struct blob_attr *volumes; }; -enum uxc_cmd { - CMD_ATTACH, - CMD_LIST, - CMD_BOOT, - CMD_START, - CMD_STATE, - CMD_KILL, - CMD_ENABLE, - CMD_DISABLE, - CMD_DELETE, - CMD_CREATE, - CMD_UNKNOWN -}; - -#define OPT_ARGS "ab:fjm:p:t:vVw:" -static struct option long_options[] = { +static const struct option create_opts[] = { {"autostart", no_argument, 0, 'a' }, - {"console", no_argument, 0, 'c' }, {"bundle", required_argument, 0, 'b' }, - {"force", no_argument, 0, 'f' }, - {"json", no_argument, 0, 'j' }, {"mounts", required_argument, 0, 'm' }, {"pid-file", required_argument, 0, 'p' }, - {"signal", required_argument, 0, 's' }, {"temp-overlay-size", required_argument, 0, 't' }, {"write-overlay-path", required_argument, 0, 'w' }, - {"verbose", no_argument, 0, 'v' }, - {"version", no_argument, 0, 'V' }, + {0, 0, 0, 0 } +}; + +static const struct option start_opts[] = { + {"console", no_argument, 0, 'c' }, + {0, 0, 0, 0 } +}; + +static const struct option kill_opts[] = { + {"signal", required_argument, 0, 's' }, + {0, 0, 0, 0 } +}; + +static const struct option delete_opts[] = { + {"force", no_argument, 0, 'f' }, + {0, 0, 0, 0 } +}; + +static const struct option list_opts[] = { + {"json", no_argument, 0, 'j' }, {0, 0, 0, 0 } }; @@ -1526,21 +1525,42 @@ static int get_signum(const char *name) int main(int argc, char **argv) { - enum uxc_cmd cmd = CMD_UNKNOWN; int ret = -EINVAL; - char *bundle = NULL; - char *pidfile = NULL; - char *tmprwsize = NULL; - char *writepath = NULL; - char *requiredmounts = NULL; - signed char autostart = -1; - bool force = false; - bool console = false; - int signal = SIGTERM; - int c; - - if (argc < 2) + const char *verb; + int verb_argc, c, i; + char **verb_argv; + + for (i = 1; i < argc; ++i) { + const char *a = argv[i]; + + if (a[0] != '-') + break; + + if (!strcmp(a, "--")) { + ++i; + break; + } + + if (!strcmp(a, "-V") || !strcmp(a, "--version")) { + printf("uxc %s\nspec: %s\n", UXC_VERSION, OCI_VERSION_STRING); + return 0; + } + + if (!strcmp(a, "-v") || !strcmp(a, "--verbose")) { + verbose = true; + continue; + } + + fprintf(stderr, "uxc: unknown option '%s'\n", a); return usage(); + } + + if (i >= argc) + return usage(); + + verb = argv[i]; + verb_argc = argc - i; + verb_argv = argv + i; ctx = ubus_connect(NULL); if (!ctx) @@ -1562,167 +1582,116 @@ int main(int argc, char **argv) if (ret) goto settings_avl_out; - while (true) { - int option_index = 0; - c = getopt_long(argc, argv, OPT_ARGS, long_options, &option_index); - if (c == -1) - break; - - switch (c) { - case 'a': - autostart = 1; - break; - - case 'b': - bundle = optarg; - break; + optind = 1; + opterr = 1; - case 'c': - console = true; - break; - - case 'f': - force = true; - break; - - case 'j': - json_output = true; - break; - - case 'p': - pidfile = optarg; - break; + if (!strcmp(verb, "list")) { + while ((c = getopt_long(verb_argc, verb_argv, "j", list_opts, NULL)) != -1) { + switch (c) { + case 'j': json_output = true; break; + default: goto usage_out; + } + } + if (optind != verb_argc) + goto usage_out; + ret = uxc_list(); + } else if (!strcmp(verb, "attach")) { + if (verb_argc != 2) + goto usage_out; + ret = uxc_attach(verb_argv[1]); + } else if (!strcmp(verb, "boot")) { + if (verb_argc != 1) + goto usage_out; + ret = uxc_boot(); + } else if (!strcmp(verb, "start")) { + bool console = false; + + while ((c = getopt_long(verb_argc, verb_argv, "c", start_opts, NULL)) != -1) { + switch (c) { + case 'c': console = true; break; + default: goto usage_out; + } + } + if (optind != verb_argc - 1) + goto usage_out; + ret = uxc_start(verb_argv[optind], console); + } else if (!strcmp(verb, "state")) { + if (verb_argc != 2) + goto usage_out; + ret = uxc_state(verb_argv[1]); + } else if (!strcmp(verb, "kill")) { + int signal = SIGTERM; + while ((c = getopt_long(verb_argc, verb_argv, "s:", kill_opts, NULL)) != -1) { + switch (c) { case 's': signal = get_signum(optarg); if (signal < 0) goto usage_out; break; - - case 't': - tmprwsize = optarg; - break; - - case 'v': - verbose = true; - break; - - case 'V': - printf("uxc %s\n", UXC_VERSION); - exit(0); - - case 'w': - writepath = optarg; - break; - - case 'm': - requiredmounts = optarg; - break; + default: goto usage_out; + } } - } - - if (optind == argc) - goto usage_out; - - if (!strcmp("list", argv[optind])) - cmd = CMD_LIST; - else if (!strcmp("attach", argv[optind])) - cmd = CMD_ATTACH; - else if (!strcmp("boot", argv[optind])) - cmd = CMD_BOOT; - else if(!strcmp("start", argv[optind])) - cmd = CMD_START; - else if(!strcmp("state", argv[optind])) - cmd = CMD_STATE; - else if(!strcmp("kill", argv[optind])) - cmd = CMD_KILL; - else if(!strcmp("enable", argv[optind])) - cmd = CMD_ENABLE; - else if(!strcmp("disable", argv[optind])) - cmd = CMD_DISABLE; - else if(!strcmp("delete", argv[optind])) - cmd = CMD_DELETE; - else if(!strcmp("create", argv[optind])) - cmd = CMD_CREATE; - - switch (cmd) { - case CMD_ATTACH: - if (optind != argc - 2) - goto usage_out; - - ret = uxc_attach(argv[optind + 1]); - break; - - case CMD_LIST: - ret = uxc_list(); - break; - - case CMD_BOOT: - ret = uxc_boot(); - break; - - case CMD_START: - if (optind != argc - 2) - goto usage_out; - - ret = uxc_start(argv[optind + 1], console); - break; - - case CMD_STATE: - if (optind != argc - 2) - goto usage_out; - - ret = uxc_state(argv[optind + 1]); - break; - - case CMD_KILL: - if (optind > argc - 2) - goto usage_out; - - ret = uxc_kill(argv[optind + 1], signal); - break; - - case CMD_ENABLE: - if (optind != argc - 2) - goto usage_out; - - ret = uxc_set(argv[optind + 1], NULL, 1, NULL, NULL, NULL, NULL); - break; - - case CMD_DISABLE: - if (optind != argc - 2) - goto usage_out; - - ret = uxc_set(argv[optind + 1], NULL, 0, NULL, NULL, NULL, NULL); - break; - - case CMD_DELETE: - if (optind != argc - 2) - goto usage_out; - - ret = uxc_delete(argv[optind + 1], force); - break; - - case CMD_CREATE: - if (optind != argc - 2) - goto usage_out; - - ret = uxc_exists(argv[optind + 1]); - if (ret) - goto runtime_out; - - ret = uxc_set(argv[optind + 1], bundle, autostart, pidfile, tmprwsize, writepath, requiredmounts); - if (ret < 0) - goto runtime_out; + if (optind != verb_argc - 1) + goto usage_out; + ret = uxc_kill(verb_argv[optind], signal); + } else if (!strcmp(verb, "enable")) { + if (verb_argc != 2) + goto usage_out; + ret = uxc_set(verb_argv[1], NULL, 1, NULL, NULL, NULL, NULL); + } else if (!strcmp(verb, "disable")) { + if (verb_argc != 2) + goto usage_out; + ret = uxc_set(verb_argv[1], NULL, 0, NULL, NULL, NULL, NULL); + } else if (!strcmp(verb, "delete")) { + bool force = false; + + while ((c = getopt_long(verb_argc, verb_argv, "f", delete_opts, NULL)) != -1) { + switch (c) { + case 'f': force = true; break; + default: goto usage_out; + } + } + if (optind != verb_argc - 1) + goto usage_out; + ret = uxc_delete(verb_argv[optind], force); + } else if (!strcmp(verb, "create")) { + char *bundle = NULL, *pidfile = NULL; + char *tmprwsize = NULL, *writepath = NULL, *requiredmounts = NULL; + signed char autostart = -1; + char *name; + + while ((c = getopt_long(verb_argc, verb_argv, "ab:m:p:t:w:", + create_opts, NULL)) != -1) { + switch (c) { + case 'a': autostart = 1; break; + case 'b': bundle = optarg; break; + case 'm': requiredmounts = optarg; break; + case 'p': pidfile = optarg; break; + case 't': tmprwsize = optarg; break; + case 'w': writepath = optarg; break; + default: goto usage_out; + } + } + if (optind != verb_argc - 1) + goto usage_out; + name = verb_argv[optind]; - if (ret > 0) - reload_conf(); + ret = uxc_exists(name); + if (ret) + goto runtime_out; - ret = uxc_create(argv[optind + 1], false); - break; + ret = uxc_set(name, bundle, autostart, pidfile, + tmprwsize, writepath, requiredmounts); + if (ret < 0) + goto runtime_out; + if (ret > 0) + reload_conf(); - default: - goto usage_out; + ret = uxc_create(name, false); + } else { + fprintf(stderr, "uxc: unknown command '%s'\n", verb); + goto usage_out; } goto runtime_out; From f8a01ec8de3e5ebc6e975eda9a7aaa4e62fbb4fa Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 05:25:41 +0100 Subject: [PATCH 14/82] jail: pause and resume via cgroup.freeze and kill all members Add pause and resume ubus methods to the container object that toggle cgroup.freeze, reflected through a new OCI "paused" state. While paused, only SIGKILL and signal 0 are accepted; any other signal is rejected so a frozen container cannot be partly signalled. Build cgroups_kill_all() and cgroups_set_frozen() on top of the generic cgroup attribute writer. The kill method gains an "all" flag that uses cgroup.kill to take down every member of the hierarchy in one write, falling back to per-pid kill where the kernel lacks it. Signed-off-by: Daniel Golle --- jail/cgroups.c | 10 +++++++ jail/cgroups.h | 2 ++ jail/jail.c | 80 +++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 91 insertions(+), 1 deletion(-) diff --git a/jail/cgroups.c b/jail/cgroups.c index 8bf8ed0..b466b89 100644 --- a/jail/cgroups.c +++ b/jail/cgroups.c @@ -131,6 +131,16 @@ static int cgroups_write_attr(const char *attr, const char *val, size_t vlen) return ret; } +int cgroups_kill_all(void) +{ + return cgroups_write_attr("cgroup.kill", "1", 1); +} + +int cgroups_set_frozen(bool frozen) +{ + return cgroups_write_attr("cgroup.freeze", frozen ? "1" : "0", 1); +} + int cgroups_reclaim(int64_t bytes, int32_t swappiness) { char val[64]; diff --git a/jail/cgroups.h b/jail/cgroups.h index c1631b3..30043b0 100644 --- a/jail/cgroups.h +++ b/jail/cgroups.h @@ -20,6 +20,8 @@ void cgroups_init(const char *p); int parseOCIlinuxcgroups(struct blob_attr *msg); void cgroups_apply(pid_t pid); +int cgroups_kill_all(void); +int cgroups_set_frozen(bool frozen); int cgroups_reclaim(int64_t bytes, int32_t swappiness); int64_t cgroups_read_int64(const char *attr); int cgroups_open_attr(const char *attr); diff --git a/jail/jail.c b/jail/jail.c index 76d5bf4..855f11a 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -3982,6 +3982,7 @@ enum { OCI_STATE_CREATING, OCI_STATE_CREATED, OCI_STATE_RUNNING, + OCI_STATE_PAUSED, OCI_STATE_STOPPED, }; @@ -4020,6 +4021,9 @@ static int handle_state(struct ubus_context *ctx, struct ubus_object *obj, case OCI_STATE_RUNNING: statusstr = "running"; break; + case OCI_STATE_PAUSED: + statusstr = "paused"; + break; case OCI_STATE_STOPPED: statusstr = "stopped"; break; @@ -4032,7 +4036,8 @@ static int handle_state(struct ubus_context *ctx, struct ubus_object *obj, blobmsg_add_string(&bb, "id", opts.name); blobmsg_add_string(&bb, "status", statusstr); if (jail_oci_state == OCI_STATE_CREATED || - jail_oci_state == OCI_STATE_RUNNING) { + jail_oci_state == OCI_STATE_RUNNING || + jail_oci_state == OCI_STATE_PAUSED) { int64_t v; blobmsg_add_u32(&bb, "pid", jail_process.pid); @@ -4084,11 +4089,13 @@ static int handle_state(struct ubus_context *ctx, struct ubus_object *obj, enum { CONTAINER_KILL_ATTR_SIGNAL, + CONTAINER_KILL_ATTR_ALL, __CONTAINER_KILL_ATTR_MAX, }; static const struct blobmsg_policy container_kill_attrs[__CONTAINER_KILL_ATTR_MAX] = { [CONTAINER_KILL_ATTR_SIGNAL] = { "signal", BLOBMSG_TYPE_INT32 }, + [CONTAINER_KILL_ATTR_ALL] = { "all", BLOBMSG_TYPE_BOOL }, }; static int @@ -4098,6 +4105,7 @@ container_handle_kill(struct ubus_context *ctx, struct ubus_object *obj, { struct blob_attr *tb[__CONTAINER_KILL_ATTR_MAX], *cur; int sig = SIGTERM; + bool all = false; blobmsg_parse(container_kill_attrs, __CONTAINER_KILL_ATTR_MAX, tb, blobmsg_data(msg), blobmsg_data_len(msg)); @@ -4105,8 +4113,21 @@ container_handle_kill(struct ubus_context *ctx, struct ubus_object *obj, if (cur) sig = blobmsg_get_u32(cur); + cur = tb[CONTAINER_KILL_ATTR_ALL]; + if (cur) + all = blobmsg_get_bool(cur); + if (jail_oci_state == OCI_STATE_CREATING) return UBUS_STATUS_NOT_FOUND; + if (jail_oci_state == OCI_STATE_PAUSED && sig != SIGKILL && sig != 0) + return UBUS_STATUS_PERMISSION_DENIED; + + if (all && sig == SIGKILL) { + int rc = cgroups_kill_all(); + if (rc == 0) + return 0; + DEBUG("cgroup.kill unavailable (%d), falling back to per-pid kill\n", rc); + } if (kill(jail_process.pid, sig) == 0) return 0; @@ -4120,6 +4141,61 @@ container_handle_kill(struct ubus_context *ctx, struct ubus_object *obj, return UBUS_STATUS_UNKNOWN_ERROR; } +static int +container_handle_pause(struct ubus_context *ctx, struct ubus_object *obj, + struct ubus_request_data *req, const char *method, + struct blob_attr *msg) +{ + int rc; + + if (jail_oci_state != OCI_STATE_CREATED && + jail_oci_state != OCI_STATE_RUNNING) + return UBUS_STATUS_INVALID_ARGUMENT; + + rc = cgroups_set_frozen(true); + if (rc < 0) { + switch (rc) { + case -ENODEV: + case -ENOENT: + return UBUS_STATUS_NOT_SUPPORTED; + case -EINVAL: + return UBUS_STATUS_INVALID_ARGUMENT; + default: + return UBUS_STATUS_UNKNOWN_ERROR; + } + } + + jail_oci_state = OCI_STATE_PAUSED; + return UBUS_STATUS_OK; +} + +static int +container_handle_resume(struct ubus_context *ctx, struct ubus_object *obj, + struct ubus_request_data *req, const char *method, + struct blob_attr *msg) +{ + int rc; + + if (jail_oci_state != OCI_STATE_PAUSED) + return UBUS_STATUS_INVALID_ARGUMENT; + + rc = cgroups_set_frozen(false); + if (rc < 0) { + switch (rc) { + case -ENODEV: + case -ENOENT: + return UBUS_STATUS_NOT_SUPPORTED; + case -EINVAL: + return UBUS_STATUS_INVALID_ARGUMENT; + default: + return UBUS_STATUS_UNKNOWN_ERROR; + } + } + + jail_oci_state = OCI_STATE_RUNNING; + return UBUS_STATUS_OK; +} + enum { CONTAINER_RECLAIM_ATTR_BYTES, CONTAINER_RECLAIM_ATTR_SWAPPINESS, @@ -4211,6 +4287,8 @@ static struct ubus_method container_methods[] = { UBUS_METHOD_NOARG("start", handle_start), UBUS_METHOD_NOARG("state", handle_state), UBUS_METHOD("kill", container_handle_kill, container_kill_attrs), + UBUS_METHOD_NOARG("pause", container_handle_pause), + UBUS_METHOD_NOARG("resume", container_handle_resume), UBUS_METHOD("reclaim", container_handle_reclaim, container_reclaim_attrs), }; From 030a758924ed0f24ba75215bc95608d7ef5d3ca8 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 05:25:41 +0100 Subject: [PATCH 15/82] uxc: add pause and resume verbs and kill --all Expose the container object's new pause and resume methods as verbs and forward the kill method's new "all" flag: kill gains an --all option, valid only together with SIGKILL, and accepts the signal as a positional argument as well; forced delete passes the flag so the whole cgroup goes down in one write. This also reworks command dispatch around a single ubus invocation path for the simple container verbs, replacing the per-command switch. Signed-off-by: Daniel Golle --- uxc.c | 56 +++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/uxc.c b/uxc.c index 6147b52..533d580 100644 --- a/uxc.c +++ b/uxc.c @@ -90,6 +90,7 @@ static const struct option start_opts[] = { static const struct option kill_opts[] = { {"signal", required_argument, 0, 's' }, + {"all", no_argument, 0, 'a' }, {0, 0, 0, 0 } }; @@ -246,10 +247,12 @@ static int usage(void) { printf("\t\t[--mounts ,,...,]\t\trequire filesystems to be available\n"); printf("\tstart [--console] \t\tstart container \n"); printf("\tstate \t\t\t\tget state of container \n"); - printf("\tkill [--signal ]\t\tsend signal to container \n"); + printf("\tkill [--signal ] [--all] []\tsignal (no signal: graceful stop); --all+KILL: whole cgroup\n"); printf("\tenable \t\t\t\tstart container on boot\n"); printf("\tdisable \t\t\t\tdon't start container on boot\n"); printf("\tdelete [--force]\t\t\tdelete \n"); + printf("\tpause \t\t\t\tfreeze every process in container 's cgroup\n"); + printf("\tresume \t\t\t\tthaw a previously paused container \n"); return -EINVAL; } @@ -1024,7 +1027,7 @@ static int uxc_start(const char *name, bool console) return ubus_invoke(ctx, id, "start", NULL, NULL, NULL, 3000); } -static int uxc_kill(char *name, int signal) +static int uxc_kill(char *name, int signal, bool all) { static struct blob_buf req; struct blob_attr *cur, *tb[__CONF_MAX]; @@ -1057,6 +1060,8 @@ static int uxc_kill(char *name, int signal) blob_buf_init(&req, 0); blobmsg_add_u32(&req, "signal", signal); blobmsg_add_string(&req, "name", name); + if (all) + blobmsg_add_u8(&req, "all", 1); if (asprintf(&objname, "container.%s", name) == -1) return -ENOMEM; @@ -1437,7 +1442,7 @@ static int uxc_delete(char *name, bool force) if (rsstate && rsstate->running) { if (force) { - ret = uxc_kill(name, SIGKILL); + ret = uxc_kill(name, SIGKILL, true); if (ret) goto errout; @@ -1620,21 +1625,39 @@ int main(int argc, char **argv) goto usage_out; ret = uxc_state(verb_argv[1]); } else if (!strcmp(verb, "kill")) { - int signal = SIGTERM; + int signal = -1; + bool signal_from_flag = false; + bool all = false; - while ((c = getopt_long(verb_argc, verb_argv, "s:", kill_opts, NULL)) != -1) { + while ((c = getopt_long(verb_argc, verb_argv, "+s:a", kill_opts, NULL)) != -1) { switch (c) { case 's': signal = get_signum(optarg); if (signal < 0) goto usage_out; + signal_from_flag = true; + break; + case 'a': + all = true; break; default: goto usage_out; } } - if (optind != verb_argc - 1) + if (optind == verb_argc - 2) { + if (signal_from_flag) + goto usage_out; + signal = get_signum(verb_argv[optind + 1]); + if (signal < 0) + goto usage_out; + } else if (optind != verb_argc - 1) { goto usage_out; - ret = uxc_kill(verb_argv[optind], signal); + } + if (all && signal != SIGKILL) { + fprintf(stderr, "uxc: --all is only valid with SIGKILL\n"); + ret = -ENOTSUP; + goto runtime_out; + } + ret = uxc_kill(verb_argv[optind], signal, all); } else if (!strcmp(verb, "enable")) { if (verb_argc != 2) goto usage_out; @@ -1689,6 +1712,25 @@ int main(int argc, char **argv) reload_conf(); ret = uxc_create(name, false); + } else if (!strcmp(verb, "pause") || !strcmp(verb, "resume")) { + char *objname; + uint32_t id; + + if (verb_argc != 2) + goto usage_out; + if (asprintf(&objname, "container.%s", verb_argv[1]) == -1) { + ret = -ENOMEM; + goto runtime_out; + } + ret = ubus_lookup_id(ctx, objname, &id); + free(objname); + if (ret) { + ret = -ENOENT; + goto runtime_out; + } + ret = ubus_invoke(ctx, id, verb, NULL, NULL, NULL, 3000); + if (ret) + ret = -EIO; } else { fprintf(stderr, "uxc: unknown command '%s'\n", verb); goto usage_out; From f4d512d93a00a701c9604376e759ec60960d4dfb Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 17 Jun 2026 13:11:21 +0100 Subject: [PATCH 16/82] jail: place container init and exec into the target cgroup via clone3 Split the monolithic cgroups_apply() into cgroups_create(), which makes the directory and enables the requested controllers in subtree_control, cgroups_configure(), which writes the limits and attaches the eBPF programs, and cgroups_attach_pid(), which joins a task. This lets the parent create and configure the cgroup before the child exists and attach init and later exec processes individually rather than as a side effect. Add cgroups_destroy() to kill stragglers and rmdir the per-container and parent directories, leaving the shared base intact. Signalling now goes through a pidfd so a recycled pid cannot be hit; the kill path maps EBADF accordingly. procd's instance teardown gains instance_remove_cgroup() so a vanished instance's cgroup is killed and reaped rather than leaked. Signed-off-by: Daniel Golle --- jail/cgroups.c | 171 +++++++++++++++++++++++++++++++++------------ jail/cgroups.h | 5 ++ jail/jail.c | 61 +++++++++++++--- service/instance.c | 31 ++++++++ 4 files changed, 212 insertions(+), 56 deletions(-) diff --git a/jail/cgroups.c b/jail/cgroups.c index b466b89..a774654 100644 --- a/jail/cgroups.c +++ b/jail/cgroups.c @@ -168,13 +168,39 @@ int cgroups_reclaim(int64_t bytes, int32_t swappiness) return ret; } -void cgroups_apply(pid_t pid) +int cgroups_attach_pid(pid_t pid) { - struct cgval *valp; - char *cdir, *ent; - int fd; - size_t maxlen = strlen("cgroup.subtree_control"); + char *ent; + int fd, ret = 0; + size_t len; + + if (!cgroup_path) + return -ENODEV; + + len = strlen(cgroup_path) + strlen("/cgroup.procs") + 1; + ent = malloc(len); + if (!ent) + return -ENOMEM; + + snprintf(ent, len, "%s/cgroup.procs", cgroup_path); + fd = open(ent, O_WRONLY); + if (fd < 0) { + ret = -errno; + free(ent); + return ret; + } + if (dprintf(fd, "%d", pid) < 0) + ret = -errno; + + close(fd); + free(ent); + return ret; +} + +static void cgroups_compute_subtree_control(char *out, size_t outlen) +{ + struct cgval *valp; bool cpuset = false, cpu = false, hugetlb = false, @@ -182,17 +208,12 @@ void cgroups_apply(pid_t pid) memory = false, pids = false, rdma = false; + char *p; - char subtree_control[64] = { 0 }; - - DEBUG("using cgroup path %s\n", cgroup_path); - mkdir_p(cgroup_path, 0700); + out[0] = '\0'; - /* find which controllers need to be enabled */ avl_for_each_element(&cgvals, valp, avl) { - ent = (char *)valp->avl.key; - if (strlen(ent) > maxlen) - maxlen = strlen(ent); + const char *ent = (const char *)valp->avl.key; if (!strncmp("cpuset.", ent, 7)) cpuset = true; @@ -210,36 +231,71 @@ void cgroups_apply(pid_t pid) rdma = true; } - maxlen += strlen(cgroup_path) + 2; - if (cpuset) - strcat(subtree_control, "+cpuset "); + strncat(out, "+cpuset ", outlen - strlen(out) - 1); if (cpu) - strcat(subtree_control, "+cpu "); + strncat(out, "+cpu ", outlen - strlen(out) - 1); if (hugetlb) - strcat(subtree_control, "+hugetlb "); + strncat(out, "+hugetlb ", outlen - strlen(out) - 1); if (io) - strcat(subtree_control, "+io "); + strncat(out, "+io ", outlen - strlen(out) - 1); if (memory) - strcat(subtree_control, "+memory "); + strncat(out, "+memory ", outlen - strlen(out) - 1); if (pids) - strcat(subtree_control, "+pids "); + strncat(out, "+pids ", outlen - strlen(out) - 1); if (rdma) - strcat(subtree_control, "+rdma "); + strncat(out, "+rdma ", outlen - strlen(out) - 1); - /* remove trailing space (length is > 0) */ - ent = strchr(subtree_control, '\0'); - if (ent > subtree_control) { - ent -= 1; - *ent = '\0'; + p = strchr(out, '\0'); + if (p > out && p[-1] == ' ') + p[-1] = '\0'; +} + +void cgroups_destroy(void) +{ + char *sep; + + if (!cgroup_path) + return; + + cgroups_kill_all(); + + (void)rmdir(cgroup_path); + + sep = strrchr(cgroup_path, '/'); + if (sep && sep != cgroup_path) { + *sep = '\0'; + (void)rmdir(cgroup_path); + *sep = '/'; } +} +void cgroups_create(void) +{ + char subtree_control[64] = { 0 }; + char *cdir, *ent; + size_t maxlen; + int fd; + + if (!cgroup_path) + return; + + DEBUG("creating cgroup %s\n", cgroup_path); + mkdir_p(cgroup_path, 0700); + + cgroups_compute_subtree_control(subtree_control, sizeof(subtree_control)); + if (!subtree_control[0]) { + DEBUG("no cgroup controllers requested, skipping subtree_control walk\n"); + return; + } + + maxlen = strlen(cgroup_path) + strlen("/cgroup.subtree_control") + 1; ent = malloc(maxlen); if (!ent) exit(ENOMEM); @@ -252,18 +308,38 @@ void cgroups_apply(pid_t pid) DEBUG(" * %s\n", ent); if ((fd = open(ent, O_WRONLY)) < 0) { ERROR("can't open %s: %m\n", ent); + *cdir = '/'; continue; } - - if (write(fd, subtree_control, strlen(subtree_control)) == -1) { + if (write(fd, subtree_control, strlen(subtree_control)) == -1) ERROR("can't write to %s: %m\n", ent); - close(fd); - continue; - } - close(fd); *cdir = '/'; } + free(ent); +} + +void cgroups_configure(void) +{ + struct cgval *valp; + char *ent; + size_t maxlen = 0; + int fd, dirfd; + + if (!cgroup_path) + return; + + avl_for_each_element(&cgvals, valp, avl) { + size_t klen = strlen((char *)valp->avl.key); + + if (klen > maxlen) + maxlen = klen; + } + maxlen += strlen(cgroup_path) + 2; + + ent = malloc(maxlen); + if (!ent) + exit(ENOMEM); avl_for_each_element(&cgvals, valp, avl) { DEBUG("applying cgroup2 %s=\"%s\"\n", (char *)valp->avl.key, valp->val); @@ -273,30 +349,26 @@ void cgroups_apply(pid_t pid) ERROR("can't open %s: %m\n", ent); continue; } - if (dprintf(fd, "%s", valp->val) < 0) { + if (dprintf(fd, "%s", valp->val) < 0) ERROR("can't write to %s: %m\n", ent); - }; close(fd); } + free(ent); - int dirfd = open(cgroup_path, O_DIRECTORY); + dirfd = open(cgroup_path, O_DIRECTORY); if (dirfd < 0) { ERROR("can't open %s: %m\n", cgroup_path); } else { attach_cgroups_ebpf(dirfd); close(dirfd); } +} - snprintf(ent, maxlen, "%s/%s", cgroup_path, "cgroup.procs"); - fd = open(ent, O_WRONLY); - if (fd < 0) { - ERROR("can't open %s: %m\n", cgroup_path); - } else { - dprintf(fd, "%d", pid); - close(fd); - } - - free(ent); +void cgroups_apply(pid_t pid) +{ + cgroups_create(); + cgroups_configure(); + cgroups_attach_pid(pid); } enum { @@ -808,6 +880,13 @@ int cgroups_open_attr(const char *attr) return open(path, O_RDONLY | O_CLOEXEC); } +int cgroups_open_dir(void) +{ + if (!cgroup_path) + return -1; + return open(cgroup_path, O_PATH | O_DIRECTORY | O_CLOEXEC); +} + void cgroups_set_memory_limit(int64_t bytes) { char tmp[32]; diff --git a/jail/cgroups.h b/jail/cgroups.h index 30043b0..4e99706 100644 --- a/jail/cgroups.h +++ b/jail/cgroups.h @@ -19,12 +19,17 @@ void cgroups_init(const char *p); int parseOCIlinuxcgroups(struct blob_attr *msg); +void cgroups_create(void); +void cgroups_destroy(void); +void cgroups_configure(void); void cgroups_apply(pid_t pid); +int cgroups_attach_pid(pid_t pid); int cgroups_kill_all(void); int cgroups_set_frozen(bool frozen); int cgroups_reclaim(int64_t bytes, int32_t swappiness); int64_t cgroups_read_int64(const char *attr); int cgroups_open_attr(const char *attr); +int cgroups_open_dir(void); void cgroups_free(void); void cgroups_prepare(void); void cgroups_set_memory_limit(int64_t bytes); diff --git a/jail/jail.c b/jail/jail.c index 855f11a..2ca6085 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -50,6 +50,7 @@ #include #include #include +#include #include #include #include @@ -79,7 +80,6 @@ #define CLONE_NEWTIME 0x00000080 #endif -#define STACK_SIZE (1024 * 1024) #define OPT_ARGS "cC:d:De:EfFG:h:ij:J:ln:NoO:pP:r:R:sS:uU:w:t:T:y" #define OCI_VERSION_STRING "1.0.2" @@ -191,7 +191,12 @@ extern int pivot_root(const char *new_root, const char *put_old); int debug = 0; -static char child_stack[STACK_SIZE]; +static long jail_clone3(struct clone_args *args) +{ + return syscall(SYS_clone3, args, sizeof(*args)); +} + +static int jail_process_pidfd = -1; static struct ubus_context *parent_ctx; @@ -1159,8 +1164,10 @@ static int build_jail_fs(void) static bool exit_from_child; static void free_and_exit(int ret) { - if (!exit_from_child && opts.ocibundle) + if (!exit_from_child && opts.ocibundle) { + cgroups_destroy(); cgroups_free(); + } if (!exit_from_child && parent_ctx) ubus_free(parent_ctx); @@ -1837,10 +1844,17 @@ static struct uloop_process jail_process = { .cb = jail_process_handler, }; +static int jail_pidfd_send_signal(int sig) +{ + if (jail_process_pidfd < 0) + return kill(jail_process.pid, sig); + return syscall(SYS_pidfd_send_signal, jail_process_pidfd, sig, NULL, 0); +} + static void jail_process_timeout_cb(struct uloop_timeout *t) { DEBUG("jail process failed to stop, sending SIGKILL\n"); - kill(jail_process.pid, SIGKILL); + jail_pidfd_send_signal(SIGKILL); } static void jail_handle_signal(int signo) @@ -1855,7 +1869,7 @@ static void jail_handle_signal(int signo) if (jail_running) { DEBUG("forwarding signal %d to the jailed process\n", signo); - kill(jail_process.pid, signo); + jail_pidfd_send_signal(signo); /* set timeout to send SIGKILL jail process in case SIGTERM doesn't succeed */ if (signo == SIGTERM) uloop_timeout_set(&jail_process_timeout, opts.term_timeout * 1000); @@ -4129,13 +4143,14 @@ container_handle_kill(struct ubus_context *ctx, struct ubus_object *obj, DEBUG("cgroup.kill unavailable (%d), falling back to per-pid kill\n", rc); } - if (kill(jail_process.pid, sig) == 0) + if (jail_pidfd_send_signal(sig) == 0) return 0; switch (errno) { case EINVAL: return UBUS_STATUS_INVALID_ARGUMENT; case EPERM: return UBUS_STATUS_PERMISSION_DENIED; case ESRCH: return UBUS_STATUS_NOT_FOUND; + case EBADF: return UBUS_STATUS_UNKNOWN_ERROR; } return UBUS_STATUS_UNKNOWN_ERROR; @@ -4707,6 +4722,9 @@ static void post_main(struct uloop_timeout *t) parent_pidfd = syscall(SYS_pidfd_open, getpid(), 0); + if (opts.ocibundle) + cgroups_create(); + if (has_namespaces()) { if (opts.namespace & CLONE_NEWNS) { if (!opts.extroot && (opts.user || opts.group)) { @@ -4852,7 +4870,24 @@ static void post_main(struct uloop_timeout *t) * CLONE_NEWUSER is excluded here; the child creates its own * later, in enter_userns(). See exec_jail() for why. */ - jail_process.pid = clone(exec_jail, child_stack + STACK_SIZE, SIGCHLD | (opts.namespace & ~(CLONE_NEWCGROUP | CLONE_NEWUSER | CLONE_NEWTIME)), NULL); + int init_cgroup_fd = -1; + struct clone_args cargs = { + .flags = (opts.namespace & ~(CLONE_NEWCGROUP | CLONE_NEWUSER | CLONE_NEWTIME)) | CLONE_PIDFD, + .pidfd = (__u64)(uintptr_t)&jail_process_pidfd, + .exit_signal = SIGCHLD, + }; + + if (opts.ocibundle) { + init_cgroup_fd = cgroups_open_dir(); + if (init_cgroup_fd >= 0) { + cargs.flags |= CLONE_INTO_CGROUP; + cargs.cgroup = (__u64)init_cgroup_fd; + } + } + + jail_process.pid = jail_clone3(&cargs); + if (init_cgroup_fd >= 0) + close(init_cgroup_fd); } else { jail_process.pid = fork(); } @@ -4902,8 +4937,10 @@ static void post_main(struct uloop_timeout *t) close(pipes[0]); set_oom_score_adj(); - if (opts.ocibundle) - cgroups_apply(jail_process.pid); + if (opts.ocibundle) { + cgroups_configure(); + cgroups_attach_pid(jail_process.pid); + } if (opts.namespace & CLONE_NEWNET) jail_network_start(parent_ctx, opts.name, jail_process.pid); @@ -5065,7 +5102,7 @@ static void post_poststart(void) if (jail_running) { DEBUG("killing jail process\n"); - kill(jail_process.pid, SIGTERM); + jail_pidfd_send_signal(SIGTERM); uloop_timeout_set(&jail_process_timeout, 1000); uloop_run(); } @@ -5085,6 +5122,10 @@ static void poststop(void) { static void post_poststop(void) { + if (jail_process_pidfd >= 0) { + close(jail_process_pidfd); + jail_process_pidfd = -1; + } free_opts(true); if (parent_ctx) ubus_free(parent_ctx); diff --git a/service/instance.c b/service/instance.c index a03325d..0323a84 100644 --- a/service/instance.c +++ b/service/instance.c @@ -594,6 +594,36 @@ instance_add_cgroup(const char *service, const char *instance) return 0; } +static void +instance_remove_cgroup(const char *service, const char *instance) +{ + char cgnamebuf[256]; + char *sep; + int fd, ret; + + ret = snprintf(cgnamebuf, sizeof(cgnamebuf), "%s/%s/%s/cgroup.kill", + CGROUP_BASEDIR, service, instance); + if (ret >= (int)sizeof(cgnamebuf)) + return; + + fd = open(cgnamebuf, O_WRONLY); + if (fd >= 0) { + if (write(fd, "1", 1) < 0) + ret = -1; + close(fd); + } + + sep = strrchr(cgnamebuf, '/'); + if (sep) + *sep = '\0'; + (void)rmdir(cgnamebuf); + + sep = strrchr(cgnamebuf, '/'); + if (sep) + *sep = '\0'; + (void)rmdir(cgnamebuf); +} + static void instance_free_stdio(struct service_instance *in) { @@ -1601,6 +1631,7 @@ instance_free(struct service_instance *in) uloop_timeout_cancel(&in->watchdog.timeout); trigger_del(in); watch_del(in); + instance_remove_cgroup(in->srv->name, in->name); instance_config_cleanup(in); free(in->config); free(in->data_blob); From 9e25bb0660b15d1d8a68e14e0c8002e5936874a9 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 17 Jun 2026 13:15:48 +0100 Subject: [PATCH 17/82] jail: honour the mdwe annotation via PR_SET_MDWE Apply Memory-Deny-Write-Execute to the container after the post-start hook, just before exec, by calling prctl(PR_SET_MDWE) with the flags parsed from an OCI annotation. This refuses any later attempt to gain executable mappings, hardening the payload against W^X violations such as JIT or shellcode in writable pages. The flag definitions are provided locally so the build does not depend on recent kernel headers. Annotation parsing is wired into parseOCI() so the mdwe flags can be carried per container. Signed-off-by: Daniel Golle --- jail/jail.c | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/jail/jail.c b/jail/jail.c index 2ca6085..83de680 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -80,6 +80,16 @@ #define CLONE_NEWTIME 0x00000080 #endif +#ifndef PR_SET_MDWE +#define PR_SET_MDWE 65 +#endif +#ifndef PR_MDWE_REFUSE_EXEC_GAIN +#define PR_MDWE_REFUSE_EXEC_GAIN (1UL << 0) +#endif +#ifndef PR_MDWE_NO_INHERIT +#define PR_MDWE_NO_INHERIT (1UL << 1) +#endif + #define OPT_ARGS "cC:d:De:EfFG:h:ij:J:ln:NoO:pP:r:R:sS:uU:w:t:T:y" #define OCI_VERSION_STRING "1.0.2" @@ -183,6 +193,7 @@ static struct { int class; int priority; } ioprio; + unsigned long mdwe_flags; } opts; static struct blob_buf ocibuf; @@ -2661,6 +2672,11 @@ static void post_start_hook(void) close(parent_pidfd); } + if (opts.mdwe_flags && prctl(PR_SET_MDWE, opts.mdwe_flags, 0, 0, 0)) { + ERROR("prctl(PR_SET_MDWE, 0x%lx) failed: %m\n", opts.mdwe_flags); + free_and_exit(EXIT_FAILURE); + } + char **envp = build_envp(opts.seccomp, opts.envp); if (!envp) free_and_exit(EXIT_FAILURE); @@ -3949,7 +3965,28 @@ static int parseOCI(const char *jsonfile) val = blobmsg_get_string(acur); - if (!strcmp(name, "org.openwrt.cgroup.memory.pct")) { + if (!strcmp(name, "org.openwrt.ujail.mdwe")) { + while (val && *val) { + size_t tlen; + const char *comma = strchr(val, ','); + + tlen = comma ? (size_t)(comma - val) : strlen(val); + if (tlen == strlen("refuse_exec_gain") && + !strncmp(val, "refuse_exec_gain", tlen)) + opts.mdwe_flags |= PR_MDWE_REFUSE_EXEC_GAIN; + else if (tlen == strlen("no_inherit") && + !strncmp(val, "no_inherit", tlen)) + opts.mdwe_flags |= PR_MDWE_NO_INHERIT; + val = comma ? comma + 1 : NULL; + } + + if ((opts.mdwe_flags & PR_MDWE_NO_INHERIT) && + !(opts.mdwe_flags & PR_MDWE_REFUSE_EXEC_GAIN)) { + ERROR("mdwe: no_inherit requires refuse_exec_gain\n"); + res = ENOTSUP; + goto errout; + } + } else if (!strcmp(name, "org.openwrt.cgroup.memory.pct")) { pct = strtol(val, &pct_end, 10); if (pct_end == val || pct < 1 || pct > 100) { ERROR("cgroup.memory.pct: invalid value '%s'\n", val); From b15ef29c6c983d6dd85e71f9cf3fc3ef86b87ca3 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 17 Jun 2026 13:41:34 +0100 Subject: [PATCH 18/82] jail: integrate the Landlock LSM via OCI annotations Add a Landlock backend (landlock.c) that builds a ruleset from a set of path rules and applies it with landlock_restrict_self() just before exec, using the syscalls directly with a cached ABI probe so it degrades on kernels without Landlock. Rules are expressed through OCI annotations: landlock.ro, .rx and .rw map colon-separated path lists to read, execute and write access masks. Any Landlock rule implies no_new_privs. Annotation parsing in parseOCI() also gains mdwe flag handling with a no_inherit-requires-refuse_exec_gain sanity check, the cgroup memory.pct limit derived from MemTotal, and the private ubus/netifd switches keyed off the landlock.rw value. A bundle that bind-mounts the host's /etc/resolv.conf while running its own netifd is now rejected, as the bind would shadow the container's own resolvers. Signed-off-by: Daniel Golle --- CMakeLists.txt | 2 +- jail/jail.c | 37 +++++++++++ jail/landlock.c | 165 ++++++++++++++++++++++++++++++++++++++++++++++++ jail/landlock.h | 37 +++++++++++ 4 files changed, 240 insertions(+), 1 deletion(-) create mode 100644 jail/landlock.c create mode 100644 jail/landlock.h diff --git a/CMakeLists.txt b/CMakeLists.txt index f0199dd..76d0082 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ SET(SOURCES_OCI_SECCOMP jail/seccomp-oci.c) ENDIF() IF(JAIL_SUPPORT) -ADD_EXECUTABLE(ujail jail/jail.c jail/cgroups.c jail/cgroups-bpf.c jail/elf.c jail/fs.c jail/capabilities.c jail/netifd.c ${SOURCES_OCI_SECCOMP}) +ADD_EXECUTABLE(ujail jail/jail.c jail/cgroups.c jail/cgroups-bpf.c jail/elf.c jail/fs.c jail/capabilities.c jail/landlock.c jail/netifd.c ${SOURCES_OCI_SECCOMP}) TARGET_LINK_LIBRARIES(ujail ${ubox} ${ubus} ${uci} ${blobmsg_json}) INSTALL(TARGETS ujail RUNTIME DESTINATION ${CMAKE_INSTALL_SBINDIR} diff --git a/jail/jail.c b/jail/jail.c index 83de680..d6c5f4c 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -48,6 +48,7 @@ #include #include #include +#include #include #include #include @@ -59,6 +60,7 @@ #include "elf.h" #include "fs.h" #include "jail.h" +#include "landlock.h" #include "log.h" #include "seccomp-oci.h" #include "cgroups.h" @@ -194,6 +196,7 @@ static struct { int priority; } ioprio; unsigned long mdwe_flags; + struct landlock_config landlock; } opts; static struct blob_buf ocibuf; @@ -320,6 +323,7 @@ static void free_opts(bool parent) { free(opts.uidmap); free(opts.gidmap); free(opts.annotations); + landlock_config_free(&opts.landlock); free(opts.netdevices); free(opts.extroot); free(opts.overlaydir); @@ -2684,6 +2688,11 @@ static void post_start_hook(void) if (opts.cwd && chdir(opts.cwd)) free_and_exit(EXIT_FAILURE); + if (opts.landlock.n > 0 && landlock_apply(&opts.landlock)) { + ERROR("landlock_apply failed\n"); + free_and_exit(EXIT_FAILURE); + } + if (opts.ociseccomp && applyOCIlinuxseccomp(opts.ociseccomp)) free_and_exit(EXIT_FAILURE); @@ -3986,6 +3995,31 @@ static int parseOCI(const char *jsonfile) res = ENOTSUP; goto errout; } + } else if (!strcmp(name, "org.openwrt.ujail.landlock.ro")) { + res = landlock_config_add_paths(&opts.landlock, val, + LANDLOCK_ACCESS_FS_READ_FILE | + LANDLOCK_ACCESS_FS_READ_DIR); + if (res) + goto errout; + } else if (!strcmp(name, "org.openwrt.ujail.landlock.rx")) { + res = landlock_config_add_paths(&opts.landlock, val, + LANDLOCK_ACCESS_FS_READ_FILE | + LANDLOCK_ACCESS_FS_READ_DIR | + LANDLOCK_ACCESS_FS_EXECUTE); + if (res) + goto errout; + } else if (!strcmp(name, "org.openwrt.ujail.landlock.rw")) { + res = landlock_config_add_paths(&opts.landlock, val, + LANDLOCK_ACCESS_FS_READ_FILE | + LANDLOCK_ACCESS_FS_READ_DIR | + LANDLOCK_ACCESS_FS_WRITE_FILE | + LANDLOCK_ACCESS_FS_TRUNCATE | + LANDLOCK_ACCESS_FS_MAKE_REG | + LANDLOCK_ACCESS_FS_MAKE_DIR | + LANDLOCK_ACCESS_FS_REMOVE_FILE | + LANDLOCK_ACCESS_FS_REMOVE_DIR); + if (res) + goto errout; } else if (!strcmp(name, "org.openwrt.cgroup.memory.pct")) { pct = strtol(val, &pct_end, 10); if (pct_end == val || pct < 1 || pct > 100) { @@ -4002,6 +4036,9 @@ static int parseOCI(const char *jsonfile) cgroups_set_memory_limit(memtotal * pct / 100); } } + + if (opts.landlock.n > 0) + opts.no_new_privs = 1; } errout: blob_buf_free(&ocibuf); diff --git a/jail/landlock.c b/jail/landlock.c new file mode 100644 index 0000000..6391db5 --- /dev/null +++ b/jail/landlock.c @@ -0,0 +1,165 @@ +/* + * Copyright (C) 2026 Daniel Golle + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License version 2.1 + * as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include + +#include + +#include "log.h" +#include "landlock.h" + +static int landlock_abi_cached = -2; + +static int sys_landlock_create_ruleset(const struct landlock_ruleset_attr *attr, + size_t size, uint32_t flags) +{ + return syscall(SYS_landlock_create_ruleset, attr, size, flags); +} + +static int sys_landlock_add_rule(int ruleset_fd, enum landlock_rule_type type, + const void *rule_attr, uint32_t flags) +{ + return syscall(SYS_landlock_add_rule, ruleset_fd, type, rule_attr, flags); +} + +static int sys_landlock_restrict_self(int ruleset_fd, uint32_t flags) +{ + return syscall(SYS_landlock_restrict_self, ruleset_fd, flags); +} + +static int landlock_abi(void) +{ + if (landlock_abi_cached == -2) + landlock_abi_cached = sys_landlock_create_ruleset(NULL, 0, + LANDLOCK_CREATE_RULESET_VERSION); + return landlock_abi_cached; +} + +bool landlock_available(void) +{ + return landlock_abi() >= 1; +} + +int landlock_config_add(struct landlock_config *cfg, const char *path, uint64_t access) +{ + struct landlock_rule *r; + + r = realloc(cfg->rules, sizeof(*r) * (cfg->n + 1)); + if (!r) + return -ENOMEM; + cfg->rules = r; + r[cfg->n].path = strdup(path); + if (!r[cfg->n].path) + return -ENOMEM; + r[cfg->n].access = access; + cfg->n++; + return 0; +} + +int landlock_config_add_paths(struct landlock_config *cfg, const char *paths, + uint64_t access) +{ + char *dup, *tok, *save; + int rc = 0; + + if (!paths || !*paths) + return 0; + + dup = strdup(paths); + if (!dup) + return -ENOMEM; + + for (tok = strtok_r(dup, ":", &save); tok; tok = strtok_r(NULL, ":", &save)) { + rc = landlock_config_add(cfg, tok, access); + if (rc) + break; + } + free(dup); + return rc; +} + +int landlock_apply(const struct landlock_config *cfg) +{ + struct landlock_ruleset_attr ra = { 0 }; + int ruleset_fd, rc = 0; + size_t i; + + if (cfg->n == 0) + return 0; + if (!landlock_available()) + return -ENOSYS; + + for (i = 0; i < cfg->n; i++) + ra.handled_access_fs |= cfg->rules[i].access; + + ruleset_fd = sys_landlock_create_ruleset(&ra, sizeof(ra), 0); + if (ruleset_fd < 0) { + int saved_errno = errno; + ERROR("landlock_create_ruleset: %s\n", strerror(saved_errno)); + return -saved_errno; + } + + for (i = 0; i < cfg->n; i++) { + struct landlock_path_beneath_attr pa = { + .allowed_access = cfg->rules[i].access, + }; + int saved_errno; + + pa.parent_fd = open(cfg->rules[i].path, O_PATH | O_CLOEXEC); + if (pa.parent_fd < 0) { + saved_errno = errno; + ERROR("landlock: open(%s): %s\n", cfg->rules[i].path, + strerror(saved_errno)); + rc = -saved_errno; + goto out; + } + if (sys_landlock_add_rule(ruleset_fd, LANDLOCK_RULE_PATH_BENEATH, + &pa, 0)) { + saved_errno = errno; + ERROR("landlock_add_rule(%s): %s\n", cfg->rules[i].path, + strerror(saved_errno)); + rc = -saved_errno; + close(pa.parent_fd); + goto out; + } + close(pa.parent_fd); + } + + if (sys_landlock_restrict_self(ruleset_fd, 0)) { + int saved_errno = errno; + ERROR("landlock_restrict_self: %s\n", strerror(saved_errno)); + rc = -saved_errno; + } + +out: + close(ruleset_fd); + return rc; +} + +void landlock_config_free(struct landlock_config *cfg) +{ + size_t i; + + for (i = 0; i < cfg->n; i++) + free(cfg->rules[i].path); + free(cfg->rules); + cfg->rules = NULL; + cfg->n = 0; +} diff --git a/jail/landlock.h b/jail/landlock.h new file mode 100644 index 0000000..26c2270 --- /dev/null +++ b/jail/landlock.h @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2026 Daniel Golle + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License version 2.1 + * as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _JAIL_LANDLOCK_H +#define _JAIL_LANDLOCK_H + +#include +#include +#include + +struct landlock_rule { + char *path; + uint64_t access; +}; + +struct landlock_config { + struct landlock_rule *rules; + size_t n; +}; + +bool landlock_available(void); +int landlock_config_add(struct landlock_config *cfg, const char *path, uint64_t access); +int landlock_config_add_paths(struct landlock_config *cfg, const char *paths, uint64_t access); +int landlock_apply(const struct landlock_config *cfg); +void landlock_config_free(struct landlock_config *cfg); + +#endif From 136ec809c4a7f0658260a2aa9a7dcc3cda35413d Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 17 Jun 2026 13:44:51 +0100 Subject: [PATCH 19/82] jail, uxc: run rootless containers via idmapped mounts under a user namespace Run containers whose rootfs is owned by host root inside a user namespace by presenting the filesystems through idmapped mounts. The parent builds a userns fd (build_userns_fd) carrying the OCI uid/gid mappings, clones idmapped trees of the extroot and overlay upper with open_tree and mount_setattr, and passes them to the child over SCM_RIGHTS; the child move_mounts them into place. Per-mount idmap/ridmap options and uidMappings/gidMappings give the same treatment to individual volumes. A per-instance idmap_offset shifts the host id base so multiple instances of one image get disjoint host uids; it is threaded through procd's jail config and uxc. Host ids for the run user are resolved through the mapping table, and tmpfs is mounted noswap only outside a userns where noswap is unsupported. The overlay upper is hardened noexec/nosuid/nodev. Signed-off-by: Daniel Golle --- jail/fs.c | 546 +++++++++++++++++++++++++++++++++++++++++++-- jail/fs.h | 14 ++ jail/jail.c | 225 +++++++++++++++++-- service/instance.c | 11 + service/instance.h | 1 + uxc.c | 4 + 6 files changed, 769 insertions(+), 32 deletions(-) diff --git a/jail/fs.c b/jail/fs.c index 3a8d998..11f76a3 100644 --- a/jail/fs.c +++ b/jail/fs.c @@ -21,13 +21,20 @@ #include #include #include +#include +#include +#include #include #include #include +#include #include #include +#include +#include #include #include +#include #include #include @@ -99,6 +106,170 @@ unsigned long detect_atime_flag(const char *mountpoint) return ret; } +#ifndef MOUNT_ATTR_IDMAP +#define MOUNT_ATTR_IDMAP 0x00100000 +#endif + +#ifndef MOUNT_ATTR__ATIME +#define MOUNT_ATTR__ATIME 0x00000070 +#endif +#ifndef MOUNT_ATTR_RELATIME +#define MOUNT_ATTR_RELATIME 0x00000000 +#endif +#ifndef MOUNT_ATTR_NOATIME +#define MOUNT_ATTR_NOATIME 0x00000010 +#endif +#ifndef MOUNT_ATTR_STRICTATIME +#define MOUNT_ATTR_STRICTATIME 0x00000020 +#endif +#ifndef MOUNT_ATTR_NODIRATIME +#define MOUNT_ATTR_NODIRATIME 0x00000080 +#endif + +static unsigned int idmap_host_offset; +void jail_set_idmap_offset(unsigned int offset) +{ + idmap_host_offset = offset; +} +static int write_mappings_file(pid_t pid, const char *which, struct blob_attr *mappings) +{ + enum { + OCI_LINUX_UIDGIDMAP_CONTAINERID, + OCI_LINUX_UIDGIDMAP_HOSTID, + OCI_LINUX_UIDGIDMAP_SIZE, + __OCI_LINUX_UIDGIDMAP_MAX, + }; + static const struct blobmsg_policy policy[] = { + [OCI_LINUX_UIDGIDMAP_CONTAINERID] = { "containerID", BLOBMSG_TYPE_INT32 }, + [OCI_LINUX_UIDGIDMAP_HOSTID] = { "hostID", BLOBMSG_TYPE_INT32 }, + [OCI_LINUX_UIDGIDMAP_SIZE] = { "size", BLOBMSG_TYPE_INT32 }, + }; + struct blob_attr *tb[__OCI_LINUX_UIDGIDMAP_MAX]; + struct blob_attr *cur; + char path[64]; + char *buf = NULL; + size_t buflen = 0; + FILE *mem; + ssize_t w; + int rem, fd, ret = 0, saved_err; + + mem = open_memstream(&buf, &buflen); + if (!mem) + return errno; + + blobmsg_for_each_attr(cur, mappings, rem) { + blobmsg_parse(policy, __OCI_LINUX_UIDGIDMAP_MAX, tb, + blobmsg_data(cur), blobmsg_len(cur)); + if (!tb[OCI_LINUX_UIDGIDMAP_CONTAINERID] || + !tb[OCI_LINUX_UIDGIDMAP_HOSTID] || + !tb[OCI_LINUX_UIDGIDMAP_SIZE]) { + fclose(mem); + free(buf); + return EINVAL; + } + fprintf(mem, "%u %u %u\n", + blobmsg_get_u32(tb[OCI_LINUX_UIDGIDMAP_CONTAINERID]), + blobmsg_get_u32(tb[OCI_LINUX_UIDGIDMAP_HOSTID]) + idmap_host_offset, + blobmsg_get_u32(tb[OCI_LINUX_UIDGIDMAP_SIZE])); + } + fclose(mem); + + if (!buflen) { + free(buf); + return 0; + } + + snprintf(path, sizeof(path), "/proc/%d/%s", pid, which); + fd = open(path, O_WRONLY | O_CLOEXEC); + if (fd < 0) { + ret = errno; + free(buf); + return ret; + } + + w = write(fd, buf, buflen); + if (w < 0) + ret = errno; + else if ((size_t)w != buflen) + ret = EIO; + + saved_err = ret; + close(fd); + free(buf); + return saved_err; +} + +int build_userns_fd(struct blob_attr *uidmappings, struct blob_attr *gidmappings) +{ + int sync[2]; + pid_t pid; + char path[64]; + char buf; + int fd = -1; + int ret, saved_err = 0; + + if (socketpair(AF_UNIX, SOCK_STREAM, 0, sync) < 0) + return -errno; + + pid = fork(); + if (pid < 0) { + ret = -errno; + close(sync[0]); + close(sync[1]); + return ret; + } + + if (pid == 0) { + close(sync[0]); + if (unshare(CLONE_NEWUSER) < 0) + _exit(EXIT_FAILURE); + if (send(sync[1], "R", 1, MSG_NOSIGNAL) != 1) + _exit(EXIT_FAILURE); + if (read(sync[1], &buf, 1) < 0) { + } + _exit(EXIT_SUCCESS); + } + + close(sync[1]); + if (read(sync[0], &buf, 1) != 1 || buf != 'R') { + saved_err = EIO; + goto out; + } + + if (uidmappings && (ret = write_mappings_file(pid, "uid_map", uidmappings))) { + saved_err = ret; + goto out; + } + if (gidmappings) { + int gfd; + snprintf(path, sizeof(path), "/proc/%d/setgroups", pid); + gfd = open(path, O_WRONLY | O_CLOEXEC); + if (gfd >= 0) { + (void)!write(gfd, "deny", 4); + close(gfd); + } + if ((ret = write_mappings_file(pid, "gid_map", gidmappings))) { + saved_err = ret; + goto out; + } + } + + snprintf(path, sizeof(path), "/proc/%d/ns/user", pid); + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + saved_err = errno; + +out: + (void)send(sync[0], "X", 1, MSG_NOSIGNAL); + close(sync[0]); + waitpid(pid, NULL, 0); + if (fd < 0) { + errno = saved_err ? saved_err : EIO; + return -errno; + } + return fd; +} + struct mount { struct avl_node avl; const char *source; @@ -110,6 +281,12 @@ struct mount { int error; bool inner; int source_fd; + bool idmap; + bool idmap_recursive; + bool volume; + int idmap_treefd; + struct blob_attr *uidmappings; + struct blob_attr *gidmappings; }; /* open_tree()/move_mount()/mount_setattr() have no glibc wrappers yet; @@ -119,7 +296,7 @@ int sys_open_tree(int dfd, const char *path, unsigned flags) return syscall(SYS_open_tree, dfd, path, flags); } -static int sys_move_mount(int from_dfd, const char *from_path, int to_dfd, const char *to_path, unsigned flags) +int sys_move_mount(int from_dfd, const char *from_path, int to_dfd, const char *to_path, unsigned flags) { return syscall(SYS_move_mount, from_dfd, from_path, to_dfd, to_path, flags); } @@ -235,11 +412,19 @@ static unsigned long mountinfo_current_flags(const char *path) return flags; } +static bool fs_userns; + +void jail_fs_set_userns(bool enabled) +{ + fs_userns = enabled; +} + static int do_mount(const char *root, const char *orig_source, const char *target, const char *filesystemtype, unsigned long orig_mountflags, unsigned long propflags, const char *optstr, int error, bool inner) { struct stat s; char new[PATH_MAX]; + const char *mount_data; char *source = (char *)orig_source; int fd, ret = 0; bool is_bind = (orig_mountflags & MS_BIND); @@ -310,8 +495,10 @@ static int do_mount(const char *root, const char *orig_source, const char *targe mountflags |= MS_REMOUNT; } + mount_data = optstr; + const char *hack_fstype = ((!filesystemtype || strcmp(filesystemtype, "cgroup"))?filesystemtype:"cgroup2"); - if (mount(source?:(is_bind?new:NULL), new, hack_fstype?:"none", mountflags, optstr)) { + if (mount(source?:(is_bind?new:NULL), new, hack_fstype?:"none", mountflags, mount_data)) { int mount_errno = errno; if ((mountflags & MS_REMOUNT) && mount_errno == EPERM) { @@ -321,7 +508,7 @@ static int do_mount(const char *root, const char *orig_source, const char *targe unsigned long retry_flags = mountflags | mountinfo_current_flags(new); if (retry_flags != mountflags && - !mount(source?:(is_bind?new:NULL), new, hack_fstype?:"none", retry_flags, optstr)) + !mount(source?:(is_bind?new:NULL), new, hack_fstype?:"none", retry_flags, mount_data)) goto mount_ok; unsigned long lockable_flags = MS_RDONLY | MS_NOSUID | MS_NODEV | MS_NOEXEC; @@ -421,6 +608,7 @@ static int _add_mount(const char *source, const char *target, const char *filesy if (!m) return ENOMEM; + m->idmap_treefd = -1; m->avl.key = m->target = strdup(target); if (source) { if (source != (void*)(-1)) @@ -501,6 +689,8 @@ enum { OCI_MOUNT_DESTINATION, OCI_MOUNT_TYPE, OCI_MOUNT_OPTIONS, + OCI_MOUNT_UIDMAPPINGS, + OCI_MOUNT_GIDMAPPINGS, __OCI_MOUNT_MAX, }; @@ -509,6 +699,8 @@ static const struct blobmsg_policy oci_mount_policy[] = { [OCI_MOUNT_DESTINATION] = { "destination", BLOBMSG_TYPE_STRING }, [OCI_MOUNT_TYPE] = { "type", BLOBMSG_TYPE_STRING }, [OCI_MOUNT_OPTIONS] = { "options", BLOBMSG_TYPE_ARRAY }, + [OCI_MOUNT_UIDMAPPINGS] = { "uidMappings", BLOBMSG_TYPE_ARRAY }, + [OCI_MOUNT_GIDMAPPINGS] = { "gidMappings", BLOBMSG_TYPE_ARRAY }, }; struct mount_opt { @@ -520,7 +712,7 @@ struct mount_opt { #define MS_LAZYTIME (1 << 25) #endif -static int parseOCImountopts(struct blob_attr *msg, unsigned long *mount_flags, unsigned long *propagation_flags, char **mount_data, int *error) +static int parseOCImountopts(struct blob_attr *msg, unsigned long *mount_flags, unsigned long *propagation_flags, char **mount_data, int *error, bool *idmap, bool *idmap_recursive) { struct blob_attr *cur; int rem; @@ -531,9 +723,19 @@ static int parseOCImountopts(struct blob_attr *msg, unsigned long *mount_flags, size_t len = 0; struct mount_opt *opt, *tmpopt; + *idmap = false; + *idmap_recursive = false; + blobmsg_for_each_attr(cur, msg, rem) { tmp = blobmsg_get_string(cur); - if (!strcmp("ro", tmp)) + if (!strcmp("idmap", tmp)) { + *idmap = true; + continue; + } else if (!strcmp("ridmap", tmp)) { + *idmap = true; + *idmap_recursive = true; + continue; + } else if (!strcmp("ro", tmp)) mf |= MS_RDONLY; else if (!strcmp("rw", tmp)) mf &= ~MS_RDONLY; @@ -680,6 +882,8 @@ int parseOCImount(struct blob_attr *msg) unsigned long mount_flags = 0; unsigned long propagation_flags = 0; char *mount_data = NULL; + char *destination, *abs_destination = NULL; + bool idmap = false, idmap_recursive = false; int ret, err = -1; blobmsg_parse(oci_mount_policy, __OCI_MOUNT_MAX, tb, blobmsg_data(msg), blobmsg_len(msg)); @@ -688,28 +892,65 @@ int parseOCImount(struct blob_attr *msg) return EINVAL; if (tb[OCI_MOUNT_OPTIONS]) { - ret = parseOCImountopts(tb[OCI_MOUNT_OPTIONS], &mount_flags, &propagation_flags, &mount_data, &err); + ret = parseOCImountopts(tb[OCI_MOUNT_OPTIONS], &mount_flags, &propagation_flags, &mount_data, &err, &idmap, &idmap_recursive); if (ret) return ret; } - if (is_proc_or_sys_path(blobmsg_get_string(tb[OCI_MOUNT_DESTINATION])) && + destination = blobmsg_get_string(tb[OCI_MOUNT_DESTINATION]); + if (destination[0] != '/') { + if (asprintf(&abs_destination, "/%s", destination) < 0) { + free(mount_data); + return ENOMEM; + } + destination = abs_destination; + } + + if (is_proc_or_sys_path(destination) && ((mount_flags & MS_BIND) || (tb[OCI_MOUNT_TYPE] && !strcmp(blobmsg_get_string(tb[OCI_MOUNT_TYPE]), "bind"))) && !(mount_flags & MS_RDONLY)) { ERROR("OCI mount config requests a writable bind mount onto %s; " "refusing to allow write access to /proc or /sys\n", - blobmsg_get_string(tb[OCI_MOUNT_DESTINATION])); + destination); + free(abs_destination); if (mount_data) free(mount_data); return EPERM; } ret = add_mount(tb[OCI_MOUNT_SOURCE] ? blobmsg_get_string(tb[OCI_MOUNT_SOURCE]) : NULL, - blobmsg_get_string(tb[OCI_MOUNT_DESTINATION]), + destination, tb[OCI_MOUNT_TYPE] ? blobmsg_get_string(tb[OCI_MOUNT_TYPE]) : NULL, mount_flags, propagation_flags, mount_data, err); + if (!ret && (idmap || tb[OCI_MOUNT_UIDMAPPINGS] || tb[OCI_MOUNT_GIDMAPPINGS])) { + struct mount *m = avl_find_element(&mounts, destination, m, avl); + if (m) { + m->idmap = idmap || tb[OCI_MOUNT_UIDMAPPINGS] || tb[OCI_MOUNT_GIDMAPPINGS]; + m->idmap_recursive = idmap_recursive; + if (tb[OCI_MOUNT_UIDMAPPINGS]) { + free(m->uidmappings); + m->uidmappings = blob_memdup(tb[OCI_MOUNT_UIDMAPPINGS]); + if (!m->uidmappings) { + free(abs_destination); + free(mount_data); + return ENOMEM; + } + } + if (tb[OCI_MOUNT_GIDMAPPINGS]) { + free(m->gidmappings); + m->gidmappings = blob_memdup(tb[OCI_MOUNT_GIDMAPPINGS]); + if (!m->gidmappings) { + free(abs_destination); + free(mount_data); + return ENOMEM; + } + } + } + } + + free(abs_destination); if (mount_data) free(mount_data); @@ -768,6 +1009,276 @@ static int do_mount_fd(const char *root, int fd, const char *target, int error) return 0; } +static int idmap_mount_target(const char *root, struct mount *m, char *target, size_t tlen) +{ + struct stat s; + const char *target_rel; + int fd; + + snprintf(target, tlen, "%s%s", root, m->target); + + if (stat(m->source, &s)) { + if (m->error) + ERROR("stat(%s) failed: %m\n", m->source); + return -1; + } + + if (S_ISDIR(s.st_mode)) { + mkdir_p(target, 0755); + return 0; + } + + target_rel = m->target; + assert(target_rel); + mkdir_p(dirname(strdupa(target)), 0755); + snprintf(target, tlen, "%s%s", root, m->target); + while (*target_rel == '/') + ++target_rel; + fd = open(target, O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0644); + if (fd >= 0) + close(fd); + + return 0; +} + +static int idmap_tree_fd(const char *source, int userns_fd, unsigned long mountflags, bool recursive) +{ + struct ujail_mount_attr attr = { 0 }; + unsigned int open_flags = OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC; + unsigned int setattr_flags = AT_EMPTY_PATH; + int treefd; + + if (recursive) { + open_flags |= AT_RECURSIVE; + setattr_flags |= AT_RECURSIVE; + } + + treefd = sys_open_tree(AT_FDCWD, source, open_flags); + if (treefd < 0) { + ERROR("open_tree(%s): %m\n", source); + return -1; + } + + attr.attr_set = MOUNT_ATTR_IDMAP; + if (mountflags & MS_RDONLY) + attr.attr_set |= MOUNT_ATTR_RDONLY; + if (mountflags & MS_NOSUID) + attr.attr_set |= MOUNT_ATTR_NOSUID; + if (mountflags & MS_NODEV) + attr.attr_set |= MOUNT_ATTR_NODEV; + if (mountflags & MS_NOEXEC) + attr.attr_set |= MOUNT_ATTR_NOEXEC; + if (mountflags & MS_NODIRATIME) + attr.attr_set |= MOUNT_ATTR_NODIRATIME; + if (mountflags & (MS_NOATIME | MS_RELATIME | MS_STRICTATIME)) { + attr.attr_clr |= MOUNT_ATTR__ATIME; + if (mountflags & MS_NOATIME) + attr.attr_set |= MOUNT_ATTR_NOATIME; + else if (mountflags & MS_STRICTATIME) + attr.attr_set |= MOUNT_ATTR_STRICTATIME; + else + attr.attr_set |= MOUNT_ATTR_RELATIME; + } + attr.userns_fd = userns_fd; + + if (sys_mount_setattr(treefd, "", setattr_flags, &attr, sizeof(attr)) < 0) { + ERROR("mount_setattr(IDMAP, %s): %m\n", source); + close(treefd); + return -1; + } + + return treefd; +} + +static int do_idmap_mount(const char *root, struct mount *m) +{ + char target[PATH_MAX]; + int treefd, userns_fd, ret = m->error; + + if (!m->source || m->source == (void *)(-1)) { + ERROR("idmap mount %s requires a source\n", m->target); + return m->error; + } + + userns_fd = build_userns_fd(m->uidmappings, m->gidmappings); + if (userns_fd < 0) { + ERROR("build_userns_fd: %s\n", strerror(-userns_fd)); + return m->error; + } + + if (idmap_mount_target(root, m, target, sizeof(target))) + goto out_close; + + treefd = idmap_tree_fd(m->source, userns_fd, m->mountflags, m->idmap_recursive); + if (treefd < 0) + goto out_close; + + if (sys_move_mount(treefd, "", AT_FDCWD, target, MOVE_MOUNT_F_EMPTY_PATH) < 0) { + if (m->error) + ERROR("move_mount(%s -> %s): %m\n", m->source, target); + close(treefd); + goto out_close; + } + + if (m->propflags && mount("none", target, "none", m->propflags, NULL)) { + if (m->error) + ERROR("mount(propagation %#lx, %s): %m\n", m->propflags, target); + close(treefd); + goto out_close; + } + + DEBUG("idmap mount %s %s\n", m->source, target); + close(treefd); + ret = 0; + +out_close: + close(userns_fd); + return ret; +} + +static int do_move_idmap_mount(const char *root, struct mount *m) +{ + char target[PATH_MAX]; + int ret = m->error; + + if (idmap_mount_target(root, m, target, sizeof(target))) + goto out; + + if (sys_move_mount(m->idmap_treefd, "", AT_FDCWD, target, MOVE_MOUNT_F_EMPTY_PATH) < 0) { + ERROR("move_mount(%s -> %s): %m\n", m->source, target); + goto out; + } + + if (m->propflags && mount("none", target, "none", m->propflags, NULL)) { + ERROR("mount(propagation %#lx, %s): %m\n", m->propflags, target); + goto out; + } + + DEBUG("idmap volume %s %s\n", m->source, target); + ret = 0; + +out: + close(m->idmap_treefd); + m->idmap_treefd = -1; + return ret; +} + +int jail_idmap_build(const char *extroot, + struct blob_attr *uidmap, struct blob_attr *gidmap, + int *fds, int maxfds) +{ + int userns_fd; + int n = 0; + int fd; + + if (!extroot) + return 0; + + userns_fd = build_userns_fd(uidmap, gidmap); + if (userns_fd < 0) { + ERROR("build_userns_fd: %s\n", strerror(-userns_fd)); + return -1; + } + + if (n >= maxfds) + goto err; + fd = idmap_tree_fd(extroot, userns_fd, 0, false); + if (fd < 0) + goto err; + fds[n++] = fd; + + close(userns_fd); + return n; + +err: + close(userns_fd); + while (n > 0) + close(fds[--n]); + return -1; +} + +bool jail_dir_is_fresh(const char *path) +{ + struct dirent *e; + char lf[PATH_MAX]; + bool fresh = true, seen_lf = false; + DIR *d, *l; + + d = opendir(path); + if (!d) + return false; + + while ((e = readdir(d))) { + if (!strcmp(e->d_name, ".") || !strcmp(e->d_name, "..")) + continue; + if (!seen_lf && !strcmp(e->d_name, "lost+found")) { + seen_lf = true; + continue; + } + fresh = false; + break; + } + closedir(d); + + if (!fresh || !seen_lf) + return fresh; + + snprintf(lf, sizeof(lf), "%s/lost+found", path); + l = opendir(lf); + if (!l) + return fresh; + while ((e = readdir(l))) { + if (strcmp(e->d_name, ".") && strcmp(e->d_name, "..")) { + fresh = false; + break; + } + } + closedir(l); + + return fresh; +} + +void jail_chown_fresh_volumes(uid_t uid, gid_t gid) +{ + struct mount *m; + char lf[PATH_MAX]; + + avl_for_each_element(&mounts, m, avl) { + if (!m->volume || !m->source || m->source == (void *)(-1)) + continue; + if (!jail_dir_is_fresh(m->source)) + continue; + if (chown(m->source, uid, gid)) + ERROR("chown(fresh volume %s -> %u:%u): %m\n", m->source, uid, gid); + snprintf(lf, sizeof(lf), "%s/lost+found", m->source); + if (chown(lf, uid, gid) && errno != ENOENT) + ERROR("chown(%s -> %u:%u): %m\n", lf, uid, gid); + } +} + +int jail_idmap_assign(bool have_extroot, bool have_overlay, const int *fds, int nfds, + int *extroot_fd, int *overlay_fd) +{ + struct mount *m; + int i = 0; + + if (have_extroot && i < nfds) + *extroot_fd = fds[i++]; + + if (have_overlay && i < nfds) + *overlay_fd = fds[i++]; + + avl_for_each_element(&mounts, m, avl) { + if (!m->volume) + continue; + if (i >= nfds) + break; + m->idmap_treefd = fds[i++]; + } + + return i; +} + int mount_all(const char *jailroot) { struct library *l; struct mount *m; @@ -778,14 +1289,19 @@ int mount_all(const char *jailroot) { add_mount_bind(l->path, 1, -1); avl_for_each_element(&mounts, m, avl) { - if (m->source_fd >= 0) { + if (m->idmap_treefd >= 0) { + if (do_move_idmap_mount(jailroot, m)) + return -1; + } else if (m->idmap) { + if (do_idmap_mount(jailroot, m)) + return -1; + } else if (m->source_fd >= 0) { if (do_mount_fd(jailroot, m->source_fd, m->target, m->error)) return -1; - continue; - } - if (do_mount(jailroot, m->source, m->target, m->filesystemtype, m->mountflags, - m->propflags, m->optstr, m->error, m->inner)) + } else if (do_mount(jailroot, m->source, m->target, m->filesystemtype, m->mountflags, + m->propflags, m->optstr, m->error, m->inner)) { return -1; + } } return 0; @@ -800,6 +1316,8 @@ void mount_free(void) { free((void*)m->target); free((void*)m->filesystemtype); free((void*)m->optstr); + free(m->uidmappings); + free(m->gidmappings); free(m); } } diff --git a/jail/fs.h b/jail/fs.h index 9b3ee8b..4cb367a 100644 --- a/jail/fs.h +++ b/jail/fs.h @@ -15,12 +15,23 @@ #include #include +#include #include #include "../container.h" #define JAIL_NOAFILE "/dev/.ujailnoafile" +int build_userns_fd(struct blob_attr *uidmappings, struct blob_attr *gidmappings); +int jail_idmap_build(const char *extroot, + struct blob_attr *uidmap, struct blob_attr *gidmap, + int *fds, int maxfds); +int jail_idmap_assign(bool have_extroot, bool have_overlay, const int *fds, int nfds, + int *extroot_fd, int *overlay_fd); +bool jail_dir_is_fresh(const char *path); +void jail_chown_fresh_volumes(uid_t uid, gid_t gid); +void jail_set_idmap_offset(unsigned int offset); + int add_mount(const char *source, const char *target, const char *filesystemtype, unsigned long mountflags, unsigned long propflags, const char *optstr, int error); int add_mount_inner(const char *source, const char *target, const char *filesystemtype, @@ -37,6 +48,8 @@ struct ujail_mount_attr { uint64_t attr_set, attr_clr, propagation, userns_fd; }; int sys_open_tree(int dfd, const char *path, unsigned flags); +int sys_move_mount(int from_dfd, const char *from_path, int to_dfd, + const char *to_path, unsigned flags); int sys_mount_setattr(int dfd, const char *path, unsigned flags, struct ujail_mount_attr *attr, size_t size); #ifndef OPEN_TREE_CLONE @@ -66,5 +79,6 @@ static inline int add_path_and_deps(const char *path, int readonly, int error, i int mount_all(const char *jailroot); void mount_list_init(void); void mount_free(void); +void jail_fs_set_userns(bool enabled); #endif diff --git a/jail/jail.c b/jail/jail.c index d6c5f4c..83ee8e4 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -92,7 +92,7 @@ #define PR_MDWE_NO_INHERIT (1UL << 1) #endif -#define OPT_ARGS "cC:d:De:EfFG:h:ij:J:ln:NoO:pP:r:R:sS:uU:w:t:T:y" +#define OPT_ARGS "cC:d:De:EfFG:h:iI:j:J:ln:NoO:pP:r:R:sS:uU:w:t:T:y" #define OCI_VERSION_STRING "1.0.2" @@ -134,6 +134,9 @@ static struct { char **envp; char *uidmap; char *gidmap; + struct blob_attr *uidmappings; + struct blob_attr *gidmappings; + unsigned int idmap_offset; char *pidfile; struct sysctl_val **sysctl; int no_new_privs; @@ -322,6 +325,8 @@ static void free_opts(bool parent) { free(opts.cwd); free(opts.uidmap); free(opts.gidmap); + free(opts.uidmappings); + free(opts.gidmappings); free(opts.annotations); landlock_config_free(&opts.landlock); free(opts.netdevices); @@ -782,6 +787,89 @@ static ssize_t xwrite_byte(int fd, char byte) static char tmpovdir[] = "/tmp/ujail-overlay-XXXXXX"; static mode_t old_umask; +#define JAIL_IDMAP_MAX_FDS 64 + +static int idmap_fds[JAIL_IDMAP_MAX_FDS]; +static int num_idmap_fds; +static int extroot_idmap_fd = -1; +static int overlay_idmap_fd = -1; + +static bool jail_idmap_active(void) +{ + return (opts.namespace & CLONE_NEWUSER) && opts.uidmap; +} + +static int sock_send_fds(int sock, char tag, const int *fds, int nfds) +{ + struct msghdr msg = { 0 }; + struct iovec iov; + char cmsgbuf[CMSG_SPACE(JAIL_IDMAP_MAX_FDS * sizeof(int))]; + struct cmsghdr *cmsg; + char data[1]; + ssize_t n; + + data[0] = tag; + iov.iov_base = data; + iov.iov_len = 1; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + + if (nfds > 0) { + msg.msg_control = cmsgbuf; + msg.msg_controllen = CMSG_SPACE(nfds * sizeof(int)); + cmsg = CMSG_FIRSTHDR(&msg); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + cmsg->cmsg_len = CMSG_LEN(nfds * sizeof(int)); + memcpy(CMSG_DATA(cmsg), fds, nfds * sizeof(int)); + } + + do { + n = sendmsg(sock, &msg, MSG_NOSIGNAL); + } while (n < 0 && errno == EINTR); + + return (n == 1) ? 0 : -1; +} + +static int sock_recv_fds(int sock, char *tag, int *fds, int maxfds) +{ + struct msghdr msg = { 0 }; + struct iovec iov; + char cmsgbuf[CMSG_SPACE(JAIL_IDMAP_MAX_FDS * sizeof(int))]; + struct cmsghdr *cmsg; + char data[1]; + ssize_t n; + int nfds = 0; + + iov.iov_base = data; + iov.iov_len = 1; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + msg.msg_control = cmsgbuf; + msg.msg_controllen = sizeof(cmsgbuf); + + do { + n = recvmsg(sock, &msg, MSG_CMSG_CLOEXEC); + } while (n < 0 && errno == EINTR); + + if (n < 1) + return -1; + + *tag = data[0]; + + for (cmsg = CMSG_FIRSTHDR(&msg); cmsg; cmsg = CMSG_NXTHDR(&msg, cmsg)) { + if (cmsg->cmsg_level != SOL_SOCKET || cmsg->cmsg_type != SCM_RIGHTS) + continue; + nfds = (cmsg->cmsg_len - CMSG_LEN(0)) / sizeof(int); + if (nfds > maxfds) + nfds = maxfds; + memcpy(fds, CMSG_DATA(cmsg), nfds * sizeof(int)); + break; + } + + return nfds; +} + static void enter_jail_fs(void); static size_t path_depth(const char *path) @@ -939,7 +1027,14 @@ static int build_jail_fs(void) } if (opts.extroot) { - if (mount(opts.extroot, jail_root, "bind", MS_BIND, NULL)) { + if (extroot_idmap_fd >= 0) { + if (sys_move_mount(extroot_idmap_fd, "", AT_FDCWD, jail_root, MOVE_MOUNT_F_EMPTY_PATH)) { + ERROR("move_mount(idmapped extroot) failed: %m\n"); + return -1; + } + close(extroot_idmap_fd); + extroot_idmap_fd = -1; + } else if (mount(opts.extroot, jail_root, "bind", MS_BIND, NULL)) { ERROR("extroot mount failed %m\n"); return -1; } @@ -971,6 +1066,18 @@ static int build_jail_fs(void) overlaydir = opts.overlaydir; if (overlaydir) { + if (overlay_idmap_fd >= 0) { + if (sys_move_mount(overlay_idmap_fd, "", AT_FDCWD, overlaydir, MOVE_MOUNT_F_EMPTY_PATH)) { + ERROR("move_mount(idmapped overlay upper) failed: %m\n"); + return -1; + } + close(overlay_idmap_fd); + overlay_idmap_fd = -1; + } else if (mount(NULL, overlaydir, NULL, + MS_BIND | MS_REMOUNT | MS_NOEXEC | MS_NOSUID | MS_NODEV, NULL)) { + WARNING("failed to harden overlay upper %s: %m\n", overlaydir); + } + ret = mount_overlay(jail_root, overlaydir); if (ret) return ret; @@ -1136,6 +1243,8 @@ static int build_jail_fs(void) } } + jail_fs_set_userns((opts.namespace & CLONE_NEWUSER) || (opts.setns.user != -1)); + if (!fail && mount_all(jail_root)) { ERROR("mount_all() failed\n"); fail = 1; @@ -2398,7 +2507,9 @@ static int parent_pidfd = -1; static int exec_jail(void *arg) { char buf[1]; - ssize_t n; + char tag; + int recv_fds[JAIL_IDMAP_MAX_FDS]; + int nrecv; exit_from_child = true; prctl(PR_SET_SECUREBITS, 0); @@ -2447,17 +2558,20 @@ static int exec_jail(void *arg) return EXIT_FAILURE; } close(pipes[1]); - do { - n = read(pipes[2], buf, 1); - } while (n < 0 && errno == EINTR); - if (n < 1) { + + nrecv = sock_recv_fds(pipes[2], &tag, recv_fds, JAIL_IDMAP_MAX_FDS); + if (nrecv < 0) { ERROR("can't read from parent\n"); return EXIT_FAILURE; } - if (buf[0] != 'O') { + if (tag != 'O') { ERROR("parent had an error, child exiting\n"); return EXIT_FAILURE; } + if (nrecv > 0) + jail_idmap_assign(jail_idmap_active() && opts.extroot, + false, + recv_fds, nrecv, &extroot_idmap_fd, &overlay_idmap_fd); if (opts.setns.user != -1 && (opts.namespace & CLONE_NEWNS) && unshare(CLONE_NEWNS)) { @@ -3399,7 +3513,7 @@ static int parseOCIuidgidmappings(struct blob_attr *msg, bool is_gidmap) /* count length */ totallen += snprintf(NULL, 0, "%d %d %d\n", blobmsg_get_u32(tb[OCI_LINUX_UIDGIDMAP_CONTAINERID]), - blobmsg_get_u32(tb[OCI_LINUX_UIDGIDMAP_HOSTID]), + blobmsg_get_u32(tb[OCI_LINUX_UIDGIDMAP_HOSTID]) + opts.idmap_offset, blobmsg_get_u32(tb[OCI_LINUX_UIDGIDMAP_SIZE])); } @@ -3413,13 +3527,13 @@ static int parseOCIuidgidmappings(struct blob_attr *msg, bool is_gidmap) blobmsg_parse(oci_linux_uidgidmap_policy, __OCI_LINUX_UIDGIDMAP_MAX, tb, blobmsg_data(cur), blobmsg_len(cur)); get_jail_root_user(is_gidmap, blobmsg_get_u32(tb[OCI_LINUX_UIDGIDMAP_CONTAINERID]), - blobmsg_get_u32(tb[OCI_LINUX_UIDGIDMAP_HOSTID]), + blobmsg_get_u32(tb[OCI_LINUX_UIDGIDMAP_HOSTID]) + opts.idmap_offset, blobmsg_get_u32(tb[OCI_LINUX_UIDGIDMAP_SIZE])); /* write mapping line into pre-allocated string */ len = snprintf(&map[pos], totallen + 1, "%d %d %d\n", blobmsg_get_u32(tb[OCI_LINUX_UIDGIDMAP_CONTAINERID]), - blobmsg_get_u32(tb[OCI_LINUX_UIDGIDMAP_HOSTID]), + blobmsg_get_u32(tb[OCI_LINUX_UIDGIDMAP_HOSTID]) + opts.idmap_offset, blobmsg_get_u32(tb[OCI_LINUX_UIDGIDMAP_SIZE])); pos += len; totallen -= len; @@ -3427,14 +3541,70 @@ static int parseOCIuidgidmappings(struct blob_attr *msg, bool is_gidmap) assert(totallen == 0); - if (is_gidmap) + if (is_gidmap) { opts.gidmap = map; - else + free(opts.gidmappings); + opts.gidmappings = blob_memdup(msg); + if (!opts.gidmappings) + return ENOMEM; + } else { opts.uidmap = map; + free(opts.uidmappings); + opts.uidmappings = blob_memdup(msg); + if (!opts.uidmappings) + return ENOMEM; + } return 0; } +static unsigned int host_id_for(struct blob_attr *mappings, unsigned int cid) +{ + struct blob_attr *tb[__OCI_LINUX_UIDGIDMAP_MAX]; + struct blob_attr *cur; + unsigned int base, host, size; + int rem; + + if (!mappings) + return (unsigned int)-1; + + blobmsg_for_each_attr(cur, mappings, rem) { + blobmsg_parse(oci_linux_uidgidmap_policy, __OCI_LINUX_UIDGIDMAP_MAX, tb, + blobmsg_data(cur), blobmsg_len(cur)); + if (!tb[OCI_LINUX_UIDGIDMAP_CONTAINERID] || + !tb[OCI_LINUX_UIDGIDMAP_HOSTID] || + !tb[OCI_LINUX_UIDGIDMAP_SIZE]) + continue; + base = blobmsg_get_u32(tb[OCI_LINUX_UIDGIDMAP_CONTAINERID]); + host = blobmsg_get_u32(tb[OCI_LINUX_UIDGIDMAP_HOSTID]); + size = blobmsg_get_u32(tb[OCI_LINUX_UIDGIDMAP_SIZE]); + if (cid >= base && cid < base + size) + return host + (cid - base) + opts.idmap_offset; + } + + return (unsigned int)-1; +} + +static void jail_chown_writable_surfaces(void) +{ + unsigned int vuid, vgid, ouid, ogid; + + vuid = host_id_for(opts.uidmappings, opts.pw_uid); + vgid = host_id_for(opts.gidmappings, opts.pw_gid); + if (vuid != (unsigned int)-1 && vgid != (unsigned int)-1) + jail_chown_fresh_volumes(vuid, vgid); + + if (!opts.overlaydir) + return; + + ouid = host_id_for(opts.uidmappings, 0); + ogid = host_id_for(opts.gidmappings, 0); + if (ouid != (unsigned int)-1 && ogid != (unsigned int)-1 && + jail_dir_is_fresh(opts.overlaydir) && + chown(opts.overlaydir, ouid, ogid)) + ERROR("chown(fresh overlay %s -> %u:%u): %m\n", opts.overlaydir, ouid, ogid); +} + enum { OCI_DEVICES_TYPE, OCI_DEVICES_PATH, @@ -4554,6 +4724,10 @@ int main(int argc, char **argv) case 'i': opts.immediately = true; break; + case 'I': + opts.idmap_offset = strtoul(optarg, NULL, 10); + jail_set_idmap_offset(opts.idmap_offset); + break; case 'P': opts.pidfile = optarg; break; @@ -4788,7 +4962,10 @@ static void post_main(struct uloop_timeout *t) if (opts.name) prctl(PR_SET_NAME, opts.name, NULL, NULL, NULL); - if (pipe(&pipes[0]) < 0 || pipe(&pipes[2]) < 0) + if (pipe(&pipes[0]) < 0) + free_and_exit(-1); + + if (socketpair(AF_UNIX, SOCK_STREAM, 0, &pipes[2]) < 0) free_and_exit(-1); if (pipe2(&userns_pipe[0], O_CLOEXEC) < 0 || pipe2(&userns_pipe[2], O_CLOEXEC) < 0) @@ -5016,6 +5193,18 @@ static void post_main(struct uloop_timeout *t) cgroups_attach_pid(jail_process.pid); } + if (jail_idmap_active()) { + num_idmap_fds = jail_idmap_build(opts.extroot, + opts.uidmappings, opts.gidmappings, + idmap_fds, JAIL_IDMAP_MAX_FDS); + if (num_idmap_fds < 0) { + ERROR("failed to build idmapped jail mounts\n"); + free_and_exit(-1); + } + + jail_chown_writable_surfaces(); + } + if (opts.namespace & CLONE_NEWNET) jail_network_start(parent_ctx, opts.name, jail_process.pid); @@ -5041,19 +5230,19 @@ static void post_main(struct uloop_timeout *t) static void post_poststart(void); static void post_create_runtime(void) { - char sig_buf[1]; - if (hook_chain_failed) { ERROR("createRuntime hook failed; aborting container\n"); free_and_exit(EXIT_FAILURE); } - sig_buf[0] = 'O'; - if (write(pipes[3], sig_buf, 1) < 0) { + if (sock_send_fds(pipes[3], 'O', idmap_fds, num_idmap_fds) < 0) { ERROR("can't write to child\n"); free_and_exit(-1); } + while (num_idmap_fds > 0) + close(idmap_fds[--num_idmap_fds]); + /* * Wait for the child to reach enter_userns() and create its own * userns before writing its uid/gid maps; see that function. diff --git a/service/instance.c b/service/instance.c index 0323a84..6c6c55f 100644 --- a/service/instance.c +++ b/service/instance.c @@ -124,6 +124,7 @@ enum { JAIL_ATTR_IMMEDIATELY, JAIL_ATTR_PIDFILE, JAIL_ATTR_SETNS, + JAIL_ATTR_IDMAP_OFFSET, __JAIL_ATTR_MAX, }; @@ -145,6 +146,7 @@ static const struct blobmsg_policy jail_attr[__JAIL_ATTR_MAX] = { [JAIL_ATTR_IMMEDIATELY] = { "immediately", BLOBMSG_TYPE_BOOL }, [JAIL_ATTR_PIDFILE] = { "pidfile", BLOBMSG_TYPE_STRING }, [JAIL_ATTR_SETNS] = { "setns", BLOBMSG_TYPE_ARRAY }, + [JAIL_ATTR_IDMAP_OFFSET] = { "idmap_offset", BLOBMSG_TYPE_STRING }, }; enum { @@ -391,6 +393,10 @@ jail_run(struct service_instance *in, char **argv) argv[argc++] = jail->pidfile; } + if (jail->idmap_offset) { + argv[argc++] = "-I"; + argv[argc++] = jail->idmap_offset; + } if (in->bundle) { argv[argc++] = "-J"; argv[argc++] = in->bundle; @@ -1252,6 +1258,10 @@ instance_jail_parse(struct service_instance *in, struct blob_attr *attr) jail->argc += 2; } + if (tb[JAIL_ATTR_IDMAP_OFFSET]) { + jail->idmap_offset = strdup(blobmsg_get_string(tb[JAIL_ATTR_IDMAP_OFFSET])); + jail->argc += 2; + } if (tb[JAIL_ATTR_SETNS]) { struct blob_attr *cur; int rem; @@ -1644,6 +1654,7 @@ instance_free(struct service_instance *in) free(in->jail.name); free(in->jail.hostname); free(in->jail.pidfile); + free(in->jail.idmap_offset); free(in->seccomp); free(in->capabilities); free(in->pidfile); diff --git a/service/instance.h b/service/instance.h index b268759..d930e84 100644 --- a/service/instance.h +++ b/service/instance.h @@ -42,6 +42,7 @@ struct jail { char *name; char *hostname; char *pidfile; + char *idmap_offset; struct blobmsg_list mount; struct blobmsg_list setns; int argc; diff --git a/uxc.c b/uxc.c index 533d580..ce7acac 100644 --- a/uxc.c +++ b/uxc.c @@ -265,6 +265,7 @@ enum { CONF_TEMP_OVERLAY_SIZE, CONF_WRITE_OVERLAY_PATH, CONF_VOLUMES, + CONF_IDMAP_OFFSET, __CONF_MAX, }; @@ -277,6 +278,7 @@ static const struct blobmsg_policy conf_policy[__CONF_MAX] = { [CONF_TEMP_OVERLAY_SIZE] = { .name = "temp-overlay-size", .type = BLOBMSG_TYPE_STRING }, [CONF_WRITE_OVERLAY_PATH] = { .name = "write-overlay-path", .type = BLOBMSG_TYPE_STRING }, [CONF_VOLUMES] = { .name = "volumes", .type = BLOBMSG_TYPE_ARRAY }, + [CONF_IDMAP_OFFSET] = { .name = "idmap-offset", .type = BLOBMSG_TYPE_STRING }, }; static int conf_load(bool load_settings) @@ -975,6 +977,8 @@ static int uxc_create(char *name, bool immediately) if (pidfile) blobmsg_add_string(&req, "pidfile", pidfile); + if (tb[CONF_IDMAP_OFFSET]) + blobmsg_add_string(&req, "idmap_offset", blobmsg_get_string(tb[CONF_IDMAP_OFFSET])); blobmsg_close_table(&req, j); if (writepath) From 61dcc566a804bb56ea844c1e9844d563146da414 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 05:27:38 +0100 Subject: [PATCH 20/82] jail: hand the console PTY master over a console socket Allocate the container PTY in the parent (host mount namespace) so the slave name resolves in the caller's namespace, then keep the slave open across the clone and hand it to the child as console_slave_fd. The child bind-mounts it as /dev/console without re-opening by name, which the new mount namespace's newinstance devpts would otherwise hide. The master is delivered over an AF_UNIX console-socket (-Y, inherited fd or path) via SCM_RIGHTS, matching the runc/conmon handover, or passed to procd as before. consoleSize from the OCI process is applied with TIOCSWINSZ. procd tracks the console socket in the jail instance config so a change triggers a restart. Signed-off-by: Daniel Golle --- jail/jail.c | 276 +++++++++++++++++++++++++++++++++++++++------ service/instance.c | 17 +++ service/instance.h | 1 + 3 files changed, 259 insertions(+), 35 deletions(-) diff --git a/jail/jail.c b/jail/jail.c index 83ee8e4..ac52874 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -47,6 +48,7 @@ #include #include #include +#include #include #include #include @@ -55,6 +57,7 @@ #include #include #include +#include #include "capabilities.h" #include "elf.h" @@ -92,7 +95,7 @@ #define PR_MDWE_NO_INHERIT (1UL << 1) #endif -#define OPT_ARGS "cC:d:De:EfFG:h:iI:j:J:ln:NoO:pP:r:R:sS:uU:w:t:T:y" +#define OPT_ARGS "cC:d:De:EfFG:h:iI:j:J:ln:NoO:pP:r:R:sS:uU:w:t:T:yY:" #define OCI_VERSION_STRING "1.0.2" @@ -155,6 +158,9 @@ static struct { int ronly; int sysfs; int console; + char *console_socket; + unsigned short console_height; + unsigned short console_width; int pw_uid; int pw_gid; int gr_gid; @@ -218,6 +224,8 @@ static int jail_process_pidfd = -1; static struct ubus_context *parent_ctx; int console_fd; +static int console_slave_fd = -1; +static char console_slave_name[64]; static inline bool has_namespaces(void) @@ -441,56 +449,143 @@ static void pass_console(int console_fd) ubus_free(child_ctx); } -static int create_dev_console(const char *jail_root) +static int parse_inherited_console_fd(const char *spec) { - char *console_fname; - char dev_console_path[PATH_MAX]; - int slave_console_fd, dev_console_dummy; + char *endptr; + long fd; + + if (!spec || !*spec) + return -1; - /* Open UNIX/98 virtual console */ - console_fd = posix_openpt(O_RDWR | O_NOCTTY); - if (console_fd < 0) + errno = 0; + fd = strtol(spec, &endptr, 10); + if (errno || *endptr || endptr == spec || fd < 0 || fd > INT_MAX) return -1; - console_fname = ptsname(console_fd); - DEBUG("got console fd %d and PTS client name %s\n", console_fd, console_fname); - if (!console_fname) - goto no_console; + if (fcntl((int)fd, F_GETFD) == -1) + return -1; + + return (int)fd; +} - grantpt(console_fd); - unlockpt(console_fd); +static int open_console_sock(const char *spec, bool *owned, bool path_only) +{ + int sock; + struct sockaddr_un addr = { .sun_family = AF_UNIX }; + + *owned = false; + if (!path_only) { + sock = parse_inherited_console_fd(spec); + if (sock >= 0) { + int dom = 0, typ = 0; + socklen_t slen = sizeof(dom); + + if (getsockopt(sock, SOL_SOCKET, SO_DOMAIN, &dom, &slen) < 0 || + dom != AF_UNIX) { + ERROR("console-socket: inherited fd %d is not AF_UNIX\n", sock); + return -1; + } + slen = sizeof(typ); + if (getsockopt(sock, SOL_SOCKET, SO_TYPE, &typ, &slen) < 0 || + typ != SOCK_STREAM) { + ERROR("console-socket: inherited fd %d is not SOCK_STREAM\n", sock); + return -1; + } + return sock; + } + } + + if (strlen(spec) >= sizeof(addr.sun_path)) { + ERROR("console-socket path too long: %s\n", spec); + return -1; + } + memcpy(addr.sun_path, spec, strlen(spec) + 1); + + sock = socket(AF_UNIX, SOCK_STREAM, 0); + if (sock < 0) { + ERROR("console-socket: socket(): %m\n"); + return -1; + } + if (connect(sock, (struct sockaddr *)&addr, sizeof(addr)) < 0) { + ERROR("console-socket: connect(%s): %m\n", spec); + close(sock); + return -1; + } + *owned = true; + return sock; +} + +static int sendmsg_console_fd(int sock, int console_fd, const char *slave_name) +{ + struct msghdr msg = { 0 }; + struct cmsghdr *cmsg; + struct iovec iov; + char cbuf[CMSG_SPACE(sizeof(int))] = { 0 }; + + iov.iov_base = (void *)slave_name; + iov.iov_len = strlen(slave_name); + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + msg.msg_control = cbuf; + msg.msg_controllen = sizeof(cbuf); + + cmsg = CMSG_FIRSTHDR(&msg); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + cmsg->cmsg_len = CMSG_LEN(sizeof(int)); + memcpy(CMSG_DATA(cmsg), &console_fd, sizeof(int)); + + if (sendmsg(sock, &msg, 0) < 0) { + ERROR("console-socket: sendmsg: %m\n"); + return -1; + } + return 0; +} + +static int send_console_fd(const char *spec, int console_fd, const char *slave_name) +{ + bool owned; + int sock, ret; + + sock = open_console_sock(spec, &owned, false); + if (sock < 0) + return -1; + ret = sendmsg_console_fd(sock, console_fd, slave_name); + if (owned) + close(sock); + return ret; +} + +static int create_dev_console(const char *jail_root) +{ + char dev_console_path[PATH_MAX]; + char fdpath[64]; + int dev_console_dummy; - /* pass PTY master to procd */ - pass_console(console_fd); + if (console_slave_fd < 0) + return 1; - /* mount-bind PTY slave to /dev/console in jail */ snprintf(dev_console_path, sizeof(dev_console_path), "%s/dev/console", jail_root); dev_console_dummy = creat(dev_console_path, 0620); if (dev_console_dummy < 0) - goto no_console; - + return 1; close(dev_console_dummy); - if (mount(console_fname, dev_console_path, "bind", MS_BIND, NULL)) - goto no_console; + snprintf(fdpath, sizeof(fdpath), "/proc/self/fd/%d", console_slave_fd); + if (mount(fdpath, dev_console_path, "bind", MS_BIND, NULL)) + return 1; - /* use PTY slave for stdio */ - slave_console_fd = open(console_fname, O_RDWR); /* | O_NOCTTY */ - if (slave_console_fd < 0) - goto no_console; + setsid(); + if (ioctl(console_slave_fd, TIOCSCTTY, 0) < 0) + WARNING("TIOCSCTTY on guest console failed: %m\n"); - dup2(slave_console_fd, 0); - dup2(slave_console_fd, 1); - dup2(slave_console_fd, 2); - close(slave_console_fd); - - INFO("using guest console %s\n", console_fname); + dup2(console_slave_fd, 0); + dup2(console_slave_fd, 1); + dup2(console_slave_fd, 2); + if (console_slave_fd > 2) + close(console_slave_fd); return 0; - -no_console: - close(console_fd); - return 1; } static int hook_running = 0; @@ -1016,6 +1111,12 @@ static int build_jail_fs(void) old_umask = umask(0); + if (opts.console && console_slave_name[0]) { + console_slave_fd = open(console_slave_name, O_RDWR); + if (console_slave_fd < 0) + WARNING("open guest console slave %s: %m\n", console_slave_name); + } + if (mkdtemp(jail_root) == NULL) { ERROR("mkdtemp(%s) failed: %m\n", jail_root); return -1; @@ -1888,6 +1989,7 @@ static void usage(void) fprintf(stderr, " -T \tuse tmpfs r/w overlayfs with \n"); fprintf(stderr, " -E\t\tfail if jail cannot be setup\n"); fprintf(stderr, " -y\t\tprovide jail console\n"); + fprintf(stderr, " -Y \tsend PTY master fd via inherited fd or AF_UNIX path\n"); fprintf(stderr, " -J \tcreate container from OCI bundle\n"); fprintf(stderr, " -i\t\tstart container immediately\n"); fprintf(stderr, " -P \tcreate \n"); @@ -3220,6 +3322,7 @@ enum { OCI_PROCESS_APPARMORPROFILE, OCI_PROCESS_ARGS, OCI_PROCESS_CAPABILITIES, + OCI_PROCESS_CONSOLESIZE, OCI_PROCESS_CWD, OCI_PROCESS_ENV, OCI_PROCESS_EXECCPUAFFINITY, @@ -3238,6 +3341,7 @@ static const struct blobmsg_policy oci_process_policy[] = { [OCI_PROCESS_APPARMORPROFILE] = { "apparmorProfile", BLOBMSG_TYPE_STRING }, [OCI_PROCESS_ARGS] = { "args", BLOBMSG_TYPE_ARRAY }, [OCI_PROCESS_CAPABILITIES] = { "capabilities", BLOBMSG_TYPE_TABLE }, + [OCI_PROCESS_CONSOLESIZE] = { "consoleSize", BLOBMSG_TYPE_TABLE }, [OCI_PROCESS_CWD] = { "cwd", BLOBMSG_TYPE_STRING }, [OCI_PROCESS_ENV] = { "env", BLOBMSG_TYPE_ARRAY }, [OCI_PROCESS_EXECCPUAFFINITY] = { "execCPUAffinity", BLOBMSG_TYPE_TABLE }, @@ -3251,6 +3355,41 @@ static const struct blobmsg_policy oci_process_policy[] = { [OCI_PROCESS_USER] = { "user", BLOBMSG_TYPE_TABLE }, }; +enum { + OCI_PROCESS_CONSOLESIZE_HEIGHT, + OCI_PROCESS_CONSOLESIZE_WIDTH, + __OCI_PROCESS_CONSOLESIZE_MAX, +}; + +static const struct blobmsg_policy oci_process_consolesize_policy[] = { + [OCI_PROCESS_CONSOLESIZE_HEIGHT] = { "height", BLOBMSG_TYPE_INT32 }, + [OCI_PROCESS_CONSOLESIZE_WIDTH] = { "width", BLOBMSG_TYPE_INT32 }, +}; + +static int parseOCIprocessconsolesize(struct blob_attr *msg) +{ + struct blob_attr *tb[__OCI_PROCESS_CONSOLESIZE_MAX]; + uint32_t height, width; + + blobmsg_parse(oci_process_consolesize_policy, __OCI_PROCESS_CONSOLESIZE_MAX, tb, + blobmsg_data(msg), blobmsg_len(msg)); + + if (!tb[OCI_PROCESS_CONSOLESIZE_HEIGHT] || !tb[OCI_PROCESS_CONSOLESIZE_WIDTH]) + return ENODATA; + + height = blobmsg_get_u32(tb[OCI_PROCESS_CONSOLESIZE_HEIGHT]); + width = blobmsg_get_u32(tb[OCI_PROCESS_CONSOLESIZE_WIDTH]); + if (!height || !width || height > USHRT_MAX || width > USHRT_MAX) { + ERROR("consoleSize: %u x %u out of range\n", height, width); + return EINVAL; + } + + opts.console_height = height; + opts.console_width = width; + + return 0; +} + static int parseOCIprocess(struct blob_attr *msg) { @@ -3279,6 +3418,12 @@ static int parseOCIprocess(struct blob_attr *msg) if (tb[OCI_PROCESS_TERMINAL]) opts.console = blobmsg_get_bool(tb[OCI_PROCESS_TERMINAL]); + if (opts.console && tb[OCI_PROCESS_CONSOLESIZE]) { + res = parseOCIprocessconsolesize(tb[OCI_PROCESS_CONSOLESIZE]); + if (res) + return res; + } + if (tb[OCI_PROCESS_SCHEDULER]) { res = parseOCIprocessscheduler(tb[OCI_PROCESS_SCHEDULER]); if (res) @@ -4731,6 +4876,22 @@ int main(int argc, char **argv) case 'P': opts.pidfile = optarg; break; + case 'Y': + opts.console_socket = optarg; + break; + } + } + + if (opts.console_socket && parse_inherited_console_fd(opts.console_socket) < 0) { + static char fdspec[16]; + bool owned; + int csfd; + + csfd = open_console_sock(opts.console_socket, &owned, false); + if (csfd >= 0) { + fcntl(csfd, F_SETFD, 0); + snprintf(fdspec, sizeof(fdspec), "%d", csfd); + opts.console_socket = fdspec; } } @@ -5117,6 +5278,51 @@ static void post_main(struct uloop_timeout *t) } } + if (opts.console) { + char *slave_name; + int parent_master; + + parent_master = posix_openpt(O_RDWR | O_NOCTTY); + if (parent_master < 0) { + ERROR("posix_openpt: %m\n"); + free_and_exit(-1); + } + if (grantpt(parent_master) || unlockpt(parent_master) || + !(slave_name = ptsname(parent_master))) { + ERROR("grantpt/unlockpt/ptsname failed\n"); + close(parent_master); + free_and_exit(-1); + } + if (opts.console_height && opts.console_width) { + struct winsize ws = { + .ws_row = opts.console_height, + .ws_col = opts.console_width, + }; + ioctl(parent_master, TIOCSWINSZ, &ws); + } + strncpy(console_slave_name, slave_name, sizeof(console_slave_name) - 1); + console_slave_name[sizeof(console_slave_name) - 1] = '\0'; + if (opts.console_socket) { + if (send_console_fd(opts.console_socket, parent_master, slave_name)) { + ERROR("send_console_fd failed\n"); + close(parent_master); + free_and_exit(-1); + } + close(parent_master); + } else { + console_fd = parent_master; + if ((opts.namespace & CLONE_NEWUSER) && seteuid(0)) { + ERROR("seteuid(0) failed: %m\n"); + free_and_exit(EXIT_FAILURE); + } + pass_console(console_fd); + if ((opts.namespace & CLONE_NEWUSER) && seteuid(opts.root_map_uid)) { + ERROR("seteuid(%d) failed: %m\n", opts.root_map_uid); + free_and_exit(EXIT_FAILURE); + } + } + } + /* * CLONE_NEWUSER is excluded here; the child creates its own * later, in enter_userns(). See exec_jail() for why. diff --git a/service/instance.c b/service/instance.c index 6c6c55f..c720d75 100644 --- a/service/instance.c +++ b/service/instance.c @@ -125,6 +125,7 @@ enum { JAIL_ATTR_PIDFILE, JAIL_ATTR_SETNS, JAIL_ATTR_IDMAP_OFFSET, + JAIL_ATTR_CONSOLESOCKET, __JAIL_ATTR_MAX, }; @@ -147,6 +148,7 @@ static const struct blobmsg_policy jail_attr[__JAIL_ATTR_MAX] = { [JAIL_ATTR_PIDFILE] = { "pidfile", BLOBMSG_TYPE_STRING }, [JAIL_ATTR_SETNS] = { "setns", BLOBMSG_TYPE_ARRAY }, [JAIL_ATTR_IDMAP_OFFSET] = { "idmap_offset", BLOBMSG_TYPE_STRING }, + [JAIL_ATTR_CONSOLESOCKET] = { "consolesocket", BLOBMSG_TYPE_STRING }, }; enum { @@ -397,6 +399,11 @@ jail_run(struct service_instance *in, char **argv) argv[argc++] = "-I"; argv[argc++] = jail->idmap_offset; } + if (jail->consolesocket) { + argv[argc++] = "-Y"; + argv[argc++] = jail->consolesocket; + } + if (in->bundle) { argv[argc++] = "-J"; argv[argc++] = in->bundle; @@ -1086,6 +1093,9 @@ instance_config_changed(struct service_instance *in, struct service_instance *in if (string_changed(in->jail.pidfile, in_new->jail.pidfile)) return true; + if (string_changed(in->jail.consolesocket, in_new->jail.consolesocket)) + return true; + if (in->jail.flags != in_new->jail.flags) return true; @@ -1262,6 +1272,11 @@ instance_jail_parse(struct service_instance *in, struct blob_attr *attr) jail->idmap_offset = strdup(blobmsg_get_string(tb[JAIL_ATTR_IDMAP_OFFSET])); jail->argc += 2; } + if (tb[JAIL_ATTR_CONSOLESOCKET]) { + jail->consolesocket = strdup(blobmsg_get_string(tb[JAIL_ATTR_CONSOLESOCKET])); + jail->argc += 2; + } + if (tb[JAIL_ATTR_SETNS]) { struct blob_attr *cur; int rem; @@ -1603,6 +1618,7 @@ instance_config_move(struct service_instance *in, struct service_instance *in_sr instance_config_move_strdup(&in->jail.name, in_src->jail.name); instance_config_move_strdup(&in->jail.hostname, in_src->jail.hostname); instance_config_move_strdup(&in->jail.pidfile, in_src->jail.pidfile); + instance_config_move_strdup(&in->jail.consolesocket, in_src->jail.consolesocket); free(in->config); in->config = in_src->config; @@ -1655,6 +1671,7 @@ instance_free(struct service_instance *in) free(in->jail.hostname); free(in->jail.pidfile); free(in->jail.idmap_offset); + free(in->jail.consolesocket); free(in->seccomp); free(in->capabilities); free(in->pidfile); diff --git a/service/instance.h b/service/instance.h index d930e84..d33900a 100644 --- a/service/instance.h +++ b/service/instance.h @@ -43,6 +43,7 @@ struct jail { char *hostname; char *pidfile; char *idmap_offset; + char *consolesocket; struct blobmsg_list mount; struct blobmsg_list setns; int argc; From d4c6ebf1e409709cfad9eb569f039157c2da00a3 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 05:27:38 +0100 Subject: [PATCH 21/82] uxc: pass --console-socket on create and rework attach cleanup create gains a --console-socket option, forwarded to procd as the jail's consolesocket so ujail delivers the PTY master there instead of to procd. attach gains a single cleanup path with explicit fd ownership, restores the caller's terminal only when it was actually switched to raw mode, and prints a clearer error when the container was created without a console. The events, checkpoint and restore verbs runc callers may issue are answered with a "not supported" message instead of usage output. Signed-off-by: Daniel Golle --- uxc.c | 94 ++++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 61 insertions(+), 33 deletions(-) diff --git a/uxc.c b/uxc.c index ce7acac..5acacef 100644 --- a/uxc.c +++ b/uxc.c @@ -73,9 +73,14 @@ struct settings { struct blob_attr *volumes; }; +enum { + OPT_CONSOLE_SOCKET = 0x100, +}; + static const struct option create_opts[] = { {"autostart", no_argument, 0, 'a' }, {"bundle", required_argument, 0, 'b' }, + {"console-socket", required_argument, 0, OPT_CONSOLE_SOCKET }, {"mounts", required_argument, 0, 'm' }, {"pid-file", required_argument, 0, 'p' }, {"temp-overlay-size", required_argument, 0, 't' }, @@ -661,8 +666,10 @@ static int uxc_attach(const char *container_name) struct ubus_context *ctx; uint32_t id; static struct blob_buf req; - int client_fd, server_fd, tty_fd; + int client_fd = -1, server_fd = -1, tty_fd = -1; struct termios oldtermios; + bool tty_raw = false; + int rc; ctx = ubus_connect(NULL); if (!ctx) { @@ -670,64 +677,64 @@ static int uxc_attach(const char *container_name) return -ECONNREFUSED; } - /* open pseudo-terminal pair */ client_fd = posix_openpt(O_RDWR | O_NOCTTY); if (client_fd < 0) { fprintf(stderr, "can't create virtual console!\n"); - ubus_free(ctx); - return -EIO; + rc = -EIO; + goto out; } - setup_tios(client_fd, &oldtermios); grantpt(client_fd); unlockpt(client_fd); server_fd = open(ptsname(client_fd), O_RDWR | O_NOCTTY); if (server_fd < 0) { fprintf(stderr, "can't open virtual console!\n"); - close(client_fd); - ubus_free(ctx); - return -EIO; + rc = -EIO; + goto out; } - setup_tios(server_fd, &oldtermios); tty_fd = open("/dev/tty", O_RDWR); if (tty_fd < 0) { fprintf(stderr, "can't open local console!\n"); - close(server_fd); - close(client_fd); - ubus_free(ctx); - return -EIO; + rc = -EIO; + goto out; } - setup_tios(tty_fd, &oldtermios); + if (!setup_tios(tty_fd, &oldtermios)) + tty_raw = true; - /* register server-side with procd */ blob_buf_init(&req, 0); blobmsg_add_string(&req, "name", container_name); blobmsg_add_string(&req, "instance", container_name); - if (ubus_lookup_id(ctx, "container", &id) || - ubus_invoke_fd(ctx, id, "console_attach", req.head, NULL, NULL, 3000, server_fd)) { - fprintf(stderr, "ubus request failed\n"); - close(tty_fd); - close(server_fd); - close(client_fd); + if (ubus_lookup_id(ctx, "container", &id)) { + fprintf(stderr, "uxc: 'container' ubus object not found\n"); blob_buf_free(&req); - ubus_free(ctx); - return -ENXIO; + rc = -ENXIO; + goto out; + } + rc = ubus_invoke_fd(ctx, id, "console_attach", req.head, NULL, NULL, 3000, server_fd); + blob_buf_free(&req); + if (rc) { + if (rc == UBUS_STATUS_NOT_SUPPORTED) + fprintf(stderr, "uxc: container '%s' has no console; " + "re-create it with console=true (OCI process.terminal=true)\n", + container_name); + else + fprintf(stderr, "uxc: console_attach failed: %s\n", ubus_strerror(rc)); + rc = -ENXIO; + goto out; } close(server_fd); - blob_buf_free(&req); + server_fd = -1; ubus_free(ctx); + ctx = NULL; uloop_init(); - /* forward between stdio and client_fd until detach is requested */ lufd.stream.notify_read = local_cb; ustream_fd_init(&lufd, tty_fd); cufd.stream.notify_read = client_cb; -/* ToDo: handle remote close and other events */ -// cufd.stream.notify_state = client_state_cb; ustream_fd_init(&cufd, client_fd); fprintf(stderr, "attaching to jail console. press [CTRL]+[B] to exit.\n"); @@ -736,12 +743,22 @@ static int uxc_attach(const char *container_name) close(2); uloop_run(); - tcsetattr(tty_fd, TCSAFLUSH, &oldtermios); ustream_free(&lufd.stream); ustream_free(&cufd.stream); - close(client_fd); + rc = 0; - return 0; +out: + if (tty_raw && tty_fd >= 0) + tcsetattr(tty_fd, TCSAFLUSH, &oldtermios); + if (tty_fd >= 0) + close(tty_fd); + if (server_fd >= 0) + close(server_fd); + if (client_fd >= 0) + close(client_fd); + if (ctx) + ubus_free(ctx); + return rc; } static int uxc_state(char *name) @@ -912,7 +929,7 @@ static int uxc_exists(char *name) return 0; } -static int uxc_create(char *name, bool immediately) +static int uxc_create(char *name, bool immediately, const char *console_socket) { static struct blob_buf req; struct blob_attr *cur, *tb[__CONF_MAX]; @@ -979,6 +996,9 @@ static int uxc_create(char *name, bool immediately) if (tb[CONF_IDMAP_OFFSET]) blobmsg_add_string(&req, "idmap_offset", blobmsg_get_string(tb[CONF_IDMAP_OFFSET])); + if (console_socket) + blobmsg_add_string(&req, "consolesocket", console_socket); + blobmsg_close_table(&req, j); if (writepath) @@ -1406,7 +1426,7 @@ static int uxc_boot(void) if (uxc_exists(name)) continue; - if (uxc_create(name, true)) + if (uxc_create(name, true, NULL)) ++ret; free(name); @@ -1687,6 +1707,7 @@ int main(int argc, char **argv) char *tmprwsize = NULL, *writepath = NULL, *requiredmounts = NULL; signed char autostart = -1; char *name; + const char *console_socket = NULL; while ((c = getopt_long(verb_argc, verb_argv, "ab:m:p:t:w:", create_opts, NULL)) != -1) { @@ -1697,6 +1718,9 @@ int main(int argc, char **argv) case 'p': pidfile = optarg; break; case 't': tmprwsize = optarg; break; case 'w': writepath = optarg; break; + case OPT_CONSOLE_SOCKET: + console_socket = optarg; + break; default: goto usage_out; } } @@ -1715,7 +1739,7 @@ int main(int argc, char **argv) if (ret > 0) reload_conf(); - ret = uxc_create(name, false); + ret = uxc_create(name, false, console_socket); } else if (!strcmp(verb, "pause") || !strcmp(verb, "resume")) { char *objname; uint32_t id; @@ -1735,6 +1759,10 @@ int main(int argc, char **argv) ret = ubus_invoke(ctx, id, verb, NULL, NULL, NULL, 3000); if (ret) ret = -EIO; + } else if (!strcmp(verb, "events") || !strcmp(verb, "checkpoint") || + !strcmp(verb, "restore")) { + fprintf(stderr, "uxc: '%s' is not supported\n", verb); + ret = -ENOTSUP; } else { fprintf(stderr, "uxc: unknown command '%s'\n", verb); goto usage_out; From 251a30cf6f271bcacbf37a5be83f153cadc31320 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 05:28:35 +0100 Subject: [PATCH 22/82] jail: add exec and update ubus methods and systemd-cgroup translation Add the ubus methods conmon and runc drive on a running container: update applies linux.resources, guarded by memory.checkBeforeUpdate against shrinking below current usage, and exec enters the container's namespaces, sets up an optional terminal over a console-socket, applies user, capabilities and rlimits and runs a command. The systemd-cgroup convention is accepted via a new -Z flag that translates a slice:prefix:name cgroupsPath to a .scope path, and procd passes the systemdcgroup jail attribute through as -Z. Entering a running container needs care with the namespaces it already shares with the runtime. Such a namespace is left alone, since setns() refuses the user namespace we are in already, and the cgroup namespace is joined only by the grandchild once clone3() has placed it, because a joined cgroup namespace hides from the kernel the very cgroup CLONE_INTO_CGROUP names. A namespace that cannot be entered, and a clone3() that fails, say so instead of leaving the caller with a bare exit code 126. The exec session's pid file holds the bare number runc writes and conmon expects to read. Container init now installs its environment as environ before execvpe() so PATH resolution uses the container's own environment rather than the runtime's. Signed-off-by: Daniel Golle --- jail/cgroups.c | 37 ++- jail/cgroups.h | 2 +- jail/jail.c | 683 ++++++++++++++++++++++++++++++++++++++++++++- service/instance.c | 10 + service/instance.h | 1 + 5 files changed, 724 insertions(+), 9 deletions(-) diff --git a/jail/cgroups.c b/jail/cgroups.c index a774654..7800ff8 100644 --- a/jail/cgroups.c +++ b/jail/cgroups.c @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -895,7 +896,7 @@ void cgroups_set_memory_limit(int64_t bytes) cgroups_set("memory.max", tmp); } -static int parseOCIlinuxcgroups_legacy_memory(struct blob_attr *msg) +static int parseOCIlinuxcgroups_legacy_memory(struct blob_attr *msg, bool is_update) { struct blob_attr *tb[__OCI_LINUX_CGROUPS_MEMORY_MAX]; char tmp[32] = { 0 }; @@ -917,6 +918,36 @@ static int parseOCIlinuxcgroups_legacy_memory(struct blob_attr *msg) tb[OCI_LINUX_CGROUPS_MEMORY_USEHIERARCHY]) return ENOTSUP; + if (is_update && tb[OCI_LINUX_CGROUPS_MEMORY_CHECKBEFOREUPDATE] && + blobmsg_get_bool(tb[OCI_LINUX_CGROUPS_MEMORY_CHECKBEFOREUPDATE])) { + char path[PATH_MAX]; + int64_t current; + + snprintf(path, sizeof(path), "%s/memory.current", cgroup_path); + current = read_int64_file(path); + if (current < 0) { + ERROR("memory.checkBeforeUpdate: cannot read %s: %m\n", path); + return EIO; + } + + if (tb[OCI_LINUX_CGROUPS_MEMORY_LIMIT]) { + int64_t new_limit = blobmsg_cast_s64(tb[OCI_LINUX_CGROUPS_MEMORY_LIMIT]); + if (new_limit != -1 && new_limit < current) { + ERROR("memory.checkBeforeUpdate: new limit %" PRId64 + " < current usage %" PRId64 "\n", new_limit, current); + return EBUSY; + } + } + if (tb[OCI_LINUX_CGROUPS_MEMORY_RESERVATION]) { + int64_t new_res = blobmsg_cast_s64(tb[OCI_LINUX_CGROUPS_MEMORY_RESERVATION]); + if (new_res != -1 && new_res < current) { + ERROR("memory.checkBeforeUpdate: new reservation %" PRId64 + " < current usage %" PRId64 "\n", new_res, current); + return EBUSY; + } + } + } + if (tb[OCI_LINUX_CGROUPS_MEMORY_LIMIT]) { limit = blobmsg_cast_s64(tb[OCI_LINUX_CGROUPS_MEMORY_LIMIT]); if (limit == -1) @@ -1037,7 +1068,7 @@ static const struct blobmsg_policy oci_linux_cgroups_policy[] = { [OCI_LINUX_CGROUPS_UNIFIED] = { "unified", BLOBMSG_TYPE_TABLE }, }; -int parseOCIlinuxcgroups(struct blob_attr *msg) +int parseOCIlinuxcgroups(struct blob_attr *msg, bool is_update) { struct blob_attr *tb[__OCI_LINUX_CGROUPS_MAX]; int ret; @@ -1069,7 +1100,7 @@ int parseOCIlinuxcgroups(struct blob_attr *msg) } if (tb[OCI_LINUX_CGROUPS_MEMORY]) { - ret = parseOCIlinuxcgroups_legacy_memory(tb[OCI_LINUX_CGROUPS_MEMORY]); + ret = parseOCIlinuxcgroups_legacy_memory(tb[OCI_LINUX_CGROUPS_MEMORY], is_update); if (ret) return ret; } diff --git a/jail/cgroups.h b/jail/cgroups.h index 4e99706..2c6f690 100644 --- a/jail/cgroups.h +++ b/jail/cgroups.h @@ -18,7 +18,7 @@ #include void cgroups_init(const char *p); -int parseOCIlinuxcgroups(struct blob_attr *msg); +int parseOCIlinuxcgroups(struct blob_attr *msg, bool is_update); void cgroups_create(void); void cgroups_destroy(void); void cgroups_configure(void); diff --git a/jail/jail.c b/jail/jail.c index ac52874..65110d4 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -95,7 +95,7 @@ #define PR_MDWE_NO_INHERIT (1UL << 1) #endif -#define OPT_ARGS "cC:d:De:EfFG:h:iI:j:J:ln:NoO:pP:r:R:sS:uU:w:t:T:yY:" +#define OPT_ARGS "cC:d:De:EfFG:h:iI:j:J:ln:NoO:pP:r:R:sS:uU:w:t:T:yY:Z" #define OCI_VERSION_STRING "1.0.2" @@ -159,6 +159,7 @@ static struct { int sysfs; int console; char *console_socket; + bool systemd_cgroup; unsigned short console_height; unsigned short console_width; int pw_uid; @@ -2915,10 +2916,12 @@ static void post_start_hook(void) uloop_end(); free_opts(false); INFO("exec-ing %s\n", *opts.jail_argv); - if (opts.envp) /* respect PATH if potentially set in ENV */ + if (opts.envp) { /* respect PATH if potentially set in ENV */ + environ = envp; execvpe(*opts.jail_argv, opts.jail_argv, envp); - else + } else { execve(*opts.jail_argv, opts.jail_argv, envp); + } /* we get there only if execve fails */ ERROR("failed to execve %s: %m\n", *opts.jail_argv); @@ -4118,7 +4121,36 @@ static int parseOCIlinux(struct blob_attr *msg) if (tb[OCI_LINUX_CGROUPSPATH]) { cgpath = blobmsg_get_string(tb[OCI_LINUX_CGROUPSPATH]); - if (cgpath[0] == '/') { + if (opts.systemd_cgroup) { + char *orig = strdupa(cgpath); + char *slice = cgpath; + char *prefix = strchr(slice, ':'); + char *id; + + if (!prefix || prefix == slice) { + ERROR("--systemd-cgroup: cgroupsPath %s is not slice:prefix:name\n", orig); + return EINVAL; + } + *prefix++ = '\0'; + id = strchr(prefix, ':'); + if (!id || id == prefix || !id[1]) { + ERROR("--systemd-cgroup: cgroupsPath %s is not slice:prefix:name\n", orig); + return EINVAL; + } + *id++ = '\0'; + + if (strlen(slice) + strlen(prefix) + strlen(id) + 9 + >= (sizeof(cgfullpath) - strlen(cgfullpath))) + return E2BIG; + + strcat(cgfullpath, "/"); + strcat(cgfullpath, slice); + strcat(cgfullpath, "/"); + strcat(cgfullpath, prefix); + strcat(cgfullpath, "-"); + strcat(cgfullpath, id); + strcat(cgfullpath, ".scope"); + } else if (cgpath[0] == '/') { if (strlen(cgpath) + 1 >= (sizeof(cgfullpath) - strlen(cgfullpath))) return E2BIG; @@ -4145,7 +4177,7 @@ static int parseOCIlinux(struct blob_attr *msg) cgroups_init(cgfullpath); if (tb[OCI_LINUX_RESOURCES]) { - res = parseOCIlinuxcgroups(tb[OCI_LINUX_RESOURCES]); + res = parseOCIlinuxcgroups(tb[OCI_LINUX_RESOURCES], false); if (res) return res; } @@ -4652,6 +4684,642 @@ container_handle_reclaim(struct ubus_context *ctx, struct ubus_object *obj, return UBUS_STATUS_UNKNOWN_ERROR; } +static int +container_handle_update(struct ubus_context *ctx, struct ubus_object *obj, + struct ubus_request_data *req, const char *method, + struct blob_attr *msg) +{ + int rc; + + if (jail_oci_state != OCI_STATE_CREATED && + jail_oci_state != OCI_STATE_RUNNING) + return UBUS_STATUS_INVALID_ARGUMENT; + + if (!msg) + return UBUS_STATUS_INVALID_ARGUMENT; + + rc = parseOCIlinuxcgroups(msg, true); + if (rc) { + switch (rc) { + case ENOTSUP: + return UBUS_STATUS_NOT_SUPPORTED; + case EBUSY: + case EINVAL: + case ENODATA: + case ERANGE: + return UBUS_STATUS_INVALID_ARGUMENT; + case EIO: + return UBUS_STATUS_UNKNOWN_ERROR; + default: + return UBUS_STATUS_UNKNOWN_ERROR; + } + } + + cgroups_apply(jail_process.pid); + return UBUS_STATUS_OK; +} + +enum { + CONTAINER_EXEC_ATTR_ARGS, + CONTAINER_EXEC_ATTR_ENV, + CONTAINER_EXEC_ATTR_CWD, + CONTAINER_EXEC_ATTR_USER, + CONTAINER_EXEC_ATTR_CAPABILITIES, + CONTAINER_EXEC_ATTR_RLIMITS, + CONTAINER_EXEC_ATTR_NO_NEW_PRIVS, + CONTAINER_EXEC_ATTR_TERMINAL, + CONTAINER_EXEC_ATTR_CONSOLE_SOCKET, + CONTAINER_EXEC_ATTR_PIDFILE, + CONTAINER_EXEC_ATTR_DETACH, + __CONTAINER_EXEC_ATTR_MAX, +}; + +static const struct blobmsg_policy container_exec_attrs[__CONTAINER_EXEC_ATTR_MAX] = { + [CONTAINER_EXEC_ATTR_ARGS] = { "args", BLOBMSG_TYPE_ARRAY }, + [CONTAINER_EXEC_ATTR_ENV] = { "env", BLOBMSG_TYPE_ARRAY }, + [CONTAINER_EXEC_ATTR_CWD] = { "cwd", BLOBMSG_TYPE_STRING }, + [CONTAINER_EXEC_ATTR_USER] = { "user", BLOBMSG_TYPE_TABLE }, + [CONTAINER_EXEC_ATTR_CAPABILITIES] = { "capabilities", BLOBMSG_TYPE_TABLE }, + [CONTAINER_EXEC_ATTR_RLIMITS] = { "rlimits", BLOBMSG_TYPE_ARRAY }, + [CONTAINER_EXEC_ATTR_NO_NEW_PRIVS] = { "noNewPrivileges", BLOBMSG_TYPE_BOOL }, + [CONTAINER_EXEC_ATTR_TERMINAL] = { "terminal", BLOBMSG_TYPE_BOOL }, + [CONTAINER_EXEC_ATTR_CONSOLE_SOCKET] = { "consolesocket", BLOBMSG_TYPE_STRING }, + [CONTAINER_EXEC_ATTR_PIDFILE] = { "pidfile", BLOBMSG_TYPE_STRING }, + [CONTAINER_EXEC_ATTR_DETACH] = { "detach", BLOBMSG_TYPE_BOOL }, +}; + +enum { + CONTAINER_EXEC_USER_UID, + CONTAINER_EXEC_USER_GID, + CONTAINER_EXEC_USER_ADDITIONAL_GIDS, + CONTAINER_EXEC_USER_UMASK, + __CONTAINER_EXEC_USER_MAX, +}; + +static const struct blobmsg_policy container_exec_user_attrs[__CONTAINER_EXEC_USER_MAX] = { + [CONTAINER_EXEC_USER_UID] = { "uid", BLOBMSG_TYPE_INT32 }, + [CONTAINER_EXEC_USER_GID] = { "gid", BLOBMSG_TYPE_INT32 }, + [CONTAINER_EXEC_USER_ADDITIONAL_GIDS] = { "additionalGids", BLOBMSG_TYPE_ARRAY }, + [CONTAINER_EXEC_USER_UMASK] = { "umask", BLOBMSG_TYPE_INT32 }, +}; + +struct container_exec { + struct ubus_context *ctx; + struct ubus_request_data req; + struct uloop_process exec_proc; +}; + +static struct container_exec *current_exec; + +static char **container_exec_strarray(struct blob_attr *arr) +{ + struct blob_attr *cur; + char **out; + int rem, n = 0; + + blobmsg_for_each_attr(cur, arr, rem) + ++n; + + out = calloc(n + 1, sizeof(char *)); + if (!out) + return NULL; + + n = 0; + blobmsg_for_each_attr(cur, arr, rem) + out[n++] = strdup(blobmsg_get_string(cur)); + out[n] = NULL; + return out; +} + +static void container_exec_free_strarray(char **a) +{ + int i; + + if (!a) + return; + for (i = 0; a[i]; i++) + free(a[i]); + free(a); +} + +static void container_exec_done_reply(struct uloop_process *p, int wstatus) +{ + struct container_exec *e = container_of(p, struct container_exec, exec_proc); + static struct blob_buf bb; + int status; + + if (WIFEXITED(wstatus)) + status = WEXITSTATUS(wstatus); + else if (WIFSIGNALED(wstatus)) + status = 128 + WTERMSIG(wstatus); + else + status = 255; + + blob_buf_init(&bb, 0); + blobmsg_add_u32(&bb, "status", status); + ubus_send_reply(e->ctx, &e->req, bb.head); + ubus_complete_deferred_request(e->ctx, &e->req, 0); + if (current_exec == e) + current_exec = NULL; + free(e); +} + +static void container_exec_done_reap(struct uloop_process *p, int wstatus) +{ + struct container_exec *e = container_of(p, struct container_exec, exec_proc); + + if (current_exec == e) + current_exec = NULL; + free(e); +} + +static int +container_handle_exec(struct ubus_context *ctx, struct ubus_object *obj, + struct ubus_request_data *req, const char *method, + struct blob_attr *msg) +{ + struct blob_attr *tb[__CONTAINER_EXEC_ATTR_MAX]; + struct blob_attr *tu[__CONTAINER_EXEC_USER_MAX] = { 0 }; + static const char * const ns_names[] = { "user", "ipc", "uts", "net", "cgroup", "pid", "mnt" }; + static const int ns_flags[] = { + CLONE_NEWUSER, CLONE_NEWIPC, CLONE_NEWUTS, CLONE_NEWNET, + CLONE_NEWCGROUP, CLONE_NEWPID, CLONE_NEWNS, + }; + int ns_fds[7] = { -1, -1, -1, -1, -1, -1, -1 }; + char **args = NULL, **env = NULL; + const char *cwd = "/", *pidfile = NULL; + const char *console_socket = NULL; + uint32_t uid, gid; + bool detach = false; + bool terminal = false; + bool exec_nnp = opts.no_new_privs; + struct jail_capset exec_capset = opts.capset; + struct rlimit exec_rlimits[RLIM_NLIMITS]; + bool exec_rlimits_set[RLIM_NLIMITS] = { 0 }; + gid_t *exec_additional_gids = NULL; + size_t exec_num_additional_gids = 0; + size_t exec_max_additional_gids; + mode_t exec_umask = opts.umask; + bool exec_set_umask = opts.set_umask; + int console_sock_fd = -1; + bool console_sock_owned = false; + int cgroup_fd = -1; + int pipe_fds[2] = { -1, -1 }; + pid_t exec_pid, grandchild = -1; + struct container_exec *e = NULL; + char nspath[64]; + int i, rc = UBUS_STATUS_UNKNOWN_ERROR; + + if (jail_oci_state != OCI_STATE_CREATED && + jail_oci_state != OCI_STATE_RUNNING) + return UBUS_STATUS_INVALID_ARGUMENT; + if (!msg) + return UBUS_STATUS_INVALID_ARGUMENT; + if (current_exec) { + ERROR("exec: another exec is already in progress\n"); + return UBUS_STATUS_PERMISSION_DENIED; + } + + uid = (opts.pw_uid > 0) ? (uint32_t)opts.pw_uid : 0; + gid = (opts.pw_gid > 0) ? (uint32_t)opts.pw_gid : 0; + + for (i = 0; i < RLIM_NLIMITS; i++) { + if (opts.rlimits[i]) { + exec_rlimits[i] = *opts.rlimits[i]; + exec_rlimits_set[i] = true; + } + } + { + long n_max = sysconf(_SC_NGROUPS_MAX); + if (n_max <= 0) + n_max = NGROUPS_MAX; + exec_max_additional_gids = (size_t)n_max; + } + if (opts.additional_gids && opts.num_additional_gids <= exec_max_additional_gids) { + exec_additional_gids = calloc(opts.num_additional_gids, sizeof(gid_t)); + if (!exec_additional_gids) { + rc = UBUS_STATUS_UNKNOWN_ERROR; + goto out; + } + memcpy(exec_additional_gids, opts.additional_gids, + opts.num_additional_gids * sizeof(gid_t)); + exec_num_additional_gids = opts.num_additional_gids; + } + + blobmsg_parse(container_exec_attrs, __CONTAINER_EXEC_ATTR_MAX, tb, + blobmsg_data(msg), blobmsg_data_len(msg)); + + if (!tb[CONTAINER_EXEC_ATTR_ARGS]) + return UBUS_STATUS_INVALID_ARGUMENT; + + args = container_exec_strarray(tb[CONTAINER_EXEC_ATTR_ARGS]); + if (!args || !args[0]) { + rc = UBUS_STATUS_INVALID_ARGUMENT; + goto out; + } + + if (tb[CONTAINER_EXEC_ATTR_ENV]) + env = container_exec_strarray(tb[CONTAINER_EXEC_ATTR_ENV]); + if (tb[CONTAINER_EXEC_ATTR_CWD]) + cwd = blobmsg_get_string(tb[CONTAINER_EXEC_ATTR_CWD]); + if (tb[CONTAINER_EXEC_ATTR_PIDFILE]) + pidfile = blobmsg_get_string(tb[CONTAINER_EXEC_ATTR_PIDFILE]); + if (tb[CONTAINER_EXEC_ATTR_DETACH]) + detach = blobmsg_get_bool(tb[CONTAINER_EXEC_ATTR_DETACH]); + if (tb[CONTAINER_EXEC_ATTR_TERMINAL]) + terminal = blobmsg_get_bool(tb[CONTAINER_EXEC_ATTR_TERMINAL]); + if (tb[CONTAINER_EXEC_ATTR_CONSOLE_SOCKET]) + console_socket = blobmsg_get_string(tb[CONTAINER_EXEC_ATTR_CONSOLE_SOCKET]); + if (tb[CONTAINER_EXEC_ATTR_NO_NEW_PRIVS]) + exec_nnp = blobmsg_get_bool(tb[CONTAINER_EXEC_ATTR_NO_NEW_PRIVS]); + if (tb[CONTAINER_EXEC_ATTR_CAPABILITIES]) { + memset(&exec_capset, 0, sizeof(exec_capset)); + if (parseOCIcapabilities(&exec_capset, + tb[CONTAINER_EXEC_ATTR_CAPABILITIES])) { + rc = UBUS_STATUS_INVALID_ARGUMENT; + goto out; + } + } + if (tb[CONTAINER_EXEC_ATTR_RLIMITS]) { + struct blob_attr *cur; + int rem; + + blobmsg_for_each_attr(cur, tb[CONTAINER_EXEC_ATTR_RLIMITS], rem) { + struct blob_attr *rl[__OCI_PROCESS_RLIMIT_MAX]; + int rlt; + + blobmsg_parse(oci_process_rlimit_policy, __OCI_PROCESS_RLIMIT_MAX, + rl, blobmsg_data(cur), blobmsg_len(cur)); + if (!rl[OCI_PROCESS_RLIMIT_TYPE] || + !rl[OCI_PROCESS_RLIMIT_SOFT] || + !rl[OCI_PROCESS_RLIMIT_HARD]) + continue; + rlt = resolve_rlimit(blobmsg_get_string(rl[OCI_PROCESS_RLIMIT_TYPE])); + if (rlt < 0) + continue; + exec_rlimits[rlt].rlim_cur = blobmsg_cast_u64(rl[OCI_PROCESS_RLIMIT_SOFT]); + exec_rlimits[rlt].rlim_max = blobmsg_cast_u64(rl[OCI_PROCESS_RLIMIT_HARD]); + exec_rlimits_set[rlt] = true; + } + } + if (tb[CONTAINER_EXEC_ATTR_USER]) { + blobmsg_parse(container_exec_user_attrs, __CONTAINER_EXEC_USER_MAX, tu, + blobmsg_data(tb[CONTAINER_EXEC_ATTR_USER]), + blobmsg_len(tb[CONTAINER_EXEC_ATTR_USER])); + if (tu[CONTAINER_EXEC_USER_UID]) + uid = blobmsg_get_u32(tu[CONTAINER_EXEC_USER_UID]); + if (tu[CONTAINER_EXEC_USER_GID]) + gid = blobmsg_get_u32(tu[CONTAINER_EXEC_USER_GID]); + if (tu[CONTAINER_EXEC_USER_UMASK]) { + exec_umask = blobmsg_get_u32(tu[CONTAINER_EXEC_USER_UMASK]); + exec_set_umask = true; + } + if (tu[CONTAINER_EXEC_USER_ADDITIONAL_GIDS]) { + struct blob_attr *cur; + int rem; + size_t count = 0; + + blobmsg_for_each_attr(cur, tu[CONTAINER_EXEC_USER_ADDITIONAL_GIDS], rem) + count++; + + if (count > exec_max_additional_gids) + count = exec_max_additional_gids; + + free(exec_additional_gids); + exec_additional_gids = count ? calloc(count, sizeof(gid_t)) : NULL; + if (count && !exec_additional_gids) { + rc = UBUS_STATUS_UNKNOWN_ERROR; + goto out; + } + + exec_num_additional_gids = 0; + blobmsg_for_each_attr(cur, tu[CONTAINER_EXEC_USER_ADDITIONAL_GIDS], rem) { + if (exec_num_additional_gids >= count) + break; + exec_additional_gids[exec_num_additional_gids++] = + blobmsg_get_u32(cur); + } + } + } + + for (i = 0; i < (int)ARRAY_SIZE(ns_names); i++) { + struct stat nsst, ownst; + + snprintf(nspath, sizeof(nspath), "/proc/%d/ns/%s", + jail_process.pid, ns_names[i]); + ns_fds[i] = open(nspath, O_RDONLY | O_CLOEXEC); + if (ns_fds[i] < 0) { + if (ns_flags[i] == CLONE_NEWCGROUP || + ns_flags[i] == CLONE_NEWUSER) + continue; + + ERROR("exec: open %s: %m\n", nspath); + goto out; + } + + snprintf(nspath, sizeof(nspath), "/proc/self/ns/%s", ns_names[i]); + if (fstat(ns_fds[i], &nsst) || stat(nspath, &ownst)) + continue; + + /* joining a namespace we are already in fails for CLONE_NEWUSER */ + if (nsst.st_dev != ownst.st_dev || nsst.st_ino != ownst.st_ino) + continue; + + close(ns_fds[i]); + ns_fds[i] = -1; + } + + if (terminal != !!console_socket) { + ERROR("exec: terminal and consolesocket must be set together\n"); + rc = UBUS_STATUS_INVALID_ARGUMENT; + goto out; + } + + if (terminal && console_socket) { + console_sock_fd = open_console_sock(console_socket, &console_sock_owned, true); + if (console_sock_fd < 0) + goto out; + } + + cgroup_fd = cgroups_open_dir(); + if (cgroup_fd < 0) + DEBUG("exec: cgroups_open_dir unavailable, will fall back to cgroups_attach_pid\n"); + + if (pipe(pipe_fds) < 0) { + ERROR("exec: pipe: %m\n"); + goto out; + } + + exec_pid = fork(); + if (exec_pid < 0) { + ERROR("exec: fork: %m\n"); + goto out; + } + + if (exec_pid == 0) { + int wstatus; + int slave_fd = -1; + + close(pipe_fds[0]); + for (i = 0; i < (int)ARRAY_SIZE(ns_names); i++) { + if (ns_fds[i] < 0) + continue; + /* + * the cgroup namespace hides our own cgroup, which the + * kernel needs to see to honour CLONE_INTO_CGROUP, so + * the grandchild joins it once it has been placed + */ + if (ns_flags[i] == CLONE_NEWCGROUP) + continue; + + if (setns(ns_fds[i], ns_flags[i]) < 0) { + ERROR("exec: setns(%s): %m\n", ns_names[i]); + _exit(126); + } + } + + if (terminal && console_sock_fd >= 0) { + int master_fd; + char *slave_name; + + master_fd = posix_openpt(O_RDWR | O_NOCTTY); + if (master_fd < 0) + _exit(126); + if (grantpt(master_fd) || unlockpt(master_fd)) + _exit(126); + slave_name = ptsname(master_fd); + if (!slave_name) + _exit(126); + slave_fd = open(slave_name, O_RDWR | O_NOCTTY); + if (slave_fd < 0) + _exit(126); + if (sendmsg_console_fd(console_sock_fd, master_fd, slave_name) < 0) + _exit(126); + close(console_sock_fd); + close(master_fd); + } + + { + struct clone_args gargs = { + .exit_signal = SIGCHLD, + }; + if (cgroup_fd >= 0) { + gargs.flags = CLONE_INTO_CGROUP; + gargs.cgroup = (__u64)cgroup_fd; + } + grandchild = jail_clone3(&gargs); + } + if (grandchild < 0) { + ERROR("exec: clone3: %m\n"); + _exit(126); + } + + if (grandchild == 0) { + int j; + + for (j = 0; j < (int)ARRAY_SIZE(ns_names); j++) { + if (ns_flags[j] != CLONE_NEWCGROUP || ns_fds[j] < 0) + continue; + if (setns(ns_fds[j], CLONE_NEWCGROUP) < 0) + _exit(127); + } + + for (j = 0; j < RLIM_NLIMITS; j++) + if (exec_rlimits_set[j] && + setrlimit(j, &exec_rlimits[j]) < 0) { + ERROR("exec: setrlimit(%d): %m\n", j); + _exit(127); + } + if (slave_fd >= 0) { + if (setsid() < 0) + _exit(127); + if (ioctl(slave_fd, TIOCSCTTY, 0) < 0) + _exit(127); + dup2(slave_fd, STDIN_FILENO); + dup2(slave_fd, STDOUT_FILENO); + dup2(slave_fd, STDERR_FILENO); + if (slave_fd > STDERR_FILENO) + close(slave_fd); + } + if (exec_set_umask) + umask(exec_umask); + + if ((uid || gid || exec_num_additional_gids) && + exec_capset.apply) { + if (prctl(PR_SET_SECUREBITS, SECBIT_NO_SETUID_FIXUP)) { + ERROR("exec: prctl(PR_SET_SECUREBITS): %m\n"); + _exit(127); + } + if (applyOCIcapabilities(exec_capset, + (1LLU << CAP_SETGID) | + (1LLU << CAP_SETUID) | + (1LLU << CAP_SETPCAP))) { + ERROR("exec: applyOCIcapabilities(pre): failed\n"); + _exit(127); + } + } + + if (setgroups(exec_num_additional_gids, + exec_num_additional_gids ? exec_additional_gids : NULL) < 0) { + ERROR("exec: setgroups: %m\n"); + _exit(127); + } + if (gid && setresgid(gid, gid, gid) < 0) { + ERROR("exec: setresgid(%u): %m\n", gid); + _exit(127); + } + if (uid && setresuid(uid, uid, uid) < 0) { + ERROR("exec: setresuid(%u): %m\n", uid); + _exit(127); + } + + if (exec_capset.apply && + applyOCIcapabilities(exec_capset, 0)) { + ERROR("exec: applyOCIcapabilities: failed\n"); + _exit(127); + } + if (exec_nnp && prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) { + ERROR("exec: prctl(PR_SET_NO_NEW_PRIVS): %m\n"); + _exit(127); + } + if (opts.mdwe_flags && + prctl(PR_SET_MDWE, opts.mdwe_flags, 0, 0, 0)) { + ERROR("exec: prctl(PR_SET_MDWE): %m\n"); + _exit(127); + } + if (chdir(cwd) < 0) { + ERROR("exec: chdir(%s): %m\n", cwd); + _exit(127); + } + if (env) + execvpe(args[0], args, env); + else + execvp(args[0], args); + ERROR("exec: execvpe(%s): %m\n", args[0]); + _exit(127); + } + + if (slave_fd >= 0) + close(slave_fd); + + (void)!write(pipe_fds[1], &grandchild, sizeof(grandchild)); + close(pipe_fds[1]); + + if (waitpid(grandchild, &wstatus, 0) < 0) { + ERROR("exec: waitpid(%d): %m\n", grandchild); + _exit(126); + } + if (WIFEXITED(wstatus)) + _exit(WEXITSTATUS(wstatus)); + _exit(128 + WTERMSIG(wstatus)); + } + + close(pipe_fds[1]); + pipe_fds[1] = -1; + { + char gcbuf[sizeof(pid_t)]; + size_t off = 0; + ssize_t n; + + while (off < sizeof(gcbuf)) { + n = read(pipe_fds[0], gcbuf + off, sizeof(gcbuf) - off); + if (n < 0) { + if (errno == EINTR) + continue; + grandchild = -1; + break; + } + if (n == 0) { + grandchild = -1; + break; + } + off += (size_t)n; + } + if (off == sizeof(gcbuf)) + memcpy(&grandchild, gcbuf, sizeof(grandchild)); + else + grandchild = -1; + } + close(pipe_fds[0]); + pipe_fds[0] = -1; + + for (i = 0; i < (int)ARRAY_SIZE(ns_names); i++) + if (ns_fds[i] >= 0) { + close(ns_fds[i]); + ns_fds[i] = -1; + } + + if (console_sock_owned && console_sock_fd >= 0) { + close(console_sock_fd); + console_sock_fd = -1; + } + + if (cgroup_fd >= 0) { + close(cgroup_fd); + cgroup_fd = -1; + } else if (grandchild > 0) { + cgroups_attach_pid(grandchild); + } + + container_exec_free_strarray(args); + container_exec_free_strarray(env); + args = env = NULL; + free(exec_additional_gids); + exec_additional_gids = NULL; + + if (pidfile && grandchild > 0) { + FILE *pf = fopen(pidfile, "w"); + if (pf) { + fprintf(pf, "%d", grandchild); + fclose(pf); + } + } + + e = calloc(1, sizeof(*e)); + if (!e) { + kill(exec_pid, SIGKILL); + return UBUS_STATUS_UNKNOWN_ERROR; + } + e->ctx = ctx; + e->exec_proc.pid = exec_pid; + current_exec = e; + + if (detach) { + static struct blob_buf bb; + + blob_buf_init(&bb, 0); + if (grandchild > 0) + blobmsg_add_u32(&bb, "pid", grandchild); + ubus_send_reply(ctx, req, bb.head); + + e->exec_proc.cb = container_exec_done_reap; + uloop_process_add(&e->exec_proc); + return UBUS_STATUS_OK; + } + + e->exec_proc.cb = container_exec_done_reply; + uloop_process_add(&e->exec_proc); + ubus_defer_request(ctx, req, &e->req); + return UBUS_STATUS_OK; + +out: + for (i = 0; i < (int)ARRAY_SIZE(ns_names); i++) + if (ns_fds[i] >= 0) + close(ns_fds[i]); + if (pipe_fds[0] >= 0) + close(pipe_fds[0]); + if (pipe_fds[1] >= 0) + close(pipe_fds[1]); + if (console_sock_owned && console_sock_fd >= 0) + close(console_sock_fd); + if (cgroup_fd >= 0) + close(cgroup_fd); + container_exec_free_strarray(args); + container_exec_free_strarray(env); + free(exec_additional_gids); + return rc; +} + static int jail_writepid(pid_t pid) { @@ -4694,6 +5362,8 @@ static struct ubus_method container_methods[] = { UBUS_METHOD_NOARG("pause", container_handle_pause), UBUS_METHOD_NOARG("resume", container_handle_resume), UBUS_METHOD("reclaim", container_handle_reclaim, container_reclaim_attrs), + UBUS_METHOD_NOARG("update", container_handle_update), + UBUS_METHOD("exec", container_handle_exec, container_exec_attrs), }; static struct ubus_object_type container_object_type = @@ -4879,6 +5549,9 @@ int main(int argc, char **argv) case 'Y': opts.console_socket = optarg; break; + case 'Z': + opts.systemd_cgroup = true; + break; } } diff --git a/service/instance.c b/service/instance.c index c720d75..38e1d2e 100644 --- a/service/instance.c +++ b/service/instance.c @@ -126,6 +126,7 @@ enum { JAIL_ATTR_SETNS, JAIL_ATTR_IDMAP_OFFSET, JAIL_ATTR_CONSOLESOCKET, + JAIL_ATTR_SYSTEMDCGROUP, __JAIL_ATTR_MAX, }; @@ -149,6 +150,7 @@ static const struct blobmsg_policy jail_attr[__JAIL_ATTR_MAX] = { [JAIL_ATTR_SETNS] = { "setns", BLOBMSG_TYPE_ARRAY }, [JAIL_ATTR_IDMAP_OFFSET] = { "idmap_offset", BLOBMSG_TYPE_STRING }, [JAIL_ATTR_CONSOLESOCKET] = { "consolesocket", BLOBMSG_TYPE_STRING }, + [JAIL_ATTR_SYSTEMDCGROUP] = { "systemdcgroup", BLOBMSG_TYPE_BOOL }, }; enum { @@ -404,6 +406,9 @@ jail_run(struct service_instance *in, char **argv) argv[argc++] = jail->consolesocket; } + if (jail->systemd_cgroup) + argv[argc++] = "-Z"; + if (in->bundle) { argv[argc++] = "-J"; argv[argc++] = in->bundle; @@ -1277,6 +1282,11 @@ instance_jail_parse(struct service_instance *in, struct blob_attr *attr) jail->argc += 2; } + if (tb[JAIL_ATTR_SYSTEMDCGROUP] && blobmsg_get_bool(tb[JAIL_ATTR_SYSTEMDCGROUP])) { + jail->systemd_cgroup = true; + jail->argc++; + } + if (tb[JAIL_ATTR_SETNS]) { struct blob_attr *cur; int rem; diff --git a/service/instance.h b/service/instance.h index d33900a..df9428e 100644 --- a/service/instance.h +++ b/service/instance.h @@ -36,6 +36,7 @@ struct jail { uint32_t userns:1; uint32_t cgroupsns:1; uint32_t console:1; + uint32_t systemd_cgroup:1; }; uint32_t flags; }; From 33d4b5e694ca2401f850d6ad9b32a9c00af19eb3 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 05:29:47 +0100 Subject: [PATCH 23/82] uxc: restructure the CLI into a runc-style dispatcher A hand-scanned pass consumes global options (--root, --log, --log-format, --systemd-cgroup, --rootless, --criu) before the verb, then each verb parses its own argv with getopt_long. Options runc callers expect but ujail does not need (--no-pivot, --no-new-keyring, --preserve-fds) are accepted and reported as ignored, and list output becomes a runc-compatible table or OCI-shaped JSON. --log redirects diagnostics to a file, --root selects the configuration directory, and --systemd-cgroup is forwarded on create as the jail's systemdcgroup attribute. Signed-off-by: Daniel Golle --- uxc.c | 255 +++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 179 insertions(+), 76 deletions(-) diff --git a/uxc.c b/uxc.c index 5acacef..dc1a4d9 100644 --- a/uxc.c +++ b/uxc.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -47,7 +48,7 @@ static bool verbose = false; static bool json_output = false; -static char *confdir = UXC_ETC_CONFDIR; +static const char *confdir = UXC_ETC_CONFDIR; static struct ustream_fd cufd; static struct ustream_fd lufd; @@ -75,17 +76,23 @@ struct settings { enum { OPT_CONSOLE_SOCKET = 0x100, + OPT_NO_PIVOT, + OPT_NO_NEW_KEYRING, + OPT_PRESERVE_FDS, }; static const struct option create_opts[] = { - {"autostart", no_argument, 0, 'a' }, - {"bundle", required_argument, 0, 'b' }, + {"autostart", no_argument, 0, 'a' }, + {"bundle", required_argument, 0, 'b' }, {"console-socket", required_argument, 0, OPT_CONSOLE_SOCKET }, - {"mounts", required_argument, 0, 'm' }, - {"pid-file", required_argument, 0, 'p' }, - {"temp-overlay-size", required_argument, 0, 't' }, - {"write-overlay-path", required_argument, 0, 'w' }, - {0, 0, 0, 0 } + {"mounts", required_argument, 0, 'm' }, + {"no-new-keyring", no_argument, 0, OPT_NO_NEW_KEYRING }, + {"no-pivot", no_argument, 0, OPT_NO_PIVOT }, + {"pid-file", required_argument, 0, 'p' }, + {"preserve-fds", required_argument, 0, OPT_PRESERVE_FDS }, + {"temp-overlay-size", required_argument, 0, 't' }, + {"write-overlay-path", required_argument, 0, 'w' }, + {0, 0, 0, 0 } }; static const struct option start_opts[] = { @@ -240,12 +247,19 @@ static struct blob_attr *fstabinfo; static struct ubus_context *ctx; static int usage(void) { - printf("syntax: uxc [parameters ...]\n"); + printf("syntax: uxc [global options] [parameters ...]\n"); + printf("global options:\n"); + printf("\t[--debug|-v] [--log ] [--log-format ]\n"); + printf("\t[--root ] [--rootless[=auto|true|false]]\n"); + printf("\t[--systemd-cgroup] [--criu ]\n"); printf("commands:\n"); - printf("\tlist [--json]\t\t\t\tlist all configured containers\n"); + printf("\tlist [--json]\t\t\t\tlist all configured containers (runc-compatible)\n"); printf("\tattach \t\t\t\tattach to container console\n"); printf("\tcreate \t\t\t\t(re-)create \n"); printf("\t\t[--bundle ]\t\t\tOCI bundle at \n"); + printf("\t\t[--pid-file ]\t\t\twrite container PID to \n"); + printf("\t\t[--console-socket ]\t\tAF_UNIX socket to receive the PTY master fd\n"); + printf("\t\t[--no-pivot|--no-new-keyring|--preserve-fds ] runc-compat, currently ignored\n"); printf("\t\t[--autostart]\t\t\t\tstart on boot\n"); printf("\t\t[--temp-overlay-size ]\t\tuse tmpfs overlay with {size}\n"); printf("\t\t[--write-overlay-path ]\t\tuse overlay on {path}\n"); @@ -296,7 +310,7 @@ static int conf_load(bool load_settings) struct stat sb; struct blob_buf *target; - if (asprintf(&globstr, "%s/%s*.json", UXC_ETC_CONFDIR, load_settings?"settings/":"") == -1) + if (asprintf(&globstr, "%s/%s*.json", confdir, load_settings?"settings/":"") == -1) return -ENOMEM; res = glob(globstr, gl_flags, NULL, &gl); @@ -810,6 +824,7 @@ static int uxc_state(char *name) blobmsg_add_string(&buf, "id", jail_name); blobmsg_add_string(&buf, "status", rsstate?"stopped":"uninitialized"); blobmsg_add_string(&buf, "bundle", bundle); + blobmsg_close_table(&buf, blobmsg_open_table(&buf, "annotations")); tmp = blobmsg_format_json_indent(buf.head, true, 0); if (!tmp) { @@ -828,79 +843,90 @@ static int uxc_state(char *name) static int uxc_list(void) { struct blob_attr *cur, *tb[__CONF_MAX], *ts[__STATE_MAX]; - int rem; + int rem, pass; struct runtime_state *rsstate = NULL; - struct settings *usettings = NULL; - char *name, *ocistatus, *status, *tmp; - int container_pid = -1; - bool autostart; + char *name, *bundle, *ocistatus, *status, *created, *tmp; + int container_pid; static struct blob_buf buf; - void *arr, *obj; + void *arr, *obj, *ann; + size_t id_w = 2, pid_w = 3, status_w = 6, bundle_w = 6, created_w = 7; + char pidstr[12]; if (json_output) { blob_buf_init(&buf, 0); arr = blobmsg_open_array(&buf, ""); } - blobmsg_for_each_attr(cur, blob_data(conf.head), rem) { - blobmsg_parse(conf_policy, __CONF_MAX, tb, blobmsg_data(cur), blobmsg_len(cur)); - if (!tb[CONF_NAME] || !tb[CONF_PATH]) - continue; - - autostart = tb[CONF_AUTOSTART] && blobmsg_get_bool(tb[CONF_AUTOSTART]); - - ocistatus = NULL; - container_pid = 0; - name = blobmsg_get_string(tb[CONF_NAME]); - rsstate = avl_find_element(&runtime, name, rsstate, avl); - - if (rsstate && rsstate->ocistate) { - blobmsg_parse(state_policy, __STATE_MAX, ts, blobmsg_data(rsstate->ocistate), blobmsg_len(rsstate->ocistate)); - ocistatus = blobmsg_get_string(ts[STATE_STATUS]); - container_pid = blobmsg_get_u32(ts[STATE_PID]); - } - - status = ocistatus?:(rsstate && rsstate->running)?"creating":"stopped"; - - usettings = avl_find_element(&settings, name, usettings, avl); - - if (usettings && (usettings->autostart >= 0)) - autostart = !!(usettings->autostart); + for (pass = json_output ? 1 : 0; pass < 2; pass++) { + if (pass == 1 && !json_output) + printf("%-*s %-*s %-*s %-*s %-*s %s\n", + (int)id_w, "ID", (int)pid_w, "PID", + (int)status_w, "STATUS", (int)bundle_w, "BUNDLE", + (int)created_w, "CREATED", "OWNER"); + + blobmsg_for_each_attr(cur, blob_data(conf.head), rem) { + blobmsg_parse(conf_policy, __CONF_MAX, tb, + blobmsg_data(cur), blobmsg_len(cur)); + if (!tb[CONF_NAME] || !tb[CONF_PATH]) + continue; - if (json_output) { - obj = blobmsg_open_table(&buf, ""); - blobmsg_add_string(&buf, "name", name); - blobmsg_add_string(&buf, "status", status); - blobmsg_add_u8(&buf, "autostart", autostart); - } else { - printf("[%c] %s %s", autostart?'*':' ', name, status); - } + name = blobmsg_get_string(tb[CONF_NAME]); + bundle = blobmsg_get_string(tb[CONF_PATH]); - if (rsstate && !rsstate->running && (rsstate->exitcode >= 0)) { - if (json_output) - blobmsg_add_u32(&buf, "exitcode", rsstate->exitcode); - else - printf(" exitcode: %d (%s)", rsstate->exitcode, strerror(rsstate->exitcode)); - } + ocistatus = NULL; + container_pid = 0; + created = "-"; + rsstate = avl_find_element(&runtime, name, rsstate, avl); + if (rsstate && rsstate->ocistate) { + blobmsg_parse(state_policy, __STATE_MAX, ts, + blobmsg_data(rsstate->ocistate), + blobmsg_len(rsstate->ocistate)); + if (ts[STATE_STATUS]) + ocistatus = blobmsg_get_string(ts[STATE_STATUS]); + if (ts[STATE_PID]) + container_pid = blobmsg_get_u32(ts[STATE_PID]); + if (ts[STATE_BUNDLE]) + bundle = blobmsg_get_string(ts[STATE_BUNDLE]); + } + status = ocistatus?:(rsstate && rsstate->running)?"creating":(rsstate?"stopped":"uninitialized"); - if (rsstate && rsstate->running && (rsstate->runtime_pid >= 0)) { - if (json_output) - blobmsg_add_u32(&buf, "runtime_pid", rsstate->runtime_pid); + if (container_pid > 0) + snprintf(pidstr, sizeof(pidstr), "%d", container_pid); else - printf(" runtime pid: %d", rsstate->runtime_pid); - } + snprintf(pidstr, sizeof(pidstr), "%s", "-"); + + if (pass == 0) { + if (strlen(name) > id_w) id_w = strlen(name); + if (strlen(pidstr) > pid_w) pid_w = strlen(pidstr); + if (strlen(status) > status_w) status_w = strlen(status); + if (strlen(bundle) > bundle_w) bundle_w = strlen(bundle); + if (strlen(created) > created_w) created_w = strlen(created); + continue; + } - if (rsstate && rsstate->running && (container_pid >= 0)) { - if (json_output) - blobmsg_add_u32(&buf, "container_pid", container_pid); - else - printf(" container pid: %d", container_pid); + if (json_output) { + obj = blobmsg_open_table(&buf, ""); + blobmsg_add_string(&buf, "ociVersion", OCI_VERSION_STRING); + blobmsg_add_string(&buf, "id", name); + if (container_pid > 0) + blobmsg_add_u32(&buf, "pid", container_pid); + blobmsg_add_string(&buf, "status", status); + blobmsg_add_string(&buf, "bundle", bundle); + if (rsstate && rsstate->ocistate && ts[STATE_ANNOTATIONS]) { + blobmsg_add_blob(&buf, ts[STATE_ANNOTATIONS]); + } else { + ann = blobmsg_open_table(&buf, "annotations"); + blobmsg_close_table(&buf, ann); + } + blobmsg_add_string(&buf, "owner", "root"); + blobmsg_close_table(&buf, obj); + } else { + printf("%-*s %-*s %-*s %-*s %-*s %s\n", + (int)id_w, name, (int)pid_w, pidstr, + (int)status_w, status, (int)bundle_w, bundle, + (int)created_w, created, "root"); + } } - - if (!json_output) - printf("\n"); - else - blobmsg_close_table(&buf, obj); } if (json_output) { @@ -913,7 +939,7 @@ static int uxc_list(void) printf("%s\n", tmp); free(tmp); blob_buf_free(&buf); - }; + } return 0; } @@ -929,7 +955,8 @@ static int uxc_exists(char *name) return 0; } -static int uxc_create(char *name, bool immediately, const char *console_socket) +static int uxc_create(char *name, bool immediately, const char *console_socket, + bool systemd_cgroup) { static struct blob_buf req; struct blob_attr *cur, *tb[__CONF_MAX]; @@ -999,6 +1026,9 @@ static int uxc_create(char *name, bool immediately, const char *console_socket) if (console_socket) blobmsg_add_string(&req, "consolesocket", console_socket); + if (systemd_cgroup) + blobmsg_add_u8(&req, "systemdcgroup", 1); + blobmsg_close_table(&req, j); if (writepath) @@ -1426,7 +1456,7 @@ static int uxc_boot(void) if (uxc_exists(name)) continue; - if (uxc_create(name, true, NULL)) + if (uxc_create(name, true, NULL, false)) ++ret; free(name); @@ -1556,11 +1586,16 @@ int main(int argc, char **argv) { int ret = -EINVAL; const char *verb; + const char *log_path = NULL; + const char *log_format = NULL; + const char *criu_path = NULL; + bool systemd_cgroup = false; int verb_argc, c, i; char **verb_argv; for (i = 1; i < argc; ++i) { const char *a = argv[i]; + const char *eq; if (a[0] != '-') break; @@ -1575,18 +1610,77 @@ int main(int argc, char **argv) return 0; } - if (!strcmp(a, "-v") || !strcmp(a, "--verbose")) { + if (!strcmp(a, "-v") || !strcmp(a, "--verbose") || !strcmp(a, "--debug")) { verbose = true; continue; } + if (!strcmp(a, "--systemd-cgroup")) { + systemd_cgroup = true; + continue; + } + + eq = strchr(a, '='); + +#define GLOBAL_OPT_VAL(name, dst) \ + do { \ + size_t _n = strlen(name); \ + if (eq && !strncmp(a, name, _n) && a[_n] == '=') { \ + dst = eq + 1; \ + goto next_global; \ + } \ + if (!strcmp(a, name)) { \ + if (i + 1 >= argc) { \ + fprintf(stderr, "uxc: %s requires an argument\n", a); \ + return -EINVAL; \ + } \ + dst = argv[++i]; \ + goto next_global; \ + } \ + } while (0) + + GLOBAL_OPT_VAL("--root", confdir); + GLOBAL_OPT_VAL("--log", log_path); + GLOBAL_OPT_VAL("--log-format", log_format); + GLOBAL_OPT_VAL("--criu", criu_path); +#undef GLOBAL_OPT_VAL + + if (eq && !strncmp(a, "--rootless=", 11)) + continue; + if (!strcmp(a, "--rootless")) + continue; + fprintf(stderr, "uxc: unknown option '%s'\n", a); return usage(); +next_global: + continue; } if (i >= argc) return usage(); + if (log_path) { + int fd = open(log_path, O_WRONLY | O_CREAT | O_APPEND, 0644); + if (fd < 0) { + fprintf(stderr, "uxc: cannot open --log path %s: %m\n", log_path); + return -EIO; + } + if (dup2(fd, STDERR_FILENO) < 0) { + dprintf(fd, "uxc: dup2(--log path) failed: %m\n"); + close(fd); + return -EIO; + } + close(fd); + } + + if (log_format && strcmp(log_format, "text")) + fprintf(stderr, "uxc: --log-format=%s accepted but only text output is emitted\n", + log_format); + + if (criu_path) + fprintf(stderr, "uxc: --criu=%s accepted but ignored (no checkpoint/restore support)\n", + criu_path); + verb = argv[i]; verb_argc = argc - i; verb_argv = argv + i; @@ -1705,9 +1799,9 @@ int main(int argc, char **argv) } else if (!strcmp(verb, "create")) { char *bundle = NULL, *pidfile = NULL; char *tmprwsize = NULL, *writepath = NULL, *requiredmounts = NULL; + char *console_socket = NULL; signed char autostart = -1; char *name; - const char *console_socket = NULL; while ((c = getopt_long(verb_argc, verb_argv, "ab:m:p:t:w:", create_opts, NULL)) != -1) { @@ -1721,6 +1815,15 @@ int main(int argc, char **argv) case OPT_CONSOLE_SOCKET: console_socket = optarg; break; + case OPT_NO_PIVOT: + fprintf(stderr, "uxc: --no-pivot accepted but ignored (ujail does not pivot_root in this mode)\n"); + break; + case OPT_NO_NEW_KEYRING: + fprintf(stderr, "uxc: --no-new-keyring accepted but ignored (ujail does not create kernel keyrings)\n"); + break; + case OPT_PRESERVE_FDS: + fprintf(stderr, "uxc: --preserve-fds=%s accepted but ignored\n", optarg); + break; default: goto usage_out; } } @@ -1739,7 +1842,7 @@ int main(int argc, char **argv) if (ret > 0) reload_conf(); - ret = uxc_create(name, false, console_socket); + ret = uxc_create(name, false, console_socket, systemd_cgroup); } else if (!strcmp(verb, "pause") || !strcmp(verb, "resume")) { char *objname; uint32_t id; From 2c6b9454b3727347c4bbd14389bf6b3f4a8bdb1c Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 05:29:47 +0100 Subject: [PATCH 24/82] uxc: add exec and update verbs Add the exec and update verbs on top of the matching container ubus methods. exec takes either an OCI process document via --process or a command line, supports --detach, --pid-file, --tty and --console-socket, insists on a console socket when a terminal is requested, and reports the command's exit status as its own. update applies a linux.resources JSON from --resources to a running container. Signed-off-by: Daniel Golle --- uxc.c | 191 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 191 insertions(+) diff --git a/uxc.c b/uxc.c index dc1a4d9..2063d12 100644 --- a/uxc.c +++ b/uxc.c @@ -79,6 +79,8 @@ enum { OPT_NO_PIVOT, OPT_NO_NEW_KEYRING, OPT_PRESERVE_FDS, + OPT_PROCESS, + OPT_RESOURCES, }; static const struct option create_opts[] = { @@ -116,6 +118,21 @@ static const struct option list_opts[] = { {0, 0, 0, 0 } }; +static const struct option exec_opts[] = { + {"console-socket", required_argument, 0, OPT_CONSOLE_SOCKET }, + {"detach", no_argument, 0, 'd' }, + {"pid-file", required_argument, 0, 'p' }, + {"preserve-fds", required_argument, 0, OPT_PRESERVE_FDS }, + {"process", required_argument, 0, OPT_PROCESS }, + {"tty", no_argument, 0, 't' }, + {0, 0, 0, 0 } +}; + +static const struct option update_opts[] = { + {"resources", required_argument, 0, OPT_RESOURCES }, + {0, 0, 0, 0 } +}; + struct signame { int signal; char name[7]; @@ -272,6 +289,9 @@ static int usage(void) { printf("\tdelete [--force]\t\t\tdelete \n"); printf("\tpause \t\t\t\tfreeze every process in container 's cgroup\n"); printf("\tresume \t\t\t\tthaw a previously paused container \n"); + printf("\texec [--process ] [-d] [-p ] [-- cmd args]\n"); + printf("\t\t\t\t\t\trun a command inside running container \n"); + printf("\tupdate --resources \tapply linux.resources from to running container \n"); return -EINVAL; } @@ -1081,6 +1101,131 @@ static int uxc_start(const char *name, bool console) return ubus_invoke(ctx, id, "start", NULL, NULL, NULL, 3000); } +struct uxc_exec_reply { + int status; +}; + +static void uxc_exec_reply_cb(struct ubus_request *req, int type, struct blob_attr *msg) +{ + enum { REPLY_STATUS, REPLY_PID, __REPLY_MAX }; + static const struct blobmsg_policy reply_policy[__REPLY_MAX] = { + [REPLY_STATUS] = { "status", BLOBMSG_TYPE_INT32 }, + [REPLY_PID] = { "pid", BLOBMSG_TYPE_INT32 }, + }; + struct blob_attr *tb[__REPLY_MAX]; + struct uxc_exec_reply *r = req->priv; + + blobmsg_parse(reply_policy, __REPLY_MAX, tb, blob_data(msg), blob_len(msg)); + if (tb[REPLY_STATUS]) + r->status = blobmsg_get_u32(tb[REPLY_STATUS]); +} + +static int uxc_exec(const char *name, const char *process_file, + const char *pid_file, bool detach, bool tty, + const char *console_socket, + char **cmd_argv, int cmd_argc) +{ + static struct blob_buf req; + struct uxc_exec_reply reply = { .status = 0 }; + char *objname; + uint32_t id; + int ret; + + if (tty && !console_socket) { + fprintf(stderr, "uxc: --tty requires --console-socket\n"); + return -EINVAL; + } + + blob_buf_init(&req, 0); + + if (process_file) { + if (!blobmsg_add_json_from_file(&req, process_file)) { + fprintf(stderr, "uxc: cannot parse %s as JSON\n", process_file); + blob_buf_free(&req); + return -EINVAL; + } + } else { + void *arr; + int i; + + if (cmd_argc < 1) { + fprintf(stderr, "uxc: exec requires --process or a command\n"); + blob_buf_free(&req); + return -EINVAL; + } + arr = blobmsg_open_array(&req, "args"); + for (i = 0; i < cmd_argc; i++) + blobmsg_add_string(&req, NULL, cmd_argv[i]); + blobmsg_close_array(&req, arr); + } + + if (pid_file) + blobmsg_add_string(&req, "pidfile", pid_file); + if (detach) + blobmsg_add_u8(&req, "detach", 1); + if (tty) + blobmsg_add_u8(&req, "terminal", 1); + if (console_socket) + blobmsg_add_string(&req, "consolesocket", console_socket); + + if (asprintf(&objname, "container.%s", name) == -1) { + blob_buf_free(&req); + return -ENOMEM; + } + + ret = ubus_lookup_id(ctx, objname, &id); + free(objname); + if (ret) { + blob_buf_free(&req); + return -ENOENT; + } + + ret = ubus_invoke(ctx, id, "exec", req.head, + uxc_exec_reply_cb, &reply, 0); + blob_buf_free(&req); + + if (ret) + return -EIO; + + return reply.status; +} + +static int uxc_update(const char *name, const char *resources_file) +{ + static struct blob_buf req; + char *objname; + uint32_t id; + int ret; + + if (!resources_file) { + fprintf(stderr, "uxc: update requires --resources \n"); + return -EINVAL; + } + + blob_buf_init(&req, 0); + if (!blobmsg_add_json_from_file(&req, resources_file)) { + fprintf(stderr, "uxc: cannot parse %s as JSON\n", resources_file); + blob_buf_free(&req); + return -EINVAL; + } + + if (asprintf(&objname, "container.%s", name) == -1) { + blob_buf_free(&req); + return -ENOMEM; + } + + ret = ubus_lookup_id(ctx, objname, &id); + free(objname); + if (ret) { + blob_buf_free(&req); + return -ENOENT; + } + + ret = ubus_invoke(ctx, id, "update", req.head, NULL, NULL, 3000); + blob_buf_free(&req); + return ret ? -EIO : 0; +} + static int uxc_kill(char *name, int signal, bool all) { static struct blob_buf req; @@ -1843,6 +1988,52 @@ int main(int argc, char **argv) reload_conf(); ret = uxc_create(name, false, console_socket, systemd_cgroup); + } else if (!strcmp(verb, "exec")) { + const char *process_file = NULL; + const char *pid_file = NULL; + const char *console_socket = NULL; + bool detach = false; + bool tty = false; + + while ((c = getopt_long(verb_argc, verb_argv, "+dp:t", + exec_opts, NULL)) != -1) { + switch (c) { + case 'd': detach = true; break; + case 'p': pid_file = optarg; break; + case 't': tty = true; break; + case OPT_PROCESS: process_file = optarg; break; + case OPT_CONSOLE_SOCKET: console_socket = optarg; break; + case OPT_PRESERVE_FDS: + fprintf(stderr, "uxc: --preserve-fds=%s accepted but ignored\n", optarg); + break; + default: goto usage_out; + } + } + if (optind >= verb_argc) + goto usage_out; + { + const char *id = verb_argv[optind]; + int cmd_start = optind + 1; + if (cmd_start < verb_argc && !strcmp(verb_argv[cmd_start], "--")) + cmd_start++; + ret = uxc_exec(id, process_file, pid_file, detach, tty, + console_socket, + verb_argv + cmd_start, + verb_argc - cmd_start); + } + } else if (!strcmp(verb, "update")) { + const char *resources_file = NULL; + + while ((c = getopt_long(verb_argc, verb_argv, "+", + update_opts, NULL)) != -1) { + switch (c) { + case OPT_RESOURCES: resources_file = optarg; break; + default: goto usage_out; + } + } + if (optind != verb_argc - 1) + goto usage_out; + ret = uxc_update(verb_argv[optind], resources_file); } else if (!strcmp(verb, "pause") || !strcmp(verb, "resume")) { char *objname; uint32_t id; From 7519638b41d4fb43bdedb4a2dfb8ff6c1a319ef0 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 17 Jun 2026 14:13:12 +0100 Subject: [PATCH 25/82] jail/seccomp: add RISC-V, MIPS-by-ABI and PPC64 architecture support Extend the seccomp BPF arch selection to RISC-V (riscv64), 64-bit PowerPC (ppc64/ppc64le by endianness) and the three MIPS ABIs. MIPS now picks AUDIT_ARCH by _MIPS_SIM, distinguishing o32, n32 and n64 and their little- and big-endian variants, where previously only the o32 audit arch was emitted. Map SCMP_ARCH_RISCV64 to AUDIT_ARCH_RISCV64 in the OCI profile architecture resolver as well. The now-unused REG_SYSCALL register-offset macros are dropped; syscall register access is handled elsewhere, leaving only the ARCH_NR selection in the header. Signed-off-by: Daniel Golle --- jail/seccomp-bpf.h | 34 ++++++++++++++++++++++++---------- jail/seccomp-oci.c | 2 ++ 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/jail/seccomp-bpf.h b/jail/seccomp-bpf.h index e6384db..820f65b 100644 --- a/jail/seccomp-bpf.h +++ b/jail/seccomp-bpf.h @@ -65,37 +65,51 @@ struct seccomp_data { #define syscall_arg(x) (offsetof(struct seccomp_data, args[x])) #if defined(__aarch64__) -# define REG_SYSCALL regs.regs[8] # define ARCH_NR AUDIT_ARCH_AARCH64 #elif defined(__amd64__) -# define REG_SYSCALL REG_RAX # define ARCH_NR AUDIT_ARCH_X86_64 #elif defined(__arm__) && (defined(__ARM_EABI__) || defined(__thumb__)) -# define REG_SYSCALL regs.uregs[7] # if __BYTE_ORDER == __LITTLE_ENDIAN # define ARCH_NR AUDIT_ARCH_ARM # else # define ARCH_NR AUDIT_ARCH_ARMEB # endif #elif defined(__i386__) -# define REG_SYSCALL REG_EAX # define ARCH_NR AUDIT_ARCH_I386 #elif defined(__loongarch_lp64) -# define REG_SYSCALL regs[11] # define ARCH_NR AUDIT_ARCH_LOONGARCH64 #elif defined(__mips__) -# define REG_SYSCALL regs[2] +# if _MIPS_SIM == _ABI64 +# if __BYTE_ORDER == __LITTLE_ENDIAN +# define ARCH_NR AUDIT_ARCH_MIPSEL64 +# else +# define ARCH_NR AUDIT_ARCH_MIPS64 +# endif +# elif _MIPS_SIM == _ABIN32 +# if __BYTE_ORDER == __LITTLE_ENDIAN +# define ARCH_NR AUDIT_ARCH_MIPSEL64N32 +# else +# define ARCH_NR AUDIT_ARCH_MIPS64N32 +# endif +# else +# if __BYTE_ORDER == __LITTLE_ENDIAN +# define ARCH_NR AUDIT_ARCH_MIPSEL +# else +# define ARCH_NR AUDIT_ARCH_MIPS +# endif +# endif +#elif defined(__powerpc64__) # if __BYTE_ORDER == __LITTLE_ENDIAN -# define ARCH_NR AUDIT_ARCH_MIPSEL +# define ARCH_NR AUDIT_ARCH_PPC64LE # else -# define ARCH_NR AUDIT_ARCH_MIPS +# define ARCH_NR AUDIT_ARCH_PPC64 # endif #elif defined(__PPC__) -# define REG_SYSCALL regs.gpr[0] # define ARCH_NR AUDIT_ARCH_PPC +#elif defined(__riscv) && __riscv_xlen == 64 +# define ARCH_NR AUDIT_ARCH_RISCV64 #else # warning "Platform does not support seccomp filter yet" -# define REG_SYSCALL 0 # define ARCH_NR 0 #endif diff --git a/jail/seccomp-oci.c b/jail/seccomp-oci.c index c279fc1..ff21a67 100644 --- a/jail/seccomp-oci.c +++ b/jail/seccomp-oci.c @@ -121,6 +121,8 @@ static uint32_t resolve_architecture(char *archname) return AUDIT_ARCH_AARCH64; else if (!strcmp(archname, "SCMP_ARCH_LOONGARCH64")) return AUDIT_ARCH_LOONGARCH64; + else if (!strcmp(archname, "SCMP_ARCH_RISCV64")) + return AUDIT_ARCH_RISCV64; else if (!strcmp(archname, "SCMP_ARCH_MIPS")) return AUDIT_ARCH_MIPS; else if (!strcmp(archname, "SCMP_ARCH_MIPS64")) From 6d12f64ae9097333404babc345dba5b7ea2ec392 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 17 Jun 2026 14:36:07 +0100 Subject: [PATCH 26/82] jail/seccomp-oci: support further OCI seccomp actions, flags and large groups Parse the remaining OCI seccomp surface: SCMP_ACT_KILL_THREAD, the SCMP_ACT_NOTIFY user-notification action, defaultErrnoRet and per-syscall errnoRet (range-checked against MAX_ERRNO), and the filter flags LOG, SPEC_ALLOW, TSYNC and WAIT_KILLABLE_RECV, with listenerPath and listenerMetadata. The default action is remembered as seccomp_default_action so staged filter deltas can revert a revoked exemption to it. When NOTIFY is used or any flag is set the filter must be installed in-process via the seccomp() syscall, which seccomp_oci_needs_inproc() reports; applyOCIlinuxseccomp then installs a NEW_LISTENER filter and forwards the notify fd, container state and metadata to listenerPath over SCM_RIGHTS. A syscalls group shares one rule body across many names via forward BPF_JEQ jumps, but the jt/jf offsets are only 8 bits, so a large group overflows the jump reaching past the names to the args and RET, producing a malformed filter. Split each group into chunks of at most SECCOMP_CHUNK_NAMES names, each re-emitting its own args and RET so the internal jumps stay in range. Fixes: ea7a790f210c ("jail: add support for running OCI bundle") Signed-off-by: Daniel Golle --- jail/jail.c | 4 +- jail/jail.h | 2 + jail/seccomp-oci.c | 441 ++++++++++++++++++++++++++++++++++++--------- jail/seccomp-oci.h | 6 +- jail/seccomp.c | 2 +- 5 files changed, 366 insertions(+), 89 deletions(-) diff --git a/jail/jail.c b/jail/jail.c index 65110d4..8d77844 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -97,8 +97,6 @@ #define OPT_ARGS "cC:d:De:EfFG:h:iI:j:J:ln:NoO:pP:r:R:sS:uU:w:t:T:yY:Z" -#define OCI_VERSION_STRING "1.0.2" - struct hook_execvpe { char *file; char **argv; @@ -2910,7 +2908,7 @@ static void post_start_hook(void) free_and_exit(EXIT_FAILURE); } - if (opts.ociseccomp && applyOCIlinuxseccomp(opts.ociseccomp)) + if (opts.ociseccomp && applyOCIlinuxseccomp(opts.ociseccomp, opts.name, opts.ocibundle)) free_and_exit(EXIT_FAILURE); uloop_end(); diff --git a/jail/jail.h b/jail/jail.h index 158d73b..2d94a2c 100644 --- a/jail/jail.h +++ b/jail/jail.h @@ -13,6 +13,8 @@ #ifndef _JAIL_JAIL_H_ #define _JAIL_JAIL_H_ +#define OCI_VERSION_STRING "1.0.2" + int mount_bind(const char *root, const char *path, int readonly, int error); int ns_open_pid(const char *nstype, const pid_t target_ns); diff --git a/jail/seccomp-oci.c b/jail/seccomp-oci.c index ff21a67..e33daa7 100644 --- a/jail/seccomp-oci.c +++ b/jail/seccomp-oci.c @@ -24,22 +24,68 @@ #include #include #include +#include #include +#include +#include +#include #include #include #include +#include + #include "log.h" +#include "jail.h" #include "seccomp-bpf.h" #include "seccomp-oci.h" #include "../syscall-names.h" #include "seccomp-syscalls-helpers.h" +#ifndef MAX_ERRNO +#define MAX_ERRNO 4095 +#endif + +#ifndef SECCOMP_SET_MODE_FILTER +#define SECCOMP_SET_MODE_FILTER 1 +#endif + +#ifndef SECCOMP_FILTER_FLAG_TSYNC +#define SECCOMP_FILTER_FLAG_TSYNC (1UL << 0) +#endif + +#ifndef SECCOMP_FILTER_FLAG_LOG +#define SECCOMP_FILTER_FLAG_LOG (1UL << 1) +#endif + +#ifndef SECCOMP_FILTER_FLAG_SPEC_ALLOW +#define SECCOMP_FILTER_FLAG_SPEC_ALLOW (1UL << 2) +#endif + +#ifndef SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV +#define SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV (1UL << 5) +#endif + +#ifndef SECCOMP_FILTER_FLAG_NEW_LISTENER +#define SECCOMP_FILTER_FLAG_NEW_LISTENER (1UL << 3) +#endif + +#ifndef SECCOMP_RET_USER_NOTIF +#define SECCOMP_RET_USER_NOTIF 0x7fc00000U +#endif + +static unsigned long seccomp_filter_flags; +static char *seccomp_listener_path; +static char *seccomp_listener_metadata; +static bool seccomp_uses_notify; + static uint32_t resolve_action(char *actname) { if (!strcmp(actname, "SCMP_ACT_KILL")) return SECCOMP_RET_KILL; + else if (!strcmp(actname, "SCMP_ACT_KILL_THREAD")) + return SECCOMP_RET_KILL; else if (!strcmp(actname, "SCMP_ACT_KILL_PROCESS")) return SECCOMP_RET_KILLPROCESS; else if (!strcmp(actname, "SCMP_ACT_TRAP")) @@ -54,6 +100,8 @@ static uint32_t resolve_action(char *actname) return SECCOMP_RET_ALLOW; else if (!strcmp(actname, "SCMP_ACT_LOG")) return SECCOMP_RET_LOGALLOW; + else if (!strcmp(actname, "SCMP_ACT_NOTIFY")) + return SECCOMP_RET_USER_NOTIF; else { ERROR("unknown seccomp action %s\n", actname); return SECCOMP_RET_KILL; @@ -157,16 +205,22 @@ static uint32_t resolve_architecture(char *archname) enum { OCI_LINUX_SECCOMP_DEFAULTACTION, + OCI_LINUX_SECCOMP_DEFAULTERRNORET, OCI_LINUX_SECCOMP_ARCHITECTURES, OCI_LINUX_SECCOMP_FLAGS, + OCI_LINUX_SECCOMP_LISTENERPATH, + OCI_LINUX_SECCOMP_LISTENERMETADATA, OCI_LINUX_SECCOMP_SYSCALLS, __OCI_LINUX_SECCOMP_MAX, }; static const struct blobmsg_policy oci_linux_seccomp_policy[] = { [OCI_LINUX_SECCOMP_DEFAULTACTION] = { "defaultAction", BLOBMSG_TYPE_STRING }, + [OCI_LINUX_SECCOMP_DEFAULTERRNORET] = { "defaultErrnoRet", BLOBMSG_TYPE_INT32 }, [OCI_LINUX_SECCOMP_ARCHITECTURES] = { "architectures", BLOBMSG_TYPE_ARRAY }, [OCI_LINUX_SECCOMP_FLAGS] = { "flags", BLOBMSG_TYPE_ARRAY }, + [OCI_LINUX_SECCOMP_LISTENERPATH] = { "listenerPath", BLOBMSG_TYPE_STRING }, + [OCI_LINUX_SECCOMP_LISTENERMETADATA] = { "listenerMetadata", BLOBMSG_TYPE_STRING }, [OCI_LINUX_SECCOMP_SYSCALLS] = { "syscalls", BLOBMSG_TYPE_ARRAY }, }; @@ -200,6 +254,8 @@ static const struct blobmsg_policy oci_linux_seccomp_syscalls_args_policy[] = { [OCI_LINUX_SECCOMP_SYSCALLS_ARGS_OP] = { "op", BLOBMSG_TYPE_STRING }, }; +#define SECCOMP_CHUNK_NAMES 240 + struct sock_fprog *parseOCIlinuxseccomp(struct blob_attr *msg) { struct blob_attr *tb[__OCI_LINUX_SECCOMP_MAX]; @@ -225,6 +281,53 @@ struct sock_fprog *parseOCIlinuxseccomp(struct blob_attr *msg) default_policy = resolve_action(blobmsg_get_string(tb[OCI_LINUX_SECCOMP_DEFAULTACTION])); + free(seccomp_listener_path); + free(seccomp_listener_metadata); + seccomp_listener_path = NULL; + seccomp_listener_metadata = NULL; + seccomp_uses_notify = (default_policy == SECCOMP_RET_USER_NOTIF); + + if (tb[OCI_LINUX_SECCOMP_LISTENERPATH]) + seccomp_listener_path = strdup(blobmsg_get_string(tb[OCI_LINUX_SECCOMP_LISTENERPATH])); + + if (tb[OCI_LINUX_SECCOMP_LISTENERMETADATA]) + seccomp_listener_metadata = strdup(blobmsg_get_string(tb[OCI_LINUX_SECCOMP_LISTENERMETADATA])); + + seccomp_filter_flags = 0; + if (tb[OCI_LINUX_SECCOMP_FLAGS]) { + blobmsg_for_each_attr(cur, tb[OCI_LINUX_SECCOMP_FLAGS], rem) { + const char *flag = blobmsg_get_string(cur); + if (!strcmp(flag, "SECCOMP_FILTER_FLAG_LOG")) + seccomp_filter_flags |= SECCOMP_FILTER_FLAG_LOG; + else if (!strcmp(flag, "SECCOMP_FILTER_FLAG_SPEC_ALLOW")) + seccomp_filter_flags |= SECCOMP_FILTER_FLAG_SPEC_ALLOW; + else if (!strcmp(flag, "SECCOMP_FILTER_FLAG_TSYNC")) + seccomp_filter_flags |= SECCOMP_FILTER_FLAG_TSYNC; + else if (!strcmp(flag, "SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV")) + seccomp_filter_flags |= SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV; + else { + ERROR("seccomp: unknown filter flag %s\n", flag); + return NULL; + } + } + } + + if (default_policy == SECCOMP_RET_ERRNO) { + uint32_t errnoret = EPERM; + if (tb[OCI_LINUX_SECCOMP_DEFAULTERRNORET]) { + errnoret = blobmsg_get_u32(tb[OCI_LINUX_SECCOMP_DEFAULTERRNORET]); + if (errnoret < 1 || errnoret > MAX_ERRNO) { + ERROR("seccomp: defaultErrnoRet %u out of range (1..%u)\n", + errnoret, MAX_ERRNO); + return NULL; + } + } + default_policy = SECCOMP_RET_ERROR(errnoret); + } else if (tb[OCI_LINUX_SECCOMP_DEFAULTERRNORET]) { + ERROR("seccomp: defaultErrnoRet only valid with SCMP_ACT_ERRNO defaultAction\n"); + return NULL; + } + /* verify architecture while ignoring the x86_64 anomaly for now */ if (tb[OCI_LINUX_SECCOMP_ARCHITECTURES]) { arch_matched = false; @@ -242,7 +345,9 @@ struct sock_fprog *parseOCIlinuxseccomp(struct blob_attr *msg) } blobmsg_for_each_attr(cur, tb[OCI_LINUX_SECCOMP_SYSCALLS], rem) { - sz += 2; /* load and return */ + int valid_names = 0; + int arg_instrs = 0; + int chunks; blobmsg_parse(oci_linux_seccomp_syscalls_policy, __OCI_LINUX_SECCOMP_SYSCALLS_MAX, @@ -254,40 +359,55 @@ struct sock_fprog *parseOCIlinuxseccomp(struct blob_attr *msg) /* TODO: support run.oci.seccomp_fail_unknown_syscall=1 annotation */ continue; } - ++sz; + ++valid_names; } if (tbn[OCI_LINUX_SECCOMP_SYSCALLS_ARGS]) { blobmsg_for_each_attr(curarg, tbn[OCI_LINUX_SECCOMP_SYSCALLS_ARGS], remargs) { - sz += 2; /* load and compare */ + arg_instrs += 2; /* load and compare */ blobmsg_parse(oci_linux_seccomp_syscalls_args_policy, __OCI_LINUX_SECCOMP_SYSCALLS_ARGS_MAX, tba, blobmsg_data(curarg), blobmsg_len(curarg)); if (!tba[OCI_LINUX_SECCOMP_SYSCALLS_ARGS_INDEX] || !tba[OCI_LINUX_SECCOMP_SYSCALLS_ARGS_VALUE] || - !tba[OCI_LINUX_SECCOMP_SYSCALLS_ARGS_OP]) + !tba[OCI_LINUX_SECCOMP_SYSCALLS_ARGS_OP]) { + ERROR("seccomp: syscall arg missing%s%s%s\n", + tba[OCI_LINUX_SECCOMP_SYSCALLS_ARGS_INDEX] ? "" : " index", + tba[OCI_LINUX_SECCOMP_SYSCALLS_ARGS_VALUE] ? "" : " value", + tba[OCI_LINUX_SECCOMP_SYSCALLS_ARGS_OP] ? "" : " op"); return NULL; + } - if (blobmsg_get_u32(tba[OCI_LINUX_SECCOMP_SYSCALLS_ARGS_INDEX]) > 5) + if (blobmsg_get_u32(tba[OCI_LINUX_SECCOMP_SYSCALLS_ARGS_INDEX]) > 5) { + ERROR("seccomp: syscall arg index %u out of range (max 5)\n", + blobmsg_get_u32(tba[OCI_LINUX_SECCOMP_SYSCALLS_ARGS_INDEX])); return NULL; + } op_str = blobmsg_get_string(tba[OCI_LINUX_SECCOMP_SYSCALLS_ARGS_OP]); if (!resolve_op_ins(op_str)) return NULL; if (resolve_op_is_masked(op_str)) - ++sz; /* SCMP_CMP_MASKED_EQ needs an extra BPF_AND op */ + ++arg_instrs; /* SCMP_CMP_MASKED_EQ needs an extra BPF_AND op */ } } + + chunks = valid_names ? (valid_names + SECCOMP_CHUNK_NAMES - 1) / SECCOMP_CHUNK_NAMES : 1; + sz += chunks * (1 + 1 + arg_instrs) + valid_names; } - if (sz < 6) + if (sz < 6) { + ERROR("seccomp: filter is empty\n"); return NULL; + } prog = malloc(sizeof(struct sock_fprog)); - if (!prog) + if (!prog) { + ERROR("seccomp: failed to allocate sock_fprog\n"); return NULL; + } filter = calloc(sz, sizeof(struct sock_filter)); if (!filter) { @@ -308,99 +428,128 @@ struct sock_fprog *parseOCIlinuxseccomp(struct blob_attr *msg) uint64_t op_val, op_val2; int start_rule_idx; int next_rule_idx; + int valid_names = 0; + int names_emitted = 0; + int chunk_size; blobmsg_parse(oci_linux_seccomp_syscalls_policy, __OCI_LINUX_SECCOMP_SYSCALLS_MAX, tbn, blobmsg_data(cur), blobmsg_len(cur)); action = resolve_action(blobmsg_get_string( tbn[OCI_LINUX_SECCOMP_SYSCALLS_ACTION])); + if (action == SECCOMP_RET_USER_NOTIF) + seccomp_uses_notify = true; if (tbn[OCI_LINUX_SECCOMP_SYSCALLS_ERRNORET]) { - if (action != SECCOMP_RET_ERRNO) + uint32_t errnoret; + + if (action != SECCOMP_RET_ERRNO) { + ERROR("seccomp: errnoRet set but action is not SCMP_ACT_ERRNO\n"); goto errout1; + } - action = SECCOMP_RET_ERROR(blobmsg_get_u32( - tbn[OCI_LINUX_SECCOMP_SYSCALLS_ERRNORET])); + errnoret = blobmsg_get_u32(tbn[OCI_LINUX_SECCOMP_SYSCALLS_ERRNORET]); + if (errnoret < 1 || errnoret > MAX_ERRNO) { + ERROR("seccomp: errnoRet %u out of range (1..%u)\n", + errnoret, MAX_ERRNO); + goto errout1; + } + action = SECCOMP_RET_ERROR(errnoret); } else if (action == SECCOMP_RET_ERRNO) action = SECCOMP_RET_ERROR(EPERM); - /* load syscall */ - set_filter(&filter[idx++], BPF_LD + BPF_W + BPF_ABS, 0, 0, syscall_nr); - - /* get number of syscall names */ - next_rule_idx = idx; blobmsg_for_each_attr(curn, tbn[OCI_LINUX_SECCOMP_SYSCALLS_NAMES], remn) { - if (find_syscall(blobmsg_get_string(curn)) == -1) - continue; + if (find_syscall(blobmsg_get_string(curn)) != -1) + ++valid_names; + } + + if (!valid_names && !tbn[OCI_LINUX_SECCOMP_SYSCALLS_ARGS]) + continue; + + while (names_emitted < valid_names || names_emitted == 0) { + int names_in_chunk; + int names_seen; + + chunk_size = valid_names - names_emitted; + if (chunk_size > SECCOMP_CHUNK_NAMES) + chunk_size = SECCOMP_CHUNK_NAMES; + names_in_chunk = chunk_size; + + set_filter(&filter[idx++], BPF_LD + BPF_W + BPF_ABS, 0, 0, syscall_nr); + + next_rule_idx = idx + names_in_chunk; + start_rule_idx = next_rule_idx; + + blobmsg_for_each_attr(curn, tbn[OCI_LINUX_SECCOMP_SYSCALLS_ARGS], remn) { + blobmsg_parse(oci_linux_seccomp_syscalls_args_policy, + __OCI_LINUX_SECCOMP_SYSCALLS_ARGS_MAX, + tba, blobmsg_data(curn), blobmsg_len(curn)); + next_rule_idx += 2; + op_str = blobmsg_get_string(tba[OCI_LINUX_SECCOMP_SYSCALLS_ARGS_OP]); + if (resolve_op_is_masked(op_str)) + ++next_rule_idx; + } ++next_rule_idx; - } - start_rule_idx = next_rule_idx; - - /* calculate length of argument filter rules */ - blobmsg_for_each_attr(curn, tbn[OCI_LINUX_SECCOMP_SYSCALLS_ARGS], remn) { - blobmsg_parse(oci_linux_seccomp_syscalls_args_policy, - __OCI_LINUX_SECCOMP_SYSCALLS_ARGS_MAX, - tba, blobmsg_data(curn), blobmsg_len(curn)); - next_rule_idx += 2; - op_str = blobmsg_get_string(tba[OCI_LINUX_SECCOMP_SYSCALLS_ARGS_OP]); - if (resolve_op_is_masked(op_str)) - ++next_rule_idx; - } - ++next_rule_idx; /* account for return action */ + names_seen = 0; + blobmsg_for_each_attr(curn, tbn[OCI_LINUX_SECCOMP_SYSCALLS_NAMES], remn) { + sc = find_syscall(blobmsg_get_string(curn)); + if (sc == -1) + continue; + if (names_seen < names_emitted) { + ++names_seen; + continue; + } + if (names_seen >= names_emitted + names_in_chunk) + break; + set_filter(&filter[idx], BPF_JMP + BPF_JEQ + BPF_K, + start_rule_idx - (idx + 1), + ((idx + 1) == start_rule_idx)?(next_rule_idx - (idx + 1)):0, + sc); + ++idx; + ++names_seen; + } - blobmsg_for_each_attr(curn, tbn[OCI_LINUX_SECCOMP_SYSCALLS_NAMES], remn) { - sc = find_syscall(blobmsg_get_string(curn)); - if (sc == -1) - continue; - /* - * check syscall, skip other syscall checks if match is found. - * if no match is found, jump to next section - */ - set_filter(&filter[idx], BPF_JMP + BPF_JEQ + BPF_K, - start_rule_idx - (idx + 1), - ((idx + 1) == start_rule_idx)?(next_rule_idx - (idx + 1)):0, - sc); - ++idx; - } + assert(idx == start_rule_idx); - assert(idx = start_rule_idx); - - /* generate argument filter rules */ - blobmsg_for_each_attr(curn, tbn[OCI_LINUX_SECCOMP_SYSCALLS_ARGS], remn) { - blobmsg_parse(oci_linux_seccomp_syscalls_args_policy, - __OCI_LINUX_SECCOMP_SYSCALLS_ARGS_MAX, - tba, blobmsg_data(curn), blobmsg_len(curn)); - - op_str = blobmsg_get_string(tba[OCI_LINUX_SECCOMP_SYSCALLS_ARGS_OP]); - op_ins = resolve_op_ins(op_str); - op_inv = resolve_op_inv(op_str); - op_masked = resolve_op_is_masked(op_str); - op_idx = blobmsg_get_u32(tba[OCI_LINUX_SECCOMP_SYSCALLS_ARGS_INDEX]); - op_val = blobmsg_cast_u64(tba[OCI_LINUX_SECCOMP_SYSCALLS_ARGS_VALUE]); - if (tba[OCI_LINUX_SECCOMP_SYSCALLS_ARGS_VALUETWO]) - op_val2 = blobmsg_cast_u64(tba[OCI_LINUX_SECCOMP_SYSCALLS_ARGS_VALUETWO]); - else - op_val2 = 0; - - /* load argument */ - set_filter(&filter[idx++], BPF_LD + BPF_W + BPF_ABS, 0, 0, syscall_arg(op_idx)); - - /* apply mask */ - if (op_masked) - set_filter(&filter[idx++], BPF_ALU + BPF_K + BPF_AND, 0, 0, op_val); - - set_filter(&filter[idx], BPF_JMP + op_ins + BPF_K, - op_inv?(next_rule_idx - (idx + 1)):0, - op_inv?0:(next_rule_idx - (idx + 1)), - op_masked?op_val2:op_val); - ++idx; - } + blobmsg_for_each_attr(curn, tbn[OCI_LINUX_SECCOMP_SYSCALLS_ARGS], remn) { + blobmsg_parse(oci_linux_seccomp_syscalls_args_policy, + __OCI_LINUX_SECCOMP_SYSCALLS_ARGS_MAX, + tba, blobmsg_data(curn), blobmsg_len(curn)); + + op_str = blobmsg_get_string(tba[OCI_LINUX_SECCOMP_SYSCALLS_ARGS_OP]); + op_ins = resolve_op_ins(op_str); + op_inv = resolve_op_inv(op_str); + op_masked = resolve_op_is_masked(op_str); + op_idx = blobmsg_get_u32(tba[OCI_LINUX_SECCOMP_SYSCALLS_ARGS_INDEX]); + op_val = blobmsg_cast_u64(tba[OCI_LINUX_SECCOMP_SYSCALLS_ARGS_VALUE]); + if (tba[OCI_LINUX_SECCOMP_SYSCALLS_ARGS_VALUETWO]) + op_val2 = blobmsg_cast_u64(tba[OCI_LINUX_SECCOMP_SYSCALLS_ARGS_VALUETWO]); + else + op_val2 = 0; + + /* load argument */ + set_filter(&filter[idx++], BPF_LD + BPF_W + BPF_ABS, 0, 0, syscall_arg(op_idx)); + + /* apply mask */ + if (op_masked) + set_filter(&filter[idx++], BPF_ALU + BPF_K + BPF_AND, 0, 0, op_val); + + set_filter(&filter[idx], BPF_JMP + op_ins + BPF_K, + op_inv?(next_rule_idx - (idx + 1)):0, + op_inv?0:(next_rule_idx - (idx + 1)), + op_masked?op_val2:op_val); + ++idx; + } - /* if we have reached until here, all conditions were met and we can return */ - set_filter(&filter[idx++], BPF_RET + BPF_K, 0, 0, action); + set_filter(&filter[idx++], BPF_RET + BPF_K, 0, 0, action); - assert(idx == next_rule_idx); + assert(idx == next_rule_idx); + + names_emitted += chunk_size; + if (!valid_names) + break; + } } set_filter(&filter[idx++], BPF_RET + BPF_K, 0, 0, default_policy); @@ -431,14 +580,140 @@ struct sock_fprog *parseOCIlinuxseccomp(struct blob_attr *msg) } -int applyOCIlinuxseccomp(struct sock_fprog *prog) +static int send_seccomp_listener_fd(int listener_fd, const char *container_id, + const char *bundle_path) +{ + struct sockaddr_un addr = { .sun_family = AF_UNIX }; + struct blob_buf bb = { 0 }; + void *fds_arr, *state; + struct msghdr msg = { 0 }; + struct iovec iov; + struct cmsghdr *cmsg; + char cmsgbuf[CMSG_SPACE(sizeof(int))]; + char *json; + int sock; + int ret = 0; + int saved_err; + + if (strlen(seccomp_listener_path) >= sizeof(addr.sun_path)) { + ERROR("seccomp: listenerPath too long: %s\n", seccomp_listener_path); + return ENAMETOOLONG; + } + + blob_buf_init(&bb, 0); + blobmsg_add_string(&bb, "ociVersion", OCI_VERSION_STRING); + fds_arr = blobmsg_open_array(&bb, "fds"); + blobmsg_add_string(&bb, NULL, "seccompFd"); + blobmsg_close_array(&bb, fds_arr); + blobmsg_add_u32(&bb, "pid", getpid()); + if (seccomp_listener_metadata) + blobmsg_add_string(&bb, "metadata", seccomp_listener_metadata); + state = blobmsg_open_table(&bb, "state"); + blobmsg_add_string(&bb, "ociVersion", OCI_VERSION_STRING); + if (container_id) + blobmsg_add_string(&bb, "id", container_id); + blobmsg_add_string(&bb, "status", "creating"); + blobmsg_add_u32(&bb, "pid", getpid()); + if (bundle_path) + blobmsg_add_string(&bb, "bundle", bundle_path); + blobmsg_close_table(&bb, state); + + json = blobmsg_format_json(bb.head, true); + if (!json) { + blob_buf_free(&bb); + return ENOMEM; + } + + sock = socket(AF_UNIX, SOCK_STREAM, 0); + if (sock < 0) { + ret = errno; + ERROR("socket(AF_UNIX): %m\n"); + goto out; + } + + memcpy(addr.sun_path, seccomp_listener_path, strlen(seccomp_listener_path) + 1); + if (connect(sock, (struct sockaddr *)&addr, sizeof(addr)) < 0) { + ret = errno; + ERROR("connect(%s): %m\n", seccomp_listener_path); + saved_err = ret; + close(sock); + ret = saved_err; + goto out; + } + + iov.iov_base = json; + iov.iov_len = strlen(json); + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + msg.msg_control = cmsgbuf; + msg.msg_controllen = sizeof(cmsgbuf); + cmsg = CMSG_FIRSTHDR(&msg); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + cmsg->cmsg_len = CMSG_LEN(sizeof(int)); + memcpy(CMSG_DATA(cmsg), &listener_fd, sizeof(int)); + + if (sendmsg(sock, &msg, 0) < 0) { + ret = errno; + ERROR("sendmsg(%s): %m\n", seccomp_listener_path); + } + + saved_err = ret; + close(sock); + ret = saved_err; +out: + free(json); + blob_buf_free(&bb); + return ret; +} + +int applyOCIlinuxseccomp(struct sock_fprog *prog, const char *container_id, + const char *bundle_path) { + int listener_fd = -1; + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) { ERROR("prctl(PR_SET_NO_NEW_PRIVS) failed: %m\n"); goto errout; } - if (prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, prog)) { + if (seccomp_uses_notify) { + if (!seccomp_listener_path) { + ERROR("seccomp: SCMP_ACT_NOTIFY used without listenerPath\n"); + errno = EINVAL; + goto errout; + } + + listener_fd = syscall(SYS_seccomp, SECCOMP_SET_MODE_FILTER, + seccomp_filter_flags | SECCOMP_FILTER_FLAG_NEW_LISTENER, + prog); + if (listener_fd < 0) { + ERROR("seccomp(SET_MODE_FILTER|NEW_LISTENER): %m\n"); + goto errout; + } + + if (send_seccomp_listener_fd(listener_fd, container_id, bundle_path)) { + int saved_err = errno; + close(listener_fd); + errno = saved_err; + goto errout; + } + + close(listener_fd); + } else if (seccomp_filter_flags) { + long r = syscall(SYS_seccomp, SECCOMP_SET_MODE_FILTER, + seccomp_filter_flags, prog); + if (r < 0) { + ERROR("seccomp(SET_MODE_FILTER, %#lx): %m\n", seccomp_filter_flags); + goto errout; + } + if (r > 0) { + ERROR("seccomp(SET_MODE_FILTER, %#lx) TSYNC failed at tid %ld\n", + seccomp_filter_flags, r); + errno = EAGAIN; + goto errout; + } + } else if (prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, prog)) { ERROR("prctl(PR_SET_SECCOMP) failed: %m\n"); goto errout; } diff --git a/jail/seccomp-oci.h b/jail/seccomp-oci.h index 8cc8ae2..cec81d6 100644 --- a/jail/seccomp-oci.h +++ b/jail/seccomp-oci.h @@ -16,14 +16,16 @@ #include struct sock_fprog *parseOCIlinuxseccomp(struct blob_attr *msg); -int applyOCIlinuxseccomp(struct sock_fprog *prog); +int applyOCIlinuxseccomp(struct sock_fprog *prog, const char *container_id, + const char *bundle_path); #ifndef SECCOMP_SUPPORT struct sock_fprog *parseOCIlinuxseccomp(struct blob_attr *msg) { return NULL; } -int applyOCIlinuxseccomp(struct sock_fprog *prog) { +int applyOCIlinuxseccomp(struct sock_fprog *prog, const char *container_id, + const char *bundle_path) { return ENOTSUP; } #endif diff --git a/jail/seccomp.c b/jail/seccomp.c index 3eeb616..fb80f93 100644 --- a/jail/seccomp.c +++ b/jail/seccomp.c @@ -41,5 +41,5 @@ int install_syscall_filter(const char *argv, const char *file) return -1; } - return applyOCIlinuxseccomp(prog); + return applyOCIlinuxseccomp(prog, NULL, NULL); } From f15d0c407c09c2b10791065c3a2f2ef208448b46 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 17 Jun 2026 14:50:35 +0100 Subject: [PATCH 27/82] jail: apply OCI seccomp filters via ptrace syscall injection Arm the compiled cBPF profile by injecting prctl(NO_NEW_PRIVS) and seccomp(SET_MODE_FILTER) into the freshly-execve'd workload over ptrace. The child does PTRACE_TRACEME before its final execve; the parent catches the post-execve stop, drives the two syscalls via a temporary trap at the program counter, restores registers and code, then detaches. This installs the filter for static binaries too, closing the gap left by the dropped LD_PRELOAD installer (preload.c, seccomp.c and the shared library are removed). A dynamic binary needs extra loader and libc pre-main syscalls. They are granted once in a phase-1 app + linker_base filter, then revoked with deny deltas at the entry and main boundaries (located via AT_BASE/auxv and ELF marker addresses), each delta reverting to the profile default. Static and Go-style binaries collapse to fewer phases. NOTIFY or flag profiles still go in-process; audit and complain modes reuse the same staging to log denials. Signed-off-by: Daniel Golle --- CMakeLists.txt | 10 +- jail/elf.c | 110 ++++ jail/elf.h | 2 + jail/jail.c | 256 +++++++- jail/preload.c | 96 --- jail/seccomp-inject.c | 1450 +++++++++++++++++++++++++++++++++++++++++ jail/seccomp-inject.h | 70 ++ jail/seccomp-oci.c | 197 +++++- jail/seccomp-oci.h | 26 +- jail/seccomp.c | 45 -- jail/seccomp.h | 21 - trace/trace.c | 13 +- 12 files changed, 2090 insertions(+), 206 deletions(-) delete mode 100644 jail/preload.c create mode 100644 jail/seccomp-inject.c create mode 100644 jail/seccomp-inject.h delete mode 100644 jail/seccomp.c delete mode 100644 jail/seccomp.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 76d0082..8be6078 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -101,13 +101,7 @@ ADD_CUSTOM_TARGET(capabilities-names-h DEPENDS capabilities-names.h) IF(SECCOMP_SUPPORT) ADD_DEFINITIONS(-DSECCOMP_SUPPORT) -ADD_LIBRARY(preload-seccomp SHARED jail/preload.c jail/seccomp.c jail/seccomp-oci.c) -TARGET_LINK_LIBRARIES(preload-seccomp dl ${ubox} ${blobmsg_json}) -INSTALL(TARGETS preload-seccomp - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} -) -ADD_DEPENDENCIES(preload-seccomp syscall-names-h) -SET(SOURCES_OCI_SECCOMP jail/seccomp-oci.c) +SET(SOURCES_OCI_SECCOMP jail/seccomp-oci.c jail/seccomp-inject.c) ENDIF() IF(JAIL_SUPPORT) @@ -129,7 +123,7 @@ INSTALL(TARGETS uxc endif() IF(UTRACE_SUPPORT) -ADD_EXECUTABLE(utrace trace/trace.c) +ADD_EXECUTABLE(utrace trace/trace.c ${SOURCES_OCI_SECCOMP}) TARGET_LINK_LIBRARIES(utrace ${ubox} ${json} ${blobmsg_json}) INSTALL(TARGETS utrace RUNTIME DESTINATION ${CMAKE_INSTALL_SBINDIR} diff --git a/jail/elf.c b/jail/elf.c index d343b15..86c274b 100644 --- a/jail/elf.c +++ b/jail/elf.c @@ -15,7 +15,9 @@ #include #include +#include #include +#include #include #include #include @@ -645,6 +647,114 @@ int elf_load_deps(const char *path, const char *map, unsigned long map_size) return -1; } +static char *elf_map_file(const char *path, size_t *size) +{ + struct stat s; + void *map; + int fd; + + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return NULL; + + if (fstat(fd, &s) || s.st_size < (off_t)sizeof(Elf64_Ehdr)) { + close(fd); + return NULL; + } + + map = mmap(NULL, s.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + close(fd); + if (map == MAP_FAILED) + return NULL; + + *size = s.st_size; + return map; +} + +#define ELF_DYNSYM_VALUE(BITS) \ +static unsigned long elf##BITS##_dynsym_value(const char *map, unsigned long map_size, const char *want) \ +{ \ + Elf##BITS##_Dyn *dyn; \ + Elf##BITS##_Sym *symtab = NULL; \ + const char *strtab = NULL; \ + unsigned long dyn_off, dyn_size, load_off, load_vaddr, delta; \ + unsigned long syment = sizeof(Elf##BITS##_Sym), nsyms = 0, i; \ + \ + if (elf##BITS##_find_section(map, map_size, PT_LOAD, &load_off, NULL, &load_vaddr)) \ + return 0; \ + if (elf##BITS##_find_section(map, map_size, PT_DYNAMIC, &dyn_off, &dyn_size, NULL)) \ + return 0; \ + delta = load_vaddr - load_off; \ + \ + for (dyn = (Elf##BITS##_Dyn *)(map + dyn_off); \ + (char *)dyn < map + dyn_off + dyn_size; dyn++) { \ + if (dyn->d_tag == DT_SYMTAB) \ + symtab = (Elf##BITS##_Sym *)(map + (dyn->d_un.d_ptr - delta)); \ + else if (dyn->d_tag == DT_STRTAB) \ + strtab = map + (dyn->d_un.d_ptr - delta); \ + else if (dyn->d_tag == DT_SYMENT) \ + syment = dyn->d_un.d_val; \ + else if (dyn->d_tag == DT_HASH) \ + nsyms = ((const uint32_t *)(map + (dyn->d_un.d_ptr - delta)))[1]; \ + } \ + \ + if (!symtab || !strtab) \ + return 0; \ + if (!nsyms) \ + nsyms = ((const char *)strtab - (const char *)symtab) / syment; \ + \ + for (i = 0; i < nsyms; i++) { \ + Elf##BITS##_Sym *sym = (Elf##BITS##_Sym *)((const char *)symtab + i * syment); \ + if (sym->st_value && !strcmp(strtab + sym->st_name, want)) \ + return sym->st_value; \ + } \ + \ + return 0; \ +} +ELF_DYNSYM_VALUE(32) +ELF_DYNSYM_VALUE(64) + +unsigned long elf_dynsym_value(const char *path, const char *sym) +{ + unsigned long val = 0; + size_t size = 0; + char *map; + int clazz; + + map = elf_map_file(path, &size); + if (!map) + return 0; + + clazz = map[EI_CLASS]; + if (clazz == ELFCLASS32) + val = elf32_dynsym_value(map, size, sym); + else if (clazz == ELFCLASS64) + val = elf64_dynsym_value(map, size, sym); + + munmap(map, size); + return val; +} + +int elf_interp(const char *path, char *out, size_t outlen) +{ + unsigned long off, size_pt; + size_t size = 0; + char *map; + int ret = -1; + + map = elf_map_file(path, &size); + if (!map) + return -1; + + if (!elf_find_section(map, size, PT_INTERP, &off, &size_pt, NULL) && off < size) { + snprintf(out, outlen, "%s", map + off); + ret = 0; + } + + munmap(map, size); + return ret; +} + static void load_ldso_conf(const char *conf) { FILE* fp = fopen(conf, "r"); diff --git a/jail/elf.h b/jail/elf.h index 046b377..6f6ede2 100644 --- a/jail/elf.h +++ b/jail/elf.h @@ -31,6 +31,8 @@ extern struct avl_tree libraries; void alloc_library(const char *path, const char *name); int elf_load_deps(const char *path, const char *map, unsigned long map_size); +unsigned long elf_dynsym_value(const char *path, const char *sym); +int elf_interp(const char *path, char *out, size_t outlen); const char* find_lib(const char *file); void init_library_search(void); int lib_open(char **fullpath, const char *file); diff --git a/jail/jail.c b/jail/jail.c index 8d77844..8a900f6 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -58,6 +59,7 @@ #include #include #include +#include #include "capabilities.h" #include "elf.h" @@ -66,6 +68,7 @@ #include "landlock.h" #include "log.h" #include "seccomp-oci.h" +#include "seccomp-inject.h" #include "cgroups.h" #include "netifd.h" @@ -125,6 +128,10 @@ static struct { char *cwd; char *seccomp; struct sock_fprog *ociseccomp; + struct sock_fprog *ociseccomp_linker; + struct sock_fprog *ociseccomp_init; + struct sock_fprog *ociseccomp_delta_entry; + struct sock_fprog *ociseccomp_delta_main; char *capabilities; struct jail_capset capset; char *user; @@ -320,6 +327,26 @@ static void free_opts(bool parent) { free(opts.ociseccomp); } + if (opts.ociseccomp_linker) { + free(opts.ociseccomp_linker->filter); + free(opts.ociseccomp_linker); + } + + if (opts.ociseccomp_init) { + free(opts.ociseccomp_init->filter); + free(opts.ociseccomp_init); + } + + if (opts.ociseccomp_delta_entry) { + free(opts.ociseccomp_delta_entry->filter); + free(opts.ociseccomp_delta_entry); + } + + if (opts.ociseccomp_delta_main) { + free(opts.ociseccomp_delta_main->filter); + free(opts.ociseccomp_delta_main); + } + free_oci_envp(opts.jail_argv); free_oci_envp(opts.envp); } @@ -1917,32 +1944,15 @@ static int apply_rlimits(void) } #define MAX_ENVP 64 -static char** build_envp(const char *seccomp, char **ocienvp) +static char** build_envp(char **ocienvp) { static char *envp[MAX_ENVP]; - static char preload_var[PATH_MAX]; - static char seccomp_var[PATH_MAX]; - static char seccomp_debug_var[20]; static char debug_var[] = "LD_DEBUG=all"; static char container_var[] = "container=ujail"; - const char *preload_lib = find_lib("libpreload-seccomp.so"); char **addenv; int count = 0; - if (seccomp && !preload_lib) { - ERROR("failed to add preload-lib to env\n"); - return NULL; - } - if (seccomp) { - snprintf(seccomp_var, sizeof(seccomp_var), "SECCOMP_FILE=%s", seccomp); - envp[count++] = seccomp_var; - snprintf(seccomp_debug_var, sizeof(seccomp_debug_var), "SECCOMP_DEBUG=%2d", debug); - envp[count++] = seccomp_debug_var; - snprintf(preload_var, sizeof(preload_var), "LD_PRELOAD=%s", preload_lib); - envp[count++] = preload_var; - } - envp[count++] = container_var; if (debug > 1) @@ -1959,6 +1969,54 @@ static char** build_envp(const char *seccomp, char **ocienvp) return envp; } +static int build_oci_seccomp(struct blob_attr *msg) +{ + opts.ociseccomp = parseOCIlinuxseccomp(msg, NULL); + if (!opts.ociseccomp) + return -1; + + if (!seccomp_profile_covers(opts.ociseccomp, seccomp_init_base)) { + opts.ociseccomp_init = parseOCIlinuxseccomp(msg, seccomp_init_base); + if (!opts.ociseccomp_init) + return -1; + } + + if (!seccomp_profile_covers(opts.ociseccomp, seccomp_linker_base)) { + opts.ociseccomp_linker = parseOCIlinuxseccomp(msg, seccomp_linker_base); + if (!opts.ociseccomp_linker) + return -1; + + opts.ociseccomp_delta_entry = seccomp_deny_delta(seccomp_loader_files, + opts.ociseccomp); + opts.ociseccomp_delta_main = seccomp_deny_delta(seccomp_init_base, + opts.ociseccomp); + } + + return 0; +} + +static int seccomp_compile_file(const char *json_path) +{ + struct blob_buf b = { 0 }; + int rc; + + blob_buf_init(&b, 0); + if (!blobmsg_add_json_from_file(&b, json_path)) { + ERROR("seccomp: failed to load %s\n", json_path); + blob_buf_free(&b); + return -1; + } + + rc = build_oci_seccomp(b.head); + blob_buf_free(&b); + if (rc) { + ERROR("seccomp: failed to parse %s\n", json_path); + return -1; + } + + return 0; +} + static void usage(void) { fprintf(stderr, "ujail -- \n"); @@ -2896,7 +2954,7 @@ static void post_start_hook(void) free_and_exit(EXIT_FAILURE); } - char **envp = build_envp(opts.seccomp, opts.envp); + char **envp = build_envp(opts.envp); if (!envp) free_and_exit(EXIT_FAILURE); @@ -2908,12 +2966,18 @@ static void post_start_hook(void) free_and_exit(EXIT_FAILURE); } - if (opts.ociseccomp && applyOCIlinuxseccomp(opts.ociseccomp, opts.name, opts.ocibundle)) + if (opts.ociseccomp && seccomp_oci_needs_inproc() && + applyOCIlinuxseccomp(opts.ociseccomp_linker ?: opts.ociseccomp, opts.name, opts.ocibundle)) free_and_exit(EXIT_FAILURE); uloop_end(); free_opts(false); - INFO("exec-ing %s\n", *opts.jail_argv); + if (opts.ociseccomp && !seccomp_oci_needs_inproc() && + ptrace(PTRACE_TRACEME, 0, 0, 0)) { + ERROR("PTRACE_TRACEME failed: %m\n"); + exit(EXIT_FAILURE); + } + DEBUG("exec-ing %s\n", *opts.jail_argv); if (opts.envp) { /* respect PATH if potentially set in ENV */ environ = envp; execvpe(*opts.jail_argv, opts.jail_argv, envp); @@ -4106,8 +4170,7 @@ static int parseOCIlinux(struct blob_attr *msg) } if (tb[OCI_LINUX_SECCOMP]) { - opts.ociseccomp = parseOCIlinuxseccomp(tb[OCI_LINUX_SECCOMP]); - if (!opts.ociseccomp) + if (build_oci_seccomp(tb[OCI_LINUX_SECCOMP])) return EINVAL; } @@ -5754,8 +5817,9 @@ int main(int argc, char **argv) } } - if (opts.namespace && opts.seccomp && add_path_and_deps("libpreload-seccomp.so", 1, -1, 1)) { - ERROR("failed to load libpreload-seccomp.so\n"); + if (opts.seccomp && !opts.ociseccomp && + seccomp_compile_file(opts.seccomp)) { + ERROR("failed to compile seccomp filter %s\n", opts.seccomp); opts.seccomp = 0; if (opts.require_jail) { ret=-1; @@ -6218,6 +6282,145 @@ static void post_create_runtime(void) pipe_send_start_container(NULL); } +static bool seccomp_target_is_static(pid_t pid) +{ + unsigned long pair[2]; + bool dynamic = false; + char path[32]; + FILE *f; + + snprintf(path, sizeof(path), "/proc/%d/auxv", (int)pid); + f = fopen(path, "rb"); + if (!f) + return false; + + while (fread(pair, sizeof(pair), 1, f) == 1) { + if (pair[0] == 0) + break; + if (pair[0] == 7 && pair[1]) + dynamic = true; + } + fclose(f); + return !dynamic; +} + +static bool seccomp_main_trackable(pid_t pid) +{ + unsigned long at_entry, lsm; + int main_argidx; + + if (seccomp_marker_addrs(pid, &at_entry, &lsm, &main_argidx)) + return false; + + return lsm != 0; +} + +static int jail_seccomp_handshake(void) +{ + int status, mrc; + + if (!opts.ociseccomp || seccomp_oci_needs_inproc()) + return 0; + + while (waitpid(jail_process.pid, &status, 0) < 0) { + if (errno == EINTR) + continue; + ERROR("seccomp-inject: waitpid: %m\n"); + return -1; + } + + if (!WIFSTOPPED(status)) { + ERROR("seccomp-inject: jail exited before entrypoint exec\n"); + uloop_process_delete(&jail_process); + jail_process_handler(&jail_process, status); + return -1; + } + + if (seccomp_target_is_static(jail_process.pid)) { + if (opts.ociseccomp_init && + seccomp_main_trackable(jail_process.pid)) { + if (seccomp_inject(jail_process.pid, opts.ociseccomp_init)) { + ERROR("seccomp-inject: failed to arm init filter\n"); + ptrace(PTRACE_KILL, jail_process.pid, 0, 0); + return -1; + } + if (seccomp_run_to_main(jail_process.pid)) { + ERROR("seccomp-inject: failed to reach main\n"); + ptrace(PTRACE_KILL, jail_process.pid, 0, 0); + return -1; + } + if (opts.ociseccomp_delta_main && + seccomp_inject(jail_process.pid, opts.ociseccomp_delta_main)) { + ERROR("seccomp-inject: failed to arm main delta\n"); + ptrace(PTRACE_KILL, jail_process.pid, 0, 0); + return -1; + } + } else if (seccomp_inject(jail_process.pid, opts.ociseccomp)) { + ERROR("seccomp-inject: failed to arm filter\n"); + ptrace(PTRACE_KILL, jail_process.pid, 0, 0); + return -1; + } + if (ptrace(PTRACE_DETACH, jail_process.pid, 0, 0)) { + ERROR("seccomp-inject: PTRACE_DETACH: %m\n"); + return -1; + } + return 0; + } + + if (!opts.ociseccomp_linker) { + if (seccomp_inject(jail_process.pid, opts.ociseccomp)) { + ERROR("seccomp-inject: failed to arm filter\n"); + ptrace(PTRACE_KILL, jail_process.pid, 0, 0); + return -1; + } + if (ptrace(PTRACE_DETACH, jail_process.pid, 0, 0)) { + ERROR("seccomp-inject: PTRACE_DETACH: %m\n"); + return -1; + } + return 0; + } + + if (seccomp_inject(jail_process.pid, opts.ociseccomp_linker)) { + ERROR("seccomp-inject: failed to arm linker filter\n"); + ptrace(PTRACE_KILL, jail_process.pid, 0, 0); + return -1; + } + + if (seccomp_run_to_entry(jail_process.pid)) { + ERROR("seccomp-inject: failed to reach entry point\n"); + ptrace(PTRACE_KILL, jail_process.pid, 0, 0); + return -1; + } + + if (opts.ociseccomp_delta_entry && + seccomp_inject(jail_process.pid, opts.ociseccomp_delta_entry)) { + ERROR("seccomp-inject: failed to arm entry delta\n"); + ptrace(PTRACE_KILL, jail_process.pid, 0, 0); + return -1; + } + + mrc = seccomp_run_to_main_from_entry(jail_process.pid); + if (mrc < 0) { + ERROR("seccomp-inject: failed to reach main\n"); + ptrace(PTRACE_KILL, jail_process.pid, 0, 0); + return -1; + } + + if (opts.ociseccomp_delta_main && + seccomp_inject(jail_process.pid, opts.ociseccomp_delta_main)) { + ERROR("seccomp-inject: failed to arm main delta\n"); + ptrace(PTRACE_KILL, jail_process.pid, 0, 0); + return -1; + } + + if (ptrace(PTRACE_DETACH, jail_process.pid, 0, 0)) { + ERROR("seccomp-inject: PTRACE_DETACH: %m\n"); + return -1; + } + + return 0; +} + static void pipe_send_start_container(struct uloop_timeout *t) { char sig_buf[1]; @@ -6230,6 +6433,9 @@ static void pipe_send_start_container(struct uloop_timeout *t) } close(pipes[3]); + if (jail_seccomp_handshake()) + free_and_exit(-1); + run_hooks(opts.hooks.poststart, post_poststart); } diff --git a/jail/preload.c b/jail/preload.c deleted file mode 100644 index 351a9f8..0000000 --- a/jail/preload.c +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright (C) 2015 John Crispin - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License version 2.1 - * as published by the Free Software Foundation - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ - -#define _GNU_SOURCE -#include -#include -#include -#include -#include - -#include "log.h" -#include "seccomp.h" -#include "../preload.h" - -static main_t __main__; -int debug; - -static int __preload_main__(int argc, char **argv, char **envp) -{ - char *env_file = getenv("SECCOMP_FILE"); - char *env_debug = getenv("SECCOMP_DEBUG"); - - if (!env_file || !env_file[0]) { - ERROR("SECCOMP_FILE not specified\n"); - return -1; - } - - if (env_debug) - debug = atoi(env_debug); - else - debug = 0; - - if (install_syscall_filter(*argv, env_file)) - return -1; - - unsetenv("LD_PRELOAD"); - unsetenv("SECCOMP_DEBUG"); - unsetenv("SECCOMP_FILE"); - - return (*__main__)(argc, argv, envp); -} - -int __libc_start_main(main_t main, - int argc, - char **argv, - ElfW(auxv_t) *auxvec, - __typeof (main) init, - void (*fini) (void), - void (*rtld_fini) (void), - void *stack_end) -{ - start_main_t __start_main__; - - __start_main__ = dlsym(RTLD_NEXT, "__libc_start_main"); - if (!__start_main__) { - INFO("failed to find __libc_start_main %s\n", dlerror()); - return -1; - } - - __main__ = main; - - return (*__start_main__)(__preload_main__, argc, argv, auxvec, - init, fini, rtld_fini, stack_end); -} - -void __uClibc_main(main_t main, - int argc, - char **argv, - void (*app_init)(void), - void (*app_fini)(void), - void (*rtld_fini)(void), - void *stack_end attribute_unused) -{ - uClibc_main __start_main__; - - __start_main__ = dlsym(RTLD_NEXT, "__uClibc_main"); - if (!__start_main__) { - INFO("failed to find __uClibc_main %s\n", dlerror()); - return; - } - - __main__ = main; - - return (*__start_main__)(__preload_main__, argc, argv, - app_init, app_fini, rtld_fini, stack_end); -} diff --git a/jail/seccomp-inject.c b/jail/seccomp-inject.c new file mode 100644 index 0000000..bad0141 --- /dev/null +++ b/jail/seccomp-inject.c @@ -0,0 +1,1450 @@ +/* + * Apply a compiled cBPF seccomp filter to a freshly-execve'd workload by + * injecting prctl(PR_SET_NO_NEW_PRIVS) and seccomp(SET_MODE_FILTER) into the + * tracee via ptrace. The tracee does PTRACE_TRACEME before its final execve; + * the parent catches the post-execve stop (before the workload's first + * userspace instruction), pokes the filter into the tracee's stack, drives the + * two syscalls by single-stepping a temporary trap instruction at the program + * counter, restores the saved registers and code, then detaches. This arms the + * filter for statically and dynamically linked workloads alike, closing the + * gap left by an LD_PRELOAD-based installer (which static binaries ignore). + * + * Copyright (C) 2026 Daniel Golle + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License version 2.1 + * as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "log.h" +#include "seccomp-inject.h" +#include "elf.h" + +#ifndef PR_SET_NO_NEW_PRIVS +#define PR_SET_NO_NEW_PRIVS 38 +#endif +#ifndef SECCOMP_SET_MODE_FILTER +#define SECCOMP_SET_MODE_FILTER 1 +#endif +#ifndef AT_NULL +#define AT_NULL 0 +#endif +#ifndef AT_ENTRY +#define AT_ENTRY 9 +#endif +#ifndef AT_BASE +#define AT_BASE 7 +#endif +#ifndef NT_PRSTATUS +#define NT_PRSTATUS 1 +#endif + +#if defined(__x86_64__) +typedef struct user_regs_struct inj_regs; +#define INJ_SYSCALL_ASM "syscall" +#define INJ_BP_ASM "int3" +static unsigned long inj_pc(const inj_regs *r) +{ + return r->rip; +} + +static void inj_set_pc(inj_regs *r, unsigned long pc) +{ + r->rip = pc; +} + +static unsigned long inj_sp(const inj_regs *r) +{ + return r->rsp; +} + +static long inj_ret(const inj_regs *r) +{ + return r->rax; +} + +static void inj_clear_restart(inj_regs *r) +{ + r->orig_rax = (unsigned long)-1; +} +static void inj_set_call(inj_regs *r, long nr, long a0, long a1, long a2, + long a3, long a4, long a5) +{ + r->rax = nr; + r->orig_rax = nr; + r->rdi = a0; + r->rsi = a1; + r->rdx = a2; + r->r10 = a3; + r->r8 = a4; + r->r9 = a5; +} + +static void inj_get_call(const inj_regs *r, long *nr, long *args) +{ + *nr = r->orig_rax; + args[0] = r->rdi; + args[1] = r->rsi; + args[2] = r->rdx; + args[3] = r->r10; + args[4] = r->r8; + args[5] = r->r9; +} + +#elif defined(__i386__) +typedef struct user_regs_struct inj_regs; +#define INJ_SYSCALL_ASM "int $0x80" +#define INJ_BP_ASM "int3" +static unsigned long inj_pc(const inj_regs *r) +{ + return r->eip; +} + +static void inj_set_pc(inj_regs *r, unsigned long pc) +{ + r->eip = pc; +} + +static unsigned long inj_sp(const inj_regs *r) +{ + return r->esp; +} + +static long inj_ret(const inj_regs *r) +{ + return r->eax; +} + +static void inj_clear_restart(inj_regs *r) +{ + r->orig_eax = (unsigned long)-1; +} +static void inj_set_call(inj_regs *r, long nr, long a0, long a1, long a2, + long a3, long a4, long a5) +{ + r->eax = nr; + r->orig_eax = nr; + r->ebx = a0; + r->ecx = a1; + r->edx = a2; + r->esi = a3; + r->edi = a4; + r->ebp = a5; +} + +static void inj_get_call(const inj_regs *r, long *nr, long *args) +{ + *nr = r->orig_eax; + args[0] = r->ebx; + args[1] = r->ecx; + args[2] = r->edx; + args[3] = r->esi; + args[4] = r->edi; + args[5] = r->ebp; +} + +#elif defined(__aarch64__) +typedef struct { unsigned long long regs[31], sp, pc, pstate; } inj_regs; +#define INJ_SYSCALL_ASM "svc #0" +#define INJ_BP_ASM "brk #0" +static unsigned long inj_pc(const inj_regs *r) +{ + return r->pc; +} + +static void inj_set_pc(inj_regs *r, unsigned long pc) +{ + r->pc = pc; +} + +static unsigned long inj_sp(const inj_regs *r) +{ + return r->sp; +} + +static long inj_ret(const inj_regs *r) +{ + return r->regs[0]; +} + +static void inj_clear_restart(inj_regs *r) +{ + (void)r; +} +static void inj_set_call(inj_regs *r, long nr, long a0, long a1, long a2, + long a3, long a4, long a5) +{ + r->regs[8] = nr; + r->regs[0] = a0; + r->regs[1] = a1; + r->regs[2] = a2; + r->regs[3] = a3; + r->regs[4] = a4; + r->regs[5] = a5; +} + +static void inj_get_call(const inj_regs *r, long *nr, long *args) +{ + *nr = r->regs[8]; + args[0] = r->regs[0]; + args[1] = r->regs[1]; + args[2] = r->regs[2]; + args[3] = r->regs[3]; + args[4] = r->regs[4]; + args[5] = r->regs[5]; +} + +#elif defined(__arm__) +typedef struct { unsigned long uregs[18]; } inj_regs; +#define INJ_SYSCALL_ASM "svc #0" +static unsigned long inj_pc(const inj_regs *r) +{ + return r->uregs[15]; +} + +static void inj_set_pc(inj_regs *r, unsigned long pc) +{ + r->uregs[15] = pc; +} + +static unsigned long inj_sp(const inj_regs *r) +{ + return r->uregs[13]; +} + +static long inj_ret(const inj_regs *r) +{ + return r->uregs[0]; +} + +static void inj_clear_restart(inj_regs *r) __attribute__((unused)); +static void inj_clear_restart(inj_regs *r) +{ + (void)r; +} +static void inj_set_call(inj_regs *r, long nr, long a0, long a1, long a2, + long a3, long a4, long a5) +{ + r->uregs[7] = nr; + r->uregs[0] = a0; + r->uregs[1] = a1; + r->uregs[2] = a2; + r->uregs[3] = a3; + r->uregs[4] = a4; + r->uregs[5] = a5; +} + +static void inj_get_call(const inj_regs *r, long *nr, long *args) +{ + *nr = r->uregs[7]; + args[0] = r->uregs[0]; + args[1] = r->uregs[1]; + args[2] = r->uregs[2]; + args[3] = r->uregs[3]; + args[4] = r->uregs[4]; + args[5] = r->uregs[5]; +} + +static int inj_thumb(const inj_regs *r) +{ + return (r->uregs[16] >> 5) & 1; +} + +#elif defined(__riscv) && __riscv_xlen == 64 +typedef struct { + unsigned long pc, ra, sp, gp, tp, t0, t1, t2, s0, s1; + unsigned long a0, a1, a2, a3, a4, a5, a6, a7; + unsigned long s2, s3, s4, s5, s6, s7, s8, s9, s10, s11; + unsigned long t3, t4, t5, t6; +} inj_regs; +#define INJ_SYSCALL_ASM "ecall" +#define INJ_BP_ASM "ebreak" +#define INJ_NO_SINGLESTEP 1 +static unsigned long inj_pc(const inj_regs *r) +{ + return r->pc; +} + +static void inj_set_pc(inj_regs *r, unsigned long pc) +{ + r->pc = pc; +} + +static unsigned long inj_sp(const inj_regs *r) +{ + return r->sp; +} + +static long inj_ret(const inj_regs *r) +{ + return r->a0; +} + +static void inj_clear_restart(inj_regs *r) +{ + (void)r; +} +static void inj_set_call(inj_regs *r, long nr, long a0, long a1, long a2, + long a3, long a4, long a5) +{ + r->a7 = nr; + r->a0 = a0; + r->a1 = a1; + r->a2 = a2; + r->a3 = a3; + r->a4 = a4; + r->a5 = a5; +} + +static void inj_get_call(const inj_regs *r, long *nr, long *args) +{ + *nr = r->a7; + args[0] = r->a0; + args[1] = r->a1; + args[2] = r->a2; + args[3] = r->a3; + args[4] = r->a4; + args[5] = r->a5; +} + +#elif defined(__loongarch__) && __loongarch_grlen == 64 +typedef struct { unsigned long regs[32], orig_a0, csr_era, csr_badv, reserved[10]; } inj_regs; +#define INJ_SYSCALL_ASM "syscall 0" +#define INJ_BP_ASM "break 0" +#define INJ_NO_SINGLESTEP 1 +static unsigned long inj_pc(const inj_regs *r) +{ + return r->csr_era; +} + +static void inj_set_pc(inj_regs *r, unsigned long pc) +{ + r->csr_era = pc; +} + +static unsigned long inj_sp(const inj_regs *r) +{ + return r->regs[3]; +} + +static long inj_ret(const inj_regs *r) +{ + return r->regs[4]; +} + +static void inj_clear_restart(inj_regs *r) +{ + (void)r; +} +static void inj_set_call(inj_regs *r, long nr, long a0, long a1, long a2, + long a3, long a4, long a5) +{ + r->regs[11] = nr; + r->regs[4] = a0; + r->regs[5] = a1; + r->regs[6] = a2; + r->regs[7] = a3; + r->regs[8] = a4; + r->regs[9] = a5; +} + +static void inj_get_call(const inj_regs *r, long *nr, long *args) +{ + *nr = r->regs[11]; + args[0] = r->regs[4]; + args[1] = r->regs[5]; + args[2] = r->regs[6]; + args[3] = r->regs[7]; + args[4] = r->regs[8]; + args[5] = r->regs[9]; +} + +#elif defined(__mips__) +#ifndef ELF_NGREG +#define ELF_NGREG 45 +#endif +#if _MIPS_SIM == _ABIO32 +#define MIPS_EF_V0 8 +#define MIPS_EF_A0 10 +#define MIPS_EF_SP 35 +#define MIPS_EF_A3 13 +#define MIPS_EF_EPC 40 +#else +#define MIPS_EF_V0 2 +#define MIPS_EF_A0 4 +#define MIPS_EF_SP 29 +#define MIPS_EF_A3 7 +#define MIPS_EF_EPC 34 +#endif +typedef struct { unsigned long gregs[ELF_NGREG]; } inj_regs; +#define INJ_SYSCALL_ASM "syscall" +#define INJ_BP_ASM "break" +#define INJ_NO_SINGLESTEP 1 +static unsigned long inj_pc(const inj_regs *r) +{ + return r->gregs[MIPS_EF_EPC]; +} + +static void inj_set_pc(inj_regs *r, unsigned long pc) +{ + r->gregs[MIPS_EF_EPC] = pc; +} + +static unsigned long inj_sp(const inj_regs *r) +{ + return r->gregs[MIPS_EF_SP]; +} + +#if _MIPS_SIM == _ABIO32 +static void inj_set_sp(inj_regs *r, unsigned long sp) +{ + r->gregs[MIPS_EF_SP] = sp; +} +#endif + +static long inj_ret(const inj_regs *r) +{ + if (r->gregs[MIPS_EF_A3]) + return -(long)r->gregs[MIPS_EF_V0]; + + return r->gregs[MIPS_EF_V0]; +} + +static void inj_clear_restart(inj_regs *r) +{ + (void)r; +} +static void inj_set_call(inj_regs *r, long nr, long a0, long a1, long a2, + long a3, long a4, long a5) +{ + r->gregs[MIPS_EF_V0] = nr; + r->gregs[MIPS_EF_A0] = a0; + r->gregs[MIPS_EF_A0 + 1] = a1; + r->gregs[MIPS_EF_A0 + 2] = a2; + r->gregs[MIPS_EF_A0 + 3] = a3; +#if _MIPS_SIM != _ABIO32 + r->gregs[MIPS_EF_A0 + 4] = a4; + r->gregs[MIPS_EF_A0 + 5] = a5; + r->gregs[MIPS_EF_A0 + 6] = 0; + r->gregs[MIPS_EF_A0 + 7] = 0; +#else + (void)a4; + (void)a5; +#endif +} + +static void inj_get_call(const inj_regs *r, long *nr, long *args) +{ + *nr = r->gregs[MIPS_EF_V0]; + args[0] = r->gregs[MIPS_EF_A0]; + args[1] = r->gregs[MIPS_EF_A0 + 1]; + args[2] = r->gregs[MIPS_EF_A0 + 2]; + args[3] = r->gregs[MIPS_EF_A0 + 3]; +#if _MIPS_SIM != _ABIO32 + args[4] = r->gregs[MIPS_EF_A0 + 4]; + args[5] = r->gregs[MIPS_EF_A0 + 5]; +#else + args[4] = 0; + args[5] = 0; +#endif +} + +#elif defined(__powerpc64__) +typedef struct { + unsigned long gpr[32]; + unsigned long nip, msr, orig_gpr3, ctr, link, xer, ccr, softe; + unsigned long trap, dar, dsisr, result; +} inj_regs; +#define INJ_SYSCALL_ASM "sc" +#define INJ_BP_ASM "trap" +static unsigned long inj_pc(const inj_regs *r) +{ + return r->nip; +} + +static void inj_set_pc(inj_regs *r, unsigned long pc) +{ + r->nip = pc; +} + +static unsigned long inj_sp(const inj_regs *r) +{ + return r->gpr[1]; +} + +static long inj_ret(const inj_regs *r) +{ + if (r->ccr & 0x10000000UL) + return -(long)r->gpr[3]; + return r->gpr[3]; +} + +static void inj_clear_restart(inj_regs *r) +{ + (void)r; +} +static void inj_set_call(inj_regs *r, long nr, long a0, long a1, long a2, + long a3, long a4, long a5) +{ + r->gpr[0] = nr; + r->gpr[3] = a0; + r->gpr[4] = a1; + r->gpr[5] = a2; + r->gpr[6] = a3; + r->gpr[7] = a4; + r->gpr[8] = a5; +} + +static void inj_get_call(const inj_regs *r, long *nr, long *args) +{ + *nr = r->gpr[0]; + args[0] = r->gpr[3]; + args[1] = r->gpr[4]; + args[2] = r->gpr[5]; + args[3] = r->gpr[6]; + args[4] = r->gpr[7]; + args[5] = r->gpr[8]; +} + +#elif defined(__powerpc__) +typedef struct { + unsigned long gpr[32]; + unsigned long nip, msr, orig_gpr3, ctr, link, xer, ccr, mq; + unsigned long trap, dar, dsisr, result; +} inj_regs; +#define INJ_SYSCALL_ASM "sc" +#define INJ_BP_ASM "trap" +static unsigned long inj_pc(const inj_regs *r) +{ + return r->nip; +} + +static void inj_set_pc(inj_regs *r, unsigned long pc) +{ + r->nip = pc; +} + +static unsigned long inj_sp(const inj_regs *r) +{ + return r->gpr[1]; +} + +static long inj_ret(const inj_regs *r) +{ + if (r->ccr & 0x10000000UL) + return -(long)r->gpr[3]; + return r->gpr[3]; +} + +static void inj_clear_restart(inj_regs *r) +{ + (void)r; +} +static void inj_set_call(inj_regs *r, long nr, long a0, long a1, long a2, + long a3, long a4, long a5) +{ + r->gpr[0] = nr; + r->gpr[3] = a0; + r->gpr[4] = a1; + r->gpr[5] = a2; + r->gpr[6] = a3; + r->gpr[7] = a4; + r->gpr[8] = a5; +} + +static void inj_get_call(const inj_regs *r, long *nr, long *args) +{ + *nr = r->gpr[0]; + args[0] = r->gpr[3]; + args[1] = r->gpr[4]; + args[2] = r->gpr[5]; + args[3] = r->gpr[6]; + args[4] = r->gpr[7]; + args[5] = r->gpr[8]; +} + +#else +#error "unsupported architecture for seccomp ptrace injection" +#endif + +#if defined(__arm__) +__asm__ ( + ".pushsection .text\n" + ".arm\n" + ".globl inj_syscall_insn_arm\ninj_syscall_insn_arm:\n\t" INJ_SYSCALL_ASM "\n" + ".globl inj_syscall_insn_arm_end\ninj_syscall_insn_arm_end:\n" + ".globl inj_bp_insn_arm\ninj_bp_insn_arm:\n\t.inst 0xe7f001f0\n" + ".globl inj_bp_insn_arm_end\ninj_bp_insn_arm_end:\n" + ".thumb\n" + ".globl inj_syscall_insn_thumb\ninj_syscall_insn_thumb:\n\t" INJ_SYSCALL_ASM "\n" + ".globl inj_syscall_insn_thumb_end\ninj_syscall_insn_thumb_end:\n" + ".globl inj_bp_insn_thumb\ninj_bp_insn_thumb:\n\t.inst.n 0xde01\n" + ".globl inj_bp_insn_thumb_end\ninj_bp_insn_thumb_end:\n" + ".popsection\n" +); +extern const unsigned char inj_syscall_insn_arm[], inj_syscall_insn_arm_end[]; +extern const unsigned char inj_syscall_insn_thumb[], inj_syscall_insn_thumb_end[]; +extern const unsigned char inj_bp_insn_arm[], inj_bp_insn_arm_end[]; +extern const unsigned char inj_bp_insn_thumb[], inj_bp_insn_thumb_end[]; + +#define INJ_INSN_MAX 16 +#else +__asm__ ( + ".pushsection .text\n" + ".globl inj_syscall_insn\ninj_syscall_insn:\n\t" INJ_SYSCALL_ASM "\n" + ".globl inj_syscall_insn_end\ninj_syscall_insn_end:\n" + ".globl inj_bp_insn\ninj_bp_insn:\n\t" INJ_BP_ASM "\n" + ".globl inj_bp_insn_end\ninj_bp_insn_end:\n" + ".popsection\n" +); +extern const unsigned char inj_syscall_insn[], inj_syscall_insn_end[]; +extern const unsigned char inj_bp_insn[], inj_bp_insn_end[]; + +#define inj_syscall_len() ((size_t)(inj_syscall_insn_end - inj_syscall_insn)) +#define inj_bp_len() ((size_t)(inj_bp_insn_end - inj_bp_insn)) +#define INJ_INSN_MAX 16 +#endif + +static int inj_getregs(pid_t pid, inj_regs *regs) +{ + struct iovec iov; + + iov.iov_base = regs; + iov.iov_len = sizeof(*regs); + + return ptrace(PTRACE_GETREGSET, pid, (void *)NT_PRSTATUS, &iov); +} + +static int inj_setregs(pid_t pid, inj_regs *regs) +{ + struct iovec iov; + + iov.iov_base = regs; + iov.iov_len = sizeof(*regs); + + return ptrace(PTRACE_SETREGSET, pid, (void *)NT_PRSTATUS, &iov); +} + +int seccomp_read_syscall(pid_t pid, long *nr, long *args) +{ + inj_regs regs; + + if (inj_getregs(pid, ®s)) + return -1; + + inj_get_call(®s, nr, args); + + return 0; +} + +static int inj_poke(pid_t pid, unsigned long addr, const void *src, size_t len) +{ + unsigned long word; + size_t off, chunk; + + for (off = 0; off < len; off += sizeof(long)) { + chunk = (len - off < sizeof(long)) ? (len - off) : sizeof(long); + errno = 0; + word = ptrace(PTRACE_PEEKTEXT, pid, (void *)(addr + off), 0); + if (word == (unsigned long)-1 && errno) + return -1; + memcpy(&word, (const char *)src + off, chunk); + if (ptrace(PTRACE_POKETEXT, pid, (void *)(addr + off), (void *)word)) + return -1; + } + + return 0; +} + +static int inj_peek(pid_t pid, unsigned long addr, void *dst, size_t len) +{ + unsigned long word; + size_t off, chunk; + + for (off = 0; off < len; off += sizeof(long)) { + chunk = (len - off < sizeof(long)) ? (len - off) : sizeof(long); + errno = 0; + word = ptrace(PTRACE_PEEKTEXT, pid, (void *)(addr + off), 0); + if (word == (unsigned long)-1 && errno) + return -1; + memcpy((char *)dst + off, &word, chunk); + } + + return 0; +} + +#if !defined(__arm__) +#ifdef INJ_NO_SINGLESTEP +static int inj_step(pid_t pid, unsigned long pc, int *status) +{ + unsigned long bpaddr = pc + inj_syscall_len(); + unsigned char saved[INJ_INSN_MAX]; + size_t bplen = inj_bp_len(); + int rc = -1; + + if (inj_peek(pid, bpaddr, saved, bplen)) + return -1; + if (inj_poke(pid, bpaddr, inj_bp_insn, bplen)) + return -1; + + if (ptrace(PTRACE_CONT, pid, 0, 0)) + goto out; + if (waitpid(pid, status, 0) < 0) + goto out; + rc = 0; + +out: + inj_poke(pid, bpaddr, saved, bplen); + return rc; +} +#else +static int inj_step(pid_t pid, unsigned long pc, int *status) +{ + (void)pc; + + if (ptrace(PTRACE_SINGLESTEP, pid, 0, 0)) + return -1; + if (waitpid(pid, status, 0) < 0) + return -1; + + return 0; +} +#endif + +static int inj_call(pid_t pid, const inj_regs *base, unsigned long pc, + long nr, long a0, long a1, long a2, + long a3, long a4, long a5, long *ret) +{ + inj_regs regs; + int status; +#if defined(__mips__) && _MIPS_SIM == _ABIO32 + unsigned long scratch; + unsigned long stkargs[4]; +#endif + + for (;;) { + regs = *base; + inj_set_pc(®s, pc); + inj_set_call(®s, nr, a0, a1, a2, a3, a4, a5); +#if defined(__mips__) && _MIPS_SIM == _ABIO32 + scratch = (inj_sp(base) - 256) & ~0xfUL; + stkargs[0] = a4; + stkargs[1] = a5; + stkargs[2] = 0; + stkargs[3] = 0; + inj_set_sp(®s, scratch); + if (inj_poke(pid, scratch + 16, stkargs, sizeof(stkargs))) + return -1; +#endif + if (inj_setregs(pid, ®s)) + return -1; + + if (inj_step(pid, pc, &status)) + return -1; + if (WIFEXITED(status) || WIFSIGNALED(status)) + return -1; + if (inj_getregs(pid, ®s)) + return -1; + if (inj_pc(®s) != pc) + break; + } + + *ret = inj_ret(®s); + + return 0; +} +#endif + +static int read_auxv(pid_t pid, unsigned long type, unsigned long *val) +{ + char path[64]; + unsigned long pair[2]; + int fd; + ssize_t n; + int ret = -1; + + snprintf(path, sizeof(path), "/proc/%d/auxv", (int)pid); + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return -1; + + while ((n = read(fd, pair, sizeof(pair))) == (ssize_t)sizeof(pair)) { + if (pair[0] == type) { + *val = pair[1]; + ret = 0; + break; + } + if (pair[0] == AT_NULL) + break; + } + + close(fd); + return ret; +} + +#if defined(__arm__) + +static int arm_thumb_at(unsigned long entry) +{ + return (int)(entry & 1UL); +} + +static const unsigned char *arm_bp_insn(int thumb, size_t *len) +{ + if (thumb) { + *len = (size_t)(inj_bp_insn_thumb_end - inj_bp_insn_thumb); + return inj_bp_insn_thumb; + } + *len = (size_t)(inj_bp_insn_arm_end - inj_bp_insn_arm); + return inj_bp_insn_arm; +} + +static const unsigned char *arm_syscall_insn(int thumb, size_t *len) +{ + if (thumb) { + *len = (size_t)(inj_syscall_insn_thumb_end - inj_syscall_insn_thumb); + return inj_syscall_insn_thumb; + } + *len = (size_t)(inj_syscall_insn_arm_end - inj_syscall_insn_arm); + return inj_syscall_insn_arm; +} + +static int inj_call_arm(pid_t pid, const inj_regs *base, unsigned long pc, + int thumb, long nr, long a0, long a1, long a2, + long a3, long a4, long a5, long *ret) +{ + const unsigned char *sci, *bpi; + size_t scilen, bpilen; + unsigned char saved[INJ_INSN_MAX]; + inj_regs regs; + int status; + + sci = arm_syscall_insn(thumb, &scilen); + bpi = arm_bp_insn(thumb, &bpilen); + + if (inj_peek(pid, pc, saved, scilen + bpilen)) + return -1; + if (inj_poke(pid, pc, sci, scilen)) + return -1; + if (inj_poke(pid, pc + scilen, bpi, bpilen)) + goto restore; + + regs = *base; + inj_set_pc(®s, pc); + inj_set_call(®s, nr, a0, a1, a2, a3, a4, a5); + if (inj_setregs(pid, ®s)) + goto restore; + + if (ptrace(PTRACE_CONT, pid, 0, 0)) + goto restore; + if (waitpid(pid, &status, 0) < 0) + goto restore; + if (!WIFSTOPPED(status)) + goto restore; + + if (inj_getregs(pid, ®s)) + goto restore; + + inj_poke(pid, pc, saved, scilen + bpilen); + *ret = inj_ret(®s); + + return 0; + +restore: + inj_poke(pid, pc, saved, scilen + bpilen); + return -1; +} + +static int seccomp_run_to_entry_arm(pid_t pid) +{ + const unsigned char *bp; + size_t bplen; + unsigned char saved[INJ_INSN_MAX]; + unsigned long entry, addr; + inj_regs regs; + int status, sig = 0, thumb; + + if (read_auxv(pid, AT_ENTRY, &entry)) { + ERROR("seccomp-inject: cannot read AT_ENTRY: %m\n"); + return -1; + } + + thumb = arm_thumb_at(entry); + addr = entry & ~1UL; + bp = arm_bp_insn(thumb, &bplen); + + if (inj_peek(pid, addr, saved, bplen)) { + ERROR("seccomp-inject: read entry: %m\n"); + return -1; + } + if (inj_poke(pid, addr, bp, bplen)) { + ERROR("seccomp-inject: poke entry breakpoint: %m\n"); + return -1; + } + + for (;;) { + if (ptrace(PTRACE_CONT, pid, 0, (void *)(long)sig)) { + ERROR("seccomp-inject: PTRACE_CONT to entry: %m\n"); + goto restore; + } + if (waitpid(pid, &status, 0) < 0) { + ERROR("seccomp-inject: waitpid to entry: %m\n"); + goto restore; + } + if (!WIFSTOPPED(status)) { + ERROR("seccomp-inject: workload exited before entry\n"); + return -1; + } + if (WSTOPSIG(status) == SIGTRAP) + break; + sig = WSTOPSIG(status); + } + + if (inj_getregs(pid, ®s)) { + ERROR("seccomp-inject: GETREGSET at entry: %m\n"); + goto restore; + } + inj_set_pc(®s, addr); + if (inj_setregs(pid, ®s)) { + ERROR("seccomp-inject: SETREGSET at entry: %m\n"); + goto restore; + } + inj_poke(pid, addr, saved, bplen); + + return 0; + +restore: + inj_poke(pid, addr, saved, bplen); + return -1; +} + +static int seccomp_inject_arm(pid_t pid, struct sock_fprog *prog) +{ + inj_regs saved; + struct sock_fprog rprog; + unsigned long sp, pc, filter_addr, prog_addr; + size_t filterlen; + long ret; + int thumb; + + if (inj_getregs(pid, &saved)) { + ERROR("seccomp-inject: GETREGSET: %m\n"); + return -1; + } + + thumb = inj_thumb(&saved); + + pc = inj_pc(&saved); + sp = inj_sp(&saved); + filterlen = (size_t)prog->len * sizeof(struct sock_filter); + + filter_addr = (sp - 4096 - filterlen - sizeof(rprog)) & ~0xfUL; + prog_addr = (filter_addr + filterlen + 0xf) & ~0xfUL; + + if (inj_poke(pid, filter_addr, prog->filter, filterlen)) { + ERROR("seccomp-inject: poke filter: %m\n"); + return -1; + } + + rprog.len = prog->len; + rprog.filter = (struct sock_filter *)(uintptr_t)filter_addr; + if (inj_poke(pid, prog_addr, &rprog, sizeof(rprog))) { + ERROR("seccomp-inject: poke fprog: %m\n"); + return -1; + } + + if (inj_call_arm(pid, &saved, pc, thumb, SYS_prctl, + PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0, 0, &ret)) { + ERROR("seccomp-inject: drive prctl: %m\n"); + goto restore; + } + if (ret) { + ERROR("seccomp-inject: PR_SET_NO_NEW_PRIVS returned %ld\n", ret); + goto restore; + } + + if (inj_call_arm(pid, &saved, pc, thumb, SYS_seccomp, + SECCOMP_SET_MODE_FILTER, 0, (long)prog_addr, 0, 0, 0, &ret)) { + ERROR("seccomp-inject: drive seccomp: %m\n"); + goto restore; + } + if (ret) { + ERROR("seccomp-inject: seccomp(SET_MODE_FILTER) returned %ld\n", ret); + goto restore; + } + + if (inj_setregs(pid, &saved)) { + ERROR("seccomp-inject: restore SETREGSET: %m\n"); + return -1; + } + + return 0; + +restore: + inj_setregs(pid, &saved); + return -1; +} +#endif + +#if !defined(__arm__) +static int run_to_addr(pid_t pid, unsigned long addr) +{ + const unsigned char *bp = inj_bp_insn; + size_t bplen = inj_bp_len(); + unsigned char saved[INJ_INSN_MAX]; + inj_regs regs; + int status, sig = 0; + + if (inj_peek(pid, addr, saved, bplen)) { + ERROR("seccomp-inject: read bp target: %m\n"); + return -1; + } + if (inj_poke(pid, addr, bp, bplen)) { + ERROR("seccomp-inject: poke breakpoint: %m\n"); + return -1; + } + + for (;;) { + if (ptrace(PTRACE_CONT, pid, 0, (void *)(long)sig)) { + ERROR("seccomp-inject: PTRACE_CONT to bp: %m\n"); + goto restore; + } + if (waitpid(pid, &status, 0) < 0) { + ERROR("seccomp-inject: waitpid to bp: %m\n"); + goto restore; + } + if (!WIFSTOPPED(status)) { + ERROR("seccomp-inject: workload exited before reaching target\n"); + return -1; + } + if (WSTOPSIG(status) == SIGTRAP) + break; + sig = WSTOPSIG(status); + } + + if (inj_getregs(pid, ®s)) { + ERROR("seccomp-inject: GETREGSET at bp: %m\n"); + goto restore; + } + inj_set_pc(®s, addr); + if (inj_setregs(pid, ®s)) { + ERROR("seccomp-inject: SETREGSET at bp: %m\n"); + goto restore; + } + inj_poke(pid, addr, saved, bplen); + + return 0; + +restore: + inj_poke(pid, addr, saved, bplen); + return -1; +} +#endif + +int seccomp_run_to_entry(pid_t pid) +{ +#if defined(__arm__) + return seccomp_run_to_entry_arm(pid); +#else + unsigned long entry; + + if (read_auxv(pid, AT_ENTRY, &entry)) { + ERROR("seccomp-inject: cannot read AT_ENTRY: %m\n"); + return -1; + } + + return run_to_addr(pid, entry); +#endif +} + +#if !defined(__arm__) && !defined(__i386__) +static unsigned long find_libc_init(pid_t pid, int *main_argidx) +{ + char path[64], line[600], rooted[PATH_MAX + 64], p[PATH_MAX]; + char seen[48][256]; + unsigned long bases[48]; + unsigned long lo, hi, val; + int nseen = 0, i; + FILE *f; + + *main_argidx = 0; + snprintf(path, sizeof(path), "/proc/%d/maps", (int)pid); + f = fopen(path, "r"); + if (!f) + return 0; + + while (fgets(line, sizeof(line), f)) { + if (sscanf(line, "%lx-%lx %*s %*s %*s %*s %4095s", &lo, &hi, p) < 3) + continue; + if (p[0] != '/') + continue; + for (i = 0; i < nseen; i++) + if (!strcmp(seen[i], p)) + break; + if (i < nseen || nseen >= 48) + continue; + strncpy(seen[nseen], p, sizeof(seen[0]) - 1); + seen[nseen][sizeof(seen[0]) - 1] = '\0'; + bases[nseen] = lo; + nseen++; + } + fclose(f); + + for (i = 0; i < nseen; i++) { + snprintf(rooted, sizeof(rooted), "/proc/%d/root%s", (int)pid, seen[i]); + val = elf_dynsym_value(rooted, "__libc_start_main"); + if (val) { + *main_argidx = 0; + return bases[i] + val; + } + } + for (i = 0; i < nseen; i++) { + snprintf(rooted, sizeof(rooted), "/proc/%d/root%s", (int)pid, seen[i]); + val = elf_dynsym_value(rooted, "__libc_init"); + if (val) { + *main_argidx = 2; + return bases[i] + val; + } + } + return 0; +} +#endif + +int seccomp_marker_addrs(pid_t pid, unsigned long *at_entry, unsigned long *lsm, int *main_argidx) +{ + *at_entry = 0; + *lsm = 0; + *main_argidx = 0; + + if (read_auxv(pid, AT_ENTRY, at_entry)) + return -1; + +#if !defined(__arm__) && !defined(__i386__) + *lsm = find_libc_init(pid, main_argidx); +#endif + + return 0; +} + +int seccomp_run_to_main(pid_t pid) +{ +#if defined(__arm__) + return seccomp_run_to_entry(pid); +#else + unsigned long at_entry, lsm, mainaddr; + long nr, args[6]; + int main_argidx; + + if (seccomp_marker_addrs(pid, &at_entry, &lsm, &main_argidx)) + return -1; + if (!lsm) + return seccomp_run_to_entry(pid); + + if (run_to_addr(pid, lsm)) + return -1; + if (seccomp_read_syscall(pid, &nr, args)) + return -1; + + mainaddr = (unsigned long)args[main_argidx]; + if (!mainaddr) + return -1; + + return run_to_addr(pid, mainaddr); +#endif +} + +int seccomp_run_to_main_from_entry(pid_t pid) +{ +#if defined(__arm__) + return 1; +#else + unsigned long at_entry, lsm, mainaddr; + long nr, args[6]; + int main_argidx; + + if (seccomp_marker_addrs(pid, &at_entry, &lsm, &main_argidx)) + return -1; + if (!lsm) + return 1; + + if (run_to_addr(pid, lsm)) + return -1; + if (seccomp_read_syscall(pid, &nr, args)) + return -1; + + mainaddr = (unsigned long)args[main_argidx]; + if (!mainaddr) + return -1; + + if (run_to_addr(pid, mainaddr)) + return -1; + + return 0; +#endif +} + +int seccomp_bp_arm(pid_t pid, unsigned long addr, struct seccomp_bp *bp) +{ +#if defined(__arm__) + const unsigned char *ins; + size_t len; + int thumb; + + thumb = arm_thumb_at(addr); + bp->addr = addr & ~1UL; + ins = arm_bp_insn(thumb, &len); +#else + const unsigned char *ins = inj_bp_insn; + size_t len = inj_bp_len(); + + bp->addr = addr; +#endif + + if (len > sizeof(bp->saved)) + return -1; + if (inj_peek(pid, bp->addr, bp->saved, len)) + return -1; + if (inj_poke(pid, bp->addr, ins, len)) + return -1; + + bp->len = (unsigned char)len; + bp->armed = 1; + + return 0; +} + +int seccomp_bp_match(pid_t pid, struct seccomp_bp **bps, int n) +{ + inj_regs regs; + unsigned long pc; + int i; + + if (inj_getregs(pid, ®s)) + return -1; + + pc = inj_pc(®s); +#if defined(__arm__) + pc &= ~1UL; +#endif + + for (i = 0; i < n; i++) { + if (!bps[i] || !bps[i]->armed) + continue; + if (pc != bps[i]->addr && pc != bps[i]->addr + bps[i]->len) + continue; + + inj_poke(pid, bps[i]->addr, bps[i]->saved, bps[i]->len); + inj_set_pc(®s, bps[i]->addr); + if (inj_setregs(pid, ®s)) + return -1; + bps[i]->armed = 0; + + return i; + } + + return -1; +} + +int seccomp_force_errno(pid_t pid, int err) +{ +#if defined(__aarch64__) + inj_regs regs; + int scno = -1; + struct iovec iov; + + iov.iov_base = &scno; + iov.iov_len = sizeof(scno); + if (ptrace(PTRACE_SETREGSET, pid, (void *)NT_ARM_SYSTEM_CALL, &iov)) + return -1; + if (inj_getregs(pid, ®s)) + return -1; + regs.regs[0] = (unsigned long long)(long long)-err; + if (inj_setregs(pid, ®s)) + return -1; + + return 0; +#elif defined(__arm__) + inj_regs regs; + + if (ptrace(PTRACE_SET_SYSCALL, pid, 0, (void *)-1L)) + return -1; + if (inj_getregs(pid, ®s)) + return -1; + regs.uregs[0] = (unsigned long)(long)-err; + if (inj_setregs(pid, ®s)) + return -1; + + return 0; +#elif defined(__mips__) + inj_regs regs; + + if (inj_getregs(pid, ®s)) + return -1; + regs.gregs[MIPS_EF_V0] = (unsigned long)(long)-1; + if (inj_setregs(pid, ®s)) + return -1; + + return 1; +#elif defined(__x86_64__) || defined(__i386__) || \ + (defined(__riscv) && __riscv_xlen == 64) || \ + (defined(__loongarch__) && __loongarch_grlen == 64) || \ + defined(__powerpc64__) || defined(__powerpc__) + inj_regs regs; + + if (inj_getregs(pid, ®s)) + return -1; + +#if defined(__x86_64__) + regs.orig_rax = (unsigned long)-1; + regs.rax = (unsigned long)(long)-err; +#elif defined(__i386__) + regs.orig_eax = (unsigned long)-1; + regs.eax = (unsigned long)(long)-err; +#elif defined(__riscv) && __riscv_xlen == 64 + regs.a7 = (unsigned long)-1; + regs.a0 = (unsigned long)(long)-err; +#elif defined(__loongarch__) && __loongarch_grlen == 64 + regs.regs[11] = (unsigned long)-1; + regs.regs[4] = (unsigned long)(long)-err; +#else + regs.gpr[0] = (unsigned long)-1; + regs.gpr[3] = (unsigned long)err; + regs.ccr |= 0x10000000UL; +#endif + + if (inj_setregs(pid, ®s)) + return -1; + + return 0; +#else + (void)pid; + (void)err; + return -1; +#endif +} + +int seccomp_force_errno_exit(pid_t pid, int err) +{ +#if defined(__mips__) + inj_regs regs; + + if (inj_getregs(pid, ®s)) + return -1; + + regs.gregs[MIPS_EF_V0] = (unsigned long)(long)err; + regs.gregs[MIPS_EF_A3] = 1; + + if (inj_setregs(pid, ®s)) + return -1; + + return 0; +#else + (void)pid; + (void)err; + return -1; +#endif +} + +int seccomp_inject(pid_t pid, struct sock_fprog *prog) +{ +#if defined(__arm__) + return seccomp_inject_arm(pid, prog); +#else + inj_regs saved; + struct sock_fprog rprog; + unsigned char savedinsn[INJ_INSN_MAX]; + unsigned long sp, pc, filter_addr, prog_addr; + size_t filterlen; + long ret; + + if (inj_getregs(pid, &saved)) { + ERROR("seccomp-inject: GETREGSET: %m\n"); + return -1; + } + + pc = inj_pc(&saved); + sp = inj_sp(&saved); + filterlen = (size_t)prog->len * sizeof(struct sock_filter); + + filter_addr = (sp - 4096 - filterlen - sizeof(rprog)) & ~0xfUL; + prog_addr = (filter_addr + filterlen + 0xf) & ~0xfUL; + + if (inj_peek(pid, pc, savedinsn, inj_syscall_len())) { + ERROR("seccomp-inject: read pc: %m\n"); + return -1; + } + if (inj_poke(pid, pc, inj_syscall_insn, inj_syscall_len())) { + ERROR("seccomp-inject: poke trap: %m\n"); + return -1; + } + if (inj_poke(pid, filter_addr, prog->filter, filterlen)) { + ERROR("seccomp-inject: poke filter: %m\n"); + goto restore; + } + + rprog.len = prog->len; + rprog.filter = (struct sock_filter *)(uintptr_t)filter_addr; + if (inj_poke(pid, prog_addr, &rprog, sizeof(rprog))) { + ERROR("seccomp-inject: poke fprog: %m\n"); + goto restore; + } + + if (inj_call(pid, &saved, pc, SYS_prctl, + PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0, 0, &ret)) { + ERROR("seccomp-inject: drive prctl: %m\n"); + goto restore; + } + if (ret) { + ERROR("seccomp-inject: PR_SET_NO_NEW_PRIVS returned %ld\n", ret); + goto restore; + } + + if (inj_call(pid, &saved, pc, SYS_seccomp, + SECCOMP_SET_MODE_FILTER, 0, (long)prog_addr, 0, 0, 0, &ret)) { + ERROR("seccomp-inject: drive seccomp: %m\n"); + goto restore; + } + if (ret) { + ERROR("seccomp-inject: seccomp(SET_MODE_FILTER) returned %ld\n", ret); + goto restore; + } + + inj_poke(pid, pc, savedinsn, inj_syscall_len()); + inj_clear_restart(&saved); + if (inj_setregs(pid, &saved)) { + ERROR("seccomp-inject: restore SETREGSET: %m\n"); + return -1; + } + + return 0; + +restore: + inj_poke(pid, pc, savedinsn, inj_syscall_len()); + inj_clear_restart(&saved); + inj_setregs(pid, &saved); + + return -1; +#endif +} diff --git a/jail/seccomp-inject.h b/jail/seccomp-inject.h new file mode 100644 index 0000000..a4618fb --- /dev/null +++ b/jail/seccomp-inject.h @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2026 Daniel Golle + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License version 2.1 + * as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#ifndef _JAIL_SECCOMP_INJECT_H_ +#define _JAIL_SECCOMP_INJECT_H_ + +#include +#include + +struct seccomp_bp { + unsigned long addr; + unsigned char saved[16]; + unsigned char len; + unsigned char armed; +}; + +#ifdef SECCOMP_SUPPORT +int seccomp_inject(pid_t pid, struct sock_fprog *prog); +int seccomp_run_to_entry(pid_t pid); +int seccomp_run_to_main(pid_t pid); +int seccomp_run_to_main_from_entry(pid_t pid); +int seccomp_read_syscall(pid_t pid, long *nr, long *args); +int seccomp_marker_addrs(pid_t pid, unsigned long *at_entry, unsigned long *lsm, int *main_argidx); +int seccomp_bp_arm(pid_t pid, unsigned long addr, struct seccomp_bp *bp); +int seccomp_bp_match(pid_t pid, struct seccomp_bp **bps, int n); +int seccomp_force_errno(pid_t pid, int err); +int seccomp_force_errno_exit(pid_t pid, int err); +#else +static inline int seccomp_inject(pid_t pid, struct sock_fprog *prog) { + return -1; +} +static inline int seccomp_run_to_entry(pid_t pid) { + return -1; +} +static inline int seccomp_run_to_main(pid_t pid) { + return -1; +} +static inline int seccomp_run_to_main_from_entry(pid_t pid) { + return -1; +} +static inline int seccomp_read_syscall(pid_t pid, long *nr, long *args) { + return -1; +} +static inline int seccomp_marker_addrs(pid_t pid, unsigned long *at_entry, unsigned long *lsm, int *main_argidx) { + return -1; +} +static inline int seccomp_bp_arm(pid_t pid, unsigned long addr, struct seccomp_bp *bp) { + return -1; +} +static inline int seccomp_bp_match(pid_t pid, struct seccomp_bp **bps, int n) { + return -1; +} +static inline int seccomp_force_errno(pid_t pid, int err) { + return -1; +} +static inline int seccomp_force_errno_exit(pid_t pid, int err) { + return -1; +} +#endif + +#endif diff --git a/jail/seccomp-oci.c b/jail/seccomp-oci.c index e33daa7..7ba163f 100644 --- a/jail/seccomp-oci.c +++ b/jail/seccomp-oci.c @@ -22,6 +22,7 @@ */ #define _GNU_SOURCE 1 #include +#include #include #include #include @@ -79,6 +80,12 @@ static unsigned long seccomp_filter_flags; static char *seccomp_listener_path; static char *seccomp_listener_metadata; static bool seccomp_uses_notify; +static uint32_t seccomp_default_action; + +bool seccomp_oci_needs_inproc(void) +{ + return seccomp_uses_notify || seccomp_filter_flags != 0; +} static uint32_t resolve_action(char *actname) { @@ -203,6 +210,26 @@ static uint32_t resolve_architecture(char *archname) } } +const char * const seccomp_linker_base[] = { + "access", "arch_prctl", "brk", "close", "faccessat", "fcntl", "fstat", + "fstatfs", "futex", "getrandom", "mmap", "mprotect", "munmap", + "newfstatat", "open", "openat", "pread64", "prctl", "prlimit64", "read", + "readlinkat", "rseq", "rt_sigaction", "sched_getscheduler", + "set_robust_list", "set_tid_address", "sigaltstack", "statfs", NULL, +}; + +const char * const seccomp_init_base[] = { + "arch_prctl", "brk", "futex", "getrandom", "mmap", "mprotect", "munmap", + "prctl", "prlimit64", "rseq", "rt_sigaction", "sched_getscheduler", + "set_robust_list", "set_tid_address", "sigaltstack", NULL, +}; + +const char * const seccomp_loader_files[] = { + "access", "close", "faccessat", "fcntl", "fstat", "fstatfs", + "newfstatat", "open", "openat", "pread64", "read", "readlinkat", + "statfs", NULL, +}; + enum { OCI_LINUX_SECCOMP_DEFAULTACTION, OCI_LINUX_SECCOMP_DEFAULTERRNORET, @@ -256,7 +283,139 @@ static const struct blobmsg_policy oci_linux_seccomp_syscalls_args_policy[] = { #define SECCOMP_CHUNK_NAMES 240 -struct sock_fprog *parseOCIlinuxseccomp(struct blob_attr *msg) +#ifndef SECCOMP_RET_ACTION_FULL +#define SECCOMP_RET_ACTION_FULL 0xffff0000U +#endif + +static bool seccomp_nr_allows(const struct sock_fprog *prog, int nr, + unsigned int pc, bool known, uint32_t acc, + int *budget) +{ + const struct sock_filter *f; + uint32_t k; + + if (--(*budget) < 0) + return false; + + while (pc < prog->len) { + f = &prog->filter[pc]; + k = f->k; + + switch (f->code) { + case BPF_LD + BPF_W + BPF_ABS: + if (k == (uint32_t)arch_nr) { + known = true; + acc = ARCH_NR; + } else if (k == (uint32_t)syscall_nr) { + known = true; + acc = (uint32_t)nr; + } else { + known = false; + } + pc++; + break; + case BPF_ALU + BPF_AND + BPF_K: + if (known) + acc &= k; + pc++; + break; + case BPF_JMP + BPF_JEQ + BPF_K: + if (!known) + return seccomp_nr_allows(prog, nr, pc + 1 + f->jt, false, 0, budget) && + seccomp_nr_allows(prog, nr, pc + 1 + f->jf, false, 0, budget); + pc += 1 + ((acc == k) ? f->jt : f->jf); + break; + case BPF_JMP + BPF_JGE + BPF_K: + if (!known) + return seccomp_nr_allows(prog, nr, pc + 1 + f->jt, false, 0, budget) && + seccomp_nr_allows(prog, nr, pc + 1 + f->jf, false, 0, budget); + pc += 1 + ((acc >= k) ? f->jt : f->jf); + break; + case BPF_JMP + BPF_JGT + BPF_K: + if (!known) + return seccomp_nr_allows(prog, nr, pc + 1 + f->jt, false, 0, budget) && + seccomp_nr_allows(prog, nr, pc + 1 + f->jf, false, 0, budget); + pc += 1 + ((acc > k) ? f->jt : f->jf); + break; + case BPF_RET + BPF_K: + return (k & SECCOMP_RET_ACTION_FULL) == SECCOMP_RET_ALLOW; + default: + return false; + } + } + + return false; +} + +bool seccomp_profile_covers(const struct sock_fprog *prog, const char * const *names) +{ + int i, sc, budget; + + if (!prog) + return false; + + for (i = 0; names && names[i]; i++) { + sc = find_syscall(names[i]); + if (sc == -1) + continue; + budget = 4096; + if (!seccomp_nr_allows(prog, sc, 0, false, 0, &budget)) + return false; + } + + return true; +} + +struct sock_fprog *seccomp_deny_delta(const char * const *names, + const struct sock_fprog *app) +{ + struct sock_filter *filter; + struct sock_fprog *prog; + int deny[64], ndeny = 0; + int i, sc, budget, idx = 0, sz; + + for (i = 0; names && names[i]; i++) { + if (ndeny >= (int)(sizeof(deny) / sizeof(deny[0]))) + break; + sc = find_syscall(names[i]); + if (sc == -1) + continue; + if (app) { + budget = 4096; + if (seccomp_nr_allows(app, sc, 0, false, 0, &budget)) + continue; + } + deny[ndeny++] = sc; + } + + if (!ndeny) + return NULL; + + sz = ndeny + 5; + filter = calloc(sz, sizeof(*filter)); + if (!filter) + return NULL; + + filter[idx++] = (struct sock_filter)BPF_STMT(BPF_LD + BPF_W + BPF_ABS, arch_nr); + filter[idx++] = (struct sock_filter)BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, ARCH_NR, 0, ndeny + 1); + filter[idx++] = (struct sock_filter)BPF_STMT(BPF_LD + BPF_W + BPF_ABS, syscall_nr); + for (i = 0; i < ndeny; i++) + filter[idx++] = (struct sock_filter)BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, deny[i], ndeny - i, 0); + filter[idx++] = (struct sock_filter)BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_ALLOW); + filter[idx++] = (struct sock_filter)BPF_STMT(BPF_RET + BPF_K, seccomp_default_action); + + prog = calloc(1, sizeof(*prog)); + if (!prog) { + free(filter); + return NULL; + } + prog->len = sz; + prog->filter = filter; + return prog; +} + +struct sock_fprog *parseOCIlinuxseccomp(struct blob_attr *msg, + const char * const *extra_allow) { struct blob_attr *tb[__OCI_LINUX_SECCOMP_MAX]; struct blob_attr *tbn[__OCI_LINUX_SECCOMP_SYSCALLS_MAX]; @@ -270,6 +429,7 @@ struct sock_fprog *parseOCIlinuxseccomp(struct blob_attr *msg) uint32_t seccomp_arch; bool arch_matched; char *op_str; + int m = 0, i, emitted = 0; blobmsg_parse(oci_linux_seccomp_policy, __OCI_LINUX_SECCOMP_MAX, tb, blobmsg_data(msg), blobmsg_len(msg)); @@ -328,6 +488,8 @@ struct sock_fprog *parseOCIlinuxseccomp(struct blob_attr *msg) return NULL; } + seccomp_default_action = default_policy; + /* verify architecture while ignoring the x86_64 anomaly for now */ if (tb[OCI_LINUX_SECCOMP_ARCHITECTURES]) { arch_matched = false; @@ -398,6 +560,16 @@ struct sock_fprog *parseOCIlinuxseccomp(struct blob_attr *msg) sz += chunks * (1 + 1 + arg_instrs) + valid_names; } + for (i = 0; extra_allow && extra_allow[i]; i++) + if (find_syscall(extra_allow[i]) != -1) + ++m; + + if (extra_allow && find_syscall("seccomp") != -1) + ++m; + + if (m > 0) + sz += m + 2; + if (sz < 6) { ERROR("seccomp: filter is empty\n"); return NULL; @@ -552,8 +724,31 @@ struct sock_fprog *parseOCIlinuxseccomp(struct blob_attr *msg) } } + if (m > 0) { + set_filter(&filter[idx++], BPF_LD + BPF_W + BPF_ABS, 0, 0, syscall_nr); + + for (i = 0; extra_allow[i]; i++) { + sc = find_syscall(extra_allow[i]); + if (sc == -1) + continue; + set_filter(&filter[idx++], BPF_JMP + BPF_JEQ + BPF_K, + m - emitted, 0, sc); + ++emitted; + } + + sc = find_syscall("seccomp"); + if (sc != -1) { + set_filter(&filter[idx++], BPF_JMP + BPF_JEQ + BPF_K, + m - emitted, 0, sc); + ++emitted; + } + } + set_filter(&filter[idx++], BPF_RET + BPF_K, 0, 0, default_policy); + if (m > 0) + set_filter(&filter[idx++], BPF_RET + BPF_K, 0, 0, SECCOMP_RET_ALLOW); + assert(idx == sz); prog->len = (unsigned short) idx; diff --git a/jail/seccomp-oci.h b/jail/seccomp-oci.h index cec81d6..5720be1 100644 --- a/jail/seccomp-oci.h +++ b/jail/seccomp-oci.h @@ -13,14 +13,32 @@ #ifndef _JAIL_SECCOMP_OCI_H_ #define _JAIL_SECCOMP_OCI_H_ +#include +#include #include -struct sock_fprog *parseOCIlinuxseccomp(struct blob_attr *msg); +#ifdef SECCOMP_SUPPORT +extern const char * const seccomp_linker_base[]; +extern const char * const seccomp_init_base[]; +extern const char * const seccomp_loader_files[]; +#else +static const char * const seccomp_linker_base[] = { NULL }; +static const char * const seccomp_init_base[] = { NULL }; +static const char * const seccomp_loader_files[] = { NULL }; +#endif + +struct sock_fprog *parseOCIlinuxseccomp(struct blob_attr *msg, + const char * const *extra_allow); +struct sock_fprog *seccomp_deny_delta(const char * const *names, + const struct sock_fprog *app); int applyOCIlinuxseccomp(struct sock_fprog *prog, const char *container_id, const char *bundle_path); +bool seccomp_oci_needs_inproc(void); +bool seccomp_profile_covers(const struct sock_fprog *prog, const char * const *names); #ifndef SECCOMP_SUPPORT -struct sock_fprog *parseOCIlinuxseccomp(struct blob_attr *msg) { +struct sock_fprog *parseOCIlinuxseccomp(struct blob_attr *msg, + const char * const *extra_allow) { return NULL; } @@ -28,6 +46,10 @@ int applyOCIlinuxseccomp(struct sock_fprog *prog, const char *container_id, const char *bundle_path) { return ENOTSUP; } + +bool seccomp_oci_needs_inproc(void) { + return false; +} #endif #endif diff --git a/jail/seccomp.c b/jail/seccomp.c deleted file mode 100644 index fb80f93..0000000 --- a/jail/seccomp.c +++ /dev/null @@ -1,45 +0,0 @@ -/* - * seccomp example with syscall reporting - * - * Copyright (c) 2012 The Chromium OS Authors - * Authors: - * Kees Cook - * Will Drewry - * - * Use of this source code is governed by a BSD-style license that can be - * found in the LICENSE file. - */ -#define _GNU_SOURCE 1 -#include -#include -#include - -#include -#include -#include - -#include "log.h" -#include "seccomp.h" -#include "seccomp-oci.h" - -int install_syscall_filter(const char *argv, const char *file) -{ - struct blob_buf b = { 0 }; - struct sock_fprog *prog = NULL; - - DEBUG("%s: setting up syscall filter\n", argv); - - blob_buf_init(&b, 0); - if (!blobmsg_add_json_from_file(&b, file)) { - ERROR("%s: failed to load %s\n", argv, file); - return -1; - } - - prog = parseOCIlinuxseccomp(b.head); - if (!prog) { - ERROR("%s: failed to parse seccomp filter rules %s\n", argv, file); - return -1; - } - - return applyOCIlinuxseccomp(prog, NULL, NULL); -} diff --git a/jail/seccomp.h b/jail/seccomp.h deleted file mode 100644 index b0c8d30..0000000 --- a/jail/seccomp.h +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright (C) 2015 John Crispin - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License version 2.1 - * as published by the Free Software Foundation - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ -#ifndef _JAIL_SECCOMP_H_ -#define _JAIL_SECCOMP_H_ - -#include -#include - -int install_syscall_filter(const char *argv, const char *file); - -#endif diff --git a/trace/trace.c b/trace/trace.c index 47c2fef..99cc5d2 100644 --- a/trace/trace.c +++ b/trace/trace.c @@ -43,6 +43,7 @@ #include #include +#include "../jail/seccomp-oci.h" #include "../syscall-names.h" #define _offsetof(a, b) __builtin_offsetof(a,b) @@ -86,7 +87,7 @@ static struct tracee tracer; static int syscall_count[SYSCALL_COUNT]; static int violation_count; static struct blob_buf b; -static int debug; +int debug; char *json = NULL; int ptrace_restart; @@ -362,14 +363,10 @@ int main(int argc, char **argv, char **envp) newenv = 1; break; case SECCOMP_TRACE: - preload = "/lib/libpreload-seccomp.so"; - newenv = 2; - if (asprintf(&_envp[1], "SECCOMP_FILE=%s", json ? json : "") < 0) - ULOG_ERR("failed to allocate SECCOMP_FILE env: %m\n"); - - kill(getpid(), SIGSTOP); - break; + ULOG_ERR("seccomp-trace is no longer supported\n"); + return -1; } + if (asprintf(&_envp[0], "LD_PRELOAD=%s%s%s", preload, old_preload ? ":" : "", old_preload ? old_preload : "") < 0) From ef2d5fc5e09d72d6d40e2c1bb2d8db00c85afd16 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 05:31:30 +0100 Subject: [PATCH 28/82] jail: add seccomp trace, audit and complain modes Add a ptrace-based syscall tracer, selected via -m (enforce, trace, audit or complain) with -M naming an NDJSON log. Trace records every syscall, classifying each into a startup phase (linker, init, app) resolved from entry-point and libc breakpoints, so a generated profile can cover the application phase alone. Audit and complain run the real seccomp filter but rewrite its returns to SECCOMP_RET_TRACE, logging every denial; audit then enforces (errno or kill, including the two-stop errno arches), while complain permits and only records. Events stream to a udebug ring and, as a fallback, to the NDJSON file. procd's instance config gains seccomp_mode and seccomp_log, passed through to ujail as -m and -M. Signed-off-by: Daniel Golle --- CMakeLists.txt | 4 +- jail/jail.c | 127 ++++++++-- jail/seccomp-inject.c | 6 + jail/seccomp-oci.c | 68 ++++++ jail/seccomp-oci.h | 5 + jail/seccomp-trace.c | 532 ++++++++++++++++++++++++++++++++++++++++++ jail/seccomp-trace.h | 42 ++++ service/instance.c | 36 +++ service/instance.h | 2 + 9 files changed, 806 insertions(+), 16 deletions(-) create mode 100644 jail/seccomp-trace.c create mode 100644 jail/seccomp-trace.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 8be6078..cf4d044 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -102,10 +102,11 @@ ADD_CUSTOM_TARGET(capabilities-names-h DEPENDS capabilities-names.h) IF(SECCOMP_SUPPORT) ADD_DEFINITIONS(-DSECCOMP_SUPPORT) SET(SOURCES_OCI_SECCOMP jail/seccomp-oci.c jail/seccomp-inject.c) +SET(SOURCES_JAIL_TRACE jail/seccomp-trace.c) ENDIF() IF(JAIL_SUPPORT) -ADD_EXECUTABLE(ujail jail/jail.c jail/cgroups.c jail/cgroups-bpf.c jail/elf.c jail/fs.c jail/capabilities.c jail/landlock.c jail/netifd.c ${SOURCES_OCI_SECCOMP}) +ADD_EXECUTABLE(ujail jail/jail.c jail/cgroups.c jail/cgroups-bpf.c jail/elf.c jail/fs.c jail/capabilities.c jail/landlock.c jail/netifd.c ${SOURCES_OCI_SECCOMP} ${SOURCES_JAIL_TRACE}) TARGET_LINK_LIBRARIES(ujail ${ubox} ${ubus} ${uci} ${blobmsg_json}) INSTALL(TARGETS ujail RUNTIME DESTINATION ${CMAKE_INSTALL_SBINDIR} @@ -113,6 +114,7 @@ INSTALL(TARGETS ujail ADD_DEPENDENCIES(ujail capabilities-names-h) IF(SECCOMP_SUPPORT) ADD_DEPENDENCIES(ujail syscall-names-h) + TARGET_LINK_LIBRARIES(ujail ${udebug}) ENDIF() ADD_EXECUTABLE(uxc uxc.c) diff --git a/jail/jail.c b/jail/jail.c index 8a900f6..af9d534 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -69,6 +69,7 @@ #include "log.h" #include "seccomp-oci.h" #include "seccomp-inject.h" +#include "seccomp-trace.h" #include "cgroups.h" #include "netifd.h" @@ -98,7 +99,7 @@ #define PR_MDWE_NO_INHERIT (1UL << 1) #endif -#define OPT_ARGS "cC:d:De:EfFG:h:iI:j:J:ln:NoO:pP:r:R:sS:uU:w:t:T:yY:Z" +#define OPT_ARGS "cC:d:De:EfFG:h:iI:j:J:lm:M:n:NoO:pP:r:R:sS:uU:w:t:T:yY:Z" struct hook_execvpe { char *file; @@ -132,6 +133,8 @@ static struct { struct sock_fprog *ociseccomp_init; struct sock_fprog *ociseccomp_delta_entry; struct sock_fprog *ociseccomp_delta_main; + enum seccomp_mode seccomp_mode; + char *seccomp_log; char *capabilities; struct jail_capset capset; char *user; @@ -1413,6 +1416,8 @@ static int build_jail_fs(void) } static bool exit_from_child; +static bool jail_ptrace_seccomp(void); + static void free_and_exit(int ret) { if (!exit_from_child && opts.ocibundle) { @@ -2022,6 +2027,8 @@ static void usage(void) fprintf(stderr, "ujail -- \n"); fprintf(stderr, " -d \tshow debug log (increase num to increase verbosity)\n"); fprintf(stderr, " -S \tseccomp filter config\n"); + fprintf(stderr, " -m \tseccomp mode: enforce (default), trace, audit or complain\n"); + fprintf(stderr, " -M \tseccomp trace log (NDJSON) output path\n"); fprintf(stderr, " -C \tcapabilities drop config\n"); fprintf(stderr, " -c\t\tset PR_SET_NO_NEW_PRIVS\n"); fprintf(stderr, " -n \tthe name of the jail\n"); @@ -2972,8 +2979,7 @@ static void post_start_hook(void) uloop_end(); free_opts(false); - if (opts.ociseccomp && !seccomp_oci_needs_inproc() && - ptrace(PTRACE_TRACEME, 0, 0, 0)) { + if (jail_ptrace_seccomp() && ptrace(PTRACE_TRACEME, 0, 0, 0)) { ERROR("PTRACE_TRACEME failed: %m\n"); exit(EXIT_FAILURE); } @@ -5613,6 +5619,19 @@ int main(int argc, char **argv) case 'Z': opts.systemd_cgroup = true; break; + case 'm': + if (!strcmp(optarg, "trace")) + opts.seccomp_mode = SECCOMP_MODE_TRACE; + else if (!strcmp(optarg, "audit")) + opts.seccomp_mode = SECCOMP_MODE_AUDIT; + else if (!strcmp(optarg, "complain")) + opts.seccomp_mode = SECCOMP_MODE_COMPLAIN; + else + opts.seccomp_mode = SECCOMP_MODE_ENFORCE; + break; + case 'M': + opts.seccomp_log = optarg; + break; } } @@ -5740,6 +5759,7 @@ int main(int argc, char **argv) goto errout; } if (!(opts.ocibundle||opts.namespace||opts.capabilities||opts.seccomp|| + (opts.seccomp_mode != SECCOMP_MODE_ENFORCE) || (opts.setns.net != -1) || (opts.setns.ns != -1) || (opts.setns.ipc != -1) || @@ -6282,6 +6302,40 @@ static void post_create_runtime(void) pipe_send_start_container(NULL); } +static bool jail_ptrace_seccomp(void) +{ + if (opts.seccomp_mode == SECCOMP_MODE_TRACE) + return true; + + return opts.ociseccomp && !seccomp_oci_needs_inproc(); +} + +static void jail_seccomp_run(void) +{ + struct seccomp_trace_opts t = { + .mode = opts.seccomp_mode, + .name = opts.name ? opts.name : "trace", + .log_fd = -1, + .main_boundary = (opts.seccomp_mode == SECCOMP_MODE_TRACE), + .dedup = 0, + }; + int fd = -1; + + if (opts.seccomp_log) { + fd = open(opts.seccomp_log, O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd < 0) + ERROR("seccomp-trace: cannot open log %s: %m\n", opts.seccomp_log); + t.log_fd = fd; + } + + seccomp_trace_run(jail_process.pid, &t); + + if (fd >= 0) + close(fd); + + free_and_exit(0); +} + static bool seccomp_target_is_static(pid_t pid) { unsigned long pair[2]; @@ -6317,9 +6371,10 @@ static bool seccomp_main_trackable(pid_t pid) static int jail_seccomp_handshake(void) { - int status, mrc; + struct sock_fprog *aprog; + int status, rc, mrc; - if (!opts.ociseccomp || seccomp_oci_needs_inproc()) + if (!jail_ptrace_seccomp()) return 0; while (waitpid(jail_process.pid, &status, 0) < 0) { @@ -6330,13 +6385,22 @@ static int jail_seccomp_handshake(void) } if (!WIFSTOPPED(status)) { - ERROR("seccomp-inject: jail exited before entrypoint exec\n"); + if (WIFEXITED(status)) + ERROR("seccomp-inject: jail exited (code %d) before entrypoint exec\n", WEXITSTATUS(status)); + else if (WIFSIGNALED(status)) + ERROR("seccomp-inject: jail killed by signal %d before entrypoint exec\n", WTERMSIG(status)); + else + ERROR("seccomp-inject: jail exited before entrypoint exec\n"); uloop_process_delete(&jail_process); jail_process_handler(&jail_process, status); return -1; } - if (seccomp_target_is_static(jail_process.pid)) { + if (opts.seccomp_mode == SECCOMP_MODE_TRACE) + jail_seccomp_run(); + + if (opts.seccomp_mode == SECCOMP_MODE_ENFORCE && + seccomp_target_is_static(jail_process.pid)) { if (opts.ociseccomp_init && seccomp_main_trackable(jail_process.pid)) { if (seccomp_inject(jail_process.pid, opts.ociseccomp_init)) { @@ -6367,7 +6431,7 @@ static int jail_seccomp_handshake(void) return 0; } - if (!opts.ociseccomp_linker) { + if (opts.seccomp_mode == SECCOMP_MODE_ENFORCE && !opts.ociseccomp_linker) { if (seccomp_inject(jail_process.pid, opts.ociseccomp)) { ERROR("seccomp-inject: failed to arm filter\n"); ptrace(PTRACE_KILL, jail_process.pid, 0, 0); @@ -6380,7 +6444,8 @@ static int jail_seccomp_handshake(void) return 0; } - if (seccomp_inject(jail_process.pid, opts.ociseccomp_linker)) { + if (opts.seccomp_mode == SECCOMP_MODE_ENFORCE && opts.ociseccomp_linker && + seccomp_inject(jail_process.pid, opts.ociseccomp_linker)) { ERROR("seccomp-inject: failed to arm linker filter\n"); ptrace(PTRACE_KILL, jail_process.pid, 0, 0); return -1; @@ -6392,9 +6457,35 @@ static int jail_seccomp_handshake(void) return -1; } - if (opts.ociseccomp_delta_entry && - seccomp_inject(jail_process.pid, opts.ociseccomp_delta_entry)) { - ERROR("seccomp-inject: failed to arm entry delta\n"); + if (opts.seccomp_mode == SECCOMP_MODE_AUDIT || + opts.seccomp_mode == SECCOMP_MODE_COMPLAIN) { + aprog = seccomp_oci_audit_filter(opts.ociseccomp); + if (!aprog) { + ERROR("seccomp-trace: failed to build audit filter\n"); + ptrace(PTRACE_KILL, jail_process.pid, 0, 0); + return -1; + } + rc = seccomp_inject(jail_process.pid, aprog); + free(aprog->filter); + free(aprog); + if (rc) { + ERROR("seccomp-trace: failed to arm audit filter\n"); + ptrace(PTRACE_KILL, jail_process.pid, 0, 0); + return -1; + } + jail_seccomp_run(); + } + + if (opts.seccomp_mode == SECCOMP_MODE_ENFORCE) { + if (opts.ociseccomp_delta_entry && + seccomp_inject(jail_process.pid, opts.ociseccomp_delta_entry)) { + ERROR("seccomp-inject: failed to arm entry delta\n"); + ptrace(PTRACE_KILL, jail_process.pid, 0, 0); + return -1; + } + } else if (opts.ociseccomp_init && + seccomp_inject(jail_process.pid, opts.ociseccomp_init)) { + ERROR("seccomp-inject: failed to arm init filter\n"); ptrace(PTRACE_KILL, jail_process.pid, 0, 0); return -1; } @@ -6406,9 +6497,15 @@ static int jail_seccomp_handshake(void) return -1; } - if (opts.ociseccomp_delta_main && - seccomp_inject(jail_process.pid, opts.ociseccomp_delta_main)) { - ERROR("seccomp-inject: failed to arm main delta\n"); + if (opts.seccomp_mode == SECCOMP_MODE_ENFORCE) { + if (opts.ociseccomp_delta_main && + seccomp_inject(jail_process.pid, opts.ociseccomp_delta_main)) { + ERROR("seccomp-inject: failed to arm main delta\n"); + ptrace(PTRACE_KILL, jail_process.pid, 0, 0); + return -1; + } + } else if (seccomp_inject(jail_process.pid, opts.ociseccomp)) { + ERROR("seccomp-inject: failed to arm filter\n"); ptrace(PTRACE_KILL, jail_process.pid, 0, 0); return -1; } diff --git a/jail/seccomp-inject.c b/jail/seccomp-inject.c index bad0141..7096a5e 100644 --- a/jail/seccomp-inject.c +++ b/jail/seccomp-inject.c @@ -60,6 +60,12 @@ #ifndef NT_PRSTATUS #define NT_PRSTATUS 1 #endif +#ifndef NT_ARM_SYSTEM_CALL +#define NT_ARM_SYSTEM_CALL 0x404 +#endif +#ifndef PTRACE_SET_SYSCALL +#define PTRACE_SET_SYSCALL 23 +#endif #if defined(__x86_64__) typedef struct user_regs_struct inj_regs; diff --git a/jail/seccomp-oci.c b/jail/seccomp-oci.c index 7ba163f..7d16103 100644 --- a/jail/seccomp-oci.c +++ b/jail/seccomp-oci.c @@ -76,6 +76,22 @@ #define SECCOMP_RET_USER_NOTIF 0x7fc00000U #endif +#ifndef SECCOMP_RET_ACTION_FULL +#define SECCOMP_RET_ACTION_FULL 0xffff0000U +#endif + +#ifndef SECCOMP_RET_DATA +#define SECCOMP_RET_DATA 0x0000ffffU +#endif + +#ifndef SECCOMP_RET_KILL_PROCESS +#define SECCOMP_RET_KILL_PROCESS 0x80000000U +#endif + +#ifndef SECCOMP_RET_KILL_THREAD +#define SECCOMP_RET_KILL_THREAD 0x00000000U +#endif + static unsigned long seccomp_filter_flags; static char *seccomp_listener_path; static char *seccomp_listener_metadata; @@ -774,6 +790,58 @@ struct sock_fprog *parseOCIlinuxseccomp(struct blob_attr *msg, return NULL; } +struct sock_fprog *seccomp_oci_audit_filter(const struct sock_fprog *prog) +{ + struct sock_fprog *out; + struct sock_filter *filter; + unsigned short i; + uint32_t action, data; + + if (!prog || !prog->len) + return NULL; + + out = malloc(sizeof(*out)); + if (!out) { + ERROR("seccomp: failed to allocate audit sock_fprog\n"); + return NULL; + } + + filter = calloc(prog->len, sizeof(*filter)); + if (!filter) { + ERROR("seccomp: failed to allocate audit filter\n"); + free(out); + return NULL; + } + + memcpy(filter, prog->filter, prog->len * sizeof(*filter)); + + for (i = 3; i < prog->len; i++) { + if (BPF_CLASS(filter[i].code) != BPF_RET) + continue; + if (BPF_RVAL(filter[i].code) != BPF_K) + continue; + + action = filter[i].k & SECCOMP_RET_ACTION_FULL; + data = filter[i].k & SECCOMP_RET_DATA; + + switch (action) { + case SECCOMP_RET_ERRNO: + filter[i].k = SECCOMP_RET_TRACE | data; + break; + case SECCOMP_RET_KILL_PROCESS: + case SECCOMP_RET_KILL_THREAD: + filter[i].k = SECCOMP_RET_TRACE | SECCOMP_RET_DATA; + break; + default: + break; + } + } + + out->len = prog->len; + out->filter = filter; + return out; +} + static int send_seccomp_listener_fd(int listener_fd, const char *container_id, const char *bundle_path) diff --git a/jail/seccomp-oci.h b/jail/seccomp-oci.h index 5720be1..41a963c 100644 --- a/jail/seccomp-oci.h +++ b/jail/seccomp-oci.h @@ -29,6 +29,7 @@ static const char * const seccomp_loader_files[] = { NULL }; struct sock_fprog *parseOCIlinuxseccomp(struct blob_attr *msg, const char * const *extra_allow); +struct sock_fprog *seccomp_oci_audit_filter(const struct sock_fprog *prog); struct sock_fprog *seccomp_deny_delta(const char * const *names, const struct sock_fprog *app); int applyOCIlinuxseccomp(struct sock_fprog *prog, const char *container_id, @@ -42,6 +43,10 @@ struct sock_fprog *parseOCIlinuxseccomp(struct blob_attr *msg, return NULL; } +struct sock_fprog *seccomp_oci_audit_filter(const struct sock_fprog *prog) { + return NULL; +} + int applyOCIlinuxseccomp(struct sock_fprog *prog, const char *container_id, const char *bundle_path) { return ENOTSUP; diff --git a/jail/seccomp-trace.c b/jail/seccomp-trace.c new file mode 100644 index 0000000..fa71f37 --- /dev/null +++ b/jail/seccomp-trace.c @@ -0,0 +1,532 @@ +/* + * seccomp syscall tracer for ujail + * + * Copyright (C) 2026 Daniel Golle + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License version 2.1 + * as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + +#include "log.h" +#include "seccomp-inject.h" +#include "seccomp-trace.h" +#include "../syscall-names.h" + +#ifndef __WALL +#define __WALL 0x40000000 +#endif + +#ifndef PTRACE_EVENT_EXEC +#define PTRACE_EVENT_EXEC 4 +#endif + +#ifndef PTRACE_EVENT_SECCOMP +#define PTRACE_EVENT_SECCOMP 7 +#endif + +#define SECCOMP_TRACE_KILL_DATA 0xffffU + +enum trace_phase { + PHASE_LINKER = 0, + PHASE_INIT, + PHASE_APP, +}; + +#define TRACE_NSYS 1024 + +struct trace_proc { + struct list_head list; + pid_t pid; + int phase; + int in_syscall; + int expect_main; + int main_argidx; + int errno_pending; + int pending_errno; + char comm[24]; + struct seccomp_bp bp_at_entry; + struct seccomp_bp bp_lsm; + struct seccomp_bp bp_main; +}; + +static LIST_HEAD(trace_procs); +static int trace_nprocs; + +static struct udebug ud; +static struct udebug_buf udb; +static struct udebug_buf_meta ring_meta; +static char ring_name[64]; +static int udebug_ready; + +static struct blob_buf event_b; +static int trace_mode; +static int trace_log_fd = -1; +static int trace_dedup; +static int trace_main_boundary; + +static uint32_t seen[3][TRACE_NSYS / 32]; + +static const char *phase_name(int phase) +{ + switch (phase) { + case PHASE_LINKER: + return "linker"; + case PHASE_INIT: + return "init"; + default: + return "app"; + } +} + +static int dedup_seen(int phase, long nr) +{ + uint32_t bit; + + if (!trace_dedup) + return 0; + if (nr < 0 || nr >= TRACE_NSYS || phase < 0 || phase > 2) + return 0; + + bit = 1u << (nr & 31); + if (seen[phase][nr / 32] & bit) + return 1; + + seen[phase][nr / 32] |= bit; + return 0; +} + +static void read_comm(pid_t pid, char *buf, size_t len) +{ + char path[32]; + ssize_t rd; + int fd; + + buf[0] = '\0'; + snprintf(path, sizeof(path), "/proc/%d/comm", (int)pid); + fd = open(path, O_RDONLY); + if (fd < 0) + return; + + rd = read(fd, buf, len - 1); + close(fd); + if (rd <= 0) { + buf[0] = '\0'; + return; + } + + buf[rd] = '\0'; + if (buf[rd - 1] == '\n') + buf[rd - 1] = '\0'; +} + +static void trace_emit(void) +{ + char *json; + + if (udebug_ready) { + udebug_entry_init(&udb); + udebug_entry_append(&udb, blob_data(event_b.head), + blob_len(event_b.head)); + udebug_entry_add(&udb); + } + + if (trace_log_fd >= 0) { + json = blobmsg_format_json(event_b.head, true); + if (json) { + dprintf(trace_log_fd, "%s\n", json); + free(json); + } + } +} + +static void emit_marker(struct trace_proc *p, const char *event) +{ + blob_buf_init(&event_b, 0); + blobmsg_add_string(&event_b, "event", event); + blobmsg_add_u32(&event_b, "pid", p->pid); + if (p->comm[0]) + blobmsg_add_string(&event_b, "comm", p->comm); + trace_emit(); +} + +static void emit_event(struct trace_proc *p, long nr, long *args, + const char *action, int errnoval) +{ + const char *name; + void *arr; + int i; + + blob_buf_init(&event_b, 0); + blobmsg_add_string(&event_b, "event", "syscall"); + blobmsg_add_u32(&event_b, "pid", p->pid); + if (p->comm[0]) + blobmsg_add_string(&event_b, "comm", p->comm); + blobmsg_add_string(&event_b, "phase", phase_name(p->phase)); + blobmsg_add_u32(&event_b, "nr", (uint32_t)nr); + + name = syscall_name((unsigned)nr); + if (name) + blobmsg_add_string(&event_b, "syscall", name); + + arr = blobmsg_open_array(&event_b, "args"); + for (i = 0; i < 6; i++) + blobmsg_add_u64(&event_b, NULL, (uint64_t)(unsigned long)args[i]); + blobmsg_close_array(&event_b, arr); + + blobmsg_add_string(&event_b, "action", action); + if (errnoval >= 0) + blobmsg_add_u32(&event_b, "errno", (uint32_t)errnoval); + trace_emit(); +} + +static void emit_syscall(struct trace_proc *p) +{ + long nr, args[6]; + + if (seccomp_read_syscall(p->pid, &nr, args)) + return; + if (dedup_seen(p->phase, nr)) + return; + + emit_event(p, nr, args, "allow", -1); +} + +static struct trace_proc *proc_get(pid_t pid) +{ + struct trace_proc *p; + + list_for_each_entry(p, &trace_procs, list) + if (p->pid == pid) + return p; + + return NULL; +} + +static struct trace_proc *proc_new(pid_t pid) +{ + struct trace_proc *p; + + p = calloc(1, sizeof(*p)); + if (!p) + return NULL; + + p->pid = pid; + p->phase = PHASE_APP; + list_add_tail(&p->list, &trace_procs); + trace_nprocs++; + + return p; +} + +static void proc_del(struct trace_proc *p) +{ + list_del(&p->list); + free(p); + trace_nprocs--; +} + +static void proc_arm_markers(struct trace_proc *p) +{ + unsigned long at_entry, lsm; + + p->expect_main = 0; + p->main_argidx = 0; + p->bp_at_entry.armed = 0; + p->bp_lsm.armed = 0; + p->bp_main.armed = 0; + + if (seccomp_marker_addrs(p->pid, &at_entry, &lsm, &p->main_argidx)) + return; + + if (at_entry) + seccomp_bp_arm(p->pid, at_entry, &p->bp_at_entry); +} + +static void proc_setopts(struct trace_proc *p) +{ + unsigned long opt = PTRACE_O_TRACESYSGOOD | PTRACE_O_TRACEFORK | + PTRACE_O_TRACEVFORK | PTRACE_O_TRACECLONE | + PTRACE_O_TRACEEXEC; + + if (trace_mode != SECCOMP_MODE_TRACE) + opt |= PTRACE_O_TRACESECCOMP; + + ptrace(PTRACE_SETOPTIONS, p->pid, 0, (void *)opt); +} + +static void proc_start_root(struct trace_proc *p) +{ + proc_setopts(p); + read_comm(p->pid, p->comm, sizeof(p->comm)); + p->in_syscall = 0; + + if (trace_mode == SECCOMP_MODE_TRACE) { + p->phase = PHASE_LINKER; + proc_arm_markers(p); + } else { + p->phase = PHASE_APP; + } +} + +static void proc_start_child(struct trace_proc *p) +{ + proc_setopts(p); + read_comm(p->pid, p->comm, sizeof(p->comm)); + p->phase = PHASE_APP; + p->in_syscall = 0; +} + +static void proc_exec(struct trace_proc *p) +{ + read_comm(p->pid, p->comm, sizeof(p->comm)); + p->in_syscall = 0; + + if (trace_mode == SECCOMP_MODE_TRACE) { + p->phase = PHASE_LINKER; + proc_arm_markers(p); + } +} + +static void handle_bp(struct trace_proc *p) +{ + struct seccomp_bp *bps[3]; + long nr, args[6]; + unsigned long mainaddr, at_entry, lsm; + int hit; + + bps[0] = &p->bp_at_entry; + bps[1] = &p->bp_lsm; + bps[2] = &p->bp_main; + + hit = seccomp_bp_match(p->pid, bps, 3); + switch (hit) { + case 0: + emit_marker(p, "at_entry"); + if (trace_main_boundary && + !seccomp_marker_addrs(p->pid, &at_entry, &lsm, &p->main_argidx) && + lsm && !seccomp_bp_arm(p->pid, lsm, &p->bp_lsm)) + p->expect_main = 1; + p->phase = p->expect_main ? PHASE_INIT : PHASE_APP; + break; + case 1: + if (seccomp_read_syscall(p->pid, &nr, args)) + break; + mainaddr = (unsigned long)args[p->main_argidx]; + if (mainaddr) + seccomp_bp_arm(p->pid, mainaddr, &p->bp_main); + break; + case 2: + emit_marker(p, "main"); + p->phase = PHASE_APP; + break; + default: + break; + } +} + +enum seccomp_resume { + SECCOMP_RESUME_NORMAL = 0, + SECCOMP_RESUME_STEP_EXIT, +}; + +static int handle_seccomp(struct trace_proc *p) +{ + long nr, args[6]; + unsigned long data = 0; + int err, rc; + + ptrace(PTRACE_GETEVENTMSG, p->pid, 0, &data); + if (seccomp_read_syscall(p->pid, &nr, args)) + return SECCOMP_RESUME_NORMAL; + + if (data == SECCOMP_TRACE_KILL_DATA) { + emit_event(p, nr, args, "kill", -1); + if (trace_mode == SECCOMP_MODE_AUDIT) + kill(p->pid, SIGKILL); + return SECCOMP_RESUME_NORMAL; + } + + err = (int)data; + emit_event(p, nr, args, "deny", err); + + if (trace_mode != SECCOMP_MODE_AUDIT) + return SECCOMP_RESUME_NORMAL; + + rc = seccomp_force_errno(p->pid, err); + if (rc < 0) { + kill(p->pid, SIGKILL); + return SECCOMP_RESUME_NORMAL; + } + if (rc == 0) + return SECCOMP_RESUME_NORMAL; + + p->errno_pending = 1; + p->pending_errno = err; + + return SECCOMP_RESUME_STEP_EXIT; +} + +static void trace_resume(pid_t pid, int sig) +{ + if (trace_mode == SECCOMP_MODE_TRACE) + ptrace(PTRACE_SYSCALL, pid, 0, (void *)(long)sig); + else + ptrace(PTRACE_CONT, pid, 0, (void *)(long)sig); +} + +static int udebug_setup(const char *name) +{ + snprintf(ring_name, sizeof(ring_name), "ujail:%s", name ? name : "trace"); + ring_meta.name = ring_name; + ring_meta.format = UDEBUG_FORMAT_BLOBMSG; + + udebug_init(&ud); + udebug_auto_connect(&ud, NULL); + if (udebug_buf_init(&udb, 1024, 256 * 1024)) + return -1; + if (udebug_buf_add(&ud, &udb, &ring_meta)) + return -1; + + return 0; +} + +static void udebug_teardown(void) +{ + if (!udebug_ready) + return; + + udebug_buf_free(&udb); + udebug_free(&ud); + udebug_ready = 0; +} + +int seccomp_trace_run(pid_t pid, const struct seccomp_trace_opts *o) +{ + struct trace_proc *p, *root; + int status; + pid_t wpid; + int sig, ev; + + trace_mode = o->mode; + trace_log_fd = o->log_fd; + trace_dedup = o->dedup; + trace_main_boundary = o->main_boundary; + memset(seen, 0, sizeof(seen)); + + if (!udebug_setup(o->name)) + udebug_ready = 1; + + root = proc_new(pid); + if (!root) { + udebug_teardown(); + return -1; + } + + proc_start_root(root); + trace_resume(pid, 0); + + while (trace_nprocs > 0) { + wpid = waitpid(-1, &status, __WALL); + if (wpid < 0) { + if (errno == EINTR) + continue; + break; + } + + p = proc_get(wpid); + if (!p) { + p = proc_new(wpid); + if (!p) + continue; + proc_start_child(p); + trace_resume(wpid, 0); + continue; + } + + if (WIFEXITED(status) || WIFSIGNALED(status)) { + proc_del(p); + continue; + } + + if (!WIFSTOPPED(status)) { + trace_resume(wpid, 0); + continue; + } + + sig = WSTOPSIG(status); + ev = (status >> 16) & 0xff; + + if (ev) { + if (ev == PTRACE_EVENT_SECCOMP) { + if (handle_seccomp(p) == SECCOMP_RESUME_STEP_EXIT) { + ptrace(PTRACE_SYSCALL, wpid, 0, 0); + continue; + } + } else if (ev == PTRACE_EVENT_EXEC) { + proc_exec(p); + } + trace_resume(wpid, 0); + continue; + } + + if (sig == (SIGTRAP | 0x80)) { + if (p->errno_pending) { + if (seccomp_force_errno_exit(wpid, p->pending_errno)) + kill(wpid, SIGKILL); + p->errno_pending = 0; + trace_resume(wpid, 0); + continue; + } + if (!p->in_syscall) + emit_syscall(p); + p->in_syscall = !p->in_syscall; + trace_resume(wpid, 0); + continue; + } + + if (sig == SIGTRAP) { + handle_bp(p); + trace_resume(wpid, 0); + continue; + } + + trace_resume(wpid, (int)sig); + } + + while (!list_empty(&trace_procs)) { + p = list_first_entry(&trace_procs, struct trace_proc, list); + proc_del(p); + } + + udebug_teardown(); + + return 0; +} diff --git a/jail/seccomp-trace.h b/jail/seccomp-trace.h new file mode 100644 index 0000000..b082920 --- /dev/null +++ b/jail/seccomp-trace.h @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2026 Daniel Golle + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License version 2.1 + * as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#ifndef _JAIL_SECCOMP_TRACE_H_ +#define _JAIL_SECCOMP_TRACE_H_ + +#include + +enum seccomp_mode { + SECCOMP_MODE_ENFORCE = 0, + SECCOMP_MODE_TRACE, + SECCOMP_MODE_AUDIT, + SECCOMP_MODE_COMPLAIN, +}; + +struct seccomp_trace_opts { + enum seccomp_mode mode; + const char *name; + int log_fd; + int main_boundary; + int dedup; +}; + +#ifdef SECCOMP_SUPPORT +int seccomp_trace_run(pid_t pid, const struct seccomp_trace_opts *o); +#else +static inline int seccomp_trace_run(pid_t pid, const struct seccomp_trace_opts *o) +{ + return -1; +} +#endif + +#endif diff --git a/service/instance.c b/service/instance.c index 38e1d2e..078c2c5 100644 --- a/service/instance.c +++ b/service/instance.c @@ -61,6 +61,8 @@ enum { INSTANCE_ATTR_JAIL, INSTANCE_ATTR_TRACE, INSTANCE_ATTR_SECCOMP, + INSTANCE_ATTR_SECCOMP_MODE, + INSTANCE_ATTR_SECCOMP_LOG, INSTANCE_ATTR_CAPABILITIES, INSTANCE_ATTR_PIDFILE, INSTANCE_ATTR_RELOADSIG, @@ -94,6 +96,8 @@ static const struct blobmsg_policy instance_attr[__INSTANCE_ATTR_MAX] = { [INSTANCE_ATTR_JAIL] = { "jail", BLOBMSG_TYPE_TABLE }, [INSTANCE_ATTR_TRACE] = { "trace", BLOBMSG_TYPE_BOOL }, [INSTANCE_ATTR_SECCOMP] = { "seccomp", BLOBMSG_TYPE_STRING }, + [INSTANCE_ATTR_SECCOMP_MODE] = { "seccomp_mode", BLOBMSG_TYPE_STRING }, + [INSTANCE_ATTR_SECCOMP_LOG] = { "seccomp_log", BLOBMSG_TYPE_STRING }, [INSTANCE_ATTR_CAPABILITIES] = { "capabilities", BLOBMSG_TYPE_STRING }, [INSTANCE_ATTR_PIDFILE] = { "pidfile", BLOBMSG_TYPE_STRING }, [INSTANCE_ATTR_RELOADSIG] = { "reload_signal", BLOBMSG_TYPE_INT32 }, @@ -326,6 +330,16 @@ jail_run(struct service_instance *in, char **argv) argv[argc++] = in->seccomp; } + if (in->seccomp_mode) { + argv[argc++] = "-m"; + argv[argc++] = in->seccomp_mode; + } + + if (in->seccomp_log) { + argv[argc++] = "-M"; + argv[argc++] = in->seccomp_log; + } + if (in->user) { argv[argc++] = "-U"; argv[argc++] = in->user; @@ -1059,6 +1073,12 @@ instance_config_changed(struct service_instance *in, struct service_instance *in if (string_changed(in->seccomp, in_new->seccomp)) return true; + if (string_changed(in->seccomp_mode, in_new->seccomp_mode)) + return true; + + if (string_changed(in->seccomp_log, in_new->seccomp_log)) + return true; + if (string_changed(in->capabilities, in_new->capabilities)) return true; @@ -1312,6 +1332,12 @@ instance_jail_parse(struct service_instance *in, struct blob_attr *attr) if (in->seccomp) jail->argc += 2; + if (in->seccomp_mode) + jail->argc += 2; + + if (in->seccomp_log) + jail->argc += 2; + if (in->capabilities) jail->argc += 2; @@ -1444,6 +1470,12 @@ instance_config_parse(struct service_instance *in) if (!in->trace && tb[INSTANCE_ATTR_SECCOMP]) in->seccomp = strdup(blobmsg_get_string(tb[INSTANCE_ATTR_SECCOMP])); + if (tb[INSTANCE_ATTR_SECCOMP_MODE]) + in->seccomp_mode = strdup(blobmsg_get_string(tb[INSTANCE_ATTR_SECCOMP_MODE])); + + if (tb[INSTANCE_ATTR_SECCOMP_LOG]) + in->seccomp_log = strdup(blobmsg_get_string(tb[INSTANCE_ATTR_SECCOMP_LOG])); + if (tb[INSTANCE_ATTR_CAPABILITIES]) in->capabilities = strdup(blobmsg_get_string(tb[INSTANCE_ATTR_CAPABILITIES])); @@ -1618,6 +1650,8 @@ instance_config_move(struct service_instance *in, struct service_instance *in_sr instance_config_move_strdup(&in->pidfile, in_src->pidfile); instance_config_move_strdup(&in->seccomp, in_src->seccomp); + instance_config_move_strdup(&in->seccomp_mode, in_src->seccomp_mode); + instance_config_move_strdup(&in->seccomp_log, in_src->seccomp_log); instance_config_move_strdup(&in->capabilities, in_src->capabilities); instance_config_move_strdup(&in->bundle, in_src->bundle); instance_config_move_strdup(&in->extroot, in_src->extroot); @@ -1683,6 +1717,8 @@ instance_free(struct service_instance *in) free(in->jail.idmap_offset); free(in->jail.consolesocket); free(in->seccomp); + free(in->seccomp_mode); + free(in->seccomp_log); free(in->capabilities); free(in->pidfile); free(in); diff --git a/service/instance.h b/service/instance.h index df9428e..ec95f8d 100644 --- a/service/instance.h +++ b/service/instance.h @@ -92,6 +92,8 @@ struct service_instance { bool no_new_privs; struct jail jail; char *seccomp; + char *seccomp_mode; + char *seccomp_log; char *capabilities; char *pidfile; char *extroot; From 655d995cf2c597f81202a3f0b4243ba3a0da0923 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 05:31:30 +0100 Subject: [PATCH 29/82] uxc: add trace, audit and complain verbs Expose the seccomp modes: trace, audit and complain create the container in the chosen mode and route the log to /tmp/uxc-..json, making complain-derived profiling a first-class operation. Signed-off-by: Daniel Golle --- uxc.c | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/uxc.c b/uxc.c index 2063d12..73ff85e 100644 --- a/uxc.c +++ b/uxc.c @@ -282,6 +282,9 @@ static int usage(void) { printf("\t\t[--write-overlay-path ]\t\tuse overlay on {path}\n"); printf("\t\t[--mounts ,,...,]\t\trequire filesystems to be available\n"); printf("\tstart [--console] \t\tstart container \n"); + printf("\ttrace \t\t\t\trun logging every syscall (seccomp trace)\n"); + printf("\taudit \t\t\t\trun enforcing its seccomp filter and logging denials\n"); + printf("\tcomplain \t\t\t\trun permitting but logging seccomp denials\n"); printf("\tstate \t\t\t\tget state of container \n"); printf("\tkill [--signal ] [--all] []\tsignal (no signal: graceful stop); --all+KILL: whole cgroup\n"); printf("\tenable \t\t\t\tstart container on boot\n"); @@ -976,7 +979,7 @@ static int uxc_exists(char *name) } static int uxc_create(char *name, bool immediately, const char *console_socket, - bool systemd_cgroup) + bool systemd_cgroup, const char *seccomp_mode) { static struct blob_buf req; struct blob_attr *cur, *tb[__CONF_MAX]; @@ -984,6 +987,7 @@ static int uxc_create(char *name, bool immediately, const char *console_socket, uint32_t id; struct settings *usettings = NULL; char *path = NULL, *jailname = NULL, *pidfile = NULL, *tmprwsize = NULL, *writepath = NULL; + char *seccomp_log = NULL; void *in, *ins, *j; bool found = false; @@ -1034,6 +1038,14 @@ static int uxc_create(char *name, bool immediately, const char *console_socket, ins = blobmsg_open_table(&req, "instances"); in = blobmsg_open_table(&req, name); blobmsg_add_string(&req, "bundle", path); + if (seccomp_mode) { + blobmsg_add_string(&req, "seccomp_mode", seccomp_mode); + if (asprintf(&seccomp_log, "/tmp/uxc-%s.%s.json", name, seccomp_mode) > 0) { + blobmsg_add_string(&req, "seccomp_log", seccomp_log); + fprintf(stderr, "uxc: %s log: %s\n", seccomp_mode, seccomp_log); + free(seccomp_log); + } + } j = blobmsg_open_table(&req, "jail"); blobmsg_add_string(&req, "name", jailname?:name); blobmsg_add_u8(&req, "immediately", immediately); @@ -1601,7 +1613,7 @@ static int uxc_boot(void) if (uxc_exists(name)) continue; - if (uxc_create(name, true, NULL, false)) + if (uxc_create(name, true, NULL, false, NULL)) ++ret; free(name); @@ -1883,6 +1895,11 @@ int main(int argc, char **argv) if (optind != verb_argc - 1) goto usage_out; ret = uxc_start(verb_argv[optind], console); + } else if (!strcmp(verb, "trace") || !strcmp(verb, "audit") || + !strcmp(verb, "complain")) { + if (verb_argc != 2) + goto usage_out; + ret = uxc_create(verb_argv[1], true, NULL, false, verb); } else if (!strcmp(verb, "state")) { if (verb_argc != 2) goto usage_out; @@ -1987,7 +2004,7 @@ int main(int argc, char **argv) if (ret > 0) reload_conf(); - ret = uxc_create(name, false, console_socket, systemd_cgroup); + ret = uxc_create(name, false, console_socket, systemd_cgroup, NULL); } else if (!strcmp(verb, "exec")) { const char *process_file = NULL; const char *pid_file = NULL; From 9f32e85e7cebc618f3f1f036087a0d81f506eeb3 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 17 Jun 2026 23:31:38 +0100 Subject: [PATCH 30/82] build: add the trace2seccomp profile generator, drop the utrace target Replace the C utrace binary and its LD_PRELOAD trace helper with trace2seccomp, a ucode tool that turns a ujail seccomp trace into an OCI seccomp profile. The old preload approach only worked for dynamically linked binaries and required a private libpreload; the new tracer in ujail (-m trace) handles static binaries and reports per-phase syscall sets, so the generator can work purely from its NDJSON output. Dispatched by argv[0]: invoked as utrace or seccomp-trace it runs a program under `ujail -m trace` and emits an application-phase allow-list, matching the classic policy-generation window. Invoked as trace2seccomp it converts a previously captured NDJSON trace, optionally merging all phases or emitting a two-phase pre/post dynamic-linker profile. The CMake UTRACE_SUPPORT target now installs the script instead of building the dropped utrace and preload-trace artefacts. Signed-off-by: Daniel Golle --- CMakeLists.txt | 13 +- trace/preload.c | 82 --------- trace/trace.c | 444 ----------------------------------------------- trace2seccomp.uc | 143 +++++++++++++++ 4 files changed, 144 insertions(+), 538 deletions(-) delete mode 100644 trace/preload.c delete mode 100644 trace/trace.c create mode 100644 trace2seccomp.uc diff --git a/CMakeLists.txt b/CMakeLists.txt index cf4d044..e8c4d04 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -125,16 +125,5 @@ INSTALL(TARGETS uxc endif() IF(UTRACE_SUPPORT) -ADD_EXECUTABLE(utrace trace/trace.c ${SOURCES_OCI_SECCOMP}) -TARGET_LINK_LIBRARIES(utrace ${ubox} ${json} ${blobmsg_json}) -INSTALL(TARGETS utrace - RUNTIME DESTINATION ${CMAKE_INSTALL_SBINDIR} -) -ADD_DEPENDENCIES(utrace syscall-names-h) - -ADD_LIBRARY(preload-trace SHARED trace/preload.c) -TARGET_LINK_LIBRARIES(preload-trace dl) -INSTALL(TARGETS preload-trace - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} -) +INSTALL(PROGRAMS trace2seccomp.uc DESTINATION ${CMAKE_INSTALL_SBINDIR} RENAME trace2seccomp) endif() diff --git a/trace/preload.c b/trace/preload.c deleted file mode 100644 index 457bd82..0000000 --- a/trace/preload.c +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (C) 2015 John Crispin - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License version 2.1 - * as published by the Free Software Foundation - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ - -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../preload.h" - -#define ERROR(fmt, ...) do { \ - fprintf(stderr,"perload-jail: "fmt, ## __VA_ARGS__); \ - } while (0) - -static main_t __main__; - -static int __preload_main__(int argc, char **argv, char **envp) -{ - unsetenv("LD_PRELOAD"); - kill(getpid(), SIGSTOP); - - return (*__main__)(argc, argv, envp); -} - -int __libc_start_main(main_t main, - int argc, - char **argv, - ElfW(auxv_t) *auxvec, - __typeof (main) init, - void (*fini) (void), - void (*rtld_fini) (void), - void *stack_end) -{ - start_main_t __start_main__; - - __start_main__ = dlsym(RTLD_NEXT, "__libc_start_main"); - if (!__start_main__) { - ERROR("failed to find __libc_start_main %s\n", dlerror()); - return -1; - } - __main__ = main; - - return (*__start_main__)(__preload_main__, argc, argv, auxvec, - init, fini, rtld_fini, stack_end); -} - -void __uClibc_main(main_t main, - int argc, - char **argv, - void (*app_init)(void), - void (*app_fini)(void), - void (*rtld_fini)(void), - void *stack_end attribute_unused) -{ - uClibc_main __start_main__; - - __start_main__ = dlsym(RTLD_NEXT, "__uClibc_main"); - if (!__start_main__) { - ERROR("failed to find __uClibc_main %s\n", dlerror()); - return; - } - - __main__ = main; - - return (*__start_main__)(__preload_main__, argc, argv, - app_init, app_fini, rtld_fini, stack_end); -} diff --git a/trace/trace.c b/trace/trace.c deleted file mode 100644 index 99cc5d2..0000000 --- a/trace/trace.c +++ /dev/null @@ -1,444 +0,0 @@ -/* - * Copyright (C) 2015 John Crispin - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License version 2.1 - * as published by the Free Software Foundation - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ - -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifndef PTRACE_EVENT_STOP -/* PTRACE_EVENT_STOP is defined in linux/ptrace.h, but this header - * collides with musl's sys/ptrace.h */ -#define PTRACE_EVENT_STOP 128 -#endif - -#ifndef PTRACE_EVENT_SECCOMP -/* undefined with uClibc-ng */ -#define PTRACE_EVENT_SECCOMP 7 -#endif - -#include -#include -#include -#include - -#include "../jail/seccomp-oci.h" -#include "../syscall-names.h" - -#define _offsetof(a, b) __builtin_offsetof(a,b) -#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) - -#if defined (__aarch64__) || defined(__loongarch_lp64) -#include -#elif defined(__amd64__) -#define reg_syscall_nr _offsetof(struct user, regs.orig_rax) -#elif defined(__arm__) -#include /* for PTRACE_SET_SYSCALL */ -#define reg_syscall_nr _offsetof(struct user, regs.uregs[7]) -# if defined(__ARM_EABI__) -# define reg_retval_nr _offsetof(struct user, regs.uregs[0]) -# endif -#elif defined(__i386__) -#define reg_syscall_nr _offsetof(struct user, regs.orig_eax) -#elif defined(__mips) -# ifndef EF_REG2 -# define EF_REG2 8 -# endif -#define reg_syscall_nr (EF_REG2 / 4) -#elif defined(__PPC__) -#define reg_syscall_nr _offsetof(struct user, regs.gpr[0]) -#define reg_retval_nr _offsetof(struct user, regs.gpr[3]) -#else -#error tracing is not supported on this architecture -#endif - -enum mode { - UTRACE, - SECCOMP_TRACE, -} mode = UTRACE; - -struct tracee { - struct uloop_process proc; - int in_syscall; -}; - -static struct tracee tracer; -static int syscall_count[SYSCALL_COUNT]; -static int violation_count; -static struct blob_buf b; -int debug; -char *json = NULL; -int ptrace_restart; - -static void set_syscall(const char *name, int val) -{ - int i; - - for (i = 0; i < SYSCALL_COUNT; i++) { - int sc = syscall_index_to_number(i); - if (syscall_name(sc) && !strcmp(syscall_name(sc), name)) { - syscall_count[i] = val; - return; - } - } -} - -struct syscall { - int syscall; - int count; -}; - -static int cmp_count(const void *a, const void *b) -{ - return ((struct syscall*)b)->count - ((struct syscall*)a)->count; -} - -static void print_syscalls(int policy, const char *json) -{ - void *c, *d, *e; - int i; - char *tmp; - - if (mode == UTRACE) { - set_syscall("rt_sigaction", 1); - set_syscall("sigreturn", 1); - set_syscall("rt_sigreturn", 1); - set_syscall("exit_group", 1); - set_syscall("exit", 1); - } - - struct syscall sorted[SYSCALL_COUNT]; - - for (i = 0; i < SYSCALL_COUNT; i++) { - sorted[i].syscall = syscall_index_to_number(i); - sorted[i].count = syscall_count[i]; - } - - qsort(sorted, SYSCALL_COUNT, sizeof(sorted[0]), cmp_count); - - blob_buf_init(&b, 0); - blobmsg_add_string(&b, "defaultAction", "SCMP_ACT_KILL_PROCESS"); - c = blobmsg_open_array(&b, "syscalls"); - d = blobmsg_open_table(&b, ""); - e = blobmsg_open_array(&b, "names"); - - for (i = 0; i < SYSCALL_COUNT; i++) { - int sc = sorted[i].syscall; - if (!sorted[i].count) - break; - if (syscall_name(sc)) { - if (debug) - printf("syscall %d (%s) was called %d times\n", - sc, syscall_name(sc), sorted[i].count); - blobmsg_add_string(&b, NULL, syscall_name(sc)); - } else { - ULOG_ERR("no name found for syscall(%d)\n", sc); - } - } - blobmsg_close_array(&b, e); - blobmsg_add_string(&b, "action", "SCMP_ACT_ALLOW"); - blobmsg_close_table(&b, d); - blobmsg_close_array(&b, c); - if (json) { - FILE *fp = fopen(json, "w"); - if (fp) { - tmp = blobmsg_format_json_indent(b.head, true, 0); - if (!tmp) { - fclose(fp); - return; - } - - fprintf(fp, "%s\n", tmp); - free(tmp); - fclose(fp); - ULOG_INFO("saving syscall trace to %s\n", json); - } else { - ULOG_ERR("failed to open %s\n", json); - } - } else { - tmp = blobmsg_format_json_indent(b.head, true, 0); - if (!tmp) - return; - - printf("%s\n", tmp); - free(tmp); - } -} - -static void report_seccomp_vialation(pid_t pid, unsigned syscall) -{ - char buf[200]; - snprintf(buf, sizeof(buf), "/proc/%d/cmdline", pid); - int f = open(buf, O_RDONLY); - if (f < 0) - return; - - int r = read(f, buf, sizeof(buf) - 1); - buf[sizeof(buf) - 1] = '\0'; - - if (r >= 0) - buf[r] = 0; - else - strcpy(buf, "unknown?"); - close(f); - - if (violation_count < INT_MAX) - violation_count++; - int i = syscall_index(syscall); - if (i >= 0) { - syscall_count[i]++; - ULOG_ERR("%s[%u] tried to call non-whitelisted syscall: %s (see %s)\n", - buf, pid, syscall_name(syscall), json); - } else { - ULOG_ERR("%s[%u] tried to call non-whitelisted syscall: %d (see %s)\n", - buf, pid, syscall, json); - } -} - -static void tracer_cb(struct uloop_process *c, int ret) -{ - struct tracee *tracee = container_of(c, struct tracee, proc); - int inject_signal = 0; - - /* We explicitely check for events in upper 16 bits, because - * musl (as opposed to glibc) does not report - * PTRACE_EVENT_STOP as WIFSTOPPED */ - if (WIFSTOPPED(ret) || (ret >> 16)) { - if (WSTOPSIG(ret) & 0x80) { - if (!tracee->in_syscall) { -#if defined(__aarch64__) || defined(__loongarch_lp64) - int syscall = -1; - struct ptrace_syscall_info ptsi = {.op=PTRACE_SYSCALL_INFO_ENTRY}; - if (ptrace(PTRACE_GET_SYSCALL_INFO, c->pid, sizeof(ptsi), &ptsi) != -1) - syscall = ptsi.entry.nr; -#else - int syscall = ptrace(PTRACE_PEEKUSER, c->pid, reg_syscall_nr); -#endif - int i = syscall_index(syscall); - if (i >= 0) { - syscall_count[i]++; - if (debug) - fprintf(stderr, "%s()\n", syscall_name(syscall)); - } else if (debug) { - fprintf(stderr, "syscal(%d)\n", syscall); - } - } - tracee->in_syscall = !tracee->in_syscall; - } else if ((ret >> 8) == (SIGTRAP | (PTRACE_EVENT_FORK << 8)) || - (ret >> 8) == (SIGTRAP | (PTRACE_EVENT_VFORK << 8)) || - (ret >> 8) == (SIGTRAP | (PTRACE_EVENT_CLONE << 8))) { - struct tracee *child = calloc(1, sizeof(struct tracee)); - - unsigned long msg; - ptrace(PTRACE_GETEVENTMSG, c->pid, 0, &msg); - child->proc.pid = msg; - child->proc.cb = tracer_cb; - ptrace(ptrace_restart, child->proc.pid, 0, 0); - uloop_process_add(&child->proc); - if (debug) - fprintf(stderr, "Tracing new child %d\n", child->proc.pid); - } else if ((ret >> 16) == PTRACE_EVENT_STOP) { - /* Nothing special to do here */ - } else if ((ret >> 8) == (SIGTRAP | (PTRACE_EVENT_SECCOMP << 8))) { -#if defined(__aarch64__) || defined(__loongarch_lp64) - int syscall = -1; - struct ptrace_syscall_info ptsi = {.op=PTRACE_SYSCALL_INFO_SECCOMP}; - if (ptrace(PTRACE_GET_SYSCALL_INFO, c->pid, sizeof(ptsi), &ptsi) != -1) - syscall = ptsi.entry.nr; -#else - int syscall = ptrace(PTRACE_PEEKUSER, c->pid, reg_syscall_nr); -#if defined(__arm__) - ptrace(PTRACE_SET_SYSCALL, c->pid, 0, -1); - ptrace(PTRACE_POKEUSER, c->pid, reg_retval_nr, -ENOSYS); -#else - ptrace(PTRACE_POKEUSER, c->pid, reg_syscall_nr, -1); -#endif -#endif - report_seccomp_vialation(c->pid, syscall); - } else { - inject_signal = WSTOPSIG(ret); - if (debug) - fprintf(stderr, "Injecting signal %d into pid %d\n", - inject_signal, tracee->proc.pid); - } - } else if (WIFEXITED(ret) || (WIFSIGNALED(ret) && WTERMSIG(ret))) { - if (tracee == &tracer) { - uloop_end(); /* Main process exit */ - } else { - if (debug) - fprintf(stderr, "Child %d exited\n", tracee->proc.pid); - free(tracee); - } - return; - } - - ptrace(ptrace_restart, c->pid, 0, inject_signal); - uloop_process_add(c); -} - -static void sigterm_handler(int signum) -{ - /* When we receive SIGTERM, we forward it to the tracee. After - * the tracee exits, trace_cb() will be called and make us - * exit too. */ - kill(tracer.proc.pid, SIGTERM); -} - - -int main(int argc, char **argv, char **envp) -{ - int status, ch, policy = EPERM; - pid_t child; - - /* When invoked via seccomp-trace symlink, work as seccomp - * violation logger rather than as syscall tracer */ - if (strstr(argv[0], "seccomp-trace")) - mode = SECCOMP_TRACE; - - while ((ch = getopt(argc, argv, "f:p:")) != -1) { - switch (ch) { - case 'f': - json = optarg; - break; - case 'p': - policy = atoi(optarg); - break; - } - } - - if (!json) - json = getenv("SECCOMP_FILE"); - - argc -= optind; - argv += optind; - - if (!argc) - return -1; - - if (getenv("TRACE_DEBUG")) - debug = 1; - unsetenv("TRACE_DEBUG"); - - child = fork(); - - if (child == 0) { - char **_argv = calloc(argc + 1, sizeof(char *)); - char **_envp; - char *preload = NULL; - const char *old_preload = getenv("LD_PRELOAD"); - int newenv = 0; - int envc = 0; - int ret; - - memcpy(_argv, argv, argc * sizeof(char *)); - - while (envp[envc++]) - ; - - _envp = calloc(envc + 2, sizeof(char *)); - switch (mode) { - case UTRACE: - preload = "/lib/libpreload-trace.so"; - newenv = 1; - break; - case SECCOMP_TRACE: - ULOG_ERR("seccomp-trace is no longer supported\n"); - return -1; - } - - if (asprintf(&_envp[0], "LD_PRELOAD=%s%s%s", preload, - old_preload ? ":" : "", - old_preload ? old_preload : "") < 0) - ULOG_ERR("failed to allocate LD_PRELOAD env: %m\n"); - - memcpy(&_envp[newenv], envp, envc * sizeof(char *)); - - ret = execve(_argv[0], _argv, _envp); - ULOG_ERR("failed to exec %s: %m\n", _argv[0]); - - free(_argv); - if (_envp[0]) - free(_envp[0]); - if (newenv == 2 && _envp[1]) - free(_envp[1]); - free(_envp); - return ret; - } - - if (child < 0) - return -1; - - waitpid(child, &status, WUNTRACED); - if (!WIFSTOPPED(status)) { - ULOG_ERR("failed to start %s\n", *argv); - return -1; - } - - /* Initialize uloop to catch all ptrace stops from now on. */ - uloop_init(); - - int ptrace_options = PTRACE_O_TRACEFORK | PTRACE_O_TRACEVFORK | PTRACE_O_TRACECLONE; - switch (mode) { - case UTRACE: - ptrace_options |= PTRACE_O_TRACESYSGOOD; - ptrace_restart = PTRACE_SYSCALL; - break; - case SECCOMP_TRACE: - ptrace_options |= PTRACE_O_TRACESECCOMP; - ptrace_restart = PTRACE_CONT; - break; - } - if (ptrace(PTRACE_SEIZE, child, 0, ptrace_options) == -1) { - ULOG_ERR("PTRACE_SEIZE: %m\n"); - return -1; - } - if (ptrace(ptrace_restart, child, 0, SIGCONT) == -1) { - ULOG_ERR("ptrace_restart: %m\n"); - return -1; - } - - tracer.proc.pid = child; - tracer.proc.cb = tracer_cb; - uloop_process_add(&tracer.proc); - signal(SIGTERM, sigterm_handler); /* Override uloop's SIGTERM handler */ - uloop_run(); - uloop_done(); - - - switch (mode) { - case UTRACE: - if (!json) - if (asprintf(&json, "/tmp/%s.%u.json", basename(*argv), child) < 0) - ULOG_ERR("failed to allocate output path: %m\n"); - break; - case SECCOMP_TRACE: - if (!violation_count) - return 0; - if (asprintf(&json, "/tmp/%s.%u.violations.json", basename(*argv), child) < 0) - ULOG_ERR("failed to allocate violations output path: %m\n"); - break; - } - print_syscalls(policy, json); - return 0; -} diff --git a/trace2seccomp.uc b/trace2seccomp.uc new file mode 100644 index 0000000..6363903 --- /dev/null +++ b/trace2seccomp.uc @@ -0,0 +1,143 @@ +#!/usr/bin/ucode -R + +import { readfile, writefile, basename, unlink } from 'fs'; + +const ACT_KILL = 'SCMP_ACT_KILL_PROCESS'; +const ACT_ALLOW = 'SCMP_ACT_ALLOW'; + +function usage(name) { + if (name == 'trace2seccomp') + warn("usage: trace2seccomp [--merged|--two-phase] [-o out.json] \n"); + else + warn(sprintf("usage: %s [args...]\n", name)); + exit(1); +} + +function shq(s) { + return "'" + replace(s, "'", "'\\''") + "'"; +} + +function invoked_name() { + let cmd = readfile('/proc/self/cmdline'); + if (!cmd) + return basename(sourcepath()); + + let parts = split(cmd, '\x00'); + while (length(parts) && parts[length(parts) - 1] == '') + pop(parts); + + let idx = length(parts) - length(ARGV) - 1; + if (idx < 0) + return basename(sourcepath()); + + return basename(parts[idx]); +} + +function collect(ndjson, want_phase) { + let seen = {}; + + for (let line in split(ndjson, '\n')) { + if (!length(line)) + continue; + + let ev; + try { ev = json(line); } + catch (e) { continue; } + + if (type(ev) != 'object' || ev.event != 'syscall' || !ev.syscall) + continue; + if (want_phase && ev.phase != want_phase) + continue; + + seen[ev.syscall] = true; + } + + return sort(keys(seen)); +} + +function profile(names) { + return { + defaultAction: ACT_KILL, + syscalls: [ { names: names, action: ACT_ALLOW } ] + }; +} + +function emit(obj, outfile) { + let text = sprintf("%.J\n", obj); + + if (!outfile) { + print(text); + return; + } + + if (writefile(outfile, text) == null) + die(sprintf("cannot write %s\n", outfile)); +} + +function run_trace(prog, args) { + let tmp = sprintf("/tmp/.trace2seccomp.%d.ndjson", time()); + let cmd = "ujail -m trace -M " + shq(tmp) + " -- " + shq(prog); + + for (let a in args) + cmd += " " + shq(a); + + system(cmd); + + let nd = readfile(tmp); + unlink(tmp); + + return nd; +} + +function run_live(name) { + if (!length(ARGV)) + usage(name); + + let prog = ARGV[0]; + let nd = run_trace(prog, slice(ARGV, 1)); + if (nd == null) + die("no trace captured (did ujail run?)\n"); + + let out = sprintf("/tmp/%s.%d.json", basename(prog), time()); + emit(profile(collect(nd, 'app')), out); + warn(sprintf("seccomp profile written to %s\n", out)); +} + +function run_offline() { + let merged, two_phase, outfile, infile; + let i = 0; + + while (i < length(ARGV)) { + let a = ARGV[i]; + if (a == '--merged') + merged = true; + else if (a == '--two-phase') + two_phase = true; + else if (a == '-o') + outfile = ARGV[++i]; + else + infile = a; + i++; + } + + if (!infile) + usage('trace2seccomp'); + + let nd = readfile(infile); + if (nd == null) + die(sprintf("cannot read %s\n", infile)); + + if (two_phase) + emit({ predl: profile(collect(nd, null)), postdl: profile(collect(nd, 'app')) }, outfile); + else if (merged) + emit(profile(collect(nd, null)), outfile); + else + emit(profile(collect(nd, 'app')), outfile); +} + +let self = invoked_name(); + +if (self == 'utrace' || self == 'seccomp-trace') + run_live(self); +else + run_offline(); From 86cf36d17768cb12d39c2bb024aa625022e686c7 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 17 Jun 2026 23:31:48 +0100 Subject: [PATCH 31/82] jail: harden container mount/device setup and add -V volume binds Stage the jail's /dev as a private tmpfs in the host namespace before clone3() enters the user namespace, where device mknod is forbidden. Nodes are created and chowned to the host id that container-root maps to, the symlinks, sub-mount points and an optional console placeholder are pre-laid, then /dev is remounted read-only so the kernel locks MNT_LOCK_READONLY when the namespace copies it. This lets a read-only, content-addressed image run without a writable /dev, and pivot_root now pivots to self with a lazily detached old root rather than a writable put-old in the rootfs. Add -V to bind a host path as a noexec,nosuid,nodev volume, and prime autofs-backed sources by opening them in the host namespace so an automount triggered later from the private namespace cannot leave the overlay base empty. Bind targets are created relative to the jail root via openat2 with RESOLVE_BENEATH and RESOLVE_NO_MAGICLINKS so a symlinked target cannot escape, and OCI bundles keep ownership of their own mount namespace. Signed-off-by: Daniel Golle --- jail/fs.c | 176 ++++++++++++++++++-- jail/fs.h | 7 +- jail/jail.c | 406 ++++++++++++++++++--------------------------- service/instance.c | 4 +- 4 files changed, 331 insertions(+), 262 deletions(-) diff --git a/jail/fs.c b/jail/fs.c index 11f76a3..137c3c3 100644 --- a/jail/fs.c +++ b/jail/fs.c @@ -126,11 +126,20 @@ unsigned long detect_atime_flag(const char *mountpoint) #define MOUNT_ATTR_NODIRATIME 0x00000080 #endif +int sys_openat2(int dfd, const char *path, struct open_how *how, size_t size) +{ + return syscall(SYS_openat2, dfd, path, how, size); +} + +static int jailroot_dirfd = -1; + static unsigned int idmap_host_offset; + void jail_set_idmap_offset(unsigned int offset) { idmap_host_offset = offset; } + static int write_mappings_file(pid_t pid, const char *which, struct blob_attr *mappings) { enum { @@ -419,11 +428,14 @@ void jail_fs_set_userns(bool enabled) fs_userns = enabled; } +static bool mount_opts_has(const char *opts, const char *needle); + static int do_mount(const char *root, const char *orig_source, const char *target, const char *filesystemtype, unsigned long orig_mountflags, unsigned long propflags, const char *optstr, int error, bool inner) { struct stat s; char new[PATH_MAX]; + char tmpfs_data[512]; const char *mount_data; char *source = (char *)orig_source; int fd, ret = 0; @@ -470,9 +482,21 @@ static int do_mount(const char *root, const char *orig_source, const char *targe if (!is_bind || (source && S_ISDIR(s.st_mode))) { mkdir_p(new, 0755); } else if (is_bind && source) { + const char *target_rel = target ? target : source; + struct open_how how = { + .flags = O_CREAT | O_WRONLY | O_TRUNC | O_EXCL | O_CLOEXEC, + .mode = 0644, + .resolve = RESOLVE_BENEATH | RESOLVE_NO_MAGICLINKS, + }; + + assert(target_rel); mkdir_p(dirname(new), 0755); snprintf(new, sizeof(new), "%s%s", root, target?target:source); - fd = open(new, O_CREAT|O_WRONLY|O_TRUNC|O_EXCL, 0644); + while (*target_rel == '/') + ++target_rel; + fd = (jailroot_dirfd >= 0) + ? sys_openat2(jailroot_dirfd, target_rel, &how, sizeof(how)) + : open(new, O_CREAT|O_WRONLY|O_TRUNC|O_EXCL|O_CLOEXEC, 0644); if (fd >= 0) close(fd); @@ -496,6 +520,14 @@ static int do_mount(const char *root, const char *orig_source, const char *targe } mount_data = optstr; + if (filesystemtype && !strcmp(filesystemtype, "tmpfs") && !fs_userns && + !mount_opts_has(optstr ?: "", "swap") && !mount_opts_has(optstr ?: "", "noswap")) { + if (optstr && *optstr) + snprintf(tmpfs_data, sizeof(tmpfs_data), "%s,noswap", optstr); + else + snprintf(tmpfs_data, sizeof(tmpfs_data), "noswap"); + mount_data = tmpfs_data; + } const char *hack_fstype = ((!filesystemtype || strcmp(filesystemtype, "cgroup"))?filesystemtype:"cgroup2"); if (mount(source?:(is_bind?new:NULL), new, hack_fstype?:"none", mountflags, mount_data)) { @@ -684,6 +716,37 @@ int add_mount_fd(int fd, const char *target, int error) return 0; } +int add_mount_volume(const char *source, const char *target, int error) +{ + struct mount *m; + int ret; + + ret = add_mount(source, target, NULL, + MS_BIND | MS_NOEXEC | MS_NOSUID | MS_NODEV, 0, NULL, error); + if (ret && ret != EEXIST) + return ret; + + m = avl_find_element(&mounts, target, m, avl); + if (m) + m->volume = true; + + return ret; +} + +char *resolve_mount_source(const char *source) +{ + char *real; + + if (!source) + return NULL; + + if (source[0] != '/') + return strdup(source); + + real = realpath(source, NULL); + return real ? real : strdup(source); +} + enum { OCI_MOUNT_SOURCE, OCI_MOUNT_DESTINATION, @@ -876,6 +939,24 @@ static bool is_proc_or_sys_path(const char *path) return false; } +static bool mount_opts_has(const char *opts, const char *needle) +{ + size_t nlen = strlen(needle); + const char *p = opts; + + while (p && *p) { + const char *end = strchr(p, ','); + size_t plen = end ? (size_t)(end - p) : strlen(p); + + if (plen == nlen && !strncmp(p, needle, nlen)) + return true; + if (!end) + break; + p = end + 1; + } + return false; +} + int parseOCImount(struct blob_attr *msg) { struct blob_attr *tb[__OCI_MOUNT_MAX]; @@ -883,6 +964,7 @@ int parseOCImount(struct blob_attr *msg) unsigned long propagation_flags = 0; char *mount_data = NULL; char *destination, *abs_destination = NULL; + char *rsrc = NULL; bool idmap = false, idmap_recursive = false; int ret, err = -1; @@ -919,11 +1001,16 @@ int parseOCImount(struct blob_attr *msg) return EPERM; } - ret = add_mount(tb[OCI_MOUNT_SOURCE] ? blobmsg_get_string(tb[OCI_MOUNT_SOURCE]) : NULL, + if (tb[OCI_MOUNT_SOURCE]) + rsrc = resolve_mount_source(blobmsg_get_string(tb[OCI_MOUNT_SOURCE])); + + ret = add_mount(rsrc, destination, tb[OCI_MOUNT_TYPE] ? blobmsg_get_string(tb[OCI_MOUNT_TYPE]) : NULL, mount_flags, propagation_flags, mount_data, err); + free(rsrc); + if (!ret && (idmap || tb[OCI_MOUNT_UIDMAPPINGS] || tb[OCI_MOUNT_GIDMAPPINGS])) { struct mount *m = avl_find_element(&mounts, destination, m, avl); if (m) { @@ -1013,6 +1100,11 @@ static int idmap_mount_target(const char *root, struct mount *m, char *target, s { struct stat s; const char *target_rel; + struct open_how how = { + .flags = O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, + .mode = 0644, + .resolve = RESOLVE_BENEATH | RESOLVE_NO_MAGICLINKS, + }; int fd; snprintf(target, tlen, "%s%s", root, m->target); @@ -1034,7 +1126,9 @@ static int idmap_mount_target(const char *root, struct mount *m, char *target, s snprintf(target, tlen, "%s%s", root, m->target); while (*target_rel == '/') ++target_rel; - fd = open(target, O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0644); + fd = (jailroot_dirfd >= 0) + ? sys_openat2(jailroot_dirfd, target_rel, &how, sizeof(how)) + : open(target, O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0644); if (fd >= 0) close(fd); @@ -1279,32 +1373,88 @@ int jail_idmap_assign(bool have_extroot, bool have_overlay, const int *fds, int return i; } -int mount_all(const char *jailroot) { +void mount_stage_dev(const char *jail_dev) +{ + struct mount *m; + struct stat s; + char path[PATH_MAX]; + bool is_dir; + int fd; + + avl_for_each_element(&mounts, m, avl) { + if (strncmp(m->target, "/dev/", 5)) + continue; + if (m->source == (void *)(-1)) + continue; + + snprintf(path, sizeof(path), "%s%s", jail_dev, m->target + 4); + + is_dir = !(m->mountflags & MS_BIND) || + (m->source && !stat(m->source, &s) && S_ISDIR(s.st_mode)); + if (is_dir) { + mkdir_p(path, 0755); + } else { + mkdir_p(dirname(strdupa(path)), 0755); + fd = open(path, O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0644); + if (fd >= 0) + close(fd); + } + } +} + +int mount_all(const char *jailroot, const char *jail_dev) { struct library *l; struct mount *m; + char devtarget[PATH_MAX]; + int ret = 0; build_noafile(); + jailroot_dirfd = open(jailroot, O_PATH | O_DIRECTORY | O_CLOEXEC); + if (jailroot_dirfd < 0) + ERROR("mount_all: open(%s, O_PATH|O_DIRECTORY): %m\n", jailroot); + avl_for_each_element(&libraries, l, avl) add_mount_bind(l->path, 1, -1); avl_for_each_element(&mounts, m, avl) { - if (m->idmap_treefd >= 0) { - if (do_move_idmap_mount(jailroot, m)) - return -1; + if (jail_dev && m->filesystemtype && !strcmp(m->filesystemtype, "tmpfs") && + !strcmp(m->target, "/dev")) { + snprintf(devtarget, sizeof(devtarget), "%s%s", jailroot, m->target); + mkdir_p(devtarget, 0755); + if (mount(jail_dev, devtarget, NULL, MS_BIND | MS_REC, NULL)) { + ERROR("mount(MS_BIND, %s -> %s): %m\n", jail_dev, devtarget); + ret = -1; + goto out; + } + } else if (m->idmap_treefd >= 0) { + if (do_move_idmap_mount(jailroot, m)) { + ret = -1; + goto out; + } } else if (m->idmap) { - if (do_idmap_mount(jailroot, m)) - return -1; + if (do_idmap_mount(jailroot, m)) { + ret = -1; + goto out; + } } else if (m->source_fd >= 0) { - if (do_mount_fd(jailroot, m->source_fd, m->target, m->error)) - return -1; + if (do_mount_fd(jailroot, m->source_fd, m->target, m->error)) { + ret = -1; + goto out; + } } else if (do_mount(jailroot, m->source, m->target, m->filesystemtype, m->mountflags, m->propflags, m->optstr, m->error, m->inner)) { - return -1; + ret = -1; + goto out; } } - return 0; +out: + if (jailroot_dirfd >= 0) { + close(jailroot_dirfd); + jailroot_dirfd = -1; + } + return ret; } void mount_free(void) { diff --git a/jail/fs.h b/jail/fs.h index 4cb367a..d0f386c 100644 --- a/jail/fs.h +++ b/jail/fs.h @@ -16,12 +16,14 @@ #include #include #include +#include #include #include "../container.h" #define JAIL_NOAFILE "/dev/.ujailnoafile" +int sys_openat2(int dfd, const char *path, struct open_how *how, size_t size); int build_userns_fd(struct blob_attr *uidmappings, struct blob_attr *gidmappings); int jail_idmap_build(const char *extroot, struct blob_attr *uidmap, struct blob_attr *gidmap, @@ -37,6 +39,8 @@ int add_mount(const char *source, const char *target, const char *filesystemtype int add_mount_inner(const char *source, const char *target, const char *filesystemtype, unsigned long mountflags, unsigned long propflags, const char *optstr, int error); int add_mount_bind(const char *path, int readonly, int error); +int add_mount_volume(const char *source, const char *target, int error); +char *resolve_mount_source(const char *source); int add_mount_fd(int fd, const char *target, int error); int mask_path_now(const char *path); @@ -76,7 +80,8 @@ static inline int add_path_and_deps(const char *path, int readonly, int error, i return add_2paths_and_deps(path, path, readonly, error, lib); } -int mount_all(const char *jailroot); +void mount_stage_dev(const char *jail_dev); +int mount_all(const char *jailroot, const char *jail_dev); void mount_list_init(void); void mount_free(void); void jail_fs_set_userns(bool enabled); diff --git a/jail/jail.c b/jail/jail.c index af9d534..377d55f 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -50,6 +50,7 @@ #include #include #include +#include #include #include #include @@ -99,7 +100,7 @@ #define PR_MDWE_NO_INHERIT (1UL << 1) #endif -#define OPT_ARGS "cC:d:De:EfFG:h:iI:j:J:lm:M:n:NoO:pP:r:R:sS:uU:w:t:T:yY:Z" +#define OPT_ARGS "cC:d:De:EfFG:h:iI:j:J:lm:M:n:NoO:pP:r:R:sS:uU:V:w:t:T:yY:Z" struct hook_execvpe { char *file; @@ -219,6 +220,9 @@ static struct { static struct blob_buf ocibuf; +static char **volume_sources; +static int num_volume_sources; + extern int pivot_root(const char *new_root, const char *put_old); int debug = 0; @@ -596,9 +600,8 @@ static int create_dev_console(const char *jail_root) snprintf(dev_console_path, sizeof(dev_console_path), "%s/dev/console", jail_root); dev_console_dummy = creat(dev_console_path, 0620); - if (dev_console_dummy < 0) - return 1; - close(dev_console_dummy); + if (dev_console_dummy >= 0) + close(dev_console_dummy); snprintf(fdpath, sizeof(fdpath), "/proc/self/fd/%d", console_slave_fd); if (mount(fdpath, dev_console_path, "bind", MS_BIND, NULL)) @@ -783,6 +786,9 @@ static int apply_sysctl(const char *jail_root) (((y)&0x000000ffULL)) ) #endif +static char jail_dev[] = "/tmp/ujail-dev-XXXXXX"; +static bool jail_dev_staged; + static struct mknod_args default_devices[] = { { .path = "/dev/null", .mode = (S_IFCHR|S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH), .dev = makedev(1, 3) }, { .path = "/dev/zero", .mode = (S_IFCHR|S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH), .dev = makedev(1, 5) }, @@ -793,92 +799,105 @@ static struct mknod_args default_devices[] = { { 0 }, }; -static int create_devices(void) +static int prepare_jail_dev(void) { struct mknod_args **cur, *curdef; - char *path, *tmp; - int ret; + uid_t base = (opts.namespace & CLONE_NEWUSER) ? opts.root_map_uid : 0; + mode_t oldmask = umask(0); + char path[PATH_MAX], *tmp; + int consfd; - if (!opts.devices) - goto only_default_devices; + if (mkdtemp(jail_dev) == NULL) { + ERROR("mkdtemp(%s) failed: %m\n", jail_dev); + return errno; + } + jail_dev_staged = true; - cur = opts.devices; + if (mount("tmpfs", jail_dev, "tmpfs", MS_NOSUID | MS_NOATIME, "mode=0755")) { + ERROR("tmpfs mount for /dev failed: %m\n"); + return errno; + } - while (*cur) { - path = (*cur)->path; + if (mount(NULL, jail_dev, NULL, MS_PRIVATE, NULL)) { + ERROR("making /dev tmpfs private failed: %m\n"); + return errno; + } + + for (cur = opts.devices; cur && *cur; ++cur) { /* don't allow devices outside of /dev */ - if (strncmp(path, "/dev", 4)) + if (strncmp((*cur)->path, "/dev", 4)) return EPERM; - if (opts.setns.user != -1) { - ++cur; - continue; - } + snprintf(path, sizeof(path), "%s%s", jail_dev, (*cur)->path + 4); /* make sure parent folder exists */ tmp = strrchr(path, '/'); if (!tmp) return EINVAL; - *tmp = '\0'; - if (strcmp(path, "/dev")) { - DEBUG("creating directory %s\n", path); - - if (mkdir_p(path, 0755)) - return errno; - } + if (strcmp(path, jail_dev) && mkdir_p(path, 0755)) + return errno; *tmp = '/'; - DEBUG("creating %s (mode=%08o)\n", path, (*cur)->mode); - - /* create device */ if (mknod(path, (*cur)->mode, (*cur)->dev)) return errno; - - /* change owner, if needed */ - if (((*cur)->uid || (*cur)->gid) && - chown(path, (*cur)->uid, (*cur)->gid)) + if (chown(path, base + (*cur)->uid, base + (*cur)->gid)) return errno; - - ++cur; } -only_default_devices: - curdef = default_devices; - while(curdef->path) { - DEBUG("creating %s (mode=%08o)\n", curdef->path, curdef->mode); - if (mknod(curdef->path, curdef->mode, curdef->dev)) { - ++curdef; - continue; /* may already exist, eg. due to a bind-mount */ - } - if ((curdef->uid || curdef->gid) && - chown(curdef->path, curdef->uid, curdef->gid)) + for (curdef = default_devices; curdef->path; ++curdef) { + snprintf(path, sizeof(path), "%s%s", jail_dev, curdef->path + 4); + if (mknod(path, curdef->mode, curdef->dev)) + return errno; + if (chown(path, base + curdef->uid, base + curdef->gid)) return errno; - - ++curdef; } /* Dev symbolic links as defined in OCI spec */ - ret = symlink("/dev/pts/ptmx", "/dev/ptmx"); - if (ret < 0) + snprintf(path, sizeof(path), "%s/ptmx", jail_dev); + if (symlink("/dev/pts/ptmx", path)) WARNING("symlink() failed to create link to /dev/pts/ptmx"); - ret = symlink("/proc/self/fd", "/dev/fd"); - if (ret < 0) + snprintf(path, sizeof(path), "%s/fd", jail_dev); + if (symlink("/proc/self/fd", path)) WARNING("symlink() failed to create link to /proc/self/fd"); - ret = symlink("/proc/self/fd/0", "/dev/stdin"); - if (ret < 0) + snprintf(path, sizeof(path), "%s/stdin", jail_dev); + if (symlink("/proc/self/fd/0", path)) WARNING("symlink() failed to create link to /proc/self/fd/0"); - ret = symlink("/proc/self/fd/1", "/dev/stdout"); - if (ret < 0) + snprintf(path, sizeof(path), "%s/stdout", jail_dev); + if (symlink("/proc/self/fd/1", path)) WARNING("symlink() failed to create link to /proc/self/fd/1"); - ret = symlink("/proc/self/fd/2", "/dev/stderr"); - if (ret < 0) + snprintf(path, sizeof(path), "%s/stderr", jail_dev); + if (symlink("/proc/self/fd/2", path)) WARNING("symlink() failed to create link to /proc/self/fd/2"); + snprintf(path, sizeof(path), "%s/pts", jail_dev); + mkdir(path, 0755); + snprintf(path, sizeof(path), "%s/shm", jail_dev); + mkdir(path, 0755); + snprintf(path, sizeof(path), "%s/mqueue", jail_dev); + mkdir(path, 0755); + + if (opts.console) { + snprintf(path, sizeof(path), "%s/console", jail_dev); + consfd = creat(path, 0620); + if (consfd < 0) + WARNING("creat() failed to stage /dev/console: %m\n"); + else + close(consfd); + } + + mount_stage_dev(jail_dev); + + if (mount(NULL, jail_dev, NULL, MS_REMOUNT | MS_RDONLY | MS_NOSUID | MS_NOATIME, NULL)) { + ERROR("read-only remount of /dev staging failed: %m\n"); + return errno; + } + + umask(oldmask); return 0; } @@ -1218,174 +1237,10 @@ static int build_jail_fs(void) return -1; } - { - /* fds stay open until mount_all() performs the /proc/self/fd/N - * binds below; closing early would drop or swap the source */ - int *held_fds = NULL; - size_t n_devices = 0, n_custom = 0, i; - int fail = 0; - - if (opts.setns.user != -1) { - struct mknod_args *curdef; - - if (opts.devices) { - struct mknod_args **cur; - - for (cur = opts.devices; *cur; cur++) - n_custom++; - } - - for (curdef = default_devices; curdef->path; curdef++) - n_devices++; - - n_devices += n_custom; - - held_fds = malloc(n_devices * sizeof(int)); - if (!held_fds) { - ERROR("out of memory validating devices\n"); - return -1; - } - for (i = 0; i < n_devices; i++) - held_fds[i] = -1; - - if (opts.devices) { - struct mknod_args **cur; - - for (i = 0, cur = opts.devices; *cur && !fail; cur++, i++) { - struct stat st; - - if (strncmp((*cur)->path, "/dev", 4)) { - ERROR("custom device %s is outside of /dev; " - "refusing to bind-mount it\n", - (*cur)->path); - fail = 1; - break; - } - - held_fds[i] = open((*cur)->path, O_PATH | O_CLOEXEC); - if (held_fds[i] < 0) { - ERROR("custom device %s requested but not found " - "on the host; it cannot be created under " - "CLONE_NEWUSER (no privilege to mknod)\n", - (*cur)->path); - fail = 1; - break; - } - - if (fstat(held_fds[i], &st)) { - ERROR("custom device %s: fstat() failed: %m\n", - (*cur)->path); - fail = 1; - break; - } - - if (((*cur)->mode & S_IFMT) && - (st.st_mode & S_IFMT) != ((*cur)->mode & S_IFMT)) { - ERROR("custom device %s exists on the host but " - "is not the requested node type; its " - "major:minor/mode/owner cannot be enforced " - "under CLONE_NEWUSER\n", - (*cur)->path); - fail = 1; - break; - } - - if (((*cur)->mode & S_IFMT) == S_IFCHR || - ((*cur)->mode & S_IFMT) == S_IFBLK) { - if ((*cur)->dev && st.st_rdev != (*cur)->dev) { - ERROR("custom device %s exists on the host " - "but its major:minor (%u:%u) does not " - "match the requested %u:%u; refusing " - "to bind-mount a different device than " - "configured\n", (*cur)->path, - major(st.st_rdev), minor(st.st_rdev), - major((*cur)->dev), minor((*cur)->dev)); - fail = 1; - break; - } - } - - { - int tree; - struct ujail_mount_attr attr = { .attr_set = MOUNT_ATTR_RDONLY }; - - tree = sys_open_tree(held_fds[i], "", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC | AT_EMPTY_PATH); - if (tree < 0) { - ERROR("open_tree() on custom device %s failed: %m\n", (*cur)->path); - fail = 1; - break; - } - close(held_fds[i]); - held_fds[i] = tree; - - if (sys_mount_setattr(tree, "", AT_EMPTY_PATH, &attr, sizeof(attr))) { - ERROR("mount_setattr() on custom device %s failed: %m\n", (*cur)->path); - fail = 1; - break; - } - } - if (add_mount_fd(held_fds[i], (*cur)->path, -1)) { - ERROR("could not queue bind-mount for mandatory " - "custom device %s; refusing to start with " - "a requested device missing\n", - (*cur)->path); - fail = 1; - break; - } - } - } - - if (!fail) { - size_t j = 0; - - for (curdef = default_devices; curdef->path; curdef++, j++) { - int tree; - struct ujail_mount_attr attr = { .attr_set = MOUNT_ATTR_RDONLY }; - - held_fds[n_custom + j] = open(curdef->path, O_PATH | O_CLOEXEC); - if (held_fds[n_custom + j] < 0) { - WARNING("could not open default device %s; " - "it will be unavailable in the jail\n", - curdef->path); - continue; - } - - tree = sys_open_tree(held_fds[n_custom + j], "", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC | AT_EMPTY_PATH); - if (tree < 0) { - WARNING("open_tree() on default device %s failed; " - "it will be unavailable in the jail\n", curdef->path); - continue; - } - close(held_fds[n_custom + j]); - held_fds[n_custom + j] = tree; - - if (sys_mount_setattr(tree, "", AT_EMPTY_PATH, &attr, sizeof(attr))) { - WARNING("mount_setattr() on default device %s failed; " - "it will be unavailable in the jail\n", curdef->path); - continue; - } - - if (add_mount_fd(held_fds[n_custom + j], curdef->path, 0)) - WARNING("could not queue bind-mount for default " - "device %s; it will be unavailable in " - "the jail\n", curdef->path); - } - } - } - jail_fs_set_userns((opts.namespace & CLONE_NEWUSER) || (opts.setns.user != -1)); - if (!fail && mount_all(jail_root)) { + if (mount_all(jail_root, jail_dev)) { ERROR("mount_all() failed\n"); - fail = 1; - } - - for (i = 0; i < n_devices; i++) - if (held_fds && held_fds[i] >= 0) - close(held_fds[i]); - free(held_fds); - - if (fail) return -1; } @@ -1425,6 +1280,12 @@ static void free_and_exit(int ret) cgroups_free(); } + if (!exit_from_child && jail_dev_staged) { + umount2(jail_dev, MNT_DETACH); + rmdir(jail_dev); + jail_dev_staged = false; + } + if (!exit_from_child && parent_ctx) ubus_free(parent_ctx); @@ -1439,15 +1300,16 @@ static void remask_after_unshare(void); static void remount_proc_sys_after_unshare(void); static void enter_jail_fs(void) { - char dirbuf[sizeof(jail_root) + 4]; - - snprintf(dirbuf, sizeof(dirbuf), "%s/old", jail_root); - if (mkdir(dirbuf, 0755)) { - ERROR("mkdir(%s) failed: %m\n", dirbuf); + if (chdir(jail_root)) { + ERROR("chdir(%s) (jail_root) failed: %m\n", jail_root); + free_and_exit(-1); + } + if (pivot_root(".", ".") == -1) { + ERROR("pivot_root(%s) failed: %m\n", jail_root); free_and_exit(-1); } - if (pivot_root(jail_root, dirbuf) == -1) { - ERROR("pivot_root(%s, %s) failed: %m\n", jail_root, dirbuf); + if (umount2(".", MNT_DETACH)) { + ERROR("umount2() of the old root failed: %m\n"); free_and_exit(-1); } if (chdir("/")) { @@ -1455,23 +1317,6 @@ static void enter_jail_fs(void) free_and_exit(-1); } - snprintf(dirbuf, sizeof(dirbuf), "/old%s", jail_root); - umount2(dirbuf, MNT_DETACH); - rmdir(dirbuf); - if (opts.tmpoverlaysize) { - char tmpdirbuf[sizeof(tmpovdir) + 4]; - snprintf(tmpdirbuf, sizeof(tmpdirbuf), "/old%s", tmpovdir); - umount2(tmpdirbuf, MNT_DETACH); - rmdir(tmpdirbuf); - } - - umount2("/old", MNT_DETACH); - rmdir("/old"); - - if (create_devices()) { - ERROR("create_devices() failed\n"); - free_and_exit(-1); - } if (opts.ronly) mount(NULL, "/", "bind", MS_REMOUNT | MS_BIND | MS_RDONLY, 0); @@ -2040,6 +1885,7 @@ static void usage(void) fprintf(stderr, " -F\t\tjail has cgroups namespace\n"); fprintf(stderr, " -r \treadonly files that should be staged\n"); fprintf(stderr, " -w \twriteable files that should be staged\n"); + fprintf(stderr, " -V \tbind at as a noexec,nosuid,nodev volume\n"); fprintf(stderr, " -p\t\tjail has /proc\n"); fprintf(stderr, " -s\t\tjail has /sys\n"); fprintf(stderr, " -l\t\tjail has /dev/log\n"); @@ -2979,6 +2825,7 @@ static void post_start_hook(void) uloop_end(); free_opts(false); + syscall(SYS_close_range, 3, ~0U, CLOSE_RANGE_CLOEXEC); if (jail_ptrace_seccomp() && ptrace(PTRACE_TRACEME, 0, 0, 0)) { ERROR("PTRACE_TRACEME failed: %m\n"); exit(EXIT_FAILURE); @@ -5258,6 +5105,7 @@ container_handle_exec(struct ubus_context *ctx, struct ubus_object *obj, ERROR("exec: chdir(%s): %m\n", cwd); _exit(127); } + syscall(SYS_close_range, 3, ~0U, CLOSE_RANGE_CLOEXEC); if (env) execvpe(args[0], args, env); else @@ -5412,7 +5260,11 @@ jail_writepid(pid_t pid) static int checkpath(const char *path) { - int dirfd = open(path, O_RDONLY | O_DIRECTORY | O_CLOEXEC); + struct open_how how = { + .flags = O_RDONLY | O_DIRECTORY | O_CLOEXEC, + .resolve = RESOLVE_NO_MAGICLINKS, + }; + int dirfd = sys_openat2(AT_FDCWD, path, &how, sizeof(how)); if (dirfd < 0) { ERROR("path %s open failed %m\n", path); return -1; @@ -5422,6 +5274,45 @@ static int checkpath(const char *path) return 0; } +static void prime_jail_mount(const char *path) +{ + int fd; + + if (!path) + return; + + fd = open(path, O_RDONLY | O_DIRECTORY | O_CLOEXEC); + if (fd >= 0) + close(fd); +} + +static int add_volume_source(const char *source) +{ + char **tmp; + + tmp = realloc(volume_sources, (num_volume_sources + 1) * sizeof(*volume_sources)); + if (!tmp) + return -ENOMEM; + + volume_sources = tmp; + volume_sources[num_volume_sources] = strdup(source); + if (!volume_sources[num_volume_sources]) + return -ENOMEM; + + ++num_volume_sources; + return 0; +} + +static void add_volume(const char *source, const char *dest) +{ + char *real; + + real = resolve_mount_source(source); + add_volume_source(real ? real : source); + add_mount_volume(real ? real : source, dest, 0); + free(real); +} + static struct ubus_method container_methods[] = { UBUS_METHOD_NOARG("start", handle_start), UBUS_METHOD_NOARG("state", handle_state), @@ -5548,7 +5439,8 @@ int main(int argc, char **argv) jail_join_ns(optarg); break; case 'r': - opts.namespace |= CLONE_NEWNS; + if (!opts.ocibundle) + opts.namespace |= CLONE_NEWNS; tmp = strchr(optarg, ':'); if (tmp) { *(tmp++) = '\0'; @@ -5558,7 +5450,8 @@ int main(int argc, char **argv) } break; case 'w': - opts.namespace |= CLONE_NEWNS; + if (!opts.ocibundle) + opts.namespace |= CLONE_NEWNS; tmp = strchr(optarg, ':'); if (tmp) { *(tmp++) = '\0'; @@ -5567,6 +5460,15 @@ int main(int argc, char **argv) add_path_and_deps(optarg, 0, 0, 0); } break; + case 'V': + tmp = strchr(optarg, ':'); + if (!tmp) { + ERROR("volume needs a src:dest pair: %s\n", optarg); + return -1; + } + *(tmp++) = '\0'; + add_volume(optarg, tmp); + break; case 'u': opts.namespace |= CLONE_NEWNS; add_mount_bind(ubus, 0, -1); @@ -6015,6 +5917,11 @@ static void post_main(struct uloop_timeout *t) timens_fd = -1; } + if ((opts.namespace & CLONE_NEWNS) && prepare_jail_dev()) { + ERROR("prepare_jail_dev() failed\n"); + free_and_exit(EXIT_FAILURE); + } + if (opts.namespace & CLONE_NEWUSER) { if (opts.overlaydir) { if (chown(opts.overlaydir, opts.root_map_uid, opts.root_map_uid)) { @@ -6097,6 +6004,11 @@ static void post_main(struct uloop_timeout *t) } } + prime_jail_mount(opts.extroot); + prime_jail_mount(opts.overlaydir); + for (size_t i = 0; i < (size_t)num_volume_sources; i++) + prime_jail_mount(volume_sources[i]); + jail_process.pid = jail_clone3(&cargs); if (init_cgroup_fd >= 0) close(init_cgroup_fd); diff --git a/service/instance.c b/service/instance.c index 078c2c5..660d9d5 100644 --- a/service/instance.c +++ b/service/instance.c @@ -439,7 +439,9 @@ jail_run(struct service_instance *in, char **argv) blobmsg_list_for_each(&jail->mount, var) { const char *type = blobmsg_data(var->data); - if (*type == '1') + if (*type == '2') + argv[argc++] = "-V"; + else if (*type == '1') argv[argc++] = "-w"; else argv[argc++] = "-r"; From 24495d92cc4fbec0e41c4b1f01f2c703b6e82774 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 17 Jun 2026 15:33:26 +0100 Subject: [PATCH 32/82] jail: provide resolv.conf to containers with a private netifd A container running its own netifd needs a resolv.conf that tracks netifd's atomic replace of resolv.conf.auto on a read-only rootfs. Bind the staged /dev/resolv.conf symlink itself (open_tree with AT_SYMLINK_NOFOLLOW, not its target) onto /etc/resolv.conf, so every open re-resolves through /dev/resolv.conf.d. The target file is created only when the rootfs is writable; otherwise the staged placeholder suffices. On the host side, pre-create resolv.conf.auto in the per-jail /tmp/resolv.conf-.d before bind-mounting it at /dev/resolv.conf.d, so the mount has a target before the in-jail netifd first writes it. This wiring is now gated on the private_netifd annotation rather than merely on a new network namespace, and the org.openwrt.procd.ubus and .netifd annotations are parsed as distinct keys. Signed-off-by: Daniel Golle --- jail/fs.c | 9 +++++++ jail/fs.h | 1 + jail/jail.c | 72 +++++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 63 insertions(+), 19 deletions(-) diff --git a/jail/fs.c b/jail/fs.c index 137c3c3..1d97432 100644 --- a/jail/fs.c +++ b/jail/fs.c @@ -1044,6 +1044,15 @@ int parseOCImount(struct blob_attr *msg) return ret; } +bool mount_is_defined(const char *target) +{ + struct mount *m; + + m = avl_find_element(&mounts, target, m, avl); + + return m != NULL; +} + static void build_noafile(void) { int fd; diff --git a/jail/fs.h b/jail/fs.h index d0f386c..8bfe8e8 100644 --- a/jail/fs.h +++ b/jail/fs.h @@ -72,6 +72,7 @@ int sys_mount_setattr(int dfd, const char *path, unsigned flags, struct ujail_mo #define AT_EMPTY_PATH 0x1000 #endif int parseOCImount(struct blob_attr *msg); +bool mount_is_defined(const char *target); int add_2paths_and_deps(const char *path, const char *path2, int readonly, int error, int lib); unsigned long detect_atime_flag(const char *mountpoint); diff --git a/jail/jail.c b/jail/jail.c index 377d55f..33315af 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -216,6 +216,8 @@ static struct { } ioprio; unsigned long mdwe_flags; struct landlock_config landlock; + bool private_ubus; + bool private_netifd; } opts; static struct blob_buf ocibuf; @@ -881,6 +883,14 @@ static int prepare_jail_dev(void) snprintf(path, sizeof(path), "%s/mqueue", jail_dev); mkdir(path, 0755); + if ((opts.namespace & CLONE_NEWNET) && opts.private_netifd) { + snprintf(path, sizeof(path), "%s/resolv.conf.d", jail_dev); + mkdir(path, 0755); + snprintf(path, sizeof(path), "%s/resolv.conf", jail_dev); + if (symlink("/dev/resolv.conf.d/resolv.conf.auto", path)) + WARNING("symlink() failed to create /dev/resolv.conf: %m\n"); + } + if (opts.console) { snprintf(path, sizeof(path), "%s/console", jail_dev); consfd = creat(path, 0620); @@ -1203,7 +1213,7 @@ static int build_jail_fs(void) ERROR("mkdtemp(%s) failed: %m\n", jail_root); return -1; } - if (mount("tmpfs", tmpovdir, "tmpfs", MS_NOATIME, + if (mount("tmpfs", tmpovdir, "tmpfs", MS_NOATIME | MS_NOEXEC | MS_NOSUID | MS_NODEV, mountoptsstr)) { ERROR("failed to mount tmpfs for overlay (size=%s)\n", opts.tmpoverlaysize); return -1; @@ -1247,22 +1257,29 @@ static int build_jail_fs(void) if (opts.console) create_dev_console(jail_root); - /* make sure /etc/resolv.conf exists if in new network namespace */ - if (opts.namespace & CLONE_NEWNET) { - char jailetc[PATH_MAX], jaillink[PATH_MAX]; - - snprintf(jailetc, PATH_MAX, "%s/etc", jail_root); - if (mkdir_p(jailetc, 0755)) { - ERROR("mkdir(%s) failed: %m\n", jailetc); - return -1; + if ((opts.namespace & CLONE_NEWNET) && opts.private_netifd) { + char jailetc[PATH_MAX], devresolv[PATH_MAX], etcresolv[PATH_MAX]; + struct stat rcst; + int treefd; + + snprintf(devresolv, PATH_MAX, "%s/dev/resolv.conf", jail_root); + snprintf(etcresolv, PATH_MAX, "%s/etc/resolv.conf", jail_root); + if (stat(etcresolv, &rcst) && (overlaydir || !opts.ronly)) { + snprintf(jailetc, PATH_MAX, "%s/etc", jail_root); + mkdir_p(jailetc, 0755); + close(creat(etcresolv, 0644)); + } + if (!stat(etcresolv, &rcst)) { + treefd = sys_open_tree(AT_FDCWD, devresolv, + OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC | AT_SYMLINK_NOFOLLOW); + if (treefd < 0) { + WARNING("open_tree(/dev/resolv.conf) failed: %m\n"); + } else { + if (sys_move_mount(treefd, "", AT_FDCWD, etcresolv, MOVE_MOUNT_F_EMPTY_PATH)) + WARNING("move_mount onto /etc/resolv.conf failed: %m\n"); + close(treefd); + } } - snprintf(jaillink, PATH_MAX, "%s/etc/resolv.conf", jail_root); - if (overlaydir) - unlink(jaillink); - - ret = symlink("../dev/resolv.conf.d/resolv.conf.auto", jaillink); - if (ret < 0) - WARNING("symlink() failed to create link to ../dev/resolv.conf.d/resolv.conf.auto"); } run_hooks(opts.hooks.createContainer, enter_jail_fs); @@ -4281,6 +4298,12 @@ static int parseOCI(const char *jsonfile) LANDLOCK_ACCESS_FS_REMOVE_DIR); if (res) goto errout; + } else if (!strcmp(name, "org.openwrt.procd.ubus")) { + opts.private_ubus = !strcmp(val, "true") || !strcmp(val, "1"); + } else if (!strcmp(name, "org.openwrt.procd.netifd")) { + opts.private_netifd = !strcmp(val, "true") || !strcmp(val, "1"); + if (opts.private_netifd) + opts.private_ubus = true; } else if (!strcmp(name, "org.openwrt.cgroup.memory.pct")) { pct = strtol(val, &pct_end, 10); if (pct_end == val || pct < 1 || pct > 100) { @@ -4301,6 +4324,13 @@ static int parseOCI(const char *jsonfile) if (opts.landlock.n > 0) opts.no_new_privs = 1; } + + if (opts.private_netifd && mount_is_defined("/etc/resolv.conf")) { + ERROR("bundle bind-mounts /etc/resolv.conf but the container has its own netifd\n"); + res = EINVAL; + goto errout; + } + errout: blob_buf_free(&ocibuf); @@ -5808,15 +5838,19 @@ static void post_main(struct uloop_timeout *t) if (opts.setns.ns == -1) { if (!(opts.namespace & CLONE_NEWNET)) { add_mount_bind("/etc/resolv.conf", 1, 0); - } else { - /* new mount namespace to provide /dev/resolv.conf.d */ - char hostdir[PATH_MAX]; + } else if (opts.private_netifd) { + char hostdir[PATH_MAX], hostresolv[PATH_MAX]; + int hostresolvfd; snprintf(hostdir, PATH_MAX, "/tmp/resolv.conf-%s.d", opts.name); if (mkdir_p(hostdir, 0755)) { ERROR("mkdir(%s) failed: %m\n", hostdir); free_and_exit(-1); } + snprintf(hostresolv, PATH_MAX, "%s/resolv.conf.auto", hostdir); + hostresolvfd = open(hostresolv, O_WRONLY | O_CREAT, 0644); + if (hostresolvfd >= 0) + close(hostresolvfd); add_mount(hostdir, "/dev/resolv.conf.d", NULL, MS_BIND | MS_NOEXEC | MS_NOATIME | MS_NOSUID | MS_NODEV | MS_RDONLY, 0, NULL, 0); } From 794866611371f70882da073b27188f38ce27e024 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 17 Jun 2026 15:37:24 +0100 Subject: [PATCH 33/82] uxc-net: declarative container networking helper Add uxc-net, a ucode helper that brings a container's host-side network up and down over ubus, leaving no residue in persistent /etc/config. A veth is created in netifd and the container end handed to the jail netns via a dynamic jail interface; the host end is wired per attachment mode read from the OCI annotations (org.openwrt.network.{attach,egress,ingress,proto}). bridged: enslaves the host end into a network's bridge and the jail inherits that network's zone wholesale, auto-creating an isolated network once if it is absent. routed places the host end as the gateway of a point-to-point /31 in the container's own fw4 zone, compiling egress and ingress into explicit forwardings and DNAT redirects, defaulting to deny-all. host and none do no wiring; rollback is symmetric. Every netifd-facing identifier is an fnv1a slug of the qualified name to fit IFNAMSIZ and avoid the dot being read as a VLAN tag. The in-jail netifd config is rendered to /tmp/run/uxc-net so the container side is configured from the same declarative source. Signed-off-by: Daniel Golle --- CMakeLists.txt | 3 + jail/uxc-net | 680 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 683 insertions(+) create mode 100644 jail/uxc-net diff --git a/CMakeLists.txt b/CMakeLists.txt index e8c4d04..9efbca2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -122,6 +122,9 @@ TARGET_LINK_LIBRARIES(uxc ${ubox} ${ubus} ${blobmsg_json}) INSTALL(TARGETS uxc RUNTIME DESTINATION ${CMAKE_INSTALL_SBINDIR} ) +INSTALL(PROGRAMS jail/uxc-net + DESTINATION ${CMAKE_INSTALL_SBINDIR} +) endif() IF(UTRACE_SUPPORT) diff --git a/jail/uxc-net b/jail/uxc-net new file mode 100644 index 0000000..58199ab --- /dev/null +++ b/jail/uxc-net @@ -0,0 +1,680 @@ +#!/usr/bin/ucode -R + +let fs = require("fs"); +let ubus = require("ubus"); +let uci = require("uci"); +let uloop = require("uloop"); + +let fnv1a = function(s) { + let h = 2166136261, i; + for (i = 0; i < length(s); i++) { + h ^= ord(s, i); + h = (h * 16777619) & 0xffffffff; + } + return h; +}; +let slug = function(name) { return sprintf("%08x", fnv1a(name)); }; + +let host_veth = function(name) { return "vh-" + slug(name); }; +let cont_veth = function(name) { return "vc-" + slug(name); }; +let bh_host_veth = function(name) { return "bh-" + slug(name); }; +let bh_cont_veth = function(name) { return "bc-" + slug(name); }; +let bh_bridge = function(id) { return "bhr-" + slug(id); }; + +let accif = function(name) { return "ca" + slug(name); }; +let gwif = function(name) { return "cg" + slug(name); }; +let bhif = function(name) { return "cb" + slug(name); }; +let bhseg = function(id) { return "cs" + slug(id); }; +let contzone = function(name) { return "cz" + slug(name); }; + +let injail_dir = "/tmp/run/uxc-net"; +let injail_path = function(name) { return injail_dir + "/" + name + ".network"; }; + +let loopback_section = "config interface 'loopback'\n" + + "\toption device 'lo'\n" + + "\toption proto 'static'\n" + + "\toption ipaddr '127.0.0.1'\n" + + "\toption netmask '255.0.0.0'\n\n"; + +let write_injail = function(name, lan_section) { + fs.mkdir(injail_dir, 0755); + let f = fs.open(injail_path(name), "w"); + if (!f) + return; + f.write(loopback_section); + f.write(lan_section); + f.close(); +}; + +let call = function(object, method, data) { + ubus.call({ object: object, method: method, data: data }); + let err = ubus.error(); + if (err) + warn(sprintf("uxc-net: ubus %s.%s failed: %s\n", object, method, err)); + return err ? -1 : 0; +}; + +let sidecar_path = function(name) { return "/tmp/run/uvol/.meta/uxc/" + name + ".annotations"; }; + +let state_dir = "/tmp/run/uvol/.meta/uxc/state"; +let macstore_dir = function(name) { return state_dir + "/" + name; }; +let macstore_path = function(name) { return macstore_dir(name) + "/macaddrs"; }; + +let rand_mac = function() { + let f = fs.open("/dev/urandom", "r"); + if (!f) + return null; + let b = f.read(6); + f.close(); + if (type(b) != "string" || length(b) != 6) + return null; + return sprintf("%02x:%02x:%02x:%02x:%02x:%02x", + (ord(b, 0) & 0xfc) | 0x02, + ord(b, 1), ord(b, 2), ord(b, 3), ord(b, 4), ord(b, 5)); +}; + +let ensure_macs = function(name, roles) { + let store = {}, out = {}, changed = false, raw, s, r, m; + + raw = fs.readfile(macstore_path(name)); + if (raw) { + try { s = json(raw); } catch (e) { s = null; } + if (type(s) == "object") + store = s; + } + + for (r in roles) { + if (!store[r]) { + m = rand_mac(); + if (m) { + store[r] = m; + changed = true; + } + } + out[r] = store[r]; + } + + if (changed) { + fs.mkdir(state_dir, 0700); + fs.mkdir(macstore_dir(name), 0700); + let f = fs.open(macstore_path(name), "w"); + if (f) { + f.write(sprintf("%J\n", store)); + f.close(); + } + } + + return out; +}; + +let with_macs = function(dev, m, hostrole, peerrole) { + if (m[hostrole]) + dev.macaddr = m[hostrole]; + if (m[peerrole]) + dev.peer_macaddr = m[peerrole]; + return dev; +}; + +let read_annotations = function(bundle, name) { + let ann = {}, raw, side, cfg, s, k, v; + + if (bundle) { + raw = fs.readfile(bundle + "/config.json"); + if (!raw) + return null; + try { cfg = json(raw); } catch (e) { return null; } + ann = cfg?.annotations ?? {}; + } + + side = fs.readfile(sidecar_path(name)); + if (side) { + try { s = json(side); } catch (e) { s = null; } + if (type(s) == "object") + for (k, v in s) + ann[k] = v; + } + + return ann; +}; + +let parse_attach = function(ann) { + let attach = ann["org.openwrt.network.attach"]; + if (attach == null || attach == "" || attach == "none") + return { kind: "none" }; + if (attach == "host") + return { kind: "host" }; + if (attach == "routed") + return { kind: "routed" }; + + let m = match(attach, /^bridged:([A-Za-z0-9_]+)$/); + if (m) + return { kind: "bridged", network: m[1] }; + + return { kind: "unknown", raw: attach }; +}; + +let parse_backhaul = function(ann) { + let id = ann["org.openwrt.network.backhaul"]; + if (id == null || id == "") + return null; + return { id: id, address: ann["org.openwrt.network.backhaul-address"] }; +}; + +let iface_exists = function(iface) { + ubus.call({ object: "network.interface." + iface, method: "status", data: {} }); + return !ubus.error(); +}; + + +let append_injail = function(name, section) { + let existed, f; + fs.mkdir(injail_dir, 0755); + existed = !!fs.stat(injail_path(name)); + f = fs.open(injail_path(name), "a"); + if (!f) + return; + if (!existed) + f.write(loopback_section); + f.write(section); + f.close(); +}; + +let csv = function(val) { + let list = [], i, parts; + if (type(val) != "string") + return list; + parts = split(val, /[ ,]+/); + for (i = 0; i < length(parts); i++) + if (parts[i] != "") + push(list, parts[i]); + return list; +}; + +let zone_of = function(token) { + let m = match(token, /^(vpn|container):([A-Za-z0-9_.]+)$/); + if (m) + return m[1] == "container" ? contzone(m[2]) : m[2]; + return token; +}; + + +let ip2int = function(s) { + let p = split(s ?? "", "."); + if (length(p) != 4) + return null; + return (+p[0] * 16777216) + (+p[1] * 65536) + (+p[2] * 256) + (+p[3]); +}; + +let used_ranges = function() { + let ranges = []; + let add = function(ipint, plen) { + plen = +plen; + if (ipint == null || plen < 0 || plen > 32) + return; + let size = (plen == 0) ? 4294967296 : (1 << (32 - plen)); + let start = ipint - (ipint % size); + push(ranges, [ start, start + size - 1 ]); + }; + let d = ubus.call({ object: "network.interface", method: "dump", data: {} }); + let list = (type(d) == "object" && type(d.interface) == "array") ? d.interface : []; + for (let intf in list) { + for (let a in intf["ipv4-address"] ?? []) + add(ip2int(a.address), a.mask); + for (let r in intf.route ?? []) + if (r.target != "0.0.0.0") + add(ip2int(r.target), r.mask); + } + return ranges; +}; + +let pick_subnet = function() { + let used = used_ranges(); + let avail = function(prefix) { + let base = ip2int(prefix + ".0"); + for (let r in used) + if (base <= r[1] && r[0] <= base + 255) + return false; + return true; + }; + for (let x = 16; x <= 254; x++) + if (avail(sprintf("192.168.%d", x))) + return sprintf("192.168.%d", x); + for (let a = 16; a <= 31; a++) + for (let x = 0; x <= 255; x++) + if (avail(sprintf("172.%d.%d", a, x))) + return sprintf("172.%d.%d", a, x); + for (let a = 0; a <= 255; a++) + for (let x = 0; x <= 255; x++) + if (avail(sprintf("10.%d.%d", a, x))) + return sprintf("10.%d.%d", a, x); + return null; +}; + +let reload_and_wait = function(iface, pkgs) { + let obj = "network.interface." + iface; + let conn = ubus.connect(); + let emit = function() { + for (let p in pkgs) + ubus.call({ object: "service", method: "event", + data: { type: "config.change", data: { package: p } } }); + }; + + if (!conn) { + emit(); + return; + } + + uloop.init(); + let ready = false; + let lst = conn.listener("ubus.object.add", function(ev, data) { + if (type(data) == "object" && data.path == obj) { + ready = true; + uloop.end(); + } + }); + + emit(); + + ubus.call({ object: obj, method: "status", data: {} }); + if (!ubus.error()) + ready = true; + + if (!ready) { + let t = uloop.timer(30000, function() { uloop.end(); }); + uloop.run(); + t.cancel(); + } + + lst.remove(); + conn.disconnect(); +}; + +let ensure_network = function(net) { + if (iface_exists(net)) + return 0; + + let subnet = pick_subnet(); + if (!subnet) + return 2; + + let cursor = uci.cursor(); + cursor.load("network"); + cursor.load("dhcp"); + cursor.load("firewall"); + + let br = "br-" + net; + let dev = cursor.add("network", "device"); + cursor.set("network", dev, "name", br); + cursor.set("network", dev, "type", "bridge"); + cursor.set("network", net, "interface"); + cursor.set("network", net, "proto", "static"); + cursor.set("network", net, "device", br); + cursor.set("network", net, "ipaddr", subnet + ".1"); + cursor.set("network", net, "netmask", "255.255.255.0"); + cursor.commit("network"); + + cursor.set("dhcp", net, "dhcp"); + cursor.set("dhcp", net, "interface", net); + cursor.set("dhcp", net, "start", "100"); + cursor.set("dhcp", net, "limit", "150"); + cursor.set("dhcp", net, "leasetime", "12h"); + cursor.set("dhcp", net, "dhcpv4", "server"); + cursor.commit("dhcp"); + + let zone = cursor.add("firewall", "zone"); + cursor.set("firewall", zone, "name", net); + cursor.set("firewall", zone, "network", net); + cursor.set("firewall", zone, "input", "ACCEPT"); + cursor.set("firewall", zone, "output", "ACCEPT"); + cursor.set("firewall", zone, "forward", "REJECT"); + cursor.commit("firewall"); + + reload_and_wait(net, [ "network", "dhcp", "firewall" ]); + + return 1; +}; + +let bridged_up = function(name, ann, attach, m) { + let net = attach.network; + let vh = host_veth(name), vc = cont_veth(name); + + if (ann["org.openwrt.network.egress"] || ann["org.openwrt.network.ingress"]) + warn("uxc-net: bridged takes no egress/ingress (L2 inherits the joined network's zone); ignoring\n"); + + let created = ensure_network(net); + if (created == 2) + return 1; + + if (call("network", "create_device", + with_macs({ name: vh, type: "veth", peer_name: vc }, m, "h", "c"))) + return 1; + + if (call("network.interface." + net, "add_device", { name: vh, "link-ext": false })) + return 1; + + if (call("network", "add_dynamic", { + name: accif(name), + proto: "none", + device: vc, + jail: name, + jail_device: "eth0", + zone: net, + persistent: true, + })) + return 1; + + write_injail(name, "config interface 'lan'\n" + + "\toption device 'eth0'\n" + + "\toption proto 'dhcp'\n"); + + return 0; +}; + +let bridged_down = function(name, attach) { + let vh = host_veth(name); + + call("network.interface." + accif(name), "remove", {}); + if (attach.network) + call("network.interface." + attach.network, "remove_device", { name: vh, "link-ext": false }); + return 0; +}; + + +let routed_subnet = function(name, ann) { + let h = 0, i, a, b, c; + + for (i = 0; i < length(name); i++) + h = (h * 31 + ord(name, i)) & 0xffffff; + a = 1 + ((h & 0xff) % 254); + b = (h >> 8) & 0xff; + c = ((h >> 16) & 0x7f) * 2; + + return { + gw_cidr: ann["org.openwrt.network.gateway"] ?? sprintf("10.%d.%d.%d/31", a, b, c + 1), + container: ann["org.openwrt.network.address"] ?? sprintf("10.%d.%d.%d", a, b, c), + }; +}; + +let fw_reload = function() { + ubus.call({ object: "service", method: "event", + data: { type: "config.change", data: { package: "firewall" } } }); +}; + +let fw_uci_create = function(name, ann, net) { + let czone = contzone(name); + let cursor = uci.cursor(); + let i, list, m, sec; + + cursor.load("firewall"); + if (cursor.get("firewall", czone)) + return 0; + + cursor.set("firewall", czone, "zone"); + cursor.set("firewall", czone, "name", czone); + cursor.set("firewall", czone, "network", gwif(name)); + cursor.set("firewall", czone, "input", "DROP"); + cursor.set("firewall", czone, "output", "ACCEPT"); + cursor.set("firewall", czone, "forward", "DROP"); + + list = csv(ann["org.openwrt.network.egress"]); + for (i = 0; i < length(list); i++) { + sec = czone + "f" + i; + cursor.set("firewall", sec, "forwarding"); + cursor.set("firewall", sec, "src", czone); + cursor.set("firewall", sec, "dest", zone_of(list[i])); + } + + list = csv(ann["org.openwrt.network.ingress"]); + for (i = 0; i < length(list); i++) { + m = match(list[i], /^([A-Za-z0-9_:]+):(tcp|udp)\/([0-9]+(-[0-9]+)?)$/); + if (!m) { + warn(sprintf("uxc-net: ignoring bad ingress '%s'\n", list[i])); + continue; + } + sec = czone + "r" + i; + cursor.set("firewall", sec, "redirect"); + cursor.set("firewall", sec, "name", czone + "-in" + i); + cursor.set("firewall", sec, "src", zone_of(m[1])); + cursor.set("firewall", sec, "dest", czone); + cursor.set("firewall", sec, "proto", m[2]); + cursor.set("firewall", sec, "src_dport", m[3]); + cursor.set("firewall", sec, "dest_ip", net.container); + cursor.set("firewall", sec, "dest_port", m[3]); + cursor.set("firewall", sec, "target", "DNAT"); + } + + list = csv(ann["org.openwrt.network.host"]); + for (i = 0; i < length(list); i++) { + m = match(list[i], /^(tcp|udp)\/([0-9]+(-[0-9]+)?)$/); + if (!m) { + warn(sprintf("uxc-net: ignoring bad host port '%s'\n", list[i])); + continue; + } + sec = czone + "h" + i; + cursor.set("firewall", sec, "rule"); + cursor.set("firewall", sec, "name", czone + "-host" + i); + cursor.set("firewall", sec, "src", czone); + cursor.set("firewall", sec, "proto", m[1]); + cursor.set("firewall", sec, "dest_port", m[2]); + cursor.set("firewall", sec, "target", "ACCEPT"); + } + + cursor.commit("firewall"); + fw_reload(); + return 0; +}; + +let fw_uci_remove = function(name) { + let czone = contzone(name); + let cursor = uci.cursor(); + let kill = [], t; + + cursor.load("firewall"); + for (t in [ "zone", "forwarding", "redirect", "rule" ]) + cursor.foreach("firewall", t, function(s) { + if (index(s[".name"], czone) == 0) + push(kill, s[".name"]); + }); + for (let n in kill) + cursor.delete("firewall", n); + if (length(kill)) { + cursor.commit("firewall"); + fw_reload(); + } + return 0; +}; + +let routed_up = function(name, ann, m) { + let czone = contzone(name); + let net = routed_subnet(name, ann); + let vh = host_veth(name), vc = cont_veth(name); + let gw_iface = gwif(name); + let gw_ip = split(net.gw_cidr, "/")[0]; + + if (call("network", "create_device", + with_macs({ name: vh, type: "veth", peer_name: vc }, m, "h", "c"))) + return 1; + + if (call("network", "add_dynamic", { + name: gw_iface, + proto: "static", + device: vh, + ipaddr: [ net.gw_cidr ], + force_link: true, + persistent: true, + zone: czone, + })) + return 1; + + if (call("network", "add_dynamic", { + name: accif(name), + proto: "none", + device: vc, + jail: name, + jail_device: "eth0", + zone: czone, + persistent: true, + })) + return 1; + + fw_uci_create(name, ann, net); + + return 0; +}; + +let routed_down = function(name) { + let gw_iface = gwif(name); + + fw_uci_remove(name); + call("network.interface." + accif(name), "remove", {}); + call("network.interface." + gw_iface, "remove", {}); + return 0; +}; + +let backhaul_up = function(name, bh, m) { + let br = bh_bridge(bh.id); + let iface = bhseg(bh.id); + let vh = bh_host_veth(name), vc = bh_cont_veth(name); + + if (!iface_exists(iface)) { + if (call("network", "create_device", { name: br, type: "bridge", ipv6: false })) + return 1; + if (call("network", "add_dynamic", { + name: iface, proto: "none", device: br, persistent: true, + })) + return 1; + } + + if (call("network", "create_device", + with_macs({ name: vh, type: "veth", peer_name: vc }, m, "bh", "bc"))) + return 1; + if (call("network.interface." + iface, "add_device", { name: vh, "link-ext": false })) + return 1; + + if (call("network", "add_dynamic", { + name: bhif(name), + proto: "none", + device: vc, + jail: name, + jail_device: "bh0", + persistent: true, + })) + return 1; + + if (bh.address) + append_injail(name, "config interface 'backhaul'\n" + + "\toption device 'bh0'\n" + + "\toption proto 'static'\n" + + "\toption ipaddr '" + bh.address + "'\n" + + "\toption netmask '255.255.255.0'\n"); + + return 0; +}; + +let bridge_members = function(br) { + let r = ubus.call({ object: "network.device", method: "status", data: { name: br } }); + if (ubus.error() || type(r) != "object" || type(r["bridge-members"]) != "array") + return -1; + return length(r["bridge-members"]); +}; + +let backhaul_down = function(name, bh) { + let br = bh_bridge(bh.id); + let iface = bhseg(bh.id); + let vh = bh_host_veth(name); + + call("network.interface." + iface, "remove_device", { name: vh, "link-ext": false }); + call("network.interface." + bhif(name), "remove", {}); + + if (bridge_members(br) == 0) { + call("network.interface." + iface, "remove", {}); + call("network", "delete_device", { name: br }); + } + + return 0; +}; + + +let do_up = function(name, bundle) { + let attach, bh; + + if (!bundle) { + warn("uxc-net: 'up' needs a bundle path\n"); + return 1; + } + + let ann = read_annotations(bundle, name); + if (ann == null) { + warn("uxc-net: cannot read annotations from " + bundle + "/config.json\n"); + return 1; + } + + attach = parse_attach(ann); + bh = parse_backhaul(ann); + + let roles = []; + if (attach.kind == "bridged" || attach.kind == "routed") { + push(roles, "h"); + push(roles, "c"); + } + if (bh) { + push(roles, "bh"); + push(roles, "bc"); + } + let m = ensure_macs(name, roles); + + if (attach.kind == "bridged") { + if (bridged_up(name, ann, attach, m)) + return 1; + } else if (attach.kind == "routed") { + if (routed_up(name, ann, m)) + return 1; + } else if (attach.kind == "unknown") { + warn(sprintf("uxc-net: attach '%s' not implemented\n", attach.raw ?? attach.kind)); + return 1; + } + + if (bh) + if (backhaul_up(name, bh, m)) + return 1; + + return 0; +}; + +let do_down = function(name, bundle) { + let ann = read_annotations(bundle, name); + if (ann == null) + ann = {}; + + let attach = parse_attach(ann); + let bh = parse_backhaul(ann); + + if (bh) + backhaul_down(name, bh); + + if (attach.kind == "none" || attach.kind == "host") + return 0; + if (attach.kind == "routed") + return routed_down(name); + if (attach.kind == "bridged") + return bridged_down(name, attach); + + return routed_down(name); +}; + +let name = ARGV[0]; +let action = ARGV[1]; +let bundle = ARGV[2]; + +if (!name || !action) { + warn("usage: uxc-net [bundle]\n"); + exit(22); +} + +if (action == "up") + exit(do_up(name, bundle)); +else if (action == "down") + exit(do_down(name, bundle)); + +warn(sprintf("uxc-net: unknown action '%s'\n", action)); +exit(22); From b1407e790711e13b1c6036ac61489bdb20b316cb Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 17 Jun 2026 15:41:32 +0100 Subject: [PATCH 34/82] jail: wire container network-namespace setup to uxc-net Drive uxc-net synchronously on start and stop: "up" compiles the container's declarative annotations into ephemeral host netifd config, declaring the jail interface that the subsequent host-side device move picks up, and "down" tears it down. The host device move (jail_network_attach) now runs for every named netns container, so an annotation-less container still receives its host-managed device without a private netifd. A container-private ubusd and netifd are opt-in via annotations and default off, so OCI orchestrators managing networking on the host side are not fought by a parallel in-jail netifd. gen_jail_uci_network() drops the UCI section-rewriting in favour of copying the config uxc-net rendered, falling back to bare loopback. The netifd startup wait gains a timeout backstop, and a stale ubus socket from an unclean teardown is removed so its IN_CREATE still fires. jail_network_stop becomes the netns-agnostic jail_network_teardown, called from poststop and every error path so the per-jail ubusd and netifd never leak as procd orphans. Signed-off-by: Daniel Golle --- jail/jail.c | 67 ++++++++++--- jail/netifd.c | 255 +++++++++++++++++++++++--------------------------- jail/netifd.h | 6 +- jail/uxc-net | 13 +++ 4 files changed, 185 insertions(+), 156 deletions(-) diff --git a/jail/jail.c b/jail/jail.c index 33315af..d9498e0 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -218,6 +218,7 @@ static struct { struct landlock_config landlock; bool private_ubus; bool private_netifd; + bool jail_network_started; } opts; static struct blob_buf ocibuf; @@ -1292,6 +1293,11 @@ static bool jail_ptrace_seccomp(void); static void free_and_exit(int ret) { + if (!exit_from_child && opts.jail_network_started) { + jail_network_teardown(); + opts.jail_network_started = false; + } + if (!exit_from_child && opts.ocibundle) { cgroups_destroy(); cgroups_free(); @@ -4266,13 +4272,6 @@ static int parseOCI(const char *jsonfile) opts.mdwe_flags |= PR_MDWE_NO_INHERIT; val = comma ? comma + 1 : NULL; } - - if ((opts.mdwe_flags & PR_MDWE_NO_INHERIT) && - !(opts.mdwe_flags & PR_MDWE_REFUSE_EXEC_GAIN)) { - ERROR("mdwe: no_inherit requires refuse_exec_gain\n"); - res = ENOTSUP; - goto errout; - } } else if (!strcmp(name, "org.openwrt.ujail.landlock.ro")) { res = landlock_config_add_paths(&opts.landlock, val, LANDLOCK_ACCESS_FS_READ_FILE | @@ -4319,6 +4318,13 @@ static int parseOCI(const char *jsonfile) } cgroups_set_memory_limit(memtotal * pct / 100); } + + if ((opts.mdwe_flags & PR_MDWE_NO_INHERIT) && + !(opts.mdwe_flags & PR_MDWE_REFUSE_EXEC_GAIN)) { + ERROR("mdwe: no_inherit requires refuse_exec_gain\n"); + res = ENOTSUP; + goto errout; + } } if (opts.landlock.n > 0) @@ -5367,7 +5373,6 @@ static void post_main(struct uloop_timeout *t); static struct uloop_timeout post_main_timeout = { .cb = post_main, }; -static int netns_fd; static int pidns_fd; static int timens_fd; static void post_create_runtime(void); @@ -5800,6 +5805,30 @@ static void post_prestart(void) run_hooks(opts.hooks.createRuntime, post_create_runtime); } +static int run_uxc_net(const char *action) +{ + char *argv[] = { "/sbin/uxc-net", opts.name, (char *)action, opts.ocibundle, NULL }; + pid_t pid; + int status; + + if (!opts.ocibundle || !opts.name) + return 0; + + pid = fork(); + if (pid == 0) { + execv(argv[0], argv); + ERROR("failed to execv uxc-net: %m\n"); + _exit(127); + } else if (pid < 0) { + ERROR("uxc-net fork error: %m\n"); + return -1; + } + + while (waitpid(pid, &status, 0) < 0 && errno == EINTR); + + return (WIFEXITED(status) && WEXITSTATUS(status) == 0) ? 0 : -1; +} + static void post_main(struct uloop_timeout *t) { if (apply_rlimits()) { @@ -6112,8 +6141,17 @@ static void post_main(struct uloop_timeout *t) jail_chown_writable_surfaces(); } - if (opts.namespace & CLONE_NEWNET) - jail_network_start(parent_ctx, opts.name, jail_process.pid); + if ((opts.namespace & CLONE_NEWNET) && opts.name && opts.ocibundle) + run_uxc_net("up"); + + if ((opts.namespace & CLONE_NEWNET) && opts.name) + jail_network_attach(parent_ctx, opts.name, jail_process.pid); + + if ((opts.namespace & CLONE_NEWNET) && opts.private_ubus) { + if (!jail_network_start(parent_ctx, opts.name, jail_process.pid, + opts.private_netifd)) + opts.jail_network_started = true; + } if (opts.netdevices && ((opts.namespace & CLONE_NEWNET) || opts.setns.net != -1) && @@ -6501,11 +6539,12 @@ static void post_poststart(void) static void post_poststop(void); static void poststop(void) { - if (opts.namespace & CLONE_NEWNET) { - setns(netns_fd, CLONE_NEWNET); - jail_network_stop(); - close(netns_fd); + if (opts.jail_network_started) { + jail_network_teardown(); + opts.jail_network_started = false; } + if ((opts.namespace & CLONE_NEWNET) && opts.name && opts.ocibundle) + run_uxc_net("down"); run_hooks(opts.hooks.poststop, post_poststop); } diff --git a/jail/netifd.c b/jail/netifd.c index 54a18ab..e765827 100644 --- a/jail/netifd.c +++ b/jail/netifd.c @@ -35,7 +35,6 @@ #include #include #include -#include #include "netifd.h" #include "log.h" @@ -45,10 +44,10 @@ static const char ubusd_path[] = "/sbin/ubusd"; static const char netifd_path[] = "/sbin/netifd"; -static const char uci_net[] = "network"; static const char ubus_sock_name[] = "ubus.sock"; static char *jail_name, *ubus_sock_path, *ubus_sock_dir, *uci_config_network = NULL; +static bool netifd_start_done; static char *inotify_buffer; static struct uloop_fd fd_inotify_read; @@ -60,103 +59,47 @@ static struct ubus_context *jail_ubus_ctx = NULL; static struct ubus_subscriber config_watch_subscribe; -/* generate /etc/config/network for jail'ed netifd */ +static const char loopback_network[] = + "config interface 'loopback'\n" + "\toption device 'lo'\n" + "\toption proto 'static'\n" + "\toption ipaddr '127.0.0.1'\n" + "\toption netmask '255.0.0.0'\n"; + static int gen_jail_uci_network(void) { - struct uci_context *uci_ctx = uci_alloc_context(); - struct uci_package *pkg = NULL; - struct uci_element *e, *t; - bool has_loopback = false; + char *src = NULL, buf[4096]; + FILE *in, *out; + size_t n; int ret = 0; - FILE *ucinetf; - /* if no network configuration is active just return */ if (!uci_config_network) - goto uci_out; - - /* open output uci network config file */ - ucinetf = fopen(uci_config_network, "w"); - if (!ucinetf) { - ret = errno; - goto uci_out; - } - - /* load network uci package */ - if (uci_load(uci_ctx, uci_net, &pkg) != UCI_OK) { - char *err; - uci_get_errorstr(uci_ctx, &err, uci_net); - fprintf(stderr, "unable to load configuration (%s)\n", err); - free(err); - ret = EIO; - goto ucinetf_out; - } + return 0; - /* remove all sections which don't match jail */ - uci_foreach_element_safe(&pkg->sections, t, e) { - struct uci_section *s = uci_to_section(e); - struct uci_option *o = uci_lookup_option(uci_ctx, s, "jail"); - struct uci_ptr ptr = { .p = pkg, .s = s }; - - /* keep match, but remove 'jail' option and rename 'jail_ifname' */ - if (o && o->type == UCI_TYPE_STRING && !strcmp(o->v.string, jail_name)) { - ptr.o = o; - struct uci_option *jio = uci_lookup_option(uci_ctx, s, "jail_device"); - if (!jio) - jio = uci_lookup_option(uci_ctx, s, "jail_ifname"); - - if (jio) { - struct uci_ptr ren_ptr = { .p = pkg, .s = s, .o = jio, .value = "device" }; - struct uci_option *host_device = uci_lookup_option(uci_ctx, s, "device"); - struct uci_option *legacy_ifname = uci_lookup_option(uci_ctx, s, "ifname"); - if (host_device && legacy_ifname) { - struct uci_ptr delif_ptr = { .p = pkg, .s = s, .o = legacy_ifname }; - uci_delete(uci_ctx, &delif_ptr); - } - - struct uci_ptr renif_ptr = { .p = pkg, .s = s, .o = host_device?:legacy_ifname, .value = "host_device" }; - uci_rename(uci_ctx, &renif_ptr); - uci_rename(uci_ctx, &ren_ptr); - } - } + out = fopen(uci_config_network, "w"); + if (!out) + return errno; - uci_delete(uci_ctx, &ptr); + if (asprintf(&src, "/tmp/run/uxc-net/%s.network", jail_name) == -1) { + fclose(out); + return ENOMEM; } - /* check if device 'lo' is defined by any remaining interfaces */ - uci_foreach_element(&pkg->sections, e) { - struct uci_section *s = uci_to_section(e); - if (strcmp(s->type, "interface")) - continue; - - const char *devname = uci_lookup_option_string(uci_ctx, s, "device"); - if (devname && !strcmp(devname, "lo")) { - has_loopback = true; - break; - } - } + in = fopen(src, "r"); + free(src); - /* create loopback interface section if not defined */ - if (!has_loopback) { - struct uci_ptr ptr = { .p = pkg, .section = "loopback", .value = "interface" }; - uci_set(uci_ctx, &ptr); - uci_reorder_section(uci_ctx, ptr.s, 0); - struct uci_ptr ptr1 = { .p = pkg, .s = ptr.s, .option = "device", .value = "lo" }; - struct uci_ptr ptr2 = { .p = pkg, .s = ptr.s, .option = "proto", .value = "static" }; - struct uci_ptr ptr3 = { .p = pkg, .s = ptr.s, .option = "ipaddr", .value = "127.0.0.1" }; - struct uci_ptr ptr4 = { .p = pkg, .s = ptr.s, .option = "netmask", .value = "255.0.0.0" }; - uci_set(uci_ctx, &ptr1); - uci_set(uci_ctx, &ptr2); - uci_set(uci_ctx, &ptr3); - uci_set(uci_ctx, &ptr4); + if (in) { + while ((n = fread(buf, 1, sizeof(buf), in)) > 0) + if (fwrite(buf, 1, n, out) != n) { + ret = EIO; + break; + } + fclose(in); + } else { + fputs(loopback_network, out); } - ret = uci_export(uci_ctx, ucinetf, pkg, false); - -ucinetf_out: - fclose(ucinetf); - -uci_out: - uci_free_context(uci_ctx); + fclose(out); return ret; } @@ -323,11 +266,20 @@ static void run_netifd(struct uloop_timeout *t) netifd_out_resolvconf_dir: free(resolvconf_dir); + netifd_start_done = running; uloop_end(); } static struct uloop_timeout netifd_start_timeout = { .cb = run_netifd, }; +static void netifd_start_giveup(struct uloop_timeout *t) +{ + ERROR("timed out waiting for jail netifd to start\n"); + uloop_end(); +} + +static struct uloop_timeout netifd_giveup_timeout = { .cb = netifd_start_giveup, }; + static void inotify_read_handler(struct uloop_fd *u, unsigned int events) { int rc; @@ -376,6 +328,20 @@ static void netns_updown(struct ubus_context *ubus, const char *name, bool start blob_buf_free(&req); } +int jail_network_attach(struct ubus_context *ctx, const char *name, pid_t pid) +{ + int netns_fd; + + netns_fd = ns_open_pid("net", pid); + if (netns_fd < 0) + return ESRCH; + + netns_updown(ctx, name, true, netns_fd); + close(netns_fd); + + return 0; +} + static void jail_network_reload(struct uloop_timeout *t) { uint32_t id; @@ -439,12 +405,40 @@ static void watch_ubus_service(void) static struct uloop_timeout ubus_start_timeout = { .cb = run_ubusd, }; -int jail_network_start(struct ubus_context *new_ctx, char *new_jail_name, pid_t new_ns_pid) +static int jail_netifd_arm(void) { - ubus_pw = getpwnam("ubus"); - int ret = 0; - int netns_fd; + fd_inotify_read.fd = inotify_init1(IN_NONBLOCK | IN_CLOEXEC); + fd_inotify_read.cb = inotify_read_handler; + if (fd_inotify_read.fd == -1) { + ERROR("failed to initialize inotify handler\n"); + return EIO; + } + uloop_fd_add(&fd_inotify_read, ULOOP_READ); + + inotify_buffer = calloc(1, INOTIFY_SZ); + if (!inotify_buffer) + goto err_close; + + if (inotify_add_watch(fd_inotify_read.fd, ubus_sock_dir, IN_CREATE) == -1) { + ERROR("failed to add inotify watch on %s\n", ubus_sock_dir); + free(inotify_buffer); + goto err_close; + } + + watch_ubus_service(); + return 0; + +err_close: + close(fd_inotify_read.fd); + return EIO; +} + +int jail_network_start(struct ubus_context *new_ctx, char *new_jail_name, pid_t new_ns_pid, bool start_netifd) +{ + int ret; + + ubus_pw = getpwnam("ubus"); host_ubus_ctx = new_ctx; ns_pid = new_ns_pid; jail_name = new_jail_name; @@ -460,54 +454,33 @@ int jail_network_start(struct ubus_context *new_ctx, char *new_jail_name, pid_t } mkdir_p(ubus_sock_dir, 0755); - if (ubus_pw) { - ret = chown(ubus_sock_dir, ubus_pw->pw_uid, ubus_pw->pw_gid); - if (ret) { - ret = errno; - goto errout; - } - } - - fd_inotify_read.fd = inotify_init1(IN_NONBLOCK | IN_CLOEXEC); - fd_inotify_read.cb = inotify_read_handler; - if (fd_inotify_read.fd == -1) { - ERROR("failed to initialize inotify handler\n"); - ret = EIO; + if (ubus_pw && chown(ubus_sock_dir, ubus_pw->pw_uid, ubus_pw->pw_gid)) { + ret = errno; goto errout; } - uloop_fd_add(&fd_inotify_read, ULOOP_READ); - - inotify_buffer = calloc(1, INOTIFY_SZ); - if (!inotify_buffer) { - ret = ENOMEM; - goto errout_inotify; - } - if (inotify_add_watch(fd_inotify_read.fd, ubus_sock_dir, IN_CREATE) == -1) { - ERROR("failed to add inotify watch on %s\n", ubus_sock_dir); - free(inotify_buffer); - ret = EIO; - goto errout_inotify; - } + unlink(ubus_sock_path); - watch_ubus_service(); - - netns_fd = ns_open_pid("net", ns_pid); - if (netns_fd < 0) { - ret = ESRCH; - goto errout_inotify; + if (!start_netifd) { + run_ubusd(NULL); + return 0; } - netns_updown(host_ubus_ctx, jail_name, true, netns_fd); + ret = jail_netifd_arm(); + if (ret) + goto errout; - close(netns_fd); + netifd_start_done = false; uloop_timeout_add(&ubus_start_timeout); + uloop_timeout_set(&netifd_giveup_timeout, 5000); uloop_run(); + uloop_timeout_cancel(&netifd_giveup_timeout); + + if (!netifd_start_done) + ERROR("jail netifd did not come up; container network may be degraded\n"); return 0; -errout_inotify: - close(fd_inotify_read.fd); errout: free(ubus_sock_path); errout_path: @@ -531,17 +504,12 @@ static int jail_delete_instance(const char *instance) return ubus_invoke(host_ubus_ctx, id, "delete", req.head, NULL, NULL, 3000); } -int jail_network_stop(void) +int jail_network_teardown(void) { - int host_netns = open("/proc/self/ns/net", O_RDONLY); - - if (host_netns < 0) - return errno; - - netns_updown(jail_ubus_ctx, NULL, false, host_netns); - - close(host_netns); - ubus_free(jail_ubus_ctx); + if (jail_ubus_ctx) { + ubus_free(jail_ubus_ctx); + jail_ubus_ctx = NULL; + } jail_delete_instance("netifd"); jail_delete_instance("ubus"); @@ -550,11 +518,18 @@ int jail_network_stop(void) unlink(uci_config_network); rmdir(dirname(uci_config_network)); free(uci_config_network); + uci_config_network = NULL; } - free(ubus_sock_path); - rmdir(ubus_sock_dir); - free(ubus_sock_dir); + if (ubus_sock_path) { + free(ubus_sock_path); + ubus_sock_path = NULL; + } + if (ubus_sock_dir) { + rmdir(ubus_sock_dir); + free(ubus_sock_dir); + ubus_sock_dir = NULL; + } return 0; } diff --git a/jail/netifd.h b/jail/netifd.h index 589ed14..46816da 100644 --- a/jail/netifd.h +++ b/jail/netifd.h @@ -13,9 +13,11 @@ #ifndef _JAIL_NETIFD_H #define _JAIL_NETIFD_H +#include #include -int jail_network_start(struct ubus_context *new_ctx, char *new_jail_name, pid_t new_ns_pid); -int jail_network_stop(void); +int jail_network_attach(struct ubus_context *ctx, const char *name, pid_t pid); +int jail_network_start(struct ubus_context *new_ctx, char *new_jail_name, pid_t new_ns_pid, bool start_netifd); +int jail_network_teardown(void); #endif diff --git a/jail/uxc-net b/jail/uxc-net index 58199ab..b2aea4a 100644 --- a/jail/uxc-net +++ b/jail/uxc-net @@ -519,6 +519,14 @@ let routed_up = function(name, ann, m) { fw_uci_create(name, ann, net); + write_injail(name, "config interface 'lan'\n" + + "\toption device 'eth0'\n" + + "\toption proto 'static'\n" + + "\toption ipaddr '" + net.container + "'\n" + + "\toption netmask '255.255.255.254'\n" + + "\toption gateway '" + gw_ip + "'\n" + + "\toption dns '" + gw_ip + "'\n"); + return 0; }; @@ -646,6 +654,9 @@ let do_down = function(name, bundle) { if (ann == null) ann = {}; + let configured = !!fs.stat(injail_path(name)); + fs.unlink(injail_path(name)); + let attach = parse_attach(ann); let bh = parse_backhaul(ann); @@ -659,6 +670,8 @@ let do_down = function(name, bundle) { if (attach.kind == "bridged") return bridged_down(name, attach); + if (!configured) + return 0; return routed_down(name); }; From d1fe47f6d78986ee4bafd660a6b33921d25859aa Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 17 Jun 2026 15:55:28 +0100 Subject: [PATCH 35/82] uxc: provision and reap per-container volumes Drive the uvol backend from uxc create and delete so a container's writable storage is created on demand. A registration may declare an overlay-size for the container's own rw upper, and a data-volumes array of named volumes with mountpoints and sizes; uxc creates each via uvol (growing an existing one, keeping it if already larger), activates it, and binds it into the jail. The link to the backing store is read straight from the existing config: the image volume is the basename of the bundle path, the rw state volume the basename of write-overlay-path, so no extra field duplicates it. Image volumes are deactivated after write, so create activates the image before ujail reads the bundle. Booting a specific mountpoint defers auto-create until the uvol backend (VG) is online. On delete the rw state and per-container data volumes are reaped only when --volumes is given; the content-addressed image volume may be shared and is never removed here. uvol is invoked by fork/exec with explicit argv to keep sizes and names clear of shell quoting. Signed-off-by: Daniel Golle --- uxc.c | 275 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 267 insertions(+), 8 deletions(-) diff --git a/uxc.c b/uxc.c index 73ff85e..0c81822 100644 --- a/uxc.c +++ b/uxc.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -28,6 +29,7 @@ #include #include #include +#include #include #include @@ -110,6 +112,7 @@ static const struct option kill_opts[] = { static const struct option delete_opts[] = { {"force", no_argument, 0, 'f' }, + {"volumes", no_argument, 0, 'V' }, {0, 0, 0, 0 } }; @@ -289,7 +292,7 @@ static int usage(void) { printf("\tkill [--signal ] [--all] []\tsignal (no signal: graceful stop); --all+KILL: whole cgroup\n"); printf("\tenable \t\t\t\tstart container on boot\n"); printf("\tdisable \t\t\t\tdon't start container on boot\n"); - printf("\tdelete [--force]\t\t\tdelete \n"); + printf("\tdelete [--force] [--volumes]\tdelete ; --volumes also reaps its rw state and per-container volumes\n"); printf("\tpause \t\t\t\tfreeze every process in container 's cgroup\n"); printf("\tresume \t\t\t\tthaw a previously paused container \n"); printf("\texec [--process ] [-d] [-p ] [-- cmd args]\n"); @@ -308,6 +311,8 @@ enum { CONF_WRITE_OVERLAY_PATH, CONF_VOLUMES, CONF_IDMAP_OFFSET, + CONF_DATA_VOLUMES, + CONF_OVERLAY_SIZE, __CONF_MAX, }; @@ -321,6 +326,8 @@ static const struct blobmsg_policy conf_policy[__CONF_MAX] = { [CONF_WRITE_OVERLAY_PATH] = { .name = "write-overlay-path", .type = BLOBMSG_TYPE_STRING }, [CONF_VOLUMES] = { .name = "volumes", .type = BLOBMSG_TYPE_ARRAY }, [CONF_IDMAP_OFFSET] = { .name = "idmap-offset", .type = BLOBMSG_TYPE_STRING }, + [CONF_DATA_VOLUMES] = { .name = "data-volumes", .type = BLOBMSG_TYPE_ARRAY }, + [CONF_OVERLAY_SIZE] = { .name = "overlay-size", .type = BLOBMSG_TYPE_STRING }, }; static int conf_load(bool load_settings) @@ -978,6 +985,26 @@ static int uxc_exists(char *name) return 0; } +enum { + VOL_NAME, + VOL_MOUNTPOINT, + VOL_SIZE, + __VOL_MAX, +}; + +static const struct blobmsg_policy vol_policy[__VOL_MAX] = { + [VOL_NAME] = { .name = "name", .type = BLOBMSG_TYPE_STRING }, + [VOL_MOUNTPOINT] = { .name = "mountpoint", .type = BLOBMSG_TYPE_STRING }, + [VOL_SIZE] = { .name = "size", .type = BLOBMSG_TYPE_STRING }, +}; + +static int uvol_status(const char *vol); +static const char *uvol_volume_name(const char *path); +static int run_uvol(const char *action, const char *vol); +static int provision_rw_uvol(const char *volname, const char *size); +static int provision_data_volumes(const char *container, struct blob_attr *vols, + struct blob_buf *req); + static int uxc_create(char *name, bool immediately, const char *console_socket, bool systemd_cgroup, const char *seccomp_mode) { @@ -987,9 +1014,11 @@ static int uxc_create(char *name, bool immediately, const char *console_socket, uint32_t id; struct settings *usettings = NULL; char *path = NULL, *jailname = NULL, *pidfile = NULL, *tmprwsize = NULL, *writepath = NULL; + const char *imgvol = NULL; char *seccomp_log = NULL; + char overlaypath[PATH_MAX]; - void *in, *ins, *j; + void *in, *ins, *j, *m; bool found = false; blobmsg_for_each_attr(cur, blob_data(conf.head), rem) { @@ -1009,6 +1038,10 @@ static int uxc_create(char *name, bool immediately, const char *console_socket, path = blobmsg_get_string(tb[CONF_PATH]); + imgvol = uvol_volume_name(path); + if (imgvol) + run_uvol("up", imgvol); + if (tb[CONF_PIDFILE]) pidfile = blobmsg_get_string(tb[CONF_PIDFILE]); @@ -1061,8 +1094,28 @@ static int uxc_create(char *name, bool immediately, const char *console_socket, if (systemd_cgroup) blobmsg_add_u8(&req, "systemdcgroup", 1); + m = blobmsg_open_table(&req, "mount"); + if (tb[CONF_DATA_VOLUMES]) { + ret = provision_data_volumes(name, tb[CONF_DATA_VOLUMES], &req); + if (ret) { + blobmsg_close_table(&req, m); + blob_buf_free(&req); + return ret; + } + } + blobmsg_close_table(&req, m); + blobmsg_close_table(&req, j); + if (!writepath && !tmprwsize && tb[CONF_OVERLAY_SIZE]) { + if (provision_rw_uvol(name, blobmsg_get_string(tb[CONF_OVERLAY_SIZE]))) { + blob_buf_free(&req); + return -EIO; + } + snprintf(overlaypath, sizeof(overlaypath), "/tmp/run/uvol/%s", name); + writepath = overlaypath; + } + if (writepath) blobmsg_add_string(&req, "overlaydir", writepath); @@ -1550,7 +1603,7 @@ static void fstab_cb(struct ubus_request *req, int type, struct blob_attr *msg) fstabinfo = blob_memdup(blobmsg_data(msg)); } -static int uxc_boot(void) +static int uxc_boot(const char *mountpoint) { struct blob_attr *cur, *tb[__CONF_MAX]; struct runtime_state *rsstate = NULL; @@ -1586,6 +1639,9 @@ static int uxc_boot(void) if (!tb[CONF_NAME] || !tb[CONF_PATH]) continue; + if (mountpoint && strncmp(blobmsg_name(cur), mountpoint, strlen(mountpoint))) + continue; + rsstate = avl_find_element(&runtime, blobmsg_get_string(tb[CONF_NAME]), rsstate, avl); if (rsstate) continue; @@ -1609,6 +1665,9 @@ static int uxc_boot(void) if (checkvolumes(usettings->volumes)) continue; + if ((tb[CONF_DATA_VOLUMES] || tb[CONF_OVERLAY_SIZE]) && uvol_status(".meta")) + continue; + name = strdup(blobmsg_get_string(tb[CONF_NAME])); if (uxc_exists(name)) continue; @@ -1622,7 +1681,189 @@ static int uxc_boot(void) return ret; } -static int uxc_delete(char *name, bool force) +static const char *uvol_volume_name(const char *path) +{ + const char prefix[] = "/tmp/run/uvol/"; + const size_t plen = sizeof(prefix) - 1; + + if (!path || strncmp(path, prefix, plen)) + return NULL; + + if (!path[plen] || strchr(path + plen, '/')) + return NULL; + + return path + plen; +} + +static int run_uvol_argv(char *const argv[]) +{ + pid_t pid; + int status; + + pid = fork(); + if (pid == 0) { + execv(argv[0], argv); + _exit(127); + } else if (pid < 0) { + return -errno; + } + + while (waitpid(pid, &status, 0) < 0 && errno == EINTR); + + if (!WIFEXITED(status)) + return -EIO; + + return WEXITSTATUS(status); +} + +static int run_uvol(const char *action, const char *vol) +{ + char *argv[] = { "/usr/sbin/uvol", (char *)action, (char *)vol, NULL }; + + return run_uvol_argv(argv); +} + +static int run_uvol_create(const char *vol, const char *size, const char *type) +{ + char *argv[] = { "/usr/sbin/uvol", "create", (char *)vol, + (char *)size, (char *)type, NULL }; + + return run_uvol_argv(argv); +} + +static int run_uvol_resize(const char *vol, const char *size) +{ + char *argv[] = { "/usr/sbin/uvol", "resize", (char *)vol, (char *)size, NULL }; + + return run_uvol_argv(argv); +} + +static int uvol_status(const char *vol) +{ + char *argv[] = { "/usr/sbin/uvol", "status", (char *)vol, NULL }; + + return run_uvol_argv(argv); +} + +static long long parse_size_bytes(const char *s) +{ + char *end; + long long v; + + if (!s) + return -1; + + v = strtoll(s, &end, 10); + if (end == s || v < 0) + return -1; + + switch (*end) { + case 'g': + case 'G': + v *= 1024; + case 'm': + case 'M': + v *= 1024; + case 'k': + case 'K': + v *= 1024; + ++end; + break; + case '\0': + break; + default: + return -1; + } + + if (*end) + return -1; + + return v; +} + +static int provision_rw_uvol(const char *volname, const char *size) +{ + char sizebytes[32]; + long long bytes; + int st, rr; + + bytes = parse_size_bytes(size); + if (bytes <= 0) { + fprintf(stderr, "uxc: invalid size '%s' for volume %s\n", size, volname); + return -EINVAL; + } + snprintf(sizebytes, sizeof(sizebytes), "%lld", bytes); + + st = uvol_status(volname); + if (st == 2) { + if (run_uvol_create(volname, sizebytes, "rw")) { + fprintf(stderr, "uxc: failed to create volume %s\n", volname); + return -EIO; + } + } else { + rr = run_uvol_resize(volname, sizebytes); + if (rr == 22) + fprintf(stderr, "uxc: volume %s larger than requested, kept\n", volname); + else if (rr) { + fprintf(stderr, "uxc: failed to resize volume %s\n", volname); + return -EIO; + } + } + + if (run_uvol("up", volname)) { + fprintf(stderr, "uxc: failed to activate volume %s\n", volname); + return -EIO; + } + + return 0; +} + +static int provision_data_volumes(const char *container, struct blob_attr *vols, + struct blob_buf *req) +{ + struct blob_attr *vcur, *vt[__VOL_MAX]; + char volname[256], bindspec[PATH_MAX]; + const char *vname, *vmount, *vsize; + int vrem; + + blobmsg_for_each_attr(vcur, vols, vrem) { + blobmsg_parse(vol_policy, __VOL_MAX, vt, blobmsg_data(vcur), blobmsg_len(vcur)); + if (!vt[VOL_NAME] || !vt[VOL_MOUNTPOINT]) + continue; + + vname = blobmsg_get_string(vt[VOL_NAME]); + vmount = blobmsg_get_string(vt[VOL_MOUNTPOINT]); + vsize = vt[VOL_SIZE] ? blobmsg_get_string(vt[VOL_SIZE]) : "64m"; + + snprintf(volname, sizeof(volname), "%s.%s", container, vname); + if (provision_rw_uvol(volname, vsize)) + return -EIO; + + snprintf(bindspec, sizeof(bindspec), "/tmp/run/uvol/%s:%s", volname, vmount); + blobmsg_add_string(req, bindspec, "2"); + } + + return 0; +} + +static void reap_data_volumes(const char *container, struct blob_attr *vols) +{ + struct blob_attr *vcur, *vt[__VOL_MAX]; + char volname[256]; + int vrem; + + blobmsg_for_each_attr(vcur, vols, vrem) { + blobmsg_parse(vol_policy, __VOL_MAX, vt, blobmsg_data(vcur), blobmsg_len(vcur)); + if (!vt[VOL_NAME]) + continue; + snprintf(volname, sizeof(volname), "%s.%s", container, + blobmsg_get_string(vt[VOL_NAME])); + if (run_uvol("remove", volname)) + fprintf(stderr, "uxc: warning: could not remove volume %s\n", volname); + } +} + +static int uxc_delete(char *name, bool force, bool volumes) { struct blob_attr *cur, *tb[__CONF_MAX]; struct runtime_state *rsstate = NULL; @@ -1633,6 +1874,7 @@ static int uxc_delete(char *name, bool force) const char *cfname = NULL; const char *sfname = NULL; struct stat sb; + const char *statevol = NULL; blobmsg_for_each_attr(cur, blob_data(conf.head), rem) { blobmsg_parse(conf_policy, __CONF_MAX, tb, blobmsg_data(cur), blobmsg_len(cur)); @@ -1683,6 +1925,13 @@ static int uxc_delete(char *name, bool force) if (usettings) sfname = usettings->fname; + if (usettings && usettings->writepath) + statevol = uvol_volume_name(usettings->writepath); + else if (tb[CONF_WRITE_OVERLAY_PATH]) + statevol = uvol_volume_name(blobmsg_get_string(tb[CONF_WRITE_OVERLAY_PATH])); + else if (tb[CONF_OVERLAY_SIZE]) + statevol = name; + if (sfname) { if (stat(sfname, &sb) == -1) { ret = -ENOENT; @@ -1703,6 +1952,14 @@ static int uxc_delete(char *name, bool force) if (unlink(cfname) == -1) ret = -errno; + if (!ret && volumes && statevol) { + if (run_uvol("remove", statevol)) + fprintf(stderr, "uxc: warning: could not remove state volume %s\n", statevol); + } + + if (!ret && volumes && tb[CONF_DATA_VOLUMES]) + reap_data_volumes(name, tb[CONF_DATA_VOLUMES]); + errout: return ret; } @@ -1880,9 +2137,9 @@ int main(int argc, char **argv) goto usage_out; ret = uxc_attach(verb_argv[1]); } else if (!strcmp(verb, "boot")) { - if (verb_argc != 1) + if (verb_argc != 1 && verb_argc != 2) goto usage_out; - ret = uxc_boot(); + ret = uxc_boot(verb_argc == 2 ? verb_argv[1] : NULL); } else if (!strcmp(verb, "start")) { bool console = false; @@ -1948,16 +2205,18 @@ int main(int argc, char **argv) ret = uxc_set(verb_argv[1], NULL, 0, NULL, NULL, NULL, NULL); } else if (!strcmp(verb, "delete")) { bool force = false; + bool volumes = false; - while ((c = getopt_long(verb_argc, verb_argv, "f", delete_opts, NULL)) != -1) { + while ((c = getopt_long(verb_argc, verb_argv, "fV", delete_opts, NULL)) != -1) { switch (c) { case 'f': force = true; break; + case 'V': volumes = true; break; default: goto usage_out; } } if (optind != verb_argc - 1) goto usage_out; - ret = uxc_delete(verb_argv[optind], force); + ret = uxc_delete(verb_argv[optind], force, volumes); } else if (!strcmp(verb, "create")) { char *bundle = NULL, *pidfile = NULL; char *tmprwsize = NULL, *writepath = NULL, *requiredmounts = NULL; From a833906321edba0d0912995ef1660913c721d65d Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 05:33:07 +0100 Subject: [PATCH 36/82] jail: append per-deployment environment from a file via -x Add a -x option that appends KEY=VALUE lines from a file to the OCI process environment, parsed after the bundle's own env so a deployment can extend it without editing the image. Blank lines, comments and lines without an equals sign are skipped. procd's instance config gains an envfile attribute passed through as -x, and a changed env file triggers an instance restart. Signed-off-by: Daniel Golle --- jail/jail.c | 48 +++++++++++++++++++++++++++++++++++++++++++++- service/instance.c | 16 ++++++++++++++++ service/instance.h | 1 + 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/jail/jail.c b/jail/jail.c index d9498e0..805de7a 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -100,7 +100,7 @@ #define PR_MDWE_NO_INHERIT (1UL << 1) #endif -#define OPT_ARGS "cC:d:De:EfFG:h:iI:j:J:lm:M:n:NoO:pP:r:R:sS:uU:V:w:t:T:yY:Z" +#define OPT_ARGS "cC:d:De:EfFG:h:iI:j:J:lm:M:n:NoO:pP:r:R:sS:uU:V:w:x:t:T:yY:Z" struct hook_execvpe { char *file; @@ -144,6 +144,7 @@ static struct { char *overlaydir; char *tmpoverlaysize; char **envp; + char *envfile; char *uidmap; char *gidmap; struct blob_attr *uidmappings; @@ -1901,6 +1902,7 @@ static void usage(void) fprintf(stderr, " -c\t\tset PR_SET_NO_NEW_PRIVS\n"); fprintf(stderr, " -n \tthe name of the jail\n"); fprintf(stderr, " -e \timport environment variable\n"); + fprintf(stderr, " -x \tappend KEY=VALUE lines from to the container env\n"); fprintf(stderr, "namespace jail options:\n"); fprintf(stderr, " -h \tchange the hostname of the jail\n"); fprintf(stderr, " -N\t\tjail has network namespace\n"); @@ -2902,6 +2904,44 @@ static int parseOCIenvarray(struct blob_attr *msg, char ***envp) return 0; } +static int append_envfile(char ***envp, const char *path) +{ + char line[4096]; + char **arr = *envp; + char *nl; + int n = 0; + FILE *f; + + f = fopen(path, "r"); + if (!f) + return 0; + + while (arr && arr[n]) + ++n; + + while (fgets(line, sizeof(line), f)) { + nl = strchr(line, '\n'); + if (nl) + *nl = '\0'; + + if (line[0] == '\0' || line[0] == '#' || !strchr(line, '=')) + continue; + + arr = realloc(arr, (n + 2) * sizeof(char *)); + if (!arr) { + fclose(f); + return ENOMEM; + } + + arr[n++] = strdup(line); + arr[n] = NULL; + } + + fclose(f); + *envp = arr; + return 0; +} + enum { OCI_ROOT_PATH, OCI_ROOT_READONLY, @@ -3389,6 +3429,9 @@ static int parseOCIprocess(struct blob_attr *msg) return res; } + if (opts.envfile && (res = append_envfile(&opts.envp, opts.envfile))) + return res; + if (tb[OCI_PROCESS_USER] && (res = parseOCIprocessuser(tb[OCI_PROCESS_USER]))) return res; @@ -5429,6 +5472,9 @@ int main(int argc, char **argv) enve->envarg = optarg; list_add_tail(&enve->list, &envl); break; + case 'x': + opts.envfile = optarg; + break; case 'p': opts.namespace |= CLONE_NEWNS; opts.procfs = 1; diff --git a/service/instance.c b/service/instance.c index 660d9d5..7907dda 100644 --- a/service/instance.c +++ b/service/instance.c @@ -131,6 +131,7 @@ enum { JAIL_ATTR_IDMAP_OFFSET, JAIL_ATTR_CONSOLESOCKET, JAIL_ATTR_SYSTEMDCGROUP, + JAIL_ATTR_ENVFILE, __JAIL_ATTR_MAX, }; @@ -155,6 +156,7 @@ static const struct blobmsg_policy jail_attr[__JAIL_ATTR_MAX] = { [JAIL_ATTR_IDMAP_OFFSET] = { "idmap_offset", BLOBMSG_TYPE_STRING }, [JAIL_ATTR_CONSOLESOCKET] = { "consolesocket", BLOBMSG_TYPE_STRING }, [JAIL_ATTR_SYSTEMDCGROUP] = { "systemdcgroup", BLOBMSG_TYPE_BOOL }, + [JAIL_ATTR_ENVFILE] = { "envfile", BLOBMSG_TYPE_STRING }, }; enum { @@ -420,6 +422,11 @@ jail_run(struct service_instance *in, char **argv) argv[argc++] = jail->consolesocket; } + if (jail->envfile) { + argv[argc++] = "-x"; + argv[argc++] = jail->envfile; + } + if (jail->systemd_cgroup) argv[argc++] = "-Z"; @@ -1123,6 +1130,9 @@ instance_config_changed(struct service_instance *in, struct service_instance *in if (string_changed(in->jail.consolesocket, in_new->jail.consolesocket)) return true; + if (string_changed(in->jail.envfile, in_new->jail.envfile)) + return true; + if (in->jail.flags != in_new->jail.flags) return true; @@ -1309,6 +1319,11 @@ instance_jail_parse(struct service_instance *in, struct blob_attr *attr) jail->argc++; } + if (tb[JAIL_ATTR_ENVFILE]) { + jail->envfile = strdup(blobmsg_get_string(tb[JAIL_ATTR_ENVFILE])); + jail->argc += 2; + } + if (tb[JAIL_ATTR_SETNS]) { struct blob_attr *cur; int rem; @@ -1665,6 +1680,7 @@ instance_config_move(struct service_instance *in, struct service_instance *in_sr instance_config_move_strdup(&in->jail.hostname, in_src->jail.hostname); instance_config_move_strdup(&in->jail.pidfile, in_src->jail.pidfile); instance_config_move_strdup(&in->jail.consolesocket, in_src->jail.consolesocket); + instance_config_move_strdup(&in->jail.envfile, in_src->jail.envfile); free(in->config); in->config = in_src->config; diff --git a/service/instance.h b/service/instance.h index ec95f8d..109e52b 100644 --- a/service/instance.h +++ b/service/instance.h @@ -45,6 +45,7 @@ struct jail { char *pidfile; char *idmap_offset; char *consolesocket; + char *envfile; struct blobmsg_list mount; struct blobmsg_list setns; int argc; From 9cd4a8592c6fbae6c331e58f4409733661b4c6af Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 05:33:07 +0100 Subject: [PATCH 37/82] uxc: materialise initenv into per-container state and generate secrets Materialise a registration's initenv into per-container state once, on first create, writing an env file ujail then loads via -x. A value of "generate" yields a fresh random hex secret; "generate@" yields a secret shared across that scope, generated-or-read under a flock so whichever instance starts first seeds it and the rest read the identical value. Secrets live mode 0600 under .meta/secrets and the env file is created mode 0600. This lets a packaged container seed first-run credentials (database passwords, API tokens) deterministically, with stack members converging on one shared value without ordering constraints, and without baking secrets into the image. Signed-off-by: Daniel Golle --- uxc.c | 176 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 174 insertions(+), 2 deletions(-) diff --git a/uxc.c b/uxc.c index 0c81822..9a04c13 100644 --- a/uxc.c +++ b/uxc.c @@ -15,9 +15,11 @@ #define _GNU_SOURCE #endif +#include #include #include #include +#include #include #include #include @@ -47,6 +49,7 @@ #define OCI_VERSION_STRING "1.0.2" #define UXC_ETC_CONFDIR "/etc/uxc" #define UXC_VOL_CONFDIR "/tmp/run/uvol/.meta/uxc" +#define UXC_VOL_SECRETDIR "/tmp/run/uvol/.meta/secrets" static bool verbose = false; static bool json_output = false; @@ -313,6 +316,7 @@ enum { CONF_IDMAP_OFFSET, CONF_DATA_VOLUMES, CONF_OVERLAY_SIZE, + CONF_INITENV, __CONF_MAX, }; @@ -328,6 +332,7 @@ static const struct blobmsg_policy conf_policy[__CONF_MAX] = { [CONF_IDMAP_OFFSET] = { .name = "idmap-offset", .type = BLOBMSG_TYPE_STRING }, [CONF_DATA_VOLUMES] = { .name = "data-volumes", .type = BLOBMSG_TYPE_ARRAY }, [CONF_OVERLAY_SIZE] = { .name = "overlay-size", .type = BLOBMSG_TYPE_STRING }, + [CONF_INITENV] = { .name = "initenv", .type = BLOBMSG_TYPE_TABLE }, }; static int conf_load(bool load_settings) @@ -1005,6 +1010,164 @@ static int provision_rw_uvol(const char *volname, const char *size); static int provision_data_volumes(const char *container, struct blob_attr *vols, struct blob_buf *req); +static void gen_secret(char *out, size_t outlen) +{ + static const char hex[] = "0123456789abcdef"; + unsigned char buf[24]; + size_t i, n; + FILE *f; + + out[0] = '\0'; + f = fopen("/dev/urandom", "r"); + if (!f) + return; + n = fread(buf, 1, sizeof(buf), f); + fclose(f); + if (n != sizeof(buf) || outlen < (n * 2 + 1)) + return; + + for (i = 0; i < n; i++) { + out[i * 2] = hex[buf[i] >> 4]; + out[i * 2 + 1] = hex[buf[i] & 0xf]; + } + out[n * 2] = '\0'; +} + +static void mkdir_path(const char *path, mode_t mode) +{ + char tmp[PATH_MAX]; + char *p; + + if (strlen(path) >= sizeof(tmp)) + return; + strcpy(tmp, path); + for (p = tmp + 1; *p; p++) { + if (*p != '/') + continue; + *p = '\0'; + mkdir(tmp, mode); + *p = '/'; + } + mkdir(tmp, mode); +} + +static bool shared_secret(const char *scope, char *out, size_t outlen) +{ + char dir[PATH_MAX], path[PATH_MAX], lockpath[PATH_MAX]; + const char *p; + int lockfd; + FILE *f; + size_t n; + bool ok = false; + + if (!scope || !*scope || *scope == '/' || strstr(scope, "..")) + return false; + for (p = scope; *p; p++) + if (!isalnum((unsigned char)*p) && *p != '/' && *p != '-' && + *p != '_' && *p != '.') + return false; + + snprintf(dir, sizeof(dir), "%s/%s", UXC_VOL_SECRETDIR, scope); + snprintf(path, sizeof(path), "%s/value", dir); + snprintf(lockpath, sizeof(lockpath), "%s/.lock", UXC_VOL_SECRETDIR); + + mkdir_path(UXC_VOL_SECRETDIR, 0700); + lockfd = open(lockpath, O_CREAT | O_RDWR, 0600); + if (lockfd < 0) + return false; + flock(lockfd, LOCK_EX); + + f = fopen(path, "r"); + if (f) { + n = fread(out, 1, outlen - 1, f); + fclose(f); + out[n] = '\0'; + while (n && (out[n - 1] == '\n' || out[n - 1] == '\r')) + out[--n] = '\0'; + ok = (out[0] != '\0'); + } else { + gen_secret(out, outlen); + if (out[0]) { + mkdir_path(dir, 0700); + f = fopen(path, "w"); + if (f) { + fchmod(fileno(f), 0600); + fprintf(f, "%s", out); + fclose(f); + ok = true; + } + } + } + + flock(lockfd, LOCK_UN); + close(lockfd); + return ok; +} + +static bool envfile_has_key(const char *path, const char *key) +{ + char line[4096]; + size_t klen = strlen(key); + bool found = false; + FILE *f; + + f = fopen(path, "r"); + if (!f) + return false; + while (fgets(line, sizeof(line), f)) + if (!strncmp(line, key, klen) && line[klen] == '=') { + found = true; + break; + } + fclose(f); + return found; +} + +static void materialise_initenv(const char *name, struct blob_attr *initenv) +{ + char statedir[PATH_MAX], dir[PATH_MAX], path[PATH_MAX], secret[64]; + struct blob_attr *cur; + const char *key, *val; + int rem; + FILE *f; + + snprintf(statedir, sizeof(statedir), "%s/state", UXC_VOL_CONFDIR); + snprintf(dir, sizeof(dir), "%s/%s", statedir, name); + snprintf(path, sizeof(path), "%s/env", dir); + + blobmsg_for_each_attr(cur, initenv, rem) { + if (blobmsg_type(cur) != BLOBMSG_TYPE_STRING) + continue; + key = blobmsg_name(cur); + if (!key || !*key || envfile_has_key(path, key)) + continue; + + val = blobmsg_get_string(cur); + if (!strcmp(val, "generate")) { + gen_secret(secret, sizeof(secret)); + if (!secret[0]) + continue; + val = secret; + } else if (!strncmp(val, "generate@", 9)) { + if (!shared_secret(val + 9, secret, sizeof(secret))) + continue; + val = secret; + } + + mkdir(statedir, 0700); + mkdir(dir, 0700); + f = fopen(path, "a"); + if (!f) + return; + fchmod(fileno(f), 0600); + fprintf(f, "%s=%s\n", key, val); + fclose(f); + + if (val == secret) + fprintf(stderr, "uxc: generated %s for container %s\n", key, name); + } +} + static int uxc_create(char *name, bool immediately, const char *console_socket, bool systemd_cgroup, const char *seccomp_mode) { @@ -1017,6 +1180,7 @@ static int uxc_create(char *name, bool immediately, const char *console_socket, const char *imgvol = NULL; char *seccomp_log = NULL; char overlaypath[PATH_MAX]; + char envpath[PATH_MAX]; void *in, *ins, *j, *m; bool found = false; @@ -1086,14 +1250,22 @@ static int uxc_create(char *name, bool immediately, const char *console_socket, if (pidfile) blobmsg_add_string(&req, "pidfile", pidfile); - if (tb[CONF_IDMAP_OFFSET]) - blobmsg_add_string(&req, "idmap_offset", blobmsg_get_string(tb[CONF_IDMAP_OFFSET])); if (console_socket) blobmsg_add_string(&req, "consolesocket", console_socket); if (systemd_cgroup) blobmsg_add_u8(&req, "systemdcgroup", 1); + if (tb[CONF_IDMAP_OFFSET]) + blobmsg_add_string(&req, "idmap_offset", blobmsg_get_string(tb[CONF_IDMAP_OFFSET])); + + if (tb[CONF_INITENV]) + materialise_initenv(name, tb[CONF_INITENV]); + + snprintf(envpath, sizeof(envpath), "%s/state/%s/env", UXC_VOL_CONFDIR, name); + if (!access(envpath, R_OK)) + blobmsg_add_string(&req, "envfile", envpath); + m = blobmsg_open_table(&req, "mount"); if (tb[CONF_DATA_VOLUMES]) { ret = provision_data_volumes(name, tb[CONF_DATA_VOLUMES], &req); From 0d1c1d45a1fae5a413065b53386498706c5593e0 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 05:33:37 +0100 Subject: [PATCH 38/82] jail: provision per-instance files via idmapped credential binds Add two data-bind options that skip the executable dependency resolution of -r: -b binds a read-only data file (a rendered config or the stack's /etc/hosts), -k a read-only credential file. A provisioned "#!/bin/sh" config must not be treated as a program whose host interpreter and libraries get bound over the container's own, so neither pulls dependencies. A credential bind is idmapped once the OCI process user is known, mapping host id 0 to the container's app uid/gid, so a single root-owned secret is readable only by the container's process without a chown and without being world readable, and the same file can serve several containers. Because mount_all() runs in the child as the mapped userns uid and cannot traverse host paths under uvol's 0700 .meta, the bind source is cloned as a detached mount with open_tree in the parent (real root) and move_mount()ed into place, mirroring the existing resolv.conf handling. procd's instance runner maps mount type codes 2/3/4 to -V/-k/-b. Signed-off-by: Daniel Golle --- jail/fs.c | 107 ++++++++++++++++++++++++++++++++++++++++----- jail/fs.h | 2 + jail/jail.c | 44 ++++++++++++++++++- service/instance.c | 6 ++- 4 files changed, 147 insertions(+), 12 deletions(-) diff --git a/jail/fs.c b/jail/fs.c index 1d97432..590e255 100644 --- a/jail/fs.c +++ b/jail/fs.c @@ -431,7 +431,8 @@ void jail_fs_set_userns(bool enabled) static bool mount_opts_has(const char *opts, const char *needle); static int do_mount(const char *root, const char *orig_source, const char *target, const char *filesystemtype, - unsigned long orig_mountflags, unsigned long propflags, const char *optstr, int error, bool inner) + unsigned long orig_mountflags, unsigned long propflags, const char *optstr, int error, bool inner, + int source_fd) { struct stat s; char new[PATH_MAX]; @@ -441,15 +442,20 @@ static int do_mount(const char *root, const char *orig_source, const char *targe int fd, ret = 0; bool is_bind = (orig_mountflags & MS_BIND); bool is_mask = (source == (void *)(-1)); + bool use_fd = false; unsigned long mountflags = orig_mountflags; assert(!(inner && is_mask)); assert(!(inner && !orig_source)); if (source && is_bind && stat(source, &s)) { - if (error) - ERROR("stat(%s) failed: %m\n", source); - return error; + if (source_fd >= 0 && !fstatat(source_fd, "", &s, AT_EMPTY_PATH)) { + use_fd = true; + } else { + if (error) + ERROR("stat(%s) failed: %m\n", source); + return error; + } } if (inner) @@ -509,7 +515,14 @@ static int do_mount(const char *root, const char *orig_source, const char *targe } if (is_bind) { - if (mount(source?:new, new, filesystemtype?:"bind", MS_BIND | (mountflags & MS_REC), optstr)) { + if (use_fd) { + if (sys_move_mount(source_fd, "", AT_FDCWD, new, MOVE_MOUNT_F_EMPTY_PATH) < 0) { + if (error) + ERROR("move_mount(%s -> %s): %m\n", source, new); + ret = error; + goto free_source_out; + } + } else if (mount(source?:new, new, filesystemtype?:"bind", MS_BIND | (mountflags & MS_REC), optstr)) { if (error) ERROR("failed to mount -B %s %s: %m\n", source, new); @@ -641,6 +654,7 @@ static int _add_mount(const char *source, const char *target, const char *filesy return ENOMEM; m->idmap_treefd = -1; + m->source_fd = -1; m->avl.key = m->target = strdup(target); if (source) { if (source != (void*)(-1)) @@ -1053,6 +1067,46 @@ bool mount_is_defined(const char *target) return m != NULL; } +static struct blob_attr *single_idmap(uint32_t container_id) +{ + struct blob_buf b = {}; + void *arr, *tbl; + struct blob_attr *ret; + + blob_buf_init(&b, 0); + arr = blobmsg_open_array(&b, "m"); + tbl = blobmsg_open_table(&b, NULL); + blobmsg_add_u32(&b, "containerID", container_id); + blobmsg_add_u32(&b, "hostID", 0); + blobmsg_add_u32(&b, "size", 1); + blobmsg_close_table(&b, tbl); + blobmsg_close_array(&b, arr); + ret = blob_memdup(blobmsg_data(b.head)); + blob_buf_free(&b); + + return ret; +} + +int fs_mount_enable_idmap(const char *target, uint32_t uid, uint32_t gid) +{ + struct mount *m = avl_find_element(&mounts, target, m, avl); + + if (!m) + return ENOENT; + + free(m->uidmappings); + free(m->gidmappings); + m->uidmappings = single_idmap(uid); + m->gidmappings = single_idmap(gid); + if (!m->uidmappings || !m->gidmappings) + return ENOMEM; + + m->idmap = true; + m->idmap_recursive = false; + + return 0; +} + static void build_noafile(void) { int fd; @@ -1144,7 +1198,7 @@ static int idmap_mount_target(const char *root, struct mount *m, char *target, s return 0; } -static int idmap_tree_fd(const char *source, int userns_fd, unsigned long mountflags, bool recursive) +static int idmap_tree_fd(const char *source, int source_fd, int userns_fd, unsigned long mountflags, bool recursive) { struct ujail_mount_attr attr = { 0 }; unsigned int open_flags = OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC; @@ -1156,7 +1210,10 @@ static int idmap_tree_fd(const char *source, int userns_fd, unsigned long mountf setattr_flags |= AT_RECURSIVE; } - treefd = sys_open_tree(AT_FDCWD, source, open_flags); + if (source_fd >= 0) + treefd = sys_open_tree(source_fd, "", open_flags | AT_EMPTY_PATH); + else + treefd = sys_open_tree(AT_FDCWD, source, open_flags); if (treefd < 0) { ERROR("open_tree(%s): %m\n", source); return -1; @@ -1212,7 +1269,7 @@ static int do_idmap_mount(const char *root, struct mount *m) if (idmap_mount_target(root, m, target, sizeof(target))) goto out_close; - treefd = idmap_tree_fd(m->source, userns_fd, m->mountflags, m->idmap_recursive); + treefd = idmap_tree_fd(m->source, m->source_fd, userns_fd, m->mountflags, m->idmap_recursive); if (treefd < 0) goto out_close; @@ -1285,7 +1342,7 @@ int jail_idmap_build(const char *extroot, if (n >= maxfds) goto err; - fd = idmap_tree_fd(extroot, userns_fd, 0, false); + fd = idmap_tree_fd(extroot, -1, userns_fd, 0, false); if (fd < 0) goto err; fds[n++] = fd; @@ -1452,7 +1509,7 @@ int mount_all(const char *jailroot, const char *jail_dev) { goto out; } } else if (do_mount(jailroot, m->source, m->target, m->filesystemtype, m->mountflags, - m->propflags, m->optstr, m->error, m->inner)) { + m->propflags, m->optstr, m->error, m->inner, m->source_fd)) { ret = -1; goto out; } @@ -1472,6 +1529,8 @@ void mount_free(void) { avl_remove_all_elements(&mounts, m, avl, tmp) { if (m->source != (void*)(-1)) free((void*)m->source); + if (m->source_fd >= 0) + close(m->source_fd); free((void*)m->target); free((void*)m->filesystemtype); free((void*)m->optstr); @@ -1521,6 +1580,7 @@ int add_2paths_and_deps(const char *path, const char *path2, int readonly, int e char *map = NULL; char *fullpath = NULL; int fd, ret = -1; + struct mount *bm = NULL; if (path[0] == '/') { if (avl_find(&mounts, path2)) return 0; @@ -1528,6 +1588,10 @@ int add_2paths_and_deps(const char *path, const char *path2, int readonly, int e if (fd < 0) return error; _add_mount_bind(path, path2, readonly, error); + bm = avl_find_element(&mounts, path2, bm, avl); + if (bm && bm->source_fd < 0) + bm->source_fd = sys_open_tree(AT_FDCWD, path, + OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC); } else { if (avl_find(&libraries, path)) return 0; @@ -1587,3 +1651,26 @@ int add_2paths_and_deps(const char *path, const char *path2, int readonly, int e return ret; } + +int add_2paths_nodeps(const char *path, const char *path2, int readonly, int error) +{ + struct mount *bm = NULL; + + if (path[0] != '/') { + ERROR("%s is not an absolute path\n", path); + return error; + } + + if (avl_find(&mounts, path2)) + return 0; + + if (_add_mount_bind(path, path2, readonly, error)) + return error; + + bm = avl_find_element(&mounts, path2, bm, avl); + if (bm && bm->source_fd < 0) + bm->source_fd = sys_open_tree(AT_FDCWD, path, + OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC); + + return 0; +} diff --git a/jail/fs.h b/jail/fs.h index 8bfe8e8..d87a209 100644 --- a/jail/fs.h +++ b/jail/fs.h @@ -40,6 +40,7 @@ int add_mount_inner(const char *source, const char *target, const char *filesyst unsigned long mountflags, unsigned long propflags, const char *optstr, int error); int add_mount_bind(const char *path, int readonly, int error); int add_mount_volume(const char *source, const char *target, int error); +int fs_mount_enable_idmap(const char *target, uint32_t uid, uint32_t gid); char *resolve_mount_source(const char *source); int add_mount_fd(int fd, const char *target, int error); int mask_path_now(const char *path); @@ -75,6 +76,7 @@ int parseOCImount(struct blob_attr *msg); bool mount_is_defined(const char *target); int add_2paths_and_deps(const char *path, const char *path2, int readonly, int error, int lib); unsigned long detect_atime_flag(const char *mountpoint); +int add_2paths_nodeps(const char *path, const char *path2, int readonly, int error); static inline int add_path_and_deps(const char *path, int readonly, int error, int lib) { diff --git a/jail/jail.c b/jail/jail.c index 805de7a..22fea24 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -100,7 +100,11 @@ #define PR_MDWE_NO_INHERIT (1UL << 1) #endif -#define OPT_ARGS "cC:d:De:EfFG:h:iI:j:J:lm:M:n:NoO:pP:r:R:sS:uU:V:w:x:t:T:yY:Z" +#define OPT_ARGS "b:cC:d:De:EfFG:h:iI:j:J:k:lm:M:n:NoO:pP:r:R:sS:uU:V:w:x:t:T:yY:Z" + +#define JAIL_MAX_CREDENTIALS 16 +static const char *cred_targets[JAIL_MAX_CREDENTIALS]; +static int n_cred_targets; struct hook_execvpe { char *file; @@ -5432,6 +5436,7 @@ int main(int argc, char **argv) const char ubus[] = "/var/run/ubus/ubus.sock"; const char udebug[] = "/var/run/udebug.sock"; int ret = EXIT_FAILURE; + int credidx; int ch; char *tmp; struct list_head envl = LIST_HEAD_INIT(envl); @@ -5519,6 +5524,17 @@ int main(int argc, char **argv) case 'j': jail_join_ns(optarg); break; + case 'b': + if (!opts.ocibundle) + opts.namespace |= CLONE_NEWNS; + tmp = strchr(optarg, ':'); + if (tmp) { + *(tmp++) = '\0'; + add_2paths_nodeps(optarg, tmp, 1, 0); + } else { + add_2paths_nodeps(optarg, optarg, 1, 0); + } + break; case 'r': if (!opts.ocibundle) opts.namespace |= CLONE_NEWNS; @@ -5541,6 +5557,20 @@ int main(int argc, char **argv) add_path_and_deps(optarg, 0, 0, 0); } break; + case 'k': + if (!opts.ocibundle) + opts.namespace |= CLONE_NEWNS; + tmp = strchr(optarg, ':'); + if (!tmp) { + ERROR("credential needs a src:dest pair: %s\n", optarg); + return -1; + } + *(tmp++) = '\0'; + if (add_2paths_and_deps(optarg, tmp, 1, 0, 0)) + return -1; + if (n_cred_targets < JAIL_MAX_CREDENTIALS) + cred_targets[n_cred_targets++] = strdup(tmp); + break; case 'V': tmp = strchr(optarg, ':'); if (!tmp) { @@ -5702,6 +5732,18 @@ int main(int argc, char **argv) } } + for (credidx = 0; credidx < n_cred_targets; credidx++) { + ret = fs_mount_enable_idmap(cred_targets[credidx], + opts.pw_uid > 0 ? (uint32_t)opts.pw_uid : 0, + opts.pw_gid > 0 ? (uint32_t)opts.pw_gid : 0); + if (ret) { + ERROR("failed to idmap credential %s: %s\n", + cred_targets[credidx], strerror(ret)); + ret = -1; + goto errout; + } + } + if (opts.namespace & CLONE_NEWNET) { if (!opts.name) { ERROR("netns needs a named jail\n"); diff --git a/service/instance.c b/service/instance.c index 7907dda..acc3a2d 100644 --- a/service/instance.c +++ b/service/instance.c @@ -446,7 +446,11 @@ jail_run(struct service_instance *in, char **argv) blobmsg_list_for_each(&jail->mount, var) { const char *type = blobmsg_data(var->data); - if (*type == '2') + if (*type == '4') + argv[argc++] = "-b"; + else if (*type == '3') + argv[argc++] = "-k"; + else if (*type == '2') argv[argc++] = "-V"; else if (*type == '1') argv[argc++] = "-w"; From 6caa397c77a2d31486610c8744b1e710eb168f2b Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 05:33:37 +0100 Subject: [PATCH 39/82] uxc: render provision entries and the stack hosts file into data binds Render the provision array into the new binds (secret => idmapped -k, plain => -b) and bind the stack's shared hosts-file read-only. The hosts-file bind registers as mount type "4" only now, together with the provision handling rather than with the initenv support it was developed alongside, because procd maps type 4 to -b only from the previous commit on; registered earlier it would have been bound via -r with executable dependency resolution. Signed-off-by: Daniel Golle --- uxc.c | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/uxc.c b/uxc.c index 9a04c13..c026bee 100644 --- a/uxc.c +++ b/uxc.c @@ -317,6 +317,8 @@ enum { CONF_DATA_VOLUMES, CONF_OVERLAY_SIZE, CONF_INITENV, + CONF_HOSTS_FILE, + CONF_PROVISION, __CONF_MAX, }; @@ -333,6 +335,21 @@ static const struct blobmsg_policy conf_policy[__CONF_MAX] = { [CONF_DATA_VOLUMES] = { .name = "data-volumes", .type = BLOBMSG_TYPE_ARRAY }, [CONF_OVERLAY_SIZE] = { .name = "overlay-size", .type = BLOBMSG_TYPE_STRING }, [CONF_INITENV] = { .name = "initenv", .type = BLOBMSG_TYPE_TABLE }, + [CONF_HOSTS_FILE] = { .name = "hosts-file", .type = BLOBMSG_TYPE_STRING }, + [CONF_PROVISION] = { .name = "provision", .type = BLOBMSG_TYPE_ARRAY }, +}; + +enum { + PROV_SOURCE, + PROV_DESTINATION, + PROV_SECRET, + __PROV_MAX, +}; + +static const struct blobmsg_policy prov_policy[__PROV_MAX] = { + [PROV_SOURCE] = { .name = "source", .type = BLOBMSG_TYPE_STRING }, + [PROV_DESTINATION] = { .name = "destination", .type = BLOBMSG_TYPE_STRING }, + [PROV_SECRET] = { .name = "secret", .type = BLOBMSG_TYPE_BOOL }, }; static int conf_load(bool load_settings) @@ -1181,6 +1198,9 @@ static int uxc_create(char *name, bool immediately, const char *console_socket, char *seccomp_log = NULL; char overlaypath[PATH_MAX]; char envpath[PATH_MAX]; + char hostsbind[PATH_MAX]; + char provbind[2 * PATH_MAX]; + struct blob_attr *ptb[__PROV_MAX]; void *in, *ins, *j, *m; bool found = false; @@ -1275,6 +1295,24 @@ static int uxc_create(char *name, bool immediately, const char *console_socket, return ret; } } + if (tb[CONF_HOSTS_FILE]) { + snprintf(hostsbind, sizeof(hostsbind), "%s:/etc/hosts", + blobmsg_get_string(tb[CONF_HOSTS_FILE])); + blobmsg_add_string(&req, hostsbind, "4"); + } + if (tb[CONF_PROVISION]) { + blobmsg_for_each_attr(cur, tb[CONF_PROVISION], rem) { + blobmsg_parse(prov_policy, __PROV_MAX, ptb, + blobmsg_data(cur), blobmsg_len(cur)); + if (!ptb[PROV_SOURCE] || !ptb[PROV_DESTINATION]) + continue; + snprintf(provbind, sizeof(provbind), "%s:%s", + blobmsg_get_string(ptb[PROV_SOURCE]), + blobmsg_get_string(ptb[PROV_DESTINATION])); + blobmsg_add_string(&req, provbind, + (ptb[PROV_SECRET] && blobmsg_get_bool(ptb[PROV_SECRET])) ? "3" : "4"); + } + } blobmsg_close_table(&req, m); blobmsg_close_table(&req, j); From 5287d3b9c5f53a983a4bac92487e898faa7c2a63 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 05:33:55 +0100 Subject: [PATCH 40/82] jail: emit container lifecycle events Emit ubus events across a container's lifecycle: instance.ready once the OCI state reaches created, instance.running once init has execve'd (signalled by EOF on a CLOEXEC exec-ack pipe, which also lets the parent drop its copy of the staging /dev), and instance.stopped last in free_and_exit, after the network is removed and cgroups destroyed. procd emits no instance.stop for containers, so these events are the only lifecycle signal, and the late instance.stopped doubles as the "fully stopped, safe to recreate" marker. The error paths of main() and post_poststop() are folded into free_and_exit() so every exit emits the event. The initial OCI state becomes creating instead of created, so state queried before the runtime is set up no longer claims a created container. This makes container_handle_kill answer NOT_FOUND while creation is still in progress, so until the uxc half of this change lands, "uxc delete --force" against a container in the middle of create fails. kill learns a negative signal meaning graceful stop: SIGTERM first, with escalation to SIGKILL after UXC_STOP_TIMEOUT. A child that dies before the sync pipe handshake is diagnosed with its wait status, and a failing chdir to the OCI cwd reports the path. Signed-off-by: Daniel Golle --- jail/jail.c | 106 ++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 90 insertions(+), 16 deletions(-) diff --git a/jail/jail.c b/jail/jail.c index 22fea24..7b0683c 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -227,9 +227,15 @@ static struct { } opts; static struct blob_buf ocibuf; +static struct blob_buf notify_buf; static char **volume_sources; static int num_volume_sources; +static int exec_ack[2] = { -1, -1 }; +static void exec_ack_cb(struct uloop_fd *fd, unsigned int events); +static struct uloop_fd exec_ack_uloop = { + .cb = exec_ack_cb, +}; extern int pivot_root(const char *new_root, const char *put_old); @@ -1294,6 +1300,7 @@ static int build_jail_fs(void) } static bool exit_from_child; +static void emit_instance_event(const char *event); static bool jail_ptrace_seccomp(void); static void free_and_exit(int ret) @@ -1314,6 +1321,9 @@ static void free_and_exit(int ret) jail_dev_staged = false; } + if (!exit_from_child && opts.ocibundle && parent_ctx && opts.name) + emit_instance_event("instance.stopped"); + if (!exit_from_child && parent_ctx) ubus_free(parent_ctx); @@ -2840,8 +2850,10 @@ static void post_start_hook(void) if (!envp) free_and_exit(EXIT_FAILURE); - if (opts.cwd && chdir(opts.cwd)) + if (opts.cwd && chdir(opts.cwd)) { + ERROR("chdir(cwd=%s) failed: %m\n", opts.cwd); free_and_exit(EXIT_FAILURE); + } if (opts.landlock.n > 0 && landlock_apply(&opts.landlock)) { ERROR("landlock_apply failed\n"); @@ -4418,7 +4430,7 @@ enum { OCI_STATE_STOPPED, }; -static int jail_oci_state = OCI_STATE_CREATED; +static int jail_oci_state = OCI_STATE_CREATING; static void pipe_send_start_container(struct uloop_timeout *t); static struct uloop_timeout start_container_timeout = { .cb = pipe_send_start_container, @@ -4519,6 +4531,8 @@ static int handle_state(struct ubus_context *ctx, struct ubus_object *obj, return UBUS_STATUS_OK; } +#define UXC_STOP_TIMEOUT 120 + enum { CONTAINER_KILL_ATTR_SIGNAL, CONTAINER_KILL_ATTR_ALL, @@ -4538,12 +4552,18 @@ container_handle_kill(struct ubus_context *ctx, struct ubus_object *obj, struct blob_attr *tb[__CONTAINER_KILL_ATTR_MAX], *cur; int sig = SIGTERM; bool all = false; + bool escalate = false; blobmsg_parse(container_kill_attrs, __CONTAINER_KILL_ATTR_MAX, tb, blobmsg_data(msg), blobmsg_data_len(msg)); cur = tb[CONTAINER_KILL_ATTR_SIGNAL]; - if (cur) - sig = blobmsg_get_u32(cur); + if (cur) { + sig = (int32_t)blobmsg_get_u32(cur); + if (sig < 0) { + sig = SIGTERM; + escalate = true; + } + } cur = tb[CONTAINER_KILL_ATTR_ALL]; if (cur) @@ -4561,8 +4581,11 @@ container_handle_kill(struct ubus_context *ctx, struct ubus_object *obj, DEBUG("cgroup.kill unavailable (%d), falling back to per-pid kill\n", rc); } - if (jail_pidfd_send_signal(sig) == 0) + if (jail_pidfd_send_signal(sig) == 0) { + if (escalate) + uloop_timeout_set(&jail_process_timeout, UXC_STOP_TIMEOUT * 1000); return 0; + } switch (errno) { case EINVAL: return UBUS_STATUS_INVALID_ARGUMENT; @@ -5876,11 +5899,7 @@ int main(int argc, char **argv) uloop_run(); errout: - if (opts.ocibundle) - cgroups_free(); - - free_opts(true); - + free_and_exit(ret); return ret; } @@ -5919,6 +5938,8 @@ static int run_uxc_net(const char *action) static void post_main(struct uloop_timeout *t) { + int child_status; + if (apply_rlimits()) { ERROR("error applying resource limits\n"); free_and_exit(EXIT_FAILURE); @@ -5938,6 +5959,9 @@ static void post_main(struct uloop_timeout *t) parent_pidfd = syscall(SYS_pidfd_open, getpid(), 0); + if (pipe2(exec_ack, O_CLOEXEC) < 0) + free_and_exit(-1); + if (opts.ocibundle) cgroups_create(); @@ -6205,8 +6229,23 @@ static void post_main(struct uloop_timeout *t) close(pipes[2]); close(userns_pipe[1]); close(userns_pipe[2]); + if (exec_ack[1] >= 0) { + close(exec_ack[1]); + exec_ack[1] = -1; + } + if (exec_ack[0] >= 0) { + exec_ack_uloop.fd = exec_ack[0]; + uloop_fd_add(&exec_ack_uloop, ULOOP_READ); + } if (read(pipes[0], sig_buf, 1) < 1) { - ERROR("can't read from child\n"); + child_status = 0; + if (waitpid(jail_process.pid, &child_status, 0) == jail_process.pid && + WIFSIGNALED(child_status)) + ERROR("can't read from child: killed by signal %d\n", WTERMSIG(child_status)); + else if (WIFEXITED(child_status)) + ERROR("can't read from child: exited %d\n", WEXITSTATUS(child_status)); + else + ERROR("can't read from child\n"); free_and_exit(-1); } close(pipes[0]); @@ -6260,6 +6299,43 @@ static void post_main(struct uloop_timeout *t) run_hooks(opts.hooks.prestart, post_prestart); } +static void emit_instance_event(const char *event) +{ + if (!opts.ocibundle || !opts.name || !parent_ctx) + return; + blob_buf_init(¬ify_buf, 0); + blobmsg_add_string(¬ify_buf, "service", opts.name); + blobmsg_add_string(¬ify_buf, "instance", opts.name); + ubus_send_event(parent_ctx, event, notify_buf.head); +} + +static void exec_ack_cb(struct uloop_fd *fd, unsigned int events) +{ + char buf[8]; + ssize_t n; + + n = read(fd->fd, buf, sizeof(buf)); + if (n < 0 && errno == EINTR) + return; + + uloop_fd_delete(fd); + close(fd->fd); + exec_ack[0] = -1; + + if (n != 0) { + ERROR("container.start: exec_ack read=%zd errno=%m\n", n); + return; + } + + if (jail_dev_staged) { + umount2(jail_dev, MNT_DETACH); + rmdir(jail_dev); + jail_dev_staged = false; + } + + emit_instance_event("instance.running"); +} + static void post_poststart(void); static void post_create_runtime(void) { @@ -6368,6 +6444,8 @@ static void post_create_runtime(void) } jail_oci_state = OCI_STATE_CREATED; + emit_instance_event("instance.ready"); + if (opts.ocibundle && !opts.immediately) uloop_run(); /* wait for 'start' command via ubus */ else @@ -6642,9 +6720,5 @@ static void post_poststop(void) close(jail_process_pidfd); jail_process_pidfd = -1; } - free_opts(true); - if (parent_ctx) - ubus_free(parent_ctx); - - exit(jail_return_code); + free_and_exit(jail_return_code); } From bfbf9d42a57c5a832cc8fd55255af514e1b85a3f Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 05:33:55 +0100 Subject: [PATCH 41/82] uxc: synchronise create, start, kill and delete on lifecycle events Add an event-driven waiter (uloop plus an instance.* handler) so create, start, kill and delete block on the matching event instead of returning once the ubus call is acked. A waiter also terminates on an early instance.stopped (the container died before becoming ready) and on a timeout for an ungraceful death that never reaches free_and_exit. A kill without an explicit signal sends the negative graceful-stop signal and waits for instance.stopped. A container's stdout and stderr are routed to the system log so a crashing payload is diagnosable via logread. create reports what the events tell it. A container that exits before it is ready, or that never reaches created state at all, is a failed create, and the instance is dropped again so that the next attempt is not met with the EEXIST of a half-created container. A container that exits before instance.running arrives is not a failed start, on the other hand: a payload that simply runs to completion is what runc and crun report success for. Signed-off-by: Daniel Golle --- uxc.c | 234 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 227 insertions(+), 7 deletions(-) diff --git a/uxc.c b/uxc.c index c026bee..3115fd4 100644 --- a/uxc.c +++ b/uxc.c @@ -269,6 +269,104 @@ static struct blob_attr *blockinfo; static struct blob_attr *fstabinfo; static struct ubus_context *ctx; +#define UXC_WAIT_UNSET 0 +#define UXC_WAIT_OK 1 +#define UXC_WAIT_STOPPED 2 + +struct uxc_wait_state { + const char *service; + const char *instance; + const char *success_event; + int result; +}; + +static struct uxc_wait_state *active_wait; + +enum { + UXC_WAIT_SERVICE, + UXC_WAIT_INSTANCE, + __UXC_WAIT_INST_MAX, +}; + +static const struct blobmsg_policy uxc_wait_inst_policy[__UXC_WAIT_INST_MAX] = { + [UXC_WAIT_SERVICE] = { "service", BLOBMSG_TYPE_STRING }, + [UXC_WAIT_INSTANCE] = { "instance", BLOBMSG_TYPE_STRING }, +}; + +static void uxc_wait_timeout_cb(struct uloop_timeout *t) +{ + uloop_end(); +} + +static void uxc_wait_event_cb(struct ubus_context *uctx, + struct ubus_event_handler *ev, + const char *type, struct blob_attr *msg) +{ + struct blob_attr *tb[__UXC_WAIT_INST_MAX]; + struct uxc_wait_state *w = active_wait; + int result; + + if (!w) + return; + blobmsg_parse(uxc_wait_inst_policy, __UXC_WAIT_INST_MAX, tb, + blobmsg_data(msg), blobmsg_data_len(msg)); + if (!tb[UXC_WAIT_SERVICE] || !tb[UXC_WAIT_INSTANCE]) + return; + if (w->service && strcmp(blobmsg_get_string(tb[UXC_WAIT_SERVICE]), w->service)) + return; + if (w->instance && strcmp(blobmsg_get_string(tb[UXC_WAIT_INSTANCE]), w->instance)) + return; + + if (w->success_event && !strcmp(type, w->success_event)) + result = UXC_WAIT_OK; + else if (!strcmp(type, "instance.stopped")) + result = UXC_WAIT_STOPPED; + else + return; + + w->result = result; + uloop_end(); +} + +static struct ubus_event_handler uxc_wait_ev; +static bool uxc_wait_armed; +static bool uxc_uloop_ready; + +static int uxc_wait_arm(struct uxc_wait_state *w) +{ + if (!uxc_uloop_ready) { + uloop_init(); + ubus_add_uloop(ctx); + uxc_uloop_ready = true; + } + if (!uxc_wait_armed) { + uxc_wait_ev.cb = uxc_wait_event_cb; + if (ubus_register_event_handler(ctx, &uxc_wait_ev, "instance.*")) + return -EIO; + uxc_wait_armed = true; + } + active_wait = w; + return 0; +} + +static int uxc_wait_run(struct uxc_wait_state *w, unsigned int timeout_ms) +{ + struct uloop_timeout t = { .cb = uxc_wait_timeout_cb }; + + uloop_timeout_set(&t, timeout_ms); + if (w->result == UXC_WAIT_UNSET) + uloop_run(); + uloop_timeout_cancel(&t); + if (w->result == UXC_WAIT_UNSET) + return -ETIMEDOUT; + return 0; +} + +static void uxc_wait_disarm(void) +{ + active_wait = NULL; +} + static int usage(void) { printf("syntax: uxc [global options] [parameters ...]\n"); printf("global options:\n"); @@ -1185,6 +1283,21 @@ static void materialise_initenv(const char *name, struct blob_attr *initenv) } } +static void uxc_instance_drop(const char *name) +{ + static struct blob_buf req; + uint32_t id; + + if (ubus_lookup_id(ctx, "container", &id)) + return; + + blob_buf_init(&req, 0); + blobmsg_add_string(&req, "name", name); + blobmsg_add_string(&req, "instance", name); + ubus_invoke(ctx, id, "delete", req.head, NULL, NULL, 3000); + blob_buf_free(&req); +} + static int uxc_create(char *name, bool immediately, const char *console_socket, bool systemd_cgroup, const char *seccomp_mode) { @@ -1201,6 +1314,7 @@ static int uxc_create(char *name, bool immediately, const char *console_socket, char hostsbind[PATH_MAX]; char provbind[2 * PATH_MAX]; struct blob_attr *ptb[__PROV_MAX]; + struct uxc_wait_state wait_state; void *in, *ins, *j, *m; bool found = false; @@ -1332,6 +1446,9 @@ static int uxc_create(char *name, bool immediately, const char *console_socket, if (tmprwsize) blobmsg_add_string(&req, "tmpoverlaysize", tmprwsize); + blobmsg_add_u8(&req, "stdout", 1); + blobmsg_add_u8(&req, "stderr", 1); + blobmsg_close_table(&req, in); blobmsg_close_table(&req, ins); @@ -1345,10 +1462,36 @@ static int uxc_create(char *name, bool immediately, const char *console_socket, free(tmp); } - if (ubus_lookup_id(ctx, "container", &id) || - ubus_invoke(ctx, id, "add", req.head, NULL, NULL, 3000)) { + if (ubus_lookup_id(ctx, "container", &id)) { + blob_buf_free(&req); + return -EIO; + } + + memset(&wait_state, 0, sizeof(wait_state)); + wait_state.service = name; + wait_state.instance = name; + wait_state.success_event = "instance.ready"; + + if (uxc_wait_arm(&wait_state)) + fprintf(stderr, "uxc: warning: cannot arm instance.* watcher\n"); + + if (ubus_invoke(ctx, id, "add", req.head, NULL, NULL, 3000)) { blob_buf_free(&req); - ret = -EIO; + uxc_wait_disarm(); + return -EIO; + } + + uxc_wait_run(&wait_state, 30000); + uxc_wait_disarm(); + if (wait_state.result == UXC_WAIT_STOPPED) { + fprintf(stderr, "uxc: create %s failed: container exited before ready\n", name); + uxc_instance_drop(name); + return -EIO; + } + if (wait_state.result == UXC_WAIT_UNSET) { + fprintf(stderr, "uxc: create %s failed: the container did not reach created state\n", name); + uxc_instance_drop(name); + return -ETIMEDOUT; } return ret; @@ -1356,9 +1499,11 @@ static int uxc_create(char *name, bool immediately, const char *console_socket, static int uxc_start(const char *name, bool console) { + struct uxc_wait_state wait_state; char *objname; unsigned int id; pid_t pid; + int ret; if (console) { pid = fork(); @@ -1369,11 +1514,33 @@ static int uxc_start(const char *name, bool console) if (asprintf(&objname, "container.%s", name) == -1) return -ENOMEM; - if (ubus_lookup_id(ctx, objname, &id)) + if (ubus_lookup_id(ctx, objname, &id)) { + free(objname); return -ENOENT; - + } free(objname); - return ubus_invoke(ctx, id, "start", NULL, NULL, NULL, 3000); + + memset(&wait_state, 0, sizeof(wait_state)); + wait_state.service = name; + wait_state.instance = name; + wait_state.success_event = "instance.running"; + + if (uxc_wait_arm(&wait_state)) + fprintf(stderr, "uxc: warning: cannot arm instance.* watcher\n"); + + ret = ubus_invoke(ctx, id, "start", NULL, NULL, NULL, 3000); + if (ret) { + uxc_wait_disarm(); + return ret; + } + + uxc_wait_run(&wait_state, 30000); + uxc_wait_disarm(); + if (wait_state.result == UXC_WAIT_UNSET) { + fprintf(stderr, "uxc: warning: timed out waiting for instance.running\n"); + return -ETIMEDOUT; + } + return 0; } struct uxc_exec_reply { @@ -1505,11 +1672,13 @@ static int uxc_kill(char *name, int signal, bool all) { static struct blob_buf req; struct blob_attr *cur, *tb[__CONF_MAX]; + struct uxc_wait_state wait_state; int rem, ret; char *objname; unsigned int id; struct runtime_state *rsstate = NULL; bool found = false; + bool wait_stop = (signal < 0); blobmsg_for_each_attr(cur, blob_data(conf.head), rem) { blobmsg_parse(conf_policy, __CONF_MAX, tb, blobmsg_data(cur), blobmsg_len(cur)); @@ -1545,8 +1714,27 @@ static int uxc_kill(char *name, int signal, bool all) if (ret) return -ENOENT; - if (ubus_invoke(ctx, id, "kill", req.head, NULL, NULL, 3000)) + if (wait_stop) { + memset(&wait_state, 0, sizeof(wait_state)); + wait_state.service = name; + wait_state.instance = name; + wait_state.success_event = "instance.stopped"; + if (uxc_wait_arm(&wait_state)) + fprintf(stderr, "uxc: warning: cannot arm instance.* watcher\n"); + } + + if (ubus_invoke(ctx, id, "kill", req.head, NULL, NULL, 3000)) { + if (wait_stop) + uxc_wait_disarm(); return -EIO; + } + + if (wait_stop) { + uxc_wait_run(&wait_state, 150000); + uxc_wait_disarm(); + if (wait_state.result == UXC_WAIT_UNSET) + fprintf(stderr, "uxc: warning: timed out waiting for %s to stop\n", name); + } return 0; } @@ -2085,6 +2273,8 @@ static int uxc_delete(char *name, bool force, bool volumes) const char *sfname = NULL; struct stat sb; const char *statevol = NULL; + struct uxc_wait_state wait_state; + char *objname = NULL; blobmsg_for_each_attr(cur, blob_data(conf.head), rem) { blobmsg_parse(conf_policy, __CONF_MAX, tb, blobmsg_data(cur), blobmsg_len(cur)); @@ -2116,6 +2306,9 @@ static int uxc_delete(char *name, bool force, bool volumes) } if (rsstate) { + uint32_t cont_id; + bool have_cont_obj; + ret = ubus_lookup_id(ctx, "container", &id); if (ret) goto errout; @@ -2124,11 +2317,38 @@ static int uxc_delete(char *name, bool force, bool volumes) blobmsg_add_string(&req, "name", rsstate->container_name); blobmsg_add_string(&req, "instance", rsstate->instance_name); + if (asprintf(&objname, "container.%s", rsstate->container_name) == -1) { + blob_buf_free(&req); + ret = -ENOMEM; + goto errout; + } + + have_cont_obj = (ubus_lookup_id(ctx, objname, &cont_id) == 0); + free(objname); + objname = NULL; + + if (have_cont_obj) { + memset(&wait_state, 0, sizeof(wait_state)); + wait_state.service = rsstate->container_name; + wait_state.instance = rsstate->instance_name; + if (uxc_wait_arm(&wait_state)) + fprintf(stderr, "uxc: warning: cannot arm instance.* watcher\n"); + } + if (ubus_invoke(ctx, id, "delete", req.head, NULL, NULL, 3000)) { blob_buf_free(&req); + if (have_cont_obj) + uxc_wait_disarm(); ret = -EIO; goto errout; } + + if (have_cont_obj) { + if (uxc_wait_run(&wait_state, 30000) == -ETIMEDOUT) + fprintf(stderr, "uxc: warning: timed out waiting for container.%s removal\n", + rsstate->container_name); + uxc_wait_disarm(); + } } usettings = avl_find_element(&settings, name, usettings, avl); From 0744a2bc3c989bbc3835e18ab45a08489e539e74 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 17 Jun 2026 16:16:10 +0100 Subject: [PATCH 42/82] uxc: package-aware delete and orphan-state reconciliation Refuse to delete a container whose registration carries origin "package": such containers are owned by an apk package and must be removed with apk del, not by hand, so uxc returns EPERM and points the operator at the right command. On boot, self-heal an interrupted upgrade that left a registered container whose content-addressed image volume was never written: rather than wedging on the missing bundle, skip the container and point at "apk fix " for recovery. Combined with the orphan-state reap that purges per-container state whose registration has gone, this keeps a fleet's on-disk state converging on its registrations without manual intervention. Signed-off-by: Daniel Golle --- uxc.c | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/uxc.c b/uxc.c index 3115fd4..ab363c4 100644 --- a/uxc.c +++ b/uxc.c @@ -394,6 +394,7 @@ static int usage(void) { printf("\tenable \t\t\t\tstart container on boot\n"); printf("\tdisable \t\t\t\tdon't start container on boot\n"); printf("\tdelete [--force] [--volumes]\tdelete ; --volumes also reaps its rw state and per-container volumes\n"); + printf("\treconcile\t\t\t\treap orphaned state of containers whose package was removed (data volumes kept)\n"); printf("\tpause \t\t\t\tfreeze every process in container 's cgroup\n"); printf("\tresume \t\t\t\tthaw a previously paused container \n"); printf("\texec [--process ] [-d] [-p ] [-- cmd args]\n"); @@ -417,6 +418,7 @@ enum { CONF_INITENV, CONF_HOSTS_FILE, CONF_PROVISION, + CONF_ORIGIN, __CONF_MAX, }; @@ -435,6 +437,7 @@ static const struct blobmsg_policy conf_policy[__CONF_MAX] = { [CONF_INITENV] = { .name = "initenv", .type = BLOBMSG_TYPE_TABLE }, [CONF_HOSTS_FILE] = { .name = "hosts-file", .type = BLOBMSG_TYPE_STRING }, [CONF_PROVISION] = { .name = "provision", .type = BLOBMSG_TYPE_ARRAY }, + [CONF_ORIGIN] = { .name = "origin", .type = BLOBMSG_TYPE_STRING }, }; enum { @@ -2009,6 +2012,7 @@ static int uxc_boot(const char *mountpoint) static struct blob_buf req; int rem, ret = 0; char *name; + const char *imgvol; unsigned int id; bool autostart; @@ -2067,8 +2071,18 @@ static int uxc_boot(const char *mountpoint) continue; name = strdup(blobmsg_get_string(tb[CONF_NAME])); - if (uxc_exists(name)) + if (uxc_exists(name)) { + free(name); continue; + } + + imgvol = uvol_volume_name(blobmsg_get_string(tb[CONF_PATH])); + if (imgvol && uvol_status(imgvol)) { + ERROR("uxc: %s image %s missing (interrupted upgrade?); run 'apk fix %s'\n", + name, imgvol, name); + free(name); + continue; + } if (uxc_create(name, true, NULL, false, NULL)) ++ret; @@ -2261,6 +2275,56 @@ static void reap_data_volumes(const char *container, struct blob_attr *vols) } } +static bool uxc_registered(const char *name) +{ + struct blob_attr *cur, *tb[__CONF_MAX]; + int rem; + + blobmsg_for_each_attr(cur, blob_data(conf.head), rem) { + blobmsg_parse(conf_policy, __CONF_MAX, tb, blobmsg_data(cur), blobmsg_len(cur)); + if (tb[CONF_NAME] && !strcmp(name, blobmsg_get_string(tb[CONF_NAME]))) + return true; + } + + return false; +} + +static void reconcile_purge(const char *name, const char *statedir) +{ + char *rm[] = { "/bin/rm", "-rf", (char *)statedir, NULL }; + char path[PATH_MAX]; + + snprintf(path, sizeof(path), "%s/settings/%s.json", UXC_VOL_CONFDIR, name); + unlink(path); + + run_uvol_argv(rm); + fprintf(stderr, "uxc: reconcile: purged orphaned state for %s\n", name); +} + +static int uxc_reconcile(void) +{ + char glob_pat[PATH_MAX]; + const char *name; + glob_t gl; + int i, ret = 0; + + snprintf(glob_pat, sizeof(glob_pat), "%s/state/*", UXC_VOL_CONFDIR); + if (glob(glob_pat, GLOB_NOSORT, NULL, &gl)) + return 0; + + for (i = 0; i < gl.gl_pathc; i++) { + name = strrchr(gl.gl_pathv[i], '/'); + name = name ? name + 1 : gl.gl_pathv[i]; + if (uxc_registered(name)) + continue; + reconcile_purge(name, gl.gl_pathv[i]); + ++ret; + } + + globfree(&gl); + return ret; +} + static int uxc_delete(char *name, bool force, bool volumes) { struct blob_attr *cur, *tb[__CONF_MAX]; @@ -2291,6 +2355,11 @@ static int uxc_delete(char *name, bool force, bool volumes) if (!cfname) return -ENOENT; + if (tb[CONF_ORIGIN] && !strcmp(blobmsg_get_string(tb[CONF_ORIGIN]), "package")) { + fprintf(stderr, "uxc: %s is provided by a package; use 'apk del' to remove it\n", name); + return -EPERM; + } + rsstate = avl_find_element(&runtime, name, rsstate, avl); if (rsstate && rsstate->running) { @@ -2569,7 +2638,12 @@ int main(int argc, char **argv) } else if (!strcmp(verb, "boot")) { if (verb_argc != 1 && verb_argc != 2) goto usage_out; + uxc_reconcile(); ret = uxc_boot(verb_argc == 2 ? verb_argv[1] : NULL); + } else if (!strcmp(verb, "reconcile")) { + if (verb_argc != 1) + goto usage_out; + ret = uxc_reconcile(); } else if (!strcmp(verb, "start")) { bool console = false; From 42a8d3358c04a41fbe41e4c5008c503006e7613f Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 17 Jun 2026 16:26:51 +0100 Subject: [PATCH 43/82] uxc-stack: add the container stack engine Add uxc-stack, a ucode engine that instantiates and tears down a multi- container stack from a template. "up " loads a stack template returning compose(api), which declares named instances drawn from shared image layers, then writes one autostart registration per instance and triggers bring-up by a ubus event procd reconciles through the boot path, avoiding a synchronous create that would serialise the stack on each member's start. "down " is driven from the persistent registrations (origin "stack:"), not the template, since apk removes the template before pre-remove runs. Addressing is deterministic and state-free, derived by FNV-1a from names: a per-instance host-uid offset strided by 64k so re-created instances keep their data-volume ownership, and a backhaul /24 within RFC 2544 198.18.0.0/15. The engine writes a shared /etc/hosts (retaining localhost) for resolver-free discovery, plus per-instance annotation sidecars uxc-net merges over the image's baked annotations. Secrets are referenced via generate@ directives uxc materialises, and bound idmapped rather than copied. On teardown, data volumes and their paired generated secrets are kept and reported for manual removal. Signed-off-by: Daniel Golle --- CMakeLists.txt | 6 ++ uxc-stack | 2 + uxc-stack.uc | 250 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 258 insertions(+) create mode 100755 uxc-stack create mode 100644 uxc-stack.uc diff --git a/CMakeLists.txt b/CMakeLists.txt index 9efbca2..ef67f48 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -125,6 +125,12 @@ INSTALL(TARGETS uxc INSTALL(PROGRAMS jail/uxc-net DESTINATION ${CMAKE_INSTALL_SBINDIR} ) +INSTALL(PROGRAMS uxc-stack + DESTINATION ${CMAKE_INSTALL_SBINDIR} +) +INSTALL(FILES uxc-stack.uc + DESTINATION ${CMAKE_INSTALL_DATADIR}/uxc +) endif() IF(UTRACE_SUPPORT) diff --git a/uxc-stack b/uxc-stack new file mode 100755 index 0000000..cbe3471 --- /dev/null +++ b/uxc-stack @@ -0,0 +1,2 @@ +#!/bin/sh +exec ucode /usr/share/uxc/uxc-stack.uc "$@" diff --git a/uxc-stack.uc b/uxc-stack.uc new file mode 100644 index 0000000..ebcf002 --- /dev/null +++ b/uxc-stack.uc @@ -0,0 +1,250 @@ +'use strict'; + +import { readfile, writefile, unlink, mkdir, rmdir, lsdir, chmod } from 'fs'; +const ubus = require('ubus'); + +const REG_DIR = '/tmp/run/uvol/.meta/uxc'; +const META_DIR = '/tmp/run/uvol/.meta'; +const STACK_DIR = '/usr/share/uxc/stacks'; + +let action = ARGV[0]; +let app = ARGV[1]; +let hosts_file = null; + +if (action != 'up' && action != 'down' || !app) + die('usage: uxc-stack '); + +function read_json(path) { + let raw = readfile(path); + return raw ? json(raw) : null; +} + +function uxc(...args) { + let rc = system([ 'uxc', ...args ]); + if (rc != 0) + printf('uxc-stack: uxc %s -> %d\n', join(' ', args), rc); + return rc; +} + +function qname(name) { + return app + '.' + name; +} + + +let comp = { instances: [], secrets: {} }; + +let api = { + instance: function(name, spec) { + spec ??= {}; + spec.name = name; + push(comp.instances, spec); + return spec; + }, + + generate: function(scope) { + comp.secrets[scope] = true; + return 'generate@' + app + '/' + scope; + }, +}; + +function compose_stack() { + let tmpl_path = STACK_DIR + '/' + app + '.uc'; + let prog = loadfile(tmpl_path); + if (!prog) + die('uxc-stack: cannot load stack template ' + tmpl_path); + let compose = prog(); + if (type(compose) != 'function') + die('uxc-stack: ' + tmpl_path + ' must return a compose(api) function'); + compose(api); +} + +function stack_members() { + let out = []; + for (let e in (lsdir(REG_DIR) ?? [])) { + if (substr(e, -5) != '.json') + continue; + let reg = read_json(REG_DIR + '/' + e); + if (reg && reg.origin == 'stack:' + app) + push(out, reg); + } + return out; +} + + +function fnv1a(s) { + let h = 2166136261; + for (let i = 0; i < length(s); i++) { + h ^= ord(s, i); + h = (h * 16777619) & 0xffffffff; + } + return h; +} + +function idmap_offset(qn) { + return ((fnv1a(qn) % 0x7000) + 1) * 0x10000; +} + +function backhaul_net() { + let slot = fnv1a(app) % 512; + return sprintf('198.%d.%d', 18 + (slot >> 8), slot & 0xff); +} + +function build_registration(inst) { + if (!inst.image) + die('uxc-stack: instance ' + inst.name + ' declares no image'); + + let img = read_json(REG_DIR + '/' + inst.image + '.json'); + if (!img) + die('uxc-stack: image "' + inst.image + '" not installed (need container-' + inst.image + ')'); + + let reg = { + name: qname(inst.name), + image: img.image, + 'image-digest': img['image-digest'], + path: img.path, + origin: 'stack:' + app, + autostart: true, + }; + + if (length(inst.volumes)) { + let vols = []; + for (let v in inst.volumes) { + let p = split(v, ':'); + push(vols, { name: p[0], mountpoint: p[1], size: p[2] }); + } + reg['data-volumes'] = vols; + } + if (inst.overlay) + reg['temp-overlay-size'] = inst.overlay; + else if (img['temp-overlay-size']) + reg['temp-overlay-size'] = img['temp-overlay-size']; + else if (img['overlay-size']) + reg['overlay-size'] = img['overlay-size']; + if (inst.env) + reg.initenv = inst.env; + + let prov = []; + if (inst.provision || inst.secrets) { + let pdir = META_DIR + '/stacks/' + app + '/' + inst.name; + mkdir(META_DIR + '/stacks/' + app, 0700); + mkdir(pdir, 0700); + for (let dest in (inst.provision ?? {})) { + let src = pdir + '/' + replace(dest, /[^A-Za-z0-9._-]/g, '_'); + if (writefile(src, inst.provision[dest]) == null) + die('uxc-stack: cannot write provisioned file ' + src); + chmod(src, 0644); + push(prov, { source: src, destination: dest }); + } + for (let dest in (inst.secrets ?? {})) + push(prov, { source: META_DIR + '/secrets/' + app + '/' + + inst.secrets[dest] + '/value', destination: dest, secret: true }); + } + if (length(prov)) + reg.provision = prov; + + reg['idmap-offset'] = sprintf('%d', idmap_offset(reg.name)); + + if (hosts_file) + reg['hosts-file'] = hosts_file; + + return reg; +} + +function instance_up(inst) { + let reg = build_registration(inst); + let path = REG_DIR + '/' + reg.name + '.json'; + if (writefile(path, sprintf('%.J\n', reg)) == null) + die('uxc-stack: cannot write ' + path); +} + +function bringup() { + ubus.call({ object: 'service', method: 'event', + data: { type: 'uxc.bringup', data: {} } }); + let err = ubus.error(); + if (err) + die('uxc-stack: cannot trigger bring-up: ' + err); +} + + +function backhaul_prepare() { + let net = backhaul_net(); + let hdir = META_DIR + '/stacks/' + app; + let hosts = '127.0.0.1\tlocalhost\n::1\tlocalhost ip6-localhost ip6-loopback\n'; + let i = 0, ordered, inst, side; + + ordered = sort(comp.instances, function(a, b) { + return (a.name < b.name) ? -1 : (a.name > b.name) ? 1 : 0; + }); + for (inst in ordered) { + inst.bh_address = net + '.' + (i + 1); + hosts += inst.bh_address + '\t' + inst.name + '\n'; + i++; + } + + mkdir(META_DIR + '/stacks', 0700); + mkdir(hdir, 0700); + writefile(hdir + '/hosts', hosts); + hosts_file = hdir + '/hosts'; + + for (inst in comp.instances) { + side = { + 'org.openwrt.network.backhaul': app, + 'org.openwrt.network.backhaul-address': inst.bh_address, + }; + if (inst.access) { + let f = split(inst.access, ' '); + side['org.openwrt.network.attach'] = f[0]; + for (let n = 1; n < length(f); n++) { + let kv = split(f[n], '='); + if (kv[0] == 'egress') + side['org.openwrt.network.egress'] = kv[1]; + else if (kv[0] == 'ingress') + side['org.openwrt.network.ingress'] = kv[1]; + else if (kv[0] == 'host') + side['org.openwrt.network.host'] = kv[1]; + } + } + writefile(REG_DIR + '/' + qname(inst.name) + '.annotations', sprintf('%J\n', side)); + } +} + + +if (action == 'up') { + compose_stack(); + backhaul_prepare(); + for (let inst in comp.instances) + instance_up(inst); + bringup(); +} else { + let members = stack_members(); + let kept = []; + + for (let reg in members) { + let krc = system([ 'uxc', 'kill', reg.name ]); + if (krc != 0 && krc != 254) + printf('uxc-stack: uxc kill %s -> %d\n', reg.name, krc); + uxc('delete', reg.name); + unlink(REG_DIR + '/' + reg.name + '.json'); + unlink(REG_DIR + '/' + reg.name + '.annotations'); + system([ 'rm', '-rf', META_DIR + '/uxc/state/' + reg.name ]); + for (let v in (reg['data-volumes'] ?? [])) + push(kept, reg.name + '.' + v.name); + } + + unlink(META_DIR + '/stacks/' + app + '/hosts'); + rmdir(META_DIR + '/stacks/' + app); + + let secrets = lsdir(META_DIR + '/secrets/' + app) ?? []; + + if (length(kept)) { + warn('uxc-stack: kept data volume(s) for ' + app + + '; remove manually if the data is no longer needed:\n'); + for (let vn in kept) + warn(' uvol remove ' + vn + '\n'); + } + if (length(secrets)) { + warn('uxc-stack: kept generated secret(s) for ' + app + + ' (paired with the data above; reused on reinstall); remove with:\n'); + warn(' rm -rf ' + META_DIR + '/secrets/' + app + '\n'); + } +} From ac06cb11c0f29f00988395f6b67592837cdf1e48 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 17 Jun 2026 16:26:51 +0100 Subject: [PATCH 44/82] jail: give each stack member a unique cgroup leaf Group a container's cgroup leaf under its stack (the qualified name's first segment) and give the leaf a per-process suffix derived from the init pid. Previously the leaf was named purely from the jail name, so a respawn could land in the cgroup an exiting instance was still tearing down. That teardown writes cgroup.kill, which would then SIGKILL the newcomer mid-setup, observed as "can't read from child". The pid suffix guarantees a fresh leaf for every process, decoupling a new instance from the teardown of its predecessor. Signed-off-by: Daniel Golle --- jail/jail.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/jail/jail.c b/jail/jail.c index 7b0683c..58ecd10 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -4018,6 +4018,8 @@ static int parseOCIlinux(struct blob_attr *msg) int res = 0; char *cgpath; char cgfullpath[256] = "/sys/fs/cgroup"; + char cgleaf[200]; + char *cgsep; blobmsg_parse(oci_linux_policy, __OCI_LINUX_MAX, tb, blobmsg_data(msg), blobmsg_len(msg)); @@ -4161,13 +4163,18 @@ static int parseOCIlinux(struct blob_attr *msg) strcat(cgfullpath, cgpath); } } else { - strcat(cgfullpath, "/containers/"); - if (2 * strlen(opts.name) + 2 >= (sizeof(cgfullpath) - strlen(cgfullpath))) + cgsep = strchr(opts.name, '.'); + if (cgsep) + snprintf(cgleaf, sizeof(cgleaf), "/containers/%.*s/%s.%d", + (int)(cgsep - opts.name), opts.name, cgsep + 1, (int)getpid()); + else + snprintf(cgleaf, sizeof(cgleaf), "/containers/%s/%s.%d", + opts.name, opts.name, (int)getpid()); + + if (strlen(cgleaf) >= sizeof(cgfullpath) - strlen(cgfullpath)) return E2BIG; - strcat(cgfullpath, opts.name); /* should be container name rather than jail name */ - strcat(cgfullpath, "/"); - strcat(cgfullpath, opts.name); /* should be container instance name rather than jail name */ + strcat(cgfullpath, cgleaf); } cgroups_init(cgfullpath); From e31ce1ea9e64b3aba5d9092f49abc9f39c485975 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 17 Jun 2026 16:26:51 +0100 Subject: [PATCH 45/82] jail, uxc: declare OCI runtime-spec 1.3.0 support Advertise OCI runtime-spec 1.3.0 now that the process, linux, hook, seccomp, cgroup and namespace fields added across this series cover it. uxc stamps 1.3.0 into the config.json it generates. ujail's version check is changed from the exact "1.0" prefix to accept any 1.1, 1.2 and 1.3 spec, so bundles produced by current tooling are no longer rejected for their declared ociVersion while remaining within the major version the runtime implements. The version was stated twice, once for ujail and once for uxc. Move it to the shared container.h so the two cannot drift apart. Signed-off-by: Daniel Golle --- container.h | 2 ++ jail/jail.c | 5 +++-- jail/jail.h | 2 +- uxc.c | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/container.h b/container.h index 63b5a8c..10b9699 100644 --- a/container.h +++ b/container.h @@ -18,6 +18,8 @@ #include #include +#define OCI_VERSION_STRING "1.3.0" + #define PROCD_NOAFILE_DIR "/tmp/.ujail" #define PROCD_NOAFILE PROCD_NOAFILE_DIR "/noafile" diff --git a/jail/jail.c b/jail/jail.c index 58ecd10..ca743b1 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -4270,8 +4270,9 @@ static int parseOCI(const char *jsonfile) goto errout; } - if (strncmp("1.0", blobmsg_get_string(tb[OCI_VERSION]), 3)) { - ERROR("unsupported ociVersion %s\n", blobmsg_get_string(tb[OCI_VERSION])); + const char *ociver = blobmsg_get_string(tb[OCI_VERSION]); + if (strncmp("1.", ociver, 2) || ociver[2] < '1' || ociver[2] > '3') { + ERROR("unsupported ociVersion %s\n", ociver); res=ENOTSUP; goto errout; } diff --git a/jail/jail.h b/jail/jail.h index 2d94a2c..b4cd9bf 100644 --- a/jail/jail.h +++ b/jail/jail.h @@ -13,7 +13,7 @@ #ifndef _JAIL_JAIL_H_ #define _JAIL_JAIL_H_ -#define OCI_VERSION_STRING "1.0.2" +#include "../container.h" int mount_bind(const char *root, const char *path, int readonly, int error); int ns_open_pid(const char *nstype, const pid_t target_ns); diff --git a/uxc.c b/uxc.c index ab363c4..1b3b916 100644 --- a/uxc.c +++ b/uxc.c @@ -43,10 +43,10 @@ # define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) #endif +#include "container.h" #include "log.h" #define UXC_VERSION "0.3" -#define OCI_VERSION_STRING "1.0.2" #define UXC_ETC_CONFDIR "/etc/uxc" #define UXC_VOL_CONFDIR "/tmp/run/uvol/.meta/uxc" #define UXC_VOL_SECRETDIR "/tmp/run/uvol/.meta/secrets" From 0f72b59beedc7cf5fcf3ebb5df560132119cff37 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Tue, 18 Aug 2026 22:41:56 +0100 Subject: [PATCH 46/82] uxc: look up settings by container name uxc_set() dereferenced the parse table of the configuration loop after that loop had ended, so creating a container which has no configuration yet read uninitialised stack and crashed. The name being looked up is already available as the function argument. Fixes: df1123e668eb ("uxc: add support for user-defined settings") Signed-off-by: Daniel Golle --- uxc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uxc.c b/uxc.c index 1b3b916..3cd6050 100644 --- a/uxc.c +++ b/uxc.c @@ -1787,7 +1787,7 @@ static int uxc_set(char *name, char *path, signed char autostart, char *pidfile, return -ENOTDIR; } - usettings = avl_find_element(&settings, blobmsg_get_string(tb[CONF_NAME]), usettings, avl); + usettings = avl_find_element(&settings, name, usettings, avl); if (path && usettings) return -EIO; From 7419d653d858b0e35936c906b7ec290ea3ffbb59 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 05:34:20 +0100 Subject: [PATCH 47/82] service, jail: accept the caller's stdio for containers An OCI runtime gives the container the standard descriptors of the process that created it: that is how conmon collects a container's output and how the runtime-tools validation suite reads its results. Until now procd could only relay container output to syslog, so everything a container wrote was lost to its caller. ubus carries a single descriptor per message, so a new top-level stdio-fds.h passes all three as SCM_RIGHTS over a socket pair. procd receives the socket with the add request and installs the descriptors on the instance in place of the syslog pipes, falling back to the previous behaviour when no socket accompanies the request; ujail does the same for the processes it starts on behalf of exec. The header also carries the sending helper the uxc side uses. Signed-off-by: Daniel Golle --- jail/jail.c | 25 ++++++++++- service/instance.c | 50 ++++++++++++++++++--- service/instance.h | 2 + service/service.c | 33 +++++++++++--- stdio-fds.h | 106 +++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 203 insertions(+), 13 deletions(-) create mode 100644 stdio-fds.h diff --git a/jail/jail.c b/jail/jail.c index ca743b1..48031d8 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -73,6 +73,7 @@ #include "seccomp-trace.h" #include "cgroups.h" #include "netifd.h" +#include "../stdio-fds.h" #include #include @@ -4893,6 +4894,8 @@ container_handle_exec(struct ubus_context *ctx, struct ubus_object *obj, bool console_sock_owned = false; int cgroup_fd = -1; int pipe_fds[2] = { -1, -1 }; + int stdio_fds[STDIO_FDS_NUM] = { -1, -1, -1 }; + int stdio_sock; pid_t exec_pid, grandchild = -1; struct container_exec *e = NULL; char nspath[64]; @@ -4934,11 +4937,21 @@ container_handle_exec(struct ubus_context *ctx, struct ubus_object *obj, exec_num_additional_gids = opts.num_additional_gids; } + stdio_sock = ubus_request_get_caller_fd(req); + if (stdio_sock > -1) { + if (stdio_fds_recv(stdio_sock, stdio_fds)) + ERROR("exec: cannot receive caller stdio: %m\n"); + + close(stdio_sock); + } + blobmsg_parse(container_exec_attrs, __CONTAINER_EXEC_ATTR_MAX, tb, blobmsg_data(msg), blobmsg_data_len(msg)); - if (!tb[CONTAINER_EXEC_ATTR_ARGS]) + if (!tb[CONTAINER_EXEC_ATTR_ARGS]) { + stdio_fds_close(stdio_fds); return UBUS_STATUS_INVALID_ARGUMENT; + } args = container_exec_strarray(tb[CONTAINER_EXEC_ATTR_ARGS]); if (!args || !args[0]) { @@ -5168,6 +5181,13 @@ container_handle_exec(struct ubus_context *ctx, struct ubus_object *obj, dup2(slave_fd, STDERR_FILENO); if (slave_fd > STDERR_FILENO) close(slave_fd); + } else if (stdio_fds[1] > -1) { + for (j = 0; j < STDIO_FDS_NUM; j++) { + if (dup2(stdio_fds[j], j) < 0) + _exit(127); + if (stdio_fds[j] > STDERR_FILENO) + close(stdio_fds[j]); + } } if (exec_set_umask) umask(exec_umask); @@ -5272,6 +5292,8 @@ container_handle_exec(struct ubus_context *ctx, struct ubus_object *obj, close(pipe_fds[0]); pipe_fds[0] = -1; + stdio_fds_close(stdio_fds); + for (i = 0; i < (int)ARRAY_SIZE(ns_names); i++) if (ns_fds[i] >= 0) { close(ns_fds[i]); @@ -5332,6 +5354,7 @@ container_handle_exec(struct ubus_context *ctx, struct ubus_object *obj, return UBUS_STATUS_OK; out: + stdio_fds_close(stdio_fds); for (i = 0; i < (int)ARRAY_SIZE(ns_names); i++) if (ns_fds[i] >= 0) close(ns_fds[i]); diff --git a/service/instance.c b/service/instance.c index acc3a2d..75d94a2 100644 --- a/service/instance.c +++ b/service/instance.c @@ -567,12 +567,19 @@ instance_run(struct service_instance *in, int _stdout, int _stderr) argv[argc] = NULL; - _stdin = open("/dev/null", O_RDONLY); + if (in->stdio_fd[0] > -1) + _stdin = in->stdio_fd[0]; + else + _stdin = open("/dev/null", O_RDONLY); - if (_stdout == -1) + if (in->stdio_fd[1] > -1) + _stdout = in->stdio_fd[1]; + else if (_stdout == -1) _stdout = open("/dev/null", O_WRONLY); - if (_stderr == -1) + if (in->stdio_fd[2] > -1) + _stderr = in->stdio_fd[2]; + else if (_stderr == -1) _stderr = open("/dev/null", O_WRONLY); if (_stdin > -1) { @@ -721,14 +728,14 @@ instance_start(struct service_instance *in) } instance_free_stdio(in); - if (in->_stdout.fd.fd > -2) { + if (in->_stdout.fd.fd > -2 && in->stdio_fd[1] < 0) { if (pipe(opipe)) { ULOG_WARN("pipe() failed: %m\n"); opipe[0] = opipe[1] = -1; } } - if (in->_stderr.fd.fd > -2) { + if (in->_stderr.fd.fd > -2 && in->stdio_fd[2] < 0) { if (pipe(epipe)) { ULOG_WARN("pipe() failed: %m\n"); epipe[0] = epipe[1] = -1; @@ -1698,6 +1705,9 @@ instance_update(struct service_instance *in, struct service_instance *in_new) bool running = in->proc.pending; bool stopping = in->halt; + if (in_new->stdio_fd[1] > -1) + instance_stdio_set(in, in_new->stdio_fd); + if (!running || stopping) { instance_config_move(in, in_new); instance_start(in); @@ -1713,11 +1723,39 @@ instance_update(struct service_instance *in, struct service_instance *in_new) } } +static void +instance_free_stdio_fds(struct service_instance *in) +{ + int i; + + for (i = 0; i < 3; i++) { + if (in->stdio_fd[i] < 0) + continue; + + close(in->stdio_fd[i]); + in->stdio_fd[i] = -1; + } +} + +void +instance_stdio_set(struct service_instance *in, int *fds) +{ + int i; + + instance_free_stdio_fds(in); + + for (i = 0; i < 3; i++) { + in->stdio_fd[i] = fds[i]; + fds[i] = -1; + } +} + void instance_free(struct service_instance *in) { service_data_trigger(&in->data); instance_free_stdio(in); + instance_free_stdio_fds(in); uloop_process_delete(&in->proc); uloop_timeout_cancel(&in->timeout); uloop_timeout_cancel(&in->watchdog.timeout); @@ -1761,6 +1799,8 @@ instance_init(struct service_instance *in, struct service *s, struct blob_attr * in->require_jail = false; in->immediately = false; + in->stdio_fd[0] = in->stdio_fd[1] = in->stdio_fd[2] = -1; + in->_stdout.fd.fd = -2; in->_stdout.stream.string_data = true; in->_stdout.stream.notify_read = instance_stdout; diff --git a/service/instance.h b/service/instance.h index 109e52b..7006b74 100644 --- a/service/instance.h +++ b/service/instance.h @@ -112,6 +112,7 @@ struct service_instance { struct blob_attr *config; struct uloop_process proc; struct uloop_timeout timeout; + int stdio_fd[3]; struct ustream_fd _stdout; struct ustream_fd _stderr; struct ustream_fd console; @@ -134,6 +135,7 @@ void instance_start(struct service_instance *in); void instance_stop(struct service_instance *in, bool halt); void instance_update(struct service_instance *in, struct service_instance *in_new); void instance_init(struct service_instance *in, struct service *s, struct blob_attr *config); +void instance_stdio_set(struct service_instance *in, int *fds); void instance_free(struct service_instance *in); void instance_dump(struct blob_buf *b, struct service_instance *in, int debug); void service_event_instance_exit(const char *type, struct service_instance *in); diff --git a/service/service.c b/service/service.c index 28e9017..b146043 100644 --- a/service/service.c +++ b/service/service.c @@ -13,6 +13,7 @@ */ #include +#include #include #include #include @@ -29,6 +30,7 @@ #include "instance.h" #include "../rcS.h" +#include "../stdio-fds.h" AVL_TREE(services, avl_strcmp, false, NULL); AVL_TREE(containers, avl_strcmp, false, NULL); @@ -37,7 +39,7 @@ static struct ubus_context *ctx; static struct ubus_object main_object; static void -service_instance_add(struct service *s, struct blob_attr *attr) +service_instance_add(struct service *s, struct blob_attr *attr, int *stdio_fds) { struct service_instance *in; @@ -49,6 +51,9 @@ service_instance_add(struct service *s, struct blob_attr *attr) return; instance_init(in, s, attr); + if (stdio_fds && stdio_fds[1] > -1) + instance_stdio_set(in, stdio_fds); + vlist_add(&s->instances, &in->node, (void *) in->name); } @@ -152,7 +157,7 @@ service_update_data(struct service *s, struct blob_attr *data) } static int -service_update(struct service *s, struct blob_attr **tb, bool add, bool init) +service_update(struct service *s, struct blob_attr **tb, bool add, bool init, int *stdio_fds) { struct blob_attr *cur; int rem; @@ -182,7 +187,7 @@ service_update(struct service *s, struct blob_attr **tb, bool add, bool init) if (!add) vlist_update(&s->instances); blobmsg_for_each_attr(cur, tb[SERVICE_SET_INSTANCES], rem) { - service_instance_add(s, cur); + service_instance_add(s, cur, stdio_fds); } if (!add) vlist_flush(&s->instances); @@ -430,11 +435,12 @@ service_handle_set(struct ubus_context *ctx, struct ubus_object *obj, struct blob_attr *msg) { struct blob_attr *tb[__SERVICE_SET_MAX], *cur; + int stdio_fds[3] = { -1, -1, -1 }; struct service *s = NULL; const char *name; bool container = is_container_obj(obj); bool add = !strcmp(method, "add"); - int ret; + int sock, ret; blobmsg_parse(service_set_attrs, __SERVICE_SET_MAX, tb, blobmsg_data(msg), blobmsg_data_len(msg)); cur = tb[SERVICE_SET_NAME]; @@ -443,6 +449,14 @@ service_handle_set(struct ubus_context *ctx, struct ubus_object *obj, name = blobmsg_data(cur); + sock = req ? ubus_request_get_caller_fd(req) : -1; + if (sock > -1) { + if (stdio_fds_recv(sock, stdio_fds)) + ULOG_WARN("failed to receive stdio for %s: %m\n", name); + + close(sock); + } + if (container) s = avl_find_element(&containers, name, s, avl); else @@ -450,17 +464,22 @@ service_handle_set(struct ubus_context *ctx, struct ubus_object *obj, if (s) { P_DEBUG(2, "Update service %s\n", name); - return service_update(s, tb, add, false); + ret = service_update(s, tb, add, false, stdio_fds); + stdio_fds_close(stdio_fds); + return ret; } P_DEBUG(2, "Create service %s\n", name); s = service_alloc(name); - if (!s) + if (!s) { + stdio_fds_close(stdio_fds); return UBUS_STATUS_UNKNOWN_ERROR; + } s->container = container; - ret = service_update(s, tb, add, true); + ret = service_update(s, tb, add, true, stdio_fds); + stdio_fds_close(stdio_fds); if (ret) return ret; diff --git a/stdio-fds.h b/stdio-fds.h new file mode 100644 index 0000000..de5a5e8 --- /dev/null +++ b/stdio-fds.h @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2026 Daniel Golle + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License version 2.1 + * as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef __STDIO_FDS_H +#define __STDIO_FDS_H + +#include +#include +#include + +#define STDIO_FDS_NUM 3 + +/* + * ubus carries a single file descriptor per request, so the three standard + * descriptors travel as SCM_RIGHTS over a socket pair whose receiving end is + * what gets passed to ubus_invoke_fd(). + */ +static inline int stdio_fds_send(const int *fds) +{ + char cmsgbuf[CMSG_SPACE(STDIO_FDS_NUM * sizeof(int))]; + struct msghdr msg = { 0 }; + struct cmsghdr *cmsg; + struct iovec iov; + char dummy = 0; + int sp[2]; + + if (socketpair(AF_UNIX, SOCK_STREAM, 0, sp)) + return -1; + + iov.iov_base = &dummy; + iov.iov_len = sizeof(dummy); + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + msg.msg_control = cmsgbuf; + msg.msg_controllen = sizeof(cmsgbuf); + + cmsg = CMSG_FIRSTHDR(&msg); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + cmsg->cmsg_len = CMSG_LEN(STDIO_FDS_NUM * sizeof(int)); + memcpy(CMSG_DATA(cmsg), fds, STDIO_FDS_NUM * sizeof(int)); + + if (sendmsg(sp[0], &msg, 0) < 0) { + close(sp[0]); + close(sp[1]); + return -1; + } + + close(sp[0]); + + return sp[1]; +} + +static inline int stdio_fds_recv(int sock, int *fds) +{ + char cmsgbuf[CMSG_SPACE(STDIO_FDS_NUM * sizeof(int))]; + struct msghdr msg = { 0 }; + struct cmsghdr *cmsg; + struct iovec iov; + char dummy; + + iov.iov_base = &dummy; + iov.iov_len = sizeof(dummy); + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + msg.msg_control = cmsgbuf; + msg.msg_controllen = sizeof(cmsgbuf); + + if (recvmsg(sock, &msg, MSG_CMSG_CLOEXEC) < 0) + return -1; + + cmsg = CMSG_FIRSTHDR(&msg); + if (!cmsg || cmsg->cmsg_level != SOL_SOCKET || + cmsg->cmsg_type != SCM_RIGHTS || + cmsg->cmsg_len != CMSG_LEN(STDIO_FDS_NUM * sizeof(int))) + return -1; + + memcpy(fds, CMSG_DATA(cmsg), STDIO_FDS_NUM * sizeof(int)); + + return 0; +} + +static inline void stdio_fds_close(int *fds) +{ + int i; + + for (i = 0; i < STDIO_FDS_NUM; i++) { + if (fds[i] < 0) + continue; + + close(fds[i]); + fds[i] = -1; + } +} + +#endif From a0a898339fa27aea4fdea83af9a3c4a894f7cd68 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 05:34:20 +0100 Subject: [PATCH 48/82] uxc: hand the caller's stdio to containers Send the caller's stdin, stdout and stderr as SCM_RIGHTS over a socket pair and pass its receiving end with the create and exec requests, so a container's output reaches the process that created it and podman logs and the validation suite see it. The three descriptors are chosen explicitly rather than taken from whatever uxc happens to hold at the time, because --log redirects our own stderr into the log file and the container's error output does not belong there. The caller's stderr is kept aside before that redirection and handed over. Signed-off-by: Daniel Golle --- uxc.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/uxc.c b/uxc.c index 3cd6050..7a6b180 100644 --- a/uxc.c +++ b/uxc.c @@ -45,6 +45,7 @@ #include "container.h" #include "log.h" +#include "stdio-fds.h" #define UXC_VERSION "0.3" #define UXC_ETC_CONFDIR "/etc/uxc" @@ -53,6 +54,7 @@ static bool verbose = false; static bool json_output = false; +static int stdio_fds[STDIO_FDS_NUM] = { STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO }; static const char *confdir = UXC_ETC_CONFDIR; static struct ustream_fd cufd; static struct ustream_fd lufd; @@ -1449,9 +1451,6 @@ static int uxc_create(char *name, bool immediately, const char *console_socket, if (tmprwsize) blobmsg_add_string(&req, "tmpoverlaysize", tmprwsize); - blobmsg_add_u8(&req, "stdout", 1); - blobmsg_add_u8(&req, "stderr", 1); - blobmsg_close_table(&req, in); blobmsg_close_table(&req, ins); @@ -1478,7 +1477,8 @@ static int uxc_create(char *name, bool immediately, const char *console_socket, if (uxc_wait_arm(&wait_state)) fprintf(stderr, "uxc: warning: cannot arm instance.* watcher\n"); - if (ubus_invoke(ctx, id, "add", req.head, NULL, NULL, 3000)) { + if (ubus_invoke_fd(ctx, id, "add", req.head, NULL, NULL, 3000, + stdio_fds_send(stdio_fds))) { blob_buf_free(&req); uxc_wait_disarm(); return -EIO; @@ -1625,8 +1625,9 @@ static int uxc_exec(const char *name, const char *process_file, return -ENOENT; } - ret = ubus_invoke(ctx, id, "exec", req.head, - uxc_exec_reply_cb, &reply, 0); + ret = ubus_invoke_fd(ctx, id, "exec", req.head, + uxc_exec_reply_cb, &reply, 0, + tty ? -1 : stdio_fds_send(stdio_fds)); blob_buf_free(&req); if (ret) @@ -2578,6 +2579,12 @@ int main(int argc, char **argv) fprintf(stderr, "uxc: cannot open --log path %s: %m\n", log_path); return -EIO; } + stdio_fds[2] = dup(STDERR_FILENO); + if (stdio_fds[2] < 0) { + fprintf(stderr, "uxc: cannot preserve stderr: %m\n"); + close(fd); + return -EIO; + } if (dup2(fd, STDERR_FILENO) < 0) { dprintf(fd, "uxc: dup2(--log path) failed: %m\n"); close(fd); From 9a1abfcba19e7ca0d95e733be56b42999dc3c4b0 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 19 Aug 2026 00:02:12 +0100 Subject: [PATCH 49/82] jail: log through ulog instead of the container's stdio INFO, WARNING and DEBUG printed to stdout and ERROR to stderr, which now carry the container's own output, so every diagnostic ujail produced was mixed into the stream its caller collects. That corrupts anything structured, the TAP output of the runtime validation suite for one. Route the macros through ulog, which picks the terminal only when one is attached, and pin the OCI container case to syslog where these messages belong. Signed-off-by: Daniel Golle --- jail/jail.c | 3 +++ jail/log.h | 18 +++++------------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/jail/jail.c b/jail/jail.c index 48031d8..3fc0e76 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -5768,6 +5768,9 @@ int main(int argc, char **argv) char *jsonfile; int ocires; + /* stdout and stderr belong to the container, not to us */ + ulog_open(ULOG_SYSLOG, LOG_DAEMON, "jail"); + if (!opts.name) { ERROR("OCI bundle needs a named jail\n"); ret=-1; diff --git a/jail/log.h b/jail/log.h index a1091d4..5be19e8 100644 --- a/jail/log.h +++ b/jail/log.h @@ -14,22 +14,14 @@ #define _JAIL_LOG_H_ extern int debug; -#include #include +#include -#define INFO(fmt, ...) do { \ - printf("jail: "fmt, ## __VA_ARGS__); \ - } while (0) -#define WARNING(fmt, ...) do { \ - syslog(LOG_WARNING, "jail: "fmt, ## __VA_ARGS__); \ - printf("jail: "fmt, ## __VA_ARGS__); \ - } while (0) -#define ERROR(fmt, ...) do { \ - syslog(LOG_ERR, "jail: "fmt, ## __VA_ARGS__); \ - fprintf(stderr,"jail: "fmt, ## __VA_ARGS__); \ - } while (0) +#define INFO(fmt, ...) ULOG_INFO(fmt, ## __VA_ARGS__) +#define WARNING(fmt, ...) ULOG_WARN(fmt, ## __VA_ARGS__) +#define ERROR(fmt, ...) ULOG_ERR(fmt, ## __VA_ARGS__) #define DEBUG(fmt, ...) do { \ - if (debug) printf("jail: "fmt, ## __VA_ARGS__); \ + if (debug) ulog(LOG_DEBUG, fmt, ## __VA_ARGS__); \ } while (0) #endif From 53266e89aae6b365c3710831c960312ae0fc9c4d Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 19 Aug 2026 00:37:16 +0100 Subject: [PATCH 50/82] jail: pass the container state to hooks on stdin The runtime spec requires the state of the container to reach every hook over its standard input so the hook can act on it; ours were started with whatever stdin the runtime happened to have, and the validation suite failed all three hooks_stdin assertions with "unexpected end of JSON input". Feed each hook the same state document the state method reports, from a pipe prepared before the fork. The document construction moves to oci_state_fill() so both paths render exactly the same thing. Signed-off-by: Daniel Golle --- jail/jail.c | 77 ++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 61 insertions(+), 16 deletions(-) diff --git a/jail/jail.c b/jail/jail.c index 3fc0e76..1a4d9fb 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -648,6 +648,7 @@ static struct uloop_timeout hook_process_timeout = { }; static void run_hooklist(void); +static void oci_state_fill(struct blob_buf *b); static void hook_process_handler(struct uloop_process *c, int ret) { uloop_timeout_cancel(&hook_process_timeout); @@ -680,9 +681,38 @@ static void hook_process_timeout_cb(struct uloop_timeout *t) kill(hook_process.pid, SIGKILL); } +static int hook_state_pipe(void) +{ + static struct blob_buf sb; + int state_pipe[2]; + char *state; + size_t len; + + blob_buf_init(&sb, 0); + oci_state_fill(&sb); + state = blobmsg_format_json(sb.head, true); + if (!state) + return -1; + + if (pipe(state_pipe)) { + free(state); + return -1; + } + + len = strlen(state); + if (write(state_pipe[1], state, len) != (ssize_t)len) + WARNING("cannot pass the container state to the hook: %m\n"); + + free(state); + close(state_pipe[1]); + + return state_pipe[0]; +} + static void run_hooklist(void) { struct hook_execvpe *hook = *current_hook; + int state_fd; struct stat s; if (!hook) @@ -696,10 +726,16 @@ static void run_hooklist(void) if (!((unsigned long)s.st_mode & (S_IXUSR | S_IXGRP | S_IXOTH))) hook_process_handler(&hook_process, EPERM); + state_fd = hook_state_pipe(); + hook_running = 1; hook_process.pid = fork(); if (hook_process.pid == 0) { /* child */ + if (state_fd > -1) { + dup2(state_fd, STDIN_FILENO); + close(state_fd); + } execve(hook->file, hook->argv, hook->envp); ERROR("execve error %m\n"); _exit(errno); @@ -707,10 +743,15 @@ static void run_hooklist(void) /* fork error */ ERROR("hook fork error\n"); hook_running = 0; + if (state_fd > -1) + close(state_fd); hook_process_handler(&hook_process, errno); } /* parent */ + if (state_fd > -1) + close(state_fd); + uloop_process_add(&hook_process); if (hook->timeout > 0) @@ -4458,9 +4499,7 @@ static int handle_start(struct ubus_context *ctx, struct ubus_object *obj, } static struct blob_buf bb; -static int handle_state(struct ubus_context *ctx, struct ubus_object *obj, - struct ubus_request_data *req, const char *method, - struct blob_attr *msg) +static void oci_state_fill(struct blob_buf *b) { char *statusstr; @@ -4484,26 +4523,25 @@ static int handle_state(struct ubus_context *ctx, struct ubus_object *obj, statusstr = "unknown"; } - blob_buf_init(&bb, 0); - blobmsg_add_string(&bb, "ociVersion", OCI_VERSION_STRING); - blobmsg_add_string(&bb, "id", opts.name); - blobmsg_add_string(&bb, "status", statusstr); + blobmsg_add_string(b, "ociVersion", OCI_VERSION_STRING); + blobmsg_add_string(b, "id", opts.name); + blobmsg_add_string(b, "status", statusstr); if (jail_oci_state == OCI_STATE_CREATED || jail_oci_state == OCI_STATE_RUNNING || jail_oci_state == OCI_STATE_PAUSED) { int64_t v; - blobmsg_add_u32(&bb, "pid", jail_process.pid); + blobmsg_add_u32(b, "pid", jail_process.pid); v = cgroups_read_int64("memory.peak"); if (v >= 0) - blobmsg_add_u64(&bb, "memoryPeak", (uint64_t)v); + blobmsg_add_u64(b, "memoryPeak", (uint64_t)v); v = cgroups_read_int64("memory.swap.peak"); if (v >= 0) - blobmsg_add_u64(&bb, "memorySwapPeak", (uint64_t)v); + blobmsg_add_u64(b, "memorySwapPeak", (uint64_t)v); v = cgroups_read_int64("pids.peak"); if (v >= 0) - blobmsg_add_u64(&bb, "pidsPeak", (uint64_t)v); + blobmsg_add_u64(b, "pidsPeak", (uint64_t)v); int events_fd = cgroups_open_attr("memory.events.local"); if (events_fd >= 0) { @@ -4512,7 +4550,7 @@ static int handle_state(struct ubus_context *ctx, struct ubus_object *obj, close(events_fd); if (en > 0) { - void *sub = blobmsg_open_table(&bb, "memoryEventsLocal"); + void *sub = blobmsg_open_table(b, "memoryEventsLocal"); ebuf[en] = '\0'; next = ebuf; @@ -4522,19 +4560,26 @@ static int handle_state(struct ubus_context *ctx, struct ubus_object *obj, if (!space) continue; *space = '\0'; - blobmsg_add_u64(&bb, line, + blobmsg_add_u64(b, line, strtoull(space + 1, NULL, 10)); } - blobmsg_close_table(&bb, sub); + blobmsg_close_table(b, sub); } } } - blobmsg_add_string(&bb, "bundle", opts.ocibundle); + blobmsg_add_string(b, "bundle", opts.ocibundle); if (opts.annotations) - blobmsg_add_blob(&bb, opts.annotations); + blobmsg_add_blob(b, opts.annotations); +} +static int handle_state(struct ubus_context *ctx, struct ubus_object *obj, + struct ubus_request_data *req, const char *method, + struct blob_attr *msg) +{ + blob_buf_init(&bb, 0); + oci_state_fill(&bb); ubus_send_reply(ctx, req, bb.head); return UBUS_STATUS_OK; From 89af6831dd01eda0e3d5c182e7709c06a7a53a0e Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 19 Aug 2026 00:37:16 +0100 Subject: [PATCH 51/82] jail: write the pid file without a trailing newline runc and crun write the bare decimal into the pid file they are asked for, and that is what readers of the file expect to find. ujail appended a newline, which a caller comparing the contents byte for byte, or reading the file with a parser that accepts nothing but digits, does not agree with. Fixes: 602b8fa14a97 ("jail: add option for pidfile") Signed-off-by: Daniel Golle --- jail/jail.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jail/jail.c b/jail/jail.c index 1a4d9fb..5fe9ef7 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -5429,7 +5429,7 @@ jail_writepid(pid_t pid) if (_pidfile == NULL) return errno; - if (fprintf(_pidfile, "%d\n", pid) < 0) { + if (fprintf(_pidfile, "%d", pid) < 0) { fclose(_pidfile); return errno; } From c2becdeb9836f2d7569a2bc218b3ab0421ed5523 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 19 Aug 2026 00:58:30 +0100 Subject: [PATCH 52/82] jail: mount in the order the configuration lists Mounts were kept in a tree keyed by their target and applied while walking it, so they were established in alphabetical order of the destination rather than the order given in the configuration. The runtime spec requires the listed order, and the validation suite reported every entry that came after an alphabetically later one as out of order, /dev landing before /proc for instance. Keep the tree for lookups and add a list which preserves insertion order for applying them. Signed-off-by: Daniel Golle --- jail/fs.c | 96 ++++++++++++++++++++++++++++++++++++----------------- jail/jail.c | 18 +++++----- 2 files changed, 75 insertions(+), 39 deletions(-) diff --git a/jail/fs.c b/jail/fs.c index 590e255..e6f1dd2 100644 --- a/jail/fs.c +++ b/jail/fs.c @@ -281,6 +281,7 @@ int build_userns_fd(struct blob_attr *uidmappings, struct blob_attr *gidmappings struct mount { struct avl_node avl; + struct list_head list; const char *source; const char *target; const char *filesystemtype; @@ -293,6 +294,7 @@ struct mount { bool idmap; bool idmap_recursive; bool volume; + bool mounted; int idmap_treefd; struct blob_attr *uidmappings; struct blob_attr *gidmappings; @@ -316,6 +318,7 @@ int sys_mount_setattr(int dfd, const char *path, unsigned flags, struct ujail_mo } struct avl_tree mounts; +static LIST_HEAD(mounts_order); /* same masking as do_mount()'s is_mask branch, applied immediately * against an absolute path instead of queued through jail_root. @@ -675,6 +678,7 @@ static int _add_mount(const char *source, const char *target, const char *filesy m->source_fd = -1; avl_insert(&mounts, &m->avl); + list_add_tail(&m->list, &mounts_order); DEBUG("adding mount %s %s bind(%d) ro(%d) err(%d)\n", (m->source == (void*)(-1))?"mask":m->source, m->target, !!(m->mountflags & MS_BIND), !!(m->mountflags & MS_RDONLY), m->error != 0); @@ -725,6 +729,7 @@ int add_mount_fd(int fd, const char *target, int error) m->source_fd = fd; avl_insert(&mounts, &m->avl); + list_add_tail(&m->list, &mounts_order); DEBUG("adding mount fd:%d %s bind(1) ro(?) err(%d)\n", fd, target, error != 0); return 0; @@ -1468,10 +1473,63 @@ void mount_stage_dev(const char *jail_dev) } } +static bool mount_target_covers(const struct mount *outer, const struct mount *inner) +{ + size_t len = strlen(outer->target); + + if (!strcmp(outer->target, "/")) + return strcmp(inner->target, "/") != 0; + + return !strncmp(inner->target, outer->target, len) && inner->target[len] == '/'; +} + +static int mount_one(const char *jailroot, const char *jail_dev, struct mount *m) +{ + char devtarget[PATH_MAX]; + struct mount *outer; + + if (m->mounted) + return 0; + + m->mounted = true; + + /* whatever this one sits on has to be established first */ + list_for_each_entry(outer, &mounts_order, list) + if (!outer->mounted && mount_target_covers(outer, m) && + mount_one(jailroot, jail_dev, outer)) + return -1; + + if (jail_dev && m->filesystemtype && !strcmp(m->filesystemtype, "tmpfs") && + !strcmp(m->target, "/dev")) { + snprintf(devtarget, sizeof(devtarget), "%s%s", jailroot, m->target); + mkdir_p(devtarget, 0755); + if (mount(jail_dev, devtarget, NULL, MS_BIND | MS_REC, NULL)) { + ERROR("mount(MS_BIND, %s -> %s): %m\n", jail_dev, devtarget); + return -1; + } + + return 0; + } + + if (m->idmap_treefd >= 0) + return do_move_idmap_mount(jailroot, m) ? -1 : 0; + + if (m->idmap) + return do_idmap_mount(jailroot, m) ? -1 : 0; + + if (m->source_fd >= 0) + return do_mount_fd(jailroot, m->source_fd, m->target, m->error) ? -1 : 0; + + if (do_mount(jailroot, m->source, m->target, m->filesystemtype, m->mountflags, + m->propflags, m->optstr, m->error, m->inner, m->source_fd)) + return -1; + + return 0; +} + int mount_all(const char *jailroot, const char *jail_dev) { struct library *l; struct mount *m; - char devtarget[PATH_MAX]; int ret = 0; build_noafile(); @@ -1483,43 +1541,19 @@ int mount_all(const char *jailroot, const char *jail_dev) { avl_for_each_element(&libraries, l, avl) add_mount_bind(l->path, 1, -1); - avl_for_each_element(&mounts, m, avl) { - if (jail_dev && m->filesystemtype && !strcmp(m->filesystemtype, "tmpfs") && - !strcmp(m->target, "/dev")) { - snprintf(devtarget, sizeof(devtarget), "%s%s", jailroot, m->target); - mkdir_p(devtarget, 0755); - if (mount(jail_dev, devtarget, NULL, MS_BIND | MS_REC, NULL)) { - ERROR("mount(MS_BIND, %s -> %s): %m\n", jail_dev, devtarget); - ret = -1; - goto out; - } - } else if (m->idmap_treefd >= 0) { - if (do_move_idmap_mount(jailroot, m)) { - ret = -1; - goto out; - } - } else if (m->idmap) { - if (do_idmap_mount(jailroot, m)) { - ret = -1; - goto out; - } - } else if (m->source_fd >= 0) { - if (do_mount_fd(jailroot, m->source_fd, m->target, m->error)) { - ret = -1; - goto out; - } - } else if (do_mount(jailroot, m->source, m->target, m->filesystemtype, m->mountflags, - m->propflags, m->optstr, m->error, m->inner, m->source_fd)) { + /* the spec has the runtime establish the mounts in the order they are listed */ + list_for_each_entry(m, &mounts_order, list) { + if (mount_one(jailroot, jail_dev, m)) { ret = -1; - goto out; + break; } } -out: if (jailroot_dirfd >= 0) { close(jailroot_dirfd); jailroot_dirfd = -1; } + return ret; } @@ -1527,6 +1561,7 @@ void mount_free(void) { struct mount *m, *tmp; avl_remove_all_elements(&mounts, m, avl, tmp) { + list_del(&m->list); if (m->source != (void*)(-1)) free((void*)m->source); if (m->source_fd >= 0) @@ -1542,6 +1577,7 @@ void mount_free(void) { void mount_list_init(void) { avl_init(&mounts, avl_strcmp, false, NULL); + INIT_LIST_HEAD(&mounts_order); } static int add_script_interp(const char *path, const char *map, int size) diff --git a/jail/jail.c b/jail/jail.c index 5fe9ef7..e79bd22 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -6106,11 +6106,8 @@ static void post_main(struct uloop_timeout *t) * /proc/sys/net into (totally unrelated, but surely existing) /proc/self/net. * Then we mount-bind /proc/sys read-only and then mount-move /proc/self/net into * /proc/sys/net. - * This works because mounts are executed in incrementing strcmp() order and - * /proc/self/net appears there before /proc/sys/net and hence the operation - * succeeds as the bind-mount of /proc/self/net is performed first and then - * move-mount of /proc/sys/net follows because 'e' preceeds 'y' in the ASCII - * table (and in the alphabet). + * Mounts are established in the order they are added, so the three + * steps are registered in exactly the order they have to happen in. * * A jail that defers its own CLONE_NEWUSER applies this (and all other * default masking below) itself in phase 2, once it regains its own @@ -6120,11 +6117,14 @@ static void post_main(struct uloop_timeout *t) * still counts against the kernel's mount-visibility check for any * nested runtime's own /proc mount. */ - if (!defer_userns && - !add_mount(NULL, "/proc/sys", NULL, MS_BIND | MS_RDONLY, 0, NULL, -1)) + if (!defer_userns && !mount_is_defined("/proc/sys")) { if (opts.namespace & CLONE_NEWNET) - if (!add_mount_inner("/proc/self/net", "/proc/sys/net", NULL, MS_MOVE, 0, NULL, -1)) - add_mount_inner("/proc/sys/net", "/proc/self/net", NULL, MS_BIND, 0, NULL, -1); + add_mount_inner("/proc/sys/net", "/proc/self/net", NULL, MS_BIND, 0, NULL, -1); + + if (!add_mount(NULL, "/proc/sys", NULL, MS_BIND | MS_RDONLY, 0, NULL, -1) && + (opts.namespace & CLONE_NEWNET)) + add_mount_inner("/proc/self/net", "/proc/sys/net", NULL, MS_MOVE, 0, NULL, -1); + } } if (opts.sysfs || opts.ocibundle) From dce9f3a55a56c647fd501776e12bdfb7084106fd Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 19 Aug 2026 03:24:45 +0100 Subject: [PATCH 53/82] jail: fail when a namespace cannot be joined setns_open() reports why a join failed and every caller threw that away, so a bundle naming a namespace path of the wrong type, or one that cannot be entered at all, was created as if the path had not been given. The runtime spec requires an error there, and the validation suite checks it for each type in turn. Propagate the failures. The time namespace is still not joined at all, which is a separate gap. Fixes: c482c5de77f4 ("jail: add support for referencing existing namespaces") Signed-off-by: Daniel Golle --- jail/jail.c | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/jail/jail.c b/jail/jail.c index e79bd22..1c4b6fe 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -2603,6 +2603,7 @@ static int exec_jail(void *arg) char tag; int recv_fds[JAIL_IDMAP_MAX_FDS]; int nrecv; + int ret; exit_from_child = true; prctl(PR_SET_SECUREBITS, 0); @@ -2626,10 +2627,17 @@ static int exec_jail(void *arg) close(userns_pipe[3]); } - setns_open(CLONE_NEWNET); - setns_open(CLONE_NEWNS); - setns_open(CLONE_NEWIPC); - setns_open(CLONE_NEWUTS); + ret = setns_open(CLONE_NEWNET); + if (!ret) + ret = setns_open(CLONE_NEWNS); + if (!ret) + ret = setns_open(CLONE_NEWIPC); + if (!ret) + ret = setns_open(CLONE_NEWUTS); + if (ret) { + ERROR("failed to join namespace: %s\n", strerror(ret)); + return EXIT_FAILURE; + } /* * Must run before setns_open(CLONE_NEWUSER) below: joining an @@ -2643,7 +2651,11 @@ static int exec_jail(void *arg) return EXIT_FAILURE; } - setns_open(CLONE_NEWUSER); + ret = setns_open(CLONE_NEWUSER); + if (ret) { + ERROR("failed to join user namespace: %s\n", strerror(ret)); + return EXIT_FAILURE; + } buf[0] = 'i'; if (write(pipes[1], buf, 1) < 1) { @@ -2675,7 +2687,11 @@ static int exec_jail(void *arg) if (opts.namespace & CLONE_NEWCGROUP) unshare(CLONE_NEWCGROUP); - setns_open(CLONE_NEWCGROUP); + ret = setns_open(CLONE_NEWCGROUP); + if (ret) { + ERROR("failed to join cgroup namespace: %s\n", strerror(ret)); + free_and_exit(EXIT_FAILURE); + } /* * A join of an existing userns (opts.setns.user) can become root From 30393174b60507d355feb946f1b1ecec89ff60e6 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 05:35:52 +0100 Subject: [PATCH 54/82] jail, uxc: report the exit status and signal the invoker conmon learns a container's exit status by waiting for the pid it reads from the runtime's pid file, but with ujail that process is a child of procd and never of conmon, so conmon never obtains a status at all: podman reports 0 for a container that exited 42, and podman exec fails even when the command succeeded. A shim would give conmon something to wait for at the price of putting a process other than the container's own in the pid file, which tooling needs for introspecting namespaces, cgroups and seccomp state. Write the decoded status, WEXITSTATUS or 128 plus the terminating signal, to an exit_status file in the directory the pid file lives in. The value goes to a temporary file and is renamed into place, so a reader either sees the previous value or the whole new one, and it is written before the SIGCHLD that ends conmon's loop. Exec sessions do the same with their own pid file, so podman exec has a status to report as well. A stale value from an earlier run is removed when a pid file is written, and nothing is written when no pid file was asked for, which leaves the behaviour of a caller that does not want any of this unchanged. The status file on its own ends no wait: the container is procd's child, so its death raises no SIGCHLD in conmon, and a detached container's streams never reach end of file, leaving conmon to sit until its own timeout. The runtime therefore signals the process that asked for the container. conmon execs the runtime, so uxc runs as conmon's child and its parent is the waiting process; uxc opens a pidfd on getppid() with pidfd_open(), called through syscall() since musl wraps neither it nor pidfd_send_signal(), and sends the descriptor along with the create and exec requests. procd keeps it on the instance beside the stdio descriptors and lets ujail inherit it across the execve() that starts the jail, -a naming the descriptor number, and ujail sends SIGCHLD through it with pidfd_send_signal() once the container is gone, after the status has been written. An exec session signals through the descriptor its own request carried once the session has been reaped, and a respawned instance carries the same descriptor into the next ujail, so the invoker also learns when a later incarnation dies. This holds for any invoker instead of leaning on the conmon.pid filename, which is merely podman's default for --conmon-pidfile and silently defeated by overriding it. A pid would identify the invoker only for as long as it lives. Callers such as uxc.init and interactive shells routinely exit long before their container does, leaving behind a number the kernel may hand out again, and any scheme that re-checks the pid before the kill still leaves a window between the check and the signal. A pidfd pins the identity at the instant it is opened: from that instant on it is the only process a signal through it can ever reach, however much later it is sent. One window remains, and it is not one a descriptor can close: getppid() is read before the descriptor exists, so an invoker that died first yields the reaper instead, and a number already recycled by then names a stranger. uxc therefore re-reads getppid() after opening and sends nothing if it changed, which leaves only the two adjacent syscalls in between, against the whole lifetime of a container in the old scheme. ujail still polls the descriptor for POLLIN first, the same way exec_jail() watches the pidfd of its own parent, and stays silent for an invoker that has already gone. A caller that supplies no descriptor is never signalled; when pidfd_open() fails uxc warns and sends none rather than falling back to a pid, since a caller which then waits for a wake-up that never comes deserves to see why. The descriptor set now carries its own count, so procd and uxc must be upgraded together: the previous revision of this commit accepted exactly three descriptors and nothing else. Losing the carrier now loses the notification with it, where the pid had travelled separately in the request. Two limits are worth naming. The signal is sent as ujail begins tearing down, before cgroups and the network are dismantled, so a woken manager can observe a container whose traces have not all gone yet. And an exec session still in flight when the container itself dies is never signalled, because the supervisor exits first; such a session is left to end on stream EOF, as it was before. ubus carries a single descriptor per request, and the three standard descriptors already travel as SCM_RIGHTS over a socket pair whose receiving end goes to ubus_invoke_fd(). That carrier now takes a counted set instead of a fixed trio, the count riding in the payload byte, because the stdio descriptors are only sent when pass-through is wanted while the notification descriptor is wanted independently of that: a create sends stdio and, when the invoker could be named, the pidfd; an exec session with a terminal sends the pidfd alone. The receiver tells the layouts apart by the count, which stays unambiguous because stdio is all or nothing. The invoker's identity is thereby no longer configuration. procd used to compare the notifypid attribute like the other jail attributes, restarting a running instance when a re-add named a different invoker; the descriptor is runtime state like the stdio descriptors, so a re-add replaces the stored descriptor for the next start while the running jail keeps the one it inherited. The descriptor stays close-on-exec everywhere except across the one execve() that starts the instance's own ujail, and ujail marks it close-on-exec again as soon as it parses the option, so neither hooks nor the container itself ever inherit it. conmon ends its loop when the streams it handed the runtime reach end of file and only then looks for the status, so an exec session's descriptors stay with the session and are closed once its status has been written. Recording it from the process that waits for the session is no alternative: that one has joined the container's mount namespace, where the path the status belongs at does not exist. Signed-off-by: Daniel Golle --- jail/jail.c | 159 +++++++++++++++++++++++++++++++++++++++++---- service/instance.c | 43 +++++++++++- service/instance.h | 2 + service/service.c | 37 +++++++---- stdio-fds.h | 99 ++++++++++++++++++++++------ uxc.c | 38 ++++++++++- 6 files changed, 324 insertions(+), 54 deletions(-) diff --git a/jail/jail.c b/jail/jail.c index 1c4b6fe..6e8b8a0 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -101,7 +101,7 @@ #define PR_MDWE_NO_INHERIT (1UL << 1) #endif -#define OPT_ARGS "b:cC:d:De:EfFG:h:iI:j:J:k:lm:M:n:NoO:pP:r:R:sS:uU:V:w:x:t:T:yY:Z" +#define OPT_ARGS "a:b:cC:d:De:EfFG:h:iI:j:J:k:lm:M:n:NoO:pP:r:R:sS:uU:V:w:x:t:T:yY:Z" #define JAIL_MAX_CREDENTIALS 16 static const char *cred_targets[JAIL_MAX_CREDENTIALS]; @@ -156,6 +156,7 @@ static struct { struct blob_attr *gidmappings; unsigned int idmap_offset; char *pidfile; + int notify_fd; struct sysctl_val **sysctl; int no_new_privs; int namespace; @@ -1343,10 +1344,91 @@ static int build_jail_fs(void) static bool exit_from_child; static void emit_instance_event(const char *event); +static bool pidfile_sibling(const char *pidfile, const char *name, char *path, size_t len) +{ + char *slash; + + if (!pidfile) + return false; + + if (snprintf(path, len, "%s", pidfile) >= (int)len) + return false; + + slash = strrchr(path, '/'); + if (!slash) + return false; + + if ((size_t)(slash - path) + strlen(name) + 1 >= len) + return false; + + strcpy(slash, name); + + return true; +} + +static void jail_write_exit_status(const char *pidfile, int status) +{ + char path[PATH_MAX], tmp[PATH_MAX]; + char buf[12]; + int fd, len; + + if (!pidfile_sibling(pidfile, "/exit_status", path, sizeof(path))) + return; + + if (snprintf(tmp, sizeof(tmp), "%s.tmp", path) >= (int)sizeof(tmp)) + return; + + len = snprintf(buf, sizeof(buf), "%d", status); + if (len < 0 || len >= (int)sizeof(buf)) + return; + + fd = open(tmp, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644); + if (fd < 0) + return; + + if (write(fd, buf, len) != len) { + close(fd); + unlink(tmp); + return; + } + + if (close(fd)) { + unlink(tmp); + return; + } + + if (rename(tmp, path)) + unlink(tmp); +} + +static void jail_clear_exit_status(const char *pidfile) +{ + char path[PATH_MAX]; + + if (pidfile_sibling(pidfile, "/exit_status", path, sizeof(path))) + unlink(path); +} + +static void notify_signal(int fd) +{ + struct pollfd pfd = { .fd = fd, .events = POLLIN }; + + if (fd < 0) + return; + + if (poll(&pfd, 1, 0) > 0) + return; + + syscall(SYS_pidfd_send_signal, fd, SIGCHLD, NULL, 0); +} + static bool jail_ptrace_seccomp(void); static void free_and_exit(int ret) { + if (!exit_from_child) + notify_signal(opts.notify_fd); + if (!exit_from_child && opts.jail_network_started) { jail_network_teardown(); opts.jail_network_started = false; @@ -1984,6 +2066,7 @@ static void usage(void) fprintf(stderr, " -J \tcreate container from OCI bundle\n"); fprintf(stderr, " -i\t\tstart container immediately\n"); fprintf(stderr, " -P \tcreate \n"); + fprintf(stderr, " -a \tsend SIGCHLD through inherited pidfd once the container is gone\n"); fprintf(stderr, "\nWarning: by default root inside the jail is the same\n\ and he has the same powers as root outside the jail,\n\ thus he can escape the jail and/or break stuff.\n\ @@ -2053,6 +2136,7 @@ static void jail_process_handler(struct uloop_process *c, int ret) jail_return_code = 128 + WTERMSIG(ret); INFO("jail (%d) exited with signal: %d\n", c->pid, WTERMSIG(ret)); } + jail_write_exit_status(opts.pidfile, jail_return_code); jail_running = 0; poststop(); } @@ -4857,6 +4941,9 @@ struct container_exec { struct ubus_context *ctx; struct ubus_request_data req; struct uloop_process exec_proc; + char *pidfile; + int notify_fd; + int stdio_fds[STDIO_FDS_NUM]; }; static struct container_exec *current_exec; @@ -4892,18 +4979,28 @@ static void container_exec_free_strarray(char **a) free(a); } +static int wait_status_decode(int wstatus) +{ + if (WIFEXITED(wstatus)) + return WEXITSTATUS(wstatus); + + if (WIFSIGNALED(wstatus)) + return 128 + WTERMSIG(wstatus); + + return 255; +} + static void container_exec_done_reply(struct uloop_process *p, int wstatus) { struct container_exec *e = container_of(p, struct container_exec, exec_proc); static struct blob_buf bb; - int status; + int status = wait_status_decode(wstatus); - if (WIFEXITED(wstatus)) - status = WEXITSTATUS(wstatus); - else if (WIFSIGNALED(wstatus)) - status = 128 + WTERMSIG(wstatus); - else - status = 255; + jail_write_exit_status(e->pidfile, status); + stdio_fds_close(e->stdio_fds); + notify_signal(e->notify_fd); + if (e->notify_fd >= 0) + close(e->notify_fd); blob_buf_init(&bb, 0); blobmsg_add_u32(&bb, "status", status); @@ -4911,6 +5008,7 @@ static void container_exec_done_reply(struct uloop_process *p, int wstatus) ubus_complete_deferred_request(e->ctx, &e->req, 0); if (current_exec == e) current_exec = NULL; + free(e->pidfile); free(e); } @@ -4918,8 +5016,15 @@ static void container_exec_done_reap(struct uloop_process *p, int wstatus) { struct container_exec *e = container_of(p, struct container_exec, exec_proc); + jail_write_exit_status(e->pidfile, wait_status_decode(wstatus)); + stdio_fds_close(e->stdio_fds); + notify_signal(e->notify_fd); + if (e->notify_fd >= 0) + close(e->notify_fd); + if (current_exec == e) current_exec = NULL; + free(e->pidfile); free(e); } @@ -4957,6 +5062,7 @@ container_handle_exec(struct ubus_context *ctx, struct ubus_object *obj, int pipe_fds[2] = { -1, -1 }; int stdio_fds[STDIO_FDS_NUM] = { -1, -1, -1 }; int stdio_sock; + int notify_fd = -1; pid_t exec_pid, grandchild = -1; struct container_exec *e = NULL; char nspath[64]; @@ -5000,8 +5106,8 @@ container_handle_exec(struct ubus_context *ctx, struct ubus_object *obj, stdio_sock = ubus_request_get_caller_fd(req); if (stdio_sock > -1) { - if (stdio_fds_recv(stdio_sock, stdio_fds)) - ERROR("exec: cannot receive caller stdio: %m\n"); + if (stdio_notify_fds_recv(stdio_sock, stdio_fds, ¬ify_fd)) + ERROR("exec: cannot receive caller descriptors: %m\n"); close(stdio_sock); } @@ -5010,8 +5116,8 @@ container_handle_exec(struct ubus_context *ctx, struct ubus_object *obj, blobmsg_data(msg), blobmsg_data_len(msg)); if (!tb[CONTAINER_EXEC_ATTR_ARGS]) { - stdio_fds_close(stdio_fds); - return UBUS_STATUS_INVALID_ARGUMENT; + rc = UBUS_STATUS_INVALID_ARGUMENT; + goto out; } args = container_exec_strarray(tb[CONTAINER_EXEC_ATTR_ARGS]); @@ -5319,6 +5425,7 @@ container_handle_exec(struct ubus_context *ctx, struct ubus_object *obj, ERROR("exec: waitpid(%d): %m\n", grandchild); _exit(126); } + if (WIFEXITED(wstatus)) _exit(WEXITSTATUS(wstatus)); _exit(128 + WTERMSIG(wstatus)); @@ -5353,8 +5460,6 @@ container_handle_exec(struct ubus_context *ctx, struct ubus_object *obj, close(pipe_fds[0]); pipe_fds[0] = -1; - stdio_fds_close(stdio_fds); - for (i = 0; i < (int)ARRAY_SIZE(ns_names); i++) if (ns_fds[i] >= 0) { close(ns_fds[i]); @@ -5379,6 +5484,8 @@ container_handle_exec(struct ubus_context *ctx, struct ubus_object *obj, free(exec_additional_gids); exec_additional_gids = NULL; + jail_clear_exit_status(pidfile); + if (pidfile && grandchild > 0) { FILE *pf = fopen(pidfile, "w"); if (pf) { @@ -5389,11 +5496,23 @@ container_handle_exec(struct ubus_context *ctx, struct ubus_object *obj, e = calloc(1, sizeof(*e)); if (!e) { + stdio_fds_close(stdio_fds); + if (notify_fd >= 0) + close(notify_fd); kill(exec_pid, SIGKILL); return UBUS_STATUS_UNKNOWN_ERROR; } + + for (i = 0; i < STDIO_FDS_NUM; i++) { + e->stdio_fds[i] = stdio_fds[i]; + stdio_fds[i] = -1; + } e->ctx = ctx; e->exec_proc.pid = exec_pid; + e->notify_fd = notify_fd; + notify_fd = -1; + if (pidfile) + e->pidfile = strdup(pidfile); current_exec = e; if (detach) { @@ -5416,6 +5535,8 @@ container_handle_exec(struct ubus_context *ctx, struct ubus_object *obj, out: stdio_fds_close(stdio_fds); + if (notify_fd >= 0) + close(notify_fd); for (i = 0; i < (int)ARRAY_SIZE(ns_names); i++) if (ns_fds[i] >= 0) close(ns_fds[i]); @@ -5441,6 +5562,8 @@ jail_writepid(pid_t pid) if (!opts.pidfile) return 0; + jail_clear_exit_status(opts.pidfile); + _pidfile = fopen(opts.pidfile, "w"); if (_pidfile == NULL) return errno; @@ -5572,6 +5695,7 @@ int main(int argc, char **argv) opts.setns.user = -1; opts.setns.cgroup = -1; opts.setns.time = -1; + opts.notify_fd = -1; /* default 5 seconds timeout after SIGTERM before SIGKILL is sent */ opts.term_timeout = 5; @@ -5741,6 +5865,13 @@ int main(int argc, char **argv) case 'P': opts.pidfile = optarg; break; + case 'a': + opts.notify_fd = atoi(optarg); + if (opts.notify_fd <= STDERR_FILENO || + fcntl(opts.notify_fd, F_SETFD, FD_CLOEXEC) || + syscall(SYS_pidfd_send_signal, opts.notify_fd, 0, NULL, 0)) + opts.notify_fd = -1; + break; case 'Y': opts.console_socket = optarg; break; diff --git a/service/instance.c b/service/instance.c index 75d94a2..61fbfaa 100644 --- a/service/instance.c +++ b/service/instance.c @@ -304,6 +304,7 @@ instance_gen_setns_argstr(struct blob_attr *attr) static inline int jail_run(struct service_instance *in, char **argv) { + static char notify_fd_str[12]; char *term_timeout_str; struct blobmsg_list_node *var; struct jail *jail = &in->jail; @@ -427,6 +428,12 @@ jail_run(struct service_instance *in, char **argv) argv[argc++] = jail->envfile; } + if (in->notify_fd > -1) { + snprintf(notify_fd_str, sizeof(notify_fd_str), "%d", in->notify_fd); + argv[argc++] = "-a"; + argv[argc++] = notify_fd_str; + } + if (jail->systemd_cgroup) argv[argc++] = "-Z"; @@ -518,6 +525,7 @@ instance_run(struct service_instance *in, int _stdout, int _stderr) char **argv; int argc = 1; /* NULL terminated */ int rem, _stdin; + int jail_argc = in->jail.argc; bool seccomp = !in->trace && !in->has_jail && in->seccomp; bool setlbf = _stdout >= 0; @@ -542,7 +550,12 @@ instance_run(struct service_instance *in, int _stdout, int _stderr) if (in->trace || seccomp) argc += 1; - argv = alloca(sizeof(char *) * (argc + in->jail.argc)); + if (in->has_jail && in->notify_fd > -1) { + fcntl(in->notify_fd, F_SETFD, 0); + jail_argc += 2; + } + + argv = alloca(sizeof(char *) * (argc + jail_argc)); argc = 0; #ifdef SECCOMP_SUPPORT @@ -557,9 +570,9 @@ instance_run(struct service_instance *in, int _stdout, int _stderr) if (in->has_jail) { argc = jail_run(in, argv); - if (argc != in->jail.argc) + if (argc != jail_argc) ULOG_WARN("expected %i jail params, used %i for %s::%s\n", - in->jail.argc, argc, in->srv->name, in->name); + jail_argc, argc, in->srv->name, in->name); } blobmsg_for_each_attr(cur, in->command, rem) @@ -1708,6 +1721,9 @@ instance_update(struct service_instance *in, struct service_instance *in_new) if (in_new->stdio_fd[1] > -1) instance_stdio_set(in, in_new->stdio_fd); + if (in_new->notify_fd > -1) + instance_notify_set(in, &in_new->notify_fd); + if (!running || stopping) { instance_config_move(in, in_new); instance_start(in); @@ -1750,12 +1766,32 @@ instance_stdio_set(struct service_instance *in, int *fds) } } +static void +instance_free_notify_fd(struct service_instance *in) +{ + if (in->notify_fd < 0) + return; + + close(in->notify_fd); + in->notify_fd = -1; +} + +void +instance_notify_set(struct service_instance *in, int *fd) +{ + instance_free_notify_fd(in); + + in->notify_fd = *fd; + *fd = -1; +} + void instance_free(struct service_instance *in) { service_data_trigger(&in->data); instance_free_stdio(in); instance_free_stdio_fds(in); + instance_free_notify_fd(in); uloop_process_delete(&in->proc); uloop_timeout_cancel(&in->timeout); uloop_timeout_cancel(&in->watchdog.timeout); @@ -1800,6 +1836,7 @@ instance_init(struct service_instance *in, struct service *s, struct blob_attr * in->immediately = false; in->stdio_fd[0] = in->stdio_fd[1] = in->stdio_fd[2] = -1; + in->notify_fd = -1; in->_stdout.fd.fd = -2; in->_stdout.stream.string_data = true; diff --git a/service/instance.h b/service/instance.h index 7006b74..db6467d 100644 --- a/service/instance.h +++ b/service/instance.h @@ -113,6 +113,7 @@ struct service_instance { struct uloop_process proc; struct uloop_timeout timeout; int stdio_fd[3]; + int notify_fd; struct ustream_fd _stdout; struct ustream_fd _stderr; struct ustream_fd console; @@ -136,6 +137,7 @@ void instance_stop(struct service_instance *in, bool halt); void instance_update(struct service_instance *in, struct service_instance *in_new); void instance_init(struct service_instance *in, struct service *s, struct blob_attr *config); void instance_stdio_set(struct service_instance *in, int *fds); +void instance_notify_set(struct service_instance *in, int *fd); void instance_free(struct service_instance *in); void instance_dump(struct blob_buf *b, struct service_instance *in, int debug); void service_event_instance_exit(const char *type, struct service_instance *in); diff --git a/service/service.c b/service/service.c index b146043..3ed955f 100644 --- a/service/service.c +++ b/service/service.c @@ -39,7 +39,8 @@ static struct ubus_context *ctx; static struct ubus_object main_object; static void -service_instance_add(struct service *s, struct blob_attr *attr, int *stdio_fds) +service_instance_add(struct service *s, struct blob_attr *attr, int *stdio_fds, + int *notify_fd) { struct service_instance *in; @@ -54,6 +55,9 @@ service_instance_add(struct service *s, struct blob_attr *attr, int *stdio_fds) if (stdio_fds && stdio_fds[1] > -1) instance_stdio_set(in, stdio_fds); + if (notify_fd && *notify_fd > -1) + instance_notify_set(in, notify_fd); + vlist_add(&s->instances, &in->node, (void *) in->name); } @@ -157,7 +161,8 @@ service_update_data(struct service *s, struct blob_attr *data) } static int -service_update(struct service *s, struct blob_attr **tb, bool add, bool init, int *stdio_fds) +service_update(struct service *s, struct blob_attr **tb, bool add, bool init, + int *stdio_fds, int *notify_fd) { struct blob_attr *cur; int rem; @@ -187,7 +192,7 @@ service_update(struct service *s, struct blob_attr **tb, bool add, bool init, in if (!add) vlist_update(&s->instances); blobmsg_for_each_attr(cur, tb[SERVICE_SET_INSTANCES], rem) { - service_instance_add(s, cur, stdio_fds); + service_instance_add(s, cur, stdio_fds, notify_fd); } if (!add) vlist_flush(&s->instances); @@ -436,6 +441,7 @@ service_handle_set(struct ubus_context *ctx, struct ubus_object *obj, { struct blob_attr *tb[__SERVICE_SET_MAX], *cur; int stdio_fds[3] = { -1, -1, -1 }; + int notify_fd = -1; struct service *s = NULL; const char *name; bool container = is_container_obj(obj); @@ -451,8 +457,8 @@ service_handle_set(struct ubus_context *ctx, struct ubus_object *obj, sock = req ? ubus_request_get_caller_fd(req) : -1; if (sock > -1) { - if (stdio_fds_recv(sock, stdio_fds)) - ULOG_WARN("failed to receive stdio for %s: %m\n", name); + if (stdio_notify_fds_recv(sock, stdio_fds, ¬ify_fd)) + ULOG_WARN("failed to receive descriptors for %s: %m\n", name); close(sock); } @@ -464,24 +470,22 @@ service_handle_set(struct ubus_context *ctx, struct ubus_object *obj, if (s) { P_DEBUG(2, "Update service %s\n", name); - ret = service_update(s, tb, add, false, stdio_fds); - stdio_fds_close(stdio_fds); - return ret; + ret = service_update(s, tb, add, false, stdio_fds, ¬ify_fd); + goto out; } P_DEBUG(2, "Create service %s\n", name); s = service_alloc(name); if (!s) { - stdio_fds_close(stdio_fds); - return UBUS_STATUS_UNKNOWN_ERROR; + ret = UBUS_STATUS_UNKNOWN_ERROR; + goto out; } s->container = container; - ret = service_update(s, tb, add, true, stdio_fds); - stdio_fds_close(stdio_fds); + ret = service_update(s, tb, add, true, stdio_fds, ¬ify_fd); if (ret) - return ret; + goto out; if (container) { avl_insert(&containers, &s->avl); @@ -492,7 +496,12 @@ service_handle_set(struct ubus_context *ctx, struct ubus_object *obj, service_event("service.start", s->name, NULL); } - return 0; + +out: + stdio_fds_close(stdio_fds); + if (notify_fd > -1) + close(notify_fd); + return ret; } static void diff --git a/stdio-fds.h b/stdio-fds.h index de5a5e8..2071d9b 100644 --- a/stdio-fds.h +++ b/stdio-fds.h @@ -19,36 +19,41 @@ #include #define STDIO_FDS_NUM 3 +#define FDS_NUM_MAX (STDIO_FDS_NUM + 1) /* - * ubus carries a single file descriptor per request, so the three standard - * descriptors travel as SCM_RIGHTS over a socket pair whose receiving end is - * what gets passed to ubus_invoke_fd(). + * ubus carries a single file descriptor per request, so descriptor sets + * travel as SCM_RIGHTS over a socket pair whose receiving end is what gets + * passed to ubus_invoke_fd(). The payload byte carries the count. */ -static inline int stdio_fds_send(const int *fds) +static inline int fds_send(const int *fds, int num) { - char cmsgbuf[CMSG_SPACE(STDIO_FDS_NUM * sizeof(int))]; + char cmsgbuf[CMSG_SPACE(FDS_NUM_MAX * sizeof(int))]; struct msghdr msg = { 0 }; struct cmsghdr *cmsg; struct iovec iov; - char dummy = 0; + char count; int sp[2]; + if (num < 1 || num > FDS_NUM_MAX) + return -1; + if (socketpair(AF_UNIX, SOCK_STREAM, 0, sp)) return -1; - iov.iov_base = &dummy; - iov.iov_len = sizeof(dummy); + count = num; + iov.iov_base = &count; + iov.iov_len = sizeof(count); msg.msg_iov = &iov; msg.msg_iovlen = 1; msg.msg_control = cmsgbuf; - msg.msg_controllen = sizeof(cmsgbuf); + msg.msg_controllen = CMSG_SPACE(num * sizeof(int)); cmsg = CMSG_FIRSTHDR(&msg); cmsg->cmsg_level = SOL_SOCKET; cmsg->cmsg_type = SCM_RIGHTS; - cmsg->cmsg_len = CMSG_LEN(STDIO_FDS_NUM * sizeof(int)); - memcpy(CMSG_DATA(cmsg), fds, STDIO_FDS_NUM * sizeof(int)); + cmsg->cmsg_len = CMSG_LEN(num * sizeof(int)); + memcpy(CMSG_DATA(cmsg), fds, num * sizeof(int)); if (sendmsg(sp[0], &msg, 0) < 0) { close(sp[0]); @@ -61,33 +66,87 @@ static inline int stdio_fds_send(const int *fds) return sp[1]; } -static inline int stdio_fds_recv(int sock, int *fds) +static inline int fds_recv(int sock, int *fds, int max) { - char cmsgbuf[CMSG_SPACE(STDIO_FDS_NUM * sizeof(int))]; + char cmsgbuf[CMSG_SPACE(FDS_NUM_MAX * sizeof(int))]; + int tmp[FDS_NUM_MAX]; struct msghdr msg = { 0 }; struct cmsghdr *cmsg; struct iovec iov; - char dummy; + char count; + int num, i; - iov.iov_base = &dummy; - iov.iov_len = sizeof(dummy); + iov.iov_base = &count; + iov.iov_len = sizeof(count); msg.msg_iov = &iov; msg.msg_iovlen = 1; msg.msg_control = cmsgbuf; msg.msg_controllen = sizeof(cmsgbuf); - if (recvmsg(sock, &msg, MSG_CMSG_CLOEXEC) < 0) + if (recvmsg(sock, &msg, MSG_CMSG_CLOEXEC | MSG_DONTWAIT) < 1) return -1; cmsg = CMSG_FIRSTHDR(&msg); if (!cmsg || cmsg->cmsg_level != SOL_SOCKET || cmsg->cmsg_type != SCM_RIGHTS || - cmsg->cmsg_len != CMSG_LEN(STDIO_FDS_NUM * sizeof(int))) + cmsg->cmsg_len < CMSG_LEN(sizeof(int)) || + cmsg->cmsg_len > CMSG_LEN(FDS_NUM_MAX * sizeof(int))) return -1; - memcpy(fds, CMSG_DATA(cmsg), STDIO_FDS_NUM * sizeof(int)); + num = (cmsg->cmsg_len - CMSG_LEN(0)) / sizeof(int); + memcpy(tmp, CMSG_DATA(cmsg), num * sizeof(int)); + + if (num != count || num > max) { + for (i = 0; i < num; i++) + close(tmp[i]); + return -1; + } + + memcpy(fds, tmp, num * sizeof(int)); + + return num; +} + +static inline int stdio_notify_fds_send(const int *stdio_fds, int notify_fd) +{ + int fds[FDS_NUM_MAX]; + int num = 0; - return 0; + if (stdio_fds) { + memcpy(fds, stdio_fds, STDIO_FDS_NUM * sizeof(int)); + num = STDIO_FDS_NUM; + } + + if (notify_fd > -1) + fds[num++] = notify_fd; + + if (!num) + return -1; + + return fds_send(fds, num); +} + +static inline int stdio_notify_fds_recv(int sock, int *stdio_fds, int *notify_fd) +{ + int fds[FDS_NUM_MAX]; + int num; + + num = fds_recv(sock, fds, FDS_NUM_MAX); + switch (num) { + case 1: + *notify_fd = fds[0]; + return 0; + case STDIO_FDS_NUM + 1: + *notify_fd = fds[STDIO_FDS_NUM]; + /* fallthrough */ + case STDIO_FDS_NUM: + memcpy(stdio_fds, fds, STDIO_FDS_NUM * sizeof(int)); + return 0; + default: + while (num > 0) + close(fds[--num]); + return -1; + } } static inline void stdio_fds_close(int *fds) diff --git a/uxc.c b/uxc.c index 7a6b180..e86a173 100644 --- a/uxc.c +++ b/uxc.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include @@ -1303,12 +1304,34 @@ static void uxc_instance_drop(const char *name) blob_buf_free(&req); } +static int uxc_invoker_pidfd(void) +{ + pid_t ppid; + int fd; + + ppid = getppid(); + fd = syscall(SYS_pidfd_open, ppid, 0); + if (fd < 0) { + fprintf(stderr, "uxc: warning: cannot watch the invoker, it will not be notified: %s\n", + strerror(errno)); + return -1; + } + + if (getppid() != ppid) { + close(fd); + return -1; + } + + return fd; +} + static int uxc_create(char *name, bool immediately, const char *console_socket, bool systemd_cgroup, const char *seccomp_mode) { static struct blob_buf req; struct blob_attr *cur, *tb[__CONF_MAX]; int rem, ret = 0; + int notify_fd; uint32_t id; struct settings *usettings = NULL; char *path = NULL, *jailname = NULL, *pidfile = NULL, *tmprwsize = NULL, *writepath = NULL; @@ -1477,8 +1500,13 @@ static int uxc_create(char *name, bool immediately, const char *console_socket, if (uxc_wait_arm(&wait_state)) fprintf(stderr, "uxc: warning: cannot arm instance.* watcher\n"); - if (ubus_invoke_fd(ctx, id, "add", req.head, NULL, NULL, 3000, - stdio_fds_send(stdio_fds))) { + notify_fd = uxc_invoker_pidfd(); + ret = ubus_invoke_fd(ctx, id, "add", req.head, NULL, NULL, 3000, + stdio_notify_fds_send(stdio_fds, notify_fd)); + if (notify_fd >= 0) + close(notify_fd); + + if (ret) { blob_buf_free(&req); uxc_wait_disarm(); return -EIO; @@ -1574,6 +1602,7 @@ static int uxc_exec(const char *name, const char *process_file, struct uxc_exec_reply reply = { .status = 0 }; char *objname; uint32_t id; + int notify_fd; int ret; if (tty && !console_socket) { @@ -1625,9 +1654,12 @@ static int uxc_exec(const char *name, const char *process_file, return -ENOENT; } + notify_fd = uxc_invoker_pidfd(); ret = ubus_invoke_fd(ctx, id, "exec", req.head, uxc_exec_reply_cb, &reply, 0, - tty ? -1 : stdio_fds_send(stdio_fds)); + stdio_notify_fds_send(tty ? NULL : stdio_fds, notify_fd)); + if (notify_fd >= 0) + close(notify_fd); blob_buf_free(&req); if (ret) From 6aa23a8b197e0235b74f76d0ee6e99e89f1d5f09 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 05:36:48 +0100 Subject: [PATCH 55/82] jail: give the container's namespaces to its own user namespace 07e0f7d8 ("jail: fix /proc,/sys mounting under CLONE_NEWUSER") did two things: it replaced the hardcoded MS_NOATIME on the procfs and sysfs mounts with detect_atime_flag(), which is what actually made them mountable and which is kept, and it moved the creation of the user namespace to after the mounts were built, which left the pid, network, ipc, uts and cgroup namespaces owned by the initial user namespace. ujail itself, being privileged, never had a problem with that; the container at runtime did. The kernel resolves a container's privileges against the owner of the namespace being used, so a container holding CAP_NET_BIND_SERVICE could not bind a privileged port, __inet_bind() asking ns_capable(net->user_ns, CAP_NET_BIND_SERVICE), and it could mount neither its own procfs, which wants CAP_SYS_ADMIN in the user namespace owning the pid namespace, nor its own sysfs, which wants the same in the one owning the network namespace. runc and crun both establish the user namespace before everything else for this reason. Undo only the reordering half of that commit. Let clone() create it together with the rest, which the kernel attributes to the new user namespace since the credentials are copied before the namespaces, and have the child wait for its uid and gid maps before it does anything that needs privilege. Creating it late is kept for the one case crun also keeps it for, a container joining existing namespaces, whether an OCI bundle names them by path or -j on the command line names them by pid, because entering those needs privilege in the user namespace owning them. The time namespace is not yet handed over: CLONE_NEWTIME stays masked out of the clone3() flags and the parent still unshares it before the clone, so it remains owned by the initial user namespace and the title only fully holds once a following commit moves it into the child as well. Redefining what defers the user namespace moves work between the two phases without changing the end state. remask_after_unshare(), remount_proc_sys_after_unshare(), the oci_deferred_* bookkeeping and the JAIL_NOAFILE bind now serve only the deferred path; on the common path the default masks and the read-only /proc/sys hack are applied while the mount list is built. The deferred path in turn drops privileges with setregid(), setreuid() and setgroups() before its second unshare(CLONE_NEWNS) rather than after. The inherited-mount detach introduced by a1c56332 ("jail: detach inherited mounts under /proc,/sys before mounting own") and guarded by 3fc9d119 ("jail: run inherited-mount detach before joining an external userns") now runs in two cases only, the deferred user namespace and the join of an external one. In a mount namespace owned by our own user namespace the inherited mounts are locked and cannot be detached; for mounts sitting on the kernel's permanently empty mount points it is not needed either, since mount_too_revealing() ignores those, but a locked mount covering an ordinary path, which our own masking and OCI maskedPaths create, still disqualifies the reference mount and can no longer be detached there. The command line could combine -j :user with -f, which the old code tolerated because CLONE_NEWUSER was always stripped from the clone flags and the join simply won. Now that the flag reaching clone3() creates a namespace of its own, -f leaves it unset when a user namespace has already been joined; an OCI bundle cannot express the combination, as parseOCIlinuxns() rejects the duplicate in both directions. Comments left over from the late-creation scheme, now stating the opposite of what the code does, are dropped. One detail follows from the new order: the gid 5 the standard /dev/pts options carry cannot be resolved by a mapping that holds a single id, and devpts refuses a mount whose gid does not map, so an unmapped gid is dropped from the options. crun likewise omits gid=5 for its rootless containers, though it does not filter bundle-supplied options this way. Signed-off-by: Daniel Golle --- jail/fs.c | 81 ++++++++++++++++++++++++++++++++++ jail/jail.c | 125 +++++++++++++++++++++++++++++++--------------------- 2 files changed, 155 insertions(+), 51 deletions(-) diff --git a/jail/fs.c b/jail/fs.c index e6f1dd2..c0a1af7 100644 --- a/jail/fs.c +++ b/jail/fs.c @@ -432,6 +432,8 @@ void jail_fs_set_userns(bool enabled) } static bool mount_opts_has(const char *opts, const char *needle); +static bool mount_opts_gid_unmapped(const char *opts); +static void mount_opts_drop(const char *opts, const char *key, char *buf, size_t len); static int do_mount(const char *root, const char *orig_source, const char *target, const char *filesystemtype, unsigned long orig_mountflags, unsigned long propflags, const char *optstr, int error, bool inner, @@ -440,6 +442,7 @@ static int do_mount(const char *root, const char *orig_source, const char *targe struct stat s; char new[PATH_MAX]; char tmpfs_data[512]; + char devpts_data[512]; const char *mount_data; char *source = (char *)orig_source; int fd, ret = 0; @@ -536,6 +539,12 @@ static int do_mount(const char *root, const char *orig_source, const char *targe } mount_data = optstr; + if (filesystemtype && !strcmp(filesystemtype, "devpts") && fs_userns && + mount_opts_gid_unmapped(optstr)) { + mount_opts_drop(optstr, "gid=", devpts_data, sizeof(devpts_data)); + mount_data = devpts_data; + } + if (filesystemtype && !strcmp(filesystemtype, "tmpfs") && !fs_userns && !mount_opts_has(optstr ?: "", "swap") && !mount_opts_has(optstr ?: "", "noswap")) { if (optstr && *optstr) @@ -958,6 +967,78 @@ static bool is_proc_or_sys_path(const char *path) return false; } +static bool id_is_mapped(const char *mapfile, unsigned long id) +{ + unsigned long inside, outside, count; + char line[128]; + bool mapped = false; + FILE *f; + + f = fopen(mapfile, "r"); + if (!f) + return true; + + while (fgets(line, sizeof(line), f)) { + if (sscanf(line, "%lu %lu %lu", &inside, &outside, &count) != 3) + continue; + + if (id >= inside && id - inside < count) { + mapped = true; + break; + } + } + + fclose(f); + + return mapped; +} + +static bool mount_opts_gid_unmapped(const char *opts) +{ + const char *p; + + if (!opts) + return false; + + for (p = opts; p; p = strchr(p, ',')) { + if (*p == ',') + ++p; + + if (!strncmp(p, "gid=", 4)) + return !id_is_mapped("/proc/self/gid_map", strtoul(p + 4, NULL, 10)); + } + + return false; +} + +static void mount_opts_drop(const char *opts, const char *key, char *buf, size_t len) +{ + size_t klen = strlen(key); + const char *p, *end; + size_t used = 0; + + buf[0] = '\0'; + + for (p = opts; p && *p; p = end) { + end = strchr(p, ','); + if (end) + ++end; + + if (!strncmp(p, key, klen)) + continue; + + while (*p && used + 1 < len) { + buf[used++] = *p; + if (*p++ == ',') + break; + } + buf[used] = '\0'; + } + + if (used && buf[used - 1] == ',') + buf[used - 1] = '\0'; +} + static bool mount_opts_has(const char *opts, const char *needle) { size_t nlen = strlen(needle); diff --git a/jail/jail.c b/jail/jail.c index 6e8b8a0..8967e00 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -257,6 +257,28 @@ static int console_slave_fd = -1; static char console_slave_name[64]; +/* + * Joining a namespace by path needs privilege in the user namespace owning it, + * which our own user namespace would take away, so in that case it is created + * after the joins instead of by clone(). crun makes the same distinction. + */ +static inline bool userns_deferred(void) +{ + if (!(opts.namespace & CLONE_NEWUSER) || opts.setns.user != -1) + return false; + + return (opts.setns.pid != -1) || + (opts.setns.net != -1) || + (opts.setns.ns != -1) || + (opts.setns.ipc != -1) || + (opts.setns.uts != -1) || + (opts.setns.cgroup != -1) || +#ifdef CLONE_NEWTIME + (opts.setns.time != -1) || +#endif + false; +} + static inline bool has_namespaces(void) { return ((opts.setns.pid != -1) || @@ -1458,6 +1480,7 @@ static void free_and_exit(int ret) static void post_jail_fs(void); static void enter_userns(void); +static int userns_wait_idmaps(void); static void remask_after_unshare(void); static void remount_proc_sys_after_unshare(void); static void enter_jail_fs(void) @@ -1486,42 +1509,60 @@ static void enter_jail_fs(void) enter_userns(); } -/* - * Create our own CLONE_NEWUSER here, after /proc and /sys are already - * mounted, so the PID namespace stays owned by the initial userns - * throughout mount setup. See the comment in exec_jail() for why. - */ -static void enter_userns(void) +static int userns_wait_idmaps(void) { char buf[1]; - if (!((opts.namespace & CLONE_NEWUSER) && opts.setns.user == -1)) { - post_jail_fs(); - return; - } - - if (unshare(CLONE_NEWUSER)) { - ERROR("unshare(CLONE_NEWUSER) failed: %m\n"); - free_and_exit(-1); - } - buf[0] = 'i'; if (xwrite_byte(userns_pipe[1], buf[0]) < 1) { ERROR("can't write to parent\n"); - free_and_exit(-1); + return -1; } close(userns_pipe[1]); if (xread_byte(userns_pipe[2], buf) < 1) { ERROR("can't read from parent\n"); - free_and_exit(-1); + return -1; } close(userns_pipe[2]); if (buf[0] != 'O') { ERROR("parent had an error, child exiting\n"); + return -1; + } + + if (setregid(0, 0) < 0 || setreuid(0, 0) < 0) { + ERROR("cannot become root in our user namespace: %m\n"); + return -1; + } + if (setgroups(0, NULL) < 0) { + ERROR("setgroups: %m\n"); + return -1; + } + + if ((opts.namespace & CLONE_NEWNS) && + mount("none", "/", "none", MS_REC | MS_PRIVATE, NULL)) { + ERROR("private mount failed: %m\n"); + return -1; + } + + return 0; +} + +static void enter_userns(void) +{ + if (!userns_deferred()) { + post_jail_fs(); + return; + } + + if (unshare(CLONE_NEWUSER)) { + ERROR("unshare(CLONE_NEWUSER) failed: %m\n"); free_and_exit(-1); } + if (userns_wait_idmaps()) + free_and_exit(-1); + if ((opts.namespace & CLONE_NEWNS) && unshare(CLONE_NEWNS)) { ERROR("unshare(CLONE_NEWNS) failed: %m\n"); free_and_exit(-1); @@ -1531,19 +1572,6 @@ static void enter_userns(void) remount_proc_sys_after_unshare(); } - if (setregid(0, 0) < 0) { - ERROR("setgid\n"); - free_and_exit(-1); - } - if (setreuid(0, 0) < 0) { - ERROR("setuid\n"); - free_and_exit(-1); - } - if (setgroups(0, NULL) < 0) { - ERROR("setgroups\n"); - free_and_exit(-1); - } - post_jail_fs(); } @@ -2699,8 +2727,6 @@ static int exec_jail(void *arg) close(pipes[3]); if ((opts.namespace & CLONE_NEWUSER) && opts.setns.user == -1) { - /* CLONE_NEWUSER is deferred to enter_userns(); keep our - * ends of the handshake open, close only the parent's. */ close(userns_pipe[0]); close(userns_pipe[3]); } else { @@ -2724,12 +2750,13 @@ static int exec_jail(void *arg) } /* - * Must run before setns_open(CLONE_NEWUSER) below: joining an - * external userns drops privilege immediately, and our own userns - * is deferred to enter_userns(), so this always runs privileged. + * Joining an external userns drops privilege immediately, so this has + * to run before it. A userns of our own owns the mount namespace it + * was created with and locks everything inherited into it, so there + * the detach neither works nor is needed. */ if ((opts.namespace & CLONE_NEWNS) && - ((opts.namespace & CLONE_NEWUSER) || opts.setns.user != -1) && + (userns_deferred() || opts.setns.user != -1) && isolate_mountns_and_detach_inherited()) { ERROR("failed to detach inherited mounts\n"); return EXIT_FAILURE; @@ -2762,6 +2789,10 @@ static int exec_jail(void *arg) false, recv_fds, nrecv, &extroot_idmap_fd, &overlay_idmap_fd); + if ((opts.namespace & CLONE_NEWUSER) && !userns_deferred() && + userns_wait_idmaps()) + return EXIT_FAILURE; + if (opts.setns.user != -1 && (opts.namespace & CLONE_NEWNS) && unshare(CLONE_NEWNS)) { ERROR("unshare(CLONE_NEWNS) failed: %m\n"); @@ -2777,12 +2808,6 @@ static int exec_jail(void *arg) free_and_exit(EXIT_FAILURE); } - /* - * A join of an existing userns (opts.setns.user) can become root - * right away. Our own CLONE_NEWUSER is not created here: doing so - * before /proc,/sys are mounted ties the PID namespace to it, - * which fails mnt_already_visible() on hosts with locked /proc. - */ if (opts.setns.user != -1) { if (setregid(0, 0) < 0) { ERROR("setgid\n"); @@ -4211,7 +4236,7 @@ static int parseOCIlinux(struct blob_attr *msg) } { - bool defer_userns = (opts.namespace & CLONE_NEWUSER) && opts.setns.user == -1; + bool defer_userns = userns_deferred(); if (tb[OCI_LINUX_READONLYPATHS]) { blobmsg_for_each_attr(cur, tb[OCI_LINUX_READONLYPATHS], rem) { @@ -5728,7 +5753,8 @@ int main(int argc, char **argv) opts.ronly = 1; break; case 'f': - opts.namespace |= CLONE_NEWUSER; + if (opts.setns.user == -1) + opts.namespace |= CLONE_NEWUSER; break; case 'F': opts.namespace |= CLONE_NEWCGROUP; @@ -6233,7 +6259,7 @@ static void post_main(struct uloop_timeout *t) add_mount(NULL, "/dev/pts", "devpts", MS_NOATIME | MS_NOEXEC | MS_NOSUID, 0, ptsopts, 0); } - bool defer_userns = (opts.namespace & CLONE_NEWUSER) && opts.setns.user == -1; + bool defer_userns = userns_deferred(); if (defer_userns) add_mount(PROCD_NOAFILE, JAIL_NOAFILE, NULL, @@ -6392,7 +6418,8 @@ static void post_main(struct uloop_timeout *t) */ int init_cgroup_fd = -1; struct clone_args cargs = { - .flags = (opts.namespace & ~(CLONE_NEWCGROUP | CLONE_NEWUSER | CLONE_NEWTIME)) | CLONE_PIDFD, + .flags = (opts.namespace & ~(CLONE_NEWCGROUP | CLONE_NEWTIME | + (userns_deferred() ? CLONE_NEWUSER : 0))) | CLONE_PIDFD, .pidfd = (__u64)(uintptr_t)&jail_process_pidfd, .exit_signal = SIGCHLD, }; @@ -6578,10 +6605,6 @@ static void post_create_runtime(void) while (num_idmap_fds > 0) close(idmap_fds[--num_idmap_fds]); - /* - * Wait for the child to reach enter_userns() and create its own - * userns before writing its uid/gid maps; see that function. - */ if ((opts.namespace & CLONE_NEWUSER) && opts.setns.user == -1) { char ubuf[1]; From 0b99742ad66771d2593471ceb9f4aa20683df352 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 05:37:55 +0100 Subject: [PATCH 56/82] jail: move time namespace setup into the child 62121395 ("jail: give the container's namespaces to its own user namespace") left one namespace behind: the time namespace was still created by the parent, with unshare(CLONE_NEWTIME) before the clone, and therefore stayed owned by the initial user namespace. clone3() cannot simply take the flag either, since a child created with CLONE_NEWTIME sits in the namespace from birth, which freezes its offsets before anyone can write them. Create it in the child instead, once the uid and gid maps are in place. A namespace made by unshare() is owned by the user namespace of its creator, so it now belongs to the container's user namespace, where mapped root holds CAP_SYS_TIME, exactly what writing /proc/self/timens_offsets asks for. unshare(CLONE_NEWTIME) does not move the caller, and the offsets of an inhabited namespace are sealed, so the child writes the offsets first and then enters through /proc/self/ns/time_for_children with setns(), which the kernel allows while the process is still single-threaded. A bundle that defers its own user namespace creates the time namespace in enter_userns() for the same ownership reason; /proc is present there, as a new time namespace can only be configured through OCI and an OCI jail always mounts it. The createContainer hooks of such a bundle now run before the time namespace exists; everything else runs inside it as before. A time namespace joined by path moves to the child as well, alongside the other setns() joins and before any user namespace is created or joined, because entering needs privilege in the user namespace owning the target. The parent keeps only the probe for kernels without time namespace support, and no longer switches its own time namespace around the clone: the old join path setns()'d the supervisor into the container's time namespace and back, briefly running it on shifted clocks. Signed-off-by: Daniel Golle --- jail/jail.c | 73 ++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 52 insertions(+), 21 deletions(-) diff --git a/jail/jail.c b/jail/jail.c index 8967e00..448ab45 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -1481,6 +1481,9 @@ static void free_and_exit(int ret) static void post_jail_fs(void); static void enter_userns(void); static int userns_wait_idmaps(void); +#ifdef CLONE_NEWTIME +static int timens_create(void); +#endif static void remask_after_unshare(void); static void remount_proc_sys_after_unshare(void); static void enter_jail_fs(void) @@ -1563,6 +1566,12 @@ static void enter_userns(void) if (userns_wait_idmaps()) free_and_exit(-1); +#ifdef CLONE_NEWTIME + if ((opts.namespace & CLONE_NEWTIME) && opts.setns.time == -1 && + timens_create()) + free_and_exit(-1); +#endif + if ((opts.namespace & CLONE_NEWNS) && unshare(CLONE_NEWNS)) { ERROR("unshare(CLONE_NEWNS) failed: %m\n"); free_and_exit(-1); @@ -2702,6 +2711,35 @@ static int applyOCIlinuxtimeoffsets(void) return 0; } +static int timens_create(void) +{ + int fd; + + if (unshare(CLONE_NEWTIME)) { + ERROR("unshare(CLONE_NEWTIME) failed: %m\n"); + return -1; + } + + if ((timens_offsets.monotonic.set || timens_offsets.boottime.set) && + applyOCIlinuxtimeoffsets()) + return -1; + + fd = open("/proc/self/ns/time_for_children", O_RDONLY | O_CLOEXEC); + if (fd < 0) { + ERROR("open(/proc/self/ns/time_for_children): %m\n"); + return -1; + } + + if (setns(fd, CLONE_NEWTIME)) { + ERROR("setns(CLONE_NEWTIME): %m\n"); + close(fd); + return -1; + } + + close(fd); + return 0; +} + static void pre_exec_jail(struct uloop_timeout *t); static struct uloop_timeout pre_exec_timeout = { .cb = pre_exec_jail, @@ -2744,6 +2782,10 @@ static int exec_jail(void *arg) ret = setns_open(CLONE_NEWIPC); if (!ret) ret = setns_open(CLONE_NEWUTS); +#ifdef CLONE_NEWTIME + if (!ret) + ret = setns_open(CLONE_NEWTIME); +#endif if (ret) { ERROR("failed to join namespace: %s\n", strerror(ret)); return EXIT_FAILURE; @@ -2829,6 +2871,12 @@ static int exec_jail(void *arg) } } +#ifdef CLONE_NEWTIME + if ((opts.namespace & CLONE_NEWTIME) && opts.setns.time == -1 && + !userns_deferred() && timens_create()) + free_and_exit(EXIT_FAILURE); +#endif + if (((opts.namespace & CLONE_NEWUTS) || opts.setns.uts != -1) && opts.hostname && strlen(opts.hostname) > 0 && sethostname(opts.hostname, strlen(opts.hostname))) { @@ -5684,7 +5732,6 @@ static struct uloop_timeout post_main_timeout = { .cb = post_main, }; static int pidns_fd; -static int timens_fd; static void post_create_runtime(void); struct env_e { @@ -6328,22 +6375,6 @@ static void post_main(struct uloop_timeout *t) free_and_exit(EXIT_FAILURE); } - if (opts.setns.time != -1) { - timens_fd = ns_open_pid("time", getpid()); - setns_open(CLONE_NEWTIME); - } else if (opts.namespace & CLONE_NEWTIME) { - timens_fd = ns_open_pid("time", getpid()); - if (unshare(CLONE_NEWTIME)) { - ERROR("unshare(CLONE_NEWTIME) failed: %m\n"); - free_and_exit(EXIT_FAILURE); - } - if ((timens_offsets.monotonic.set || timens_offsets.boottime.set) && - applyOCIlinuxtimeoffsets()) - free_and_exit(EXIT_FAILURE); - } else { - timens_fd = -1; - } - if ((opts.namespace & CLONE_NEWNS) && prepare_jail_dev()) { ERROR("prepare_jail_dev() failed\n"); free_and_exit(EXIT_FAILURE); @@ -6462,10 +6493,6 @@ static void post_main(struct uloop_timeout *t) setns(pidns_fd, CLONE_NEWPID); close(pidns_fd); } - if (timens_fd != -1) { - setns(timens_fd, CLONE_NEWTIME); - close(timens_fd); - } if (opts.setns.net != -1) close(opts.setns.net); if (opts.setns.ns != -1) @@ -6478,6 +6505,10 @@ static void post_main(struct uloop_timeout *t) close(opts.setns.user); if (opts.setns.cgroup != -1) close(opts.setns.cgroup); +#ifdef CLONE_NEWTIME + if (opts.setns.time != -1) + close(opts.setns.time); +#endif close(pipes[1]); close(pipes[2]); close(userns_pipe[1]); From 3f4ff94d0f644ea4ec26dff32cc6f13f758d097d Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 18:44:03 +0100 Subject: [PATCH 57/82] uxc-stack: pass the addressing annotations through to members uxc-net now honours the org.openwrt.network.proto, proto6 and ip6ifaceid annotations to pick the in-jail IPv4 and IPv6 protocols and to pin the IPv6 interface identifier inside a delegated prefix. Let a stack definition set them per member through the new spec keys proto, proto6 and ip6ifaceid, named after the annotation leaves just like the egress, ingress and host access options, and copy them into the member's annotations sidecar alongside the existing network annotations. A key set by the stack definition overrides the same annotation in the member's own bundle, since uxc-net merges the sidecar over the bundle's config.json annotations; a key the stack leaves unset is omitted from the sidecar, so the bundle's own value stays effective. This matches the behaviour of the existing attach, egress, ingress and host annotations, whose precedence is unchanged. Signed-off-by: Daniel Golle --- uxc-stack.uc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/uxc-stack.uc b/uxc-stack.uc index ebcf002..0c143a9 100644 --- a/uxc-stack.uc +++ b/uxc-stack.uc @@ -204,6 +204,12 @@ function backhaul_prepare() { side['org.openwrt.network.host'] = kv[1]; } } + if (inst.proto) + side['org.openwrt.network.proto'] = inst.proto; + if (inst.proto6) + side['org.openwrt.network.proto6'] = inst.proto6; + if (inst.ip6ifaceid) + side['org.openwrt.network.ip6ifaceid'] = inst.ip6ifaceid; writefile(REG_DIR + '/' + qname(inst.name) + '.annotations', sprintf('%J\n', side)); } } From 7130a8e70ce981bc8949a97f93f4d77950ce02a4 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 18:46:06 +0100 Subject: [PATCH 58/82] jail: mount odhcp6c into the netifd helper jail The helper jail mounts /lib/netifd wholesale, so its netifd registers the dhcpv6 protocol handler whenever the odhcp6c package is installed on the host, but /usr/sbin/odhcp6c itself is missing from the mount list. A container can therefore never run DHCPv6 or accept router advertisements through its private netifd, while IPv4 DHCP works because udhcpc is mounted. Add odhcp6c next to udhcpc; read-only mount entries are skipped silently when the source is absent, so hosts without the odhcp6c package are unaffected. Also mount /bin/sed: proto_dhcpv6_setup() unconditionally pipes the prefix request hint through sed to extract an explicit IAID suffix, so without it every DHCPv6 interface setup logs "sed: not found". tr and hexdump stay out on purpose. Their only users are hexdump_2hex() in /lib/functions.sh and the vendorid encoding in dhcp.sh, reached via option clientid, option vendorid or a global dhcp_default_duid, and no configuration the jail can see contains any of those: the jail's /etc/config/network is compiled exclusively by uxc-net, which emits neither these options nor a globals section, the compiled config is bind-mounted read-only, and the private ubus socket is mounted only into the helper jails, so the container cannot inject dynamic interfaces carrying such options either. Both proto handlers then take the branch where clientid is empty and the dhcp_default_duid lookup returns empty before hexdump_2hex() is called. Signed-off-by: Daniel Golle --- jail/netifd.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jail/netifd.c b/jail/netifd.c index e765827..eec3d79 100644 --- a/jail/netifd.c +++ b/jail/netifd.c @@ -207,6 +207,7 @@ static void run_netifd(struct uloop_timeout *t) blobmsg_add_string(&req, "/bin/cat", "0"); blobmsg_add_string(&req, "/bin/ipcalc.sh", "0"); blobmsg_add_string(&req, "/bin/kill", "0"); + blobmsg_add_string(&req, "/bin/sed", "0"); blobmsg_add_string(&req, "/bin/ubus", "0"); blobmsg_add_string(&req, "/etc/hotplug.d", "0"); blobmsg_add_string(&req, "/lib/config/uci.sh", "0"); @@ -228,6 +229,7 @@ static void run_netifd(struct uloop_timeout *t) blobmsg_add_string(&req, "/sbin/hotplug-call", "0"); blobmsg_add_string(&req, "/sbin/uci", "0"); blobmsg_add_string(&req, "/sbin/udhcpc", "0"); + blobmsg_add_string(&req, "/usr/sbin/odhcp6c", "0"); blobmsg_close_table(&req, mount); blobmsg_add_u8(&req, "log", 1); From 0c15c7ac7e8f55b518a0df86da78b8a5dbdb1fc4 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 18:51:48 +0100 Subject: [PATCH 59/82] jail: report the container's network namespace and addresses in its state uxc list cannot show a container's addresses, nor tell a container with its own network namespace apart from one sharing the host's. Only ujail can provide that data: a bridged container obtains its address by DHCP inside its network namespace, and the in-jail netifd sits on a private ubus, so nothing on the host bus ever learns it. Extend the state reply with a reverse-DNS top-level object: "org.openwrt.network": { "namespace": "private", "attach": "routed", "interfaces": [ { "name": "eth0", "mac": "9a:...", "addresses": ["10.7.3.2/31", "fe80::1/64"] } ] } "namespace" is host, joined or private depending on whether the container requests no network namespace, joins one by path or gets its own. "attach" is the effective attachment mode: the bundle annotation merged with the sidecar file uxc-stack leaves for uxc-net, the sidecar taking precedence just as uxc-net applies it; a private namespace without any annotation is reported as none since uxc-net treats it that way, and the field is omitted when the mode is genuinely unknown. "interfaces" lists the interfaces in the container's network namespace, excluding loopback, with their MAC and their IPv4 and IPv6 addresses in CIDR form, gathered by RTM_GETLINK and RTM_GETADDR dumps over a netlink socket created inside the container's namespace; the parent enters the namespace just long enough to create the socket and returns at once, leaving the container undisturbed. The dump is only attempted while the container process is alive, so a query racing teardown simply omits the list. The runtime spec allows additional state properties and runc, crun and ujail already ship some; a reverse-DNS key can never collide with a future spec property. The object is added in handle_state() only, after oci_state_fill(), so OCI hooks keep receiving an unchanged spec-shaped document on stdin and no netlink work happens per hook invocation. Signed-off-by: Daniel Golle --- jail/jail.c | 269 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 269 insertions(+) diff --git a/jail/jail.c b/jail/jail.c index 448ab45..975967e 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -30,6 +30,7 @@ #include #include #include +#include /* musl only defined 15 limit types, make sure all 16 are supported */ #ifndef RLIMIT_RTTIME @@ -4671,6 +4672,273 @@ static int handle_start(struct ubus_context *ctx, struct ubus_object *obj, return UBUS_STATUS_OK; } +struct netns_ifinfo { + int ifindex; + char name[IF_NAMESIZE]; + char mac[18]; +}; + +struct netns_ifaddr { + int ifindex; + char cidr[INET6_ADDRSTRLEN + 4]; +}; + +static int netns_open_sock(pid_t pid) +{ + struct sockaddr_nl sa = { .nl_family = AF_NETLINK }; + struct timeval tv = { .tv_sec = 1 }; + char path[64]; + int netns_fd, self_fd, sock, saved_err; + + snprintf(path, sizeof(path), "/proc/%d/ns/net", pid); + netns_fd = open(path, O_RDONLY | O_CLOEXEC); + if (netns_fd < 0) + return -1; + + self_fd = open("/proc/self/ns/net", O_RDONLY | O_CLOEXEC); + if (self_fd < 0) { + close(netns_fd); + return -1; + } + + if (setns(netns_fd, CLONE_NEWNET)) { + close(netns_fd); + close(self_fd); + return -1; + } + + sock = socket(AF_NETLINK, SOCK_RAW | SOCK_CLOEXEC, NETLINK_ROUTE); + saved_err = errno; + if (setns(self_fd, CLONE_NEWNET)) + ERROR("cannot return to own network namespace: %m\n"); + close(netns_fd); + close(self_fd); + if (sock < 0) { + errno = saved_err; + return -1; + } + + if (bind(sock, (struct sockaddr *)&sa, sizeof(sa)) < 0) { + close(sock); + return -1; + } + + setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); + + return sock; +} + +static int netns_parse_link(struct nlmsghdr *nh, struct netns_ifinfo **ifaces, size_t *n) +{ + struct ifinfomsg *ifi = NLMSG_DATA(nh); + struct netns_ifinfo *tmp, *iface; + struct rtattr *rta; + const uint8_t *hw; + int len = nh->nlmsg_len - NLMSG_LENGTH(sizeof(*ifi)); + + if (ifi->ifi_flags & IFF_LOOPBACK) + return 0; + + tmp = realloc(*ifaces, (*n + 1) * sizeof(*tmp)); + if (!tmp) + return ENOMEM; + + *ifaces = tmp; + iface = &tmp[(*n)++]; + memset(iface, 0, sizeof(*iface)); + iface->ifindex = ifi->ifi_index; + + for (rta = IFLA_RTA(ifi); RTA_OK(rta, len); rta = RTA_NEXT(rta, len)) { + if (rta->rta_type == IFLA_IFNAME) { + strncpy(iface->name, RTA_DATA(rta), sizeof(iface->name) - 1); + } else if (rta->rta_type == IFLA_ADDRESS && RTA_PAYLOAD(rta) == 6) { + hw = RTA_DATA(rta); + snprintf(iface->mac, sizeof(iface->mac), + "%02x:%02x:%02x:%02x:%02x:%02x", + hw[0], hw[1], hw[2], hw[3], hw[4], hw[5]); + } + } + + return 0; +} + +static int netns_parse_addr(struct nlmsghdr *nh, struct netns_ifaddr **addrs, size_t *n) +{ + struct ifaddrmsg *ifa = NLMSG_DATA(nh); + struct netns_ifaddr *tmp, *addr; + struct rtattr *rta, *sel = NULL; + char abuf[INET6_ADDRSTRLEN]; + int len = nh->nlmsg_len - NLMSG_LENGTH(sizeof(*ifa)); + + if (ifa->ifa_family != AF_INET && ifa->ifa_family != AF_INET6) + return 0; + + for (rta = IFA_RTA(ifa); RTA_OK(rta, len); rta = RTA_NEXT(rta, len)) { + if (rta->rta_type == IFA_LOCAL) + sel = rta; + else if (rta->rta_type == IFA_ADDRESS && !sel) + sel = rta; + } + + if (!sel || !inet_ntop(ifa->ifa_family, RTA_DATA(sel), abuf, sizeof(abuf))) + return 0; + + tmp = realloc(*addrs, (*n + 1) * sizeof(*tmp)); + if (!tmp) + return ENOMEM; + + *addrs = tmp; + addr = &tmp[(*n)++]; + addr->ifindex = ifa->ifa_index; + snprintf(addr->cidr, sizeof(addr->cidr), "%s/%u", abuf, ifa->ifa_prefixlen); + + return 0; +} + +static int netns_dump(int sock, int type, struct netns_ifinfo **ifaces, size_t *nifaces, + struct netns_ifaddr **addrs, size_t *naddrs) +{ + struct { + struct nlmsghdr hdr; + struct rtgenmsg gen; + } req = { + .hdr = { + .nlmsg_len = NLMSG_LENGTH(sizeof(struct rtgenmsg)), + .nlmsg_type = type, + .nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP, + .nlmsg_seq = type, + }, + .gen = { .rtgen_family = AF_UNSPEC }, + }; + struct nlmsghdr *nh; + char buf[32768]; + ssize_t n; + int msglen, err; + + if (send(sock, &req, req.hdr.nlmsg_len, 0) < 0) + return errno; + + while (1) { + n = recv(sock, buf, sizeof(buf), 0); + if (n < 0) { + if (errno == EINTR) + continue; + return errno; + } + if (n == 0) + return EIO; + + msglen = n; + for (nh = (struct nlmsghdr *)buf; NLMSG_OK(nh, msglen); nh = NLMSG_NEXT(nh, msglen)) { + err = 0; + if (nh->nlmsg_type == NLMSG_DONE) + return 0; + else if (nh->nlmsg_type == NLMSG_ERROR) + return EIO; + else if (nh->nlmsg_type == RTM_NEWLINK) + err = netns_parse_link(nh, ifaces, nifaces); + else if (nh->nlmsg_type == RTM_NEWADDR) + err = netns_parse_addr(nh, addrs, naddrs); + if (err) + return err; + } + } +} + +static void netns_fill_interfaces(struct blob_buf *b, pid_t pid) +{ + struct netns_ifinfo *ifaces = NULL; + struct netns_ifaddr *addrs = NULL; + size_t nifaces = 0, naddrs = 0, i, j; + void *a, *t, *aa; + int sock; + + sock = netns_open_sock(pid); + if (sock < 0) + return; + + if (netns_dump(sock, RTM_GETLINK, &ifaces, &nifaces, &addrs, &naddrs) || + netns_dump(sock, RTM_GETADDR, &ifaces, &nifaces, &addrs, &naddrs)) + goto out; + + a = blobmsg_open_array(b, "interfaces"); + for (i = 0; i < nifaces; i++) { + t = blobmsg_open_table(b, NULL); + blobmsg_add_string(b, "name", ifaces[i].name); + if (ifaces[i].mac[0]) + blobmsg_add_string(b, "mac", ifaces[i].mac); + aa = blobmsg_open_array(b, "addresses"); + for (j = 0; j < naddrs; j++) + if (addrs[j].ifindex == ifaces[i].ifindex) + blobmsg_add_string(b, NULL, addrs[j].cidr); + blobmsg_close_array(b, aa); + blobmsg_close_table(b, t); + } + blobmsg_close_array(b, a); + +out: + free(ifaces); + free(addrs); + close(sock); +} + +static const char *annotation_get(struct blob_attr *attrs, const char *key) +{ + struct blob_attr *cur; + int rem; + + if (!attrs) + return NULL; + + blobmsg_for_each_attr(cur, attrs, rem) + if (blobmsg_type(cur) == BLOBMSG_TYPE_STRING && + !strcmp(blobmsg_name(cur), key)) + return blobmsg_get_string(cur); + + return NULL; +} + +static void oci_state_fill_network(struct blob_buf *b) +{ + struct blob_buf sidecar = { 0 }; + char path[128]; + const char *mode, *attach; + void *c; + + if (opts.setns.net != -1) + mode = "joined"; + else if (opts.namespace & CLONE_NEWNET) + mode = "private"; + else + mode = "host"; + + c = blobmsg_open_table(b, "org.openwrt.network"); + blobmsg_add_string(b, "namespace", mode); + + blob_buf_init(&sidecar, 0); + snprintf(path, sizeof(path), "/tmp/run/uvol/.meta/uxc/%s.annotations", opts.name); + blobmsg_add_json_from_file(&sidecar, path); + + attach = annotation_get(sidecar.head, "org.openwrt.network.attach"); + if (!attach) + attach = annotation_get(opts.annotations, "org.openwrt.network.attach"); + if (attach && !*attach) + attach = NULL; + if (!attach && (opts.namespace & CLONE_NEWNET)) + attach = "none"; + if (attach) + blobmsg_add_string(b, "attach", attach); + blob_buf_free(&sidecar); + + if (strcmp(mode, "host") && jail_running && + (jail_oci_state == OCI_STATE_CREATED || + jail_oci_state == OCI_STATE_RUNNING || + jail_oci_state == OCI_STATE_PAUSED)) + netns_fill_interfaces(b, jail_process.pid); + + blobmsg_close_table(b, c); +} + static struct blob_buf bb; static void oci_state_fill(struct blob_buf *b) { @@ -4753,6 +5021,7 @@ static int handle_state(struct ubus_context *ctx, struct ubus_object *obj, { blob_buf_init(&bb, 0); oci_state_fill(&bb); + oci_state_fill_network(&bb); ubus_send_reply(ctx, req, bb.head); return UBUS_STATUS_OK; From 17a6650cd060e5c4cdfeb92249083f66976a4bdc Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 18:48:51 +0100 Subject: [PATCH 60/82] uxc-net: honour org.openwrt.network.proto for the in-jail interface The in-jail addressing was hardcoded per attachment mode: bridged containers always received a DHCP interface and routed containers always received the static /31 configuration, while the documented org.openwrt.network.proto annotation was read nowhere. Implement the annotation. Replace the hardcoded configuration blobs with a renderer parameterised on the requested protocol, composed through append_injail(), and regenerate the in-jail file from scratch on every bring-up so a stale file from an earlier run cannot leak into the composition. Accepted values are 'dhcp' and 'static'. The defaults preserve today's behaviour exactly: bridged defaults to 'dhcp', routed to 'static'. An unknown value is rejected with a diagnostic instead of silently falling back. Requesting 'dhcp' for a routed container is refused as well, because routed builds a /31 point-to-point link whose gateway end runs no DHCP server, so the request could never be served. A bridged container with proto 'static' takes its address from org.openwrt.network.address in address/prefix notation, with the optional org.openwrt.network.gateway and org.openwrt.network.dns annotations rendered verbatim; the backhaul section now goes through the same renderer. Signed-off-by: Daniel Golle --- jail/uxc-net | 93 ++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 68 insertions(+), 25 deletions(-) diff --git a/jail/uxc-net b/jail/uxc-net index b2aea4a..34c543a 100644 --- a/jail/uxc-net +++ b/jail/uxc-net @@ -36,16 +36,6 @@ let loopback_section = "config interface 'loopback'\n" + "\toption ipaddr '127.0.0.1'\n" + "\toption netmask '255.0.0.0'\n\n"; -let write_injail = function(name, lan_section) { - fs.mkdir(injail_dir, 0755); - let f = fs.open(injail_path(name), "w"); - if (!f) - return; - f.write(loopback_section); - f.write(lan_section); - f.close(); -}; - let call = function(object, method, data) { ubus.call({ object: object, method: method, data: data }); let err = ubus.error(); @@ -179,6 +169,48 @@ let append_injail = function(name, section) { f.close(); }; +let render_section = function(iface, device, proto, opts) { + let s = "config interface '" + iface + "'\n" + + "\toption device '" + device + "'\n" + + "\toption proto '" + proto + "'\n", k, v; + + for (k, v in opts) + s += "\toption " + k + " '" + v + "'\n"; + return s; +}; + +let injail_proto = function(ann, kind) { + let proto = ann["org.openwrt.network.proto"]; + + if (proto == null || proto == "") + return kind == "routed" ? "static" : "dhcp"; + if (proto == "static") + return proto; + if (proto == "dhcp") { + if (kind != "routed") + return proto; + warn("uxc-net: routed cannot use proto 'dhcp': the /31 gateway link has no DHCP server\n"); + return null; + } + warn(sprintf("uxc-net: unsupported org.openwrt.network.proto '%s'\n", proto)); + return null; +}; + +let bridged_static_opts = function(ann) { + let addr = ann["org.openwrt.network.address"], opts; + + if (addr == null || index(addr, "/") < 0) { + warn("uxc-net: bridged proto 'static' needs org.openwrt.network.address as address/prefix\n"); + return null; + } + opts = { ipaddr: addr }; + if (ann["org.openwrt.network.gateway"]) + opts.gateway = ann["org.openwrt.network.gateway"]; + if (ann["org.openwrt.network.dns"]) + opts.dns = ann["org.openwrt.network.dns"]; + return opts; +}; + let csv = function(val) { let list = [], i, parts; if (type(val) != "string") @@ -337,6 +369,15 @@ let ensure_network = function(net) { let bridged_up = function(name, ann, attach, m) { let net = attach.network; let vh = host_veth(name), vc = cont_veth(name); + let proto = injail_proto(ann, "bridged"), opts = {}; + + if (!proto) + return 1; + if (proto == "static") { + opts = bridged_static_opts(ann); + if (!opts) + return 1; + } if (ann["org.openwrt.network.egress"] || ann["org.openwrt.network.ingress"]) warn("uxc-net: bridged takes no egress/ingress (L2 inherits the joined network's zone); ignoring\n"); @@ -363,9 +404,7 @@ let bridged_up = function(name, ann, attach, m) { })) return 1; - write_injail(name, "config interface 'lan'\n" + - "\toption device 'eth0'\n" + - "\toption proto 'dhcp'\n"); + append_injail(name, render_section("lan", "eth0", proto, opts)); return 0; }; @@ -490,6 +529,10 @@ let routed_up = function(name, ann, m) { let vh = host_veth(name), vc = cont_veth(name); let gw_iface = gwif(name); let gw_ip = split(net.gw_cidr, "/")[0]; + let proto = injail_proto(ann, "routed"); + + if (!proto) + return 1; if (call("network", "create_device", with_macs({ name: vh, type: "veth", peer_name: vc }, m, "h", "c"))) @@ -519,13 +562,12 @@ let routed_up = function(name, ann, m) { fw_uci_create(name, ann, net); - write_injail(name, "config interface 'lan'\n" + - "\toption device 'eth0'\n" + - "\toption proto 'static'\n" + - "\toption ipaddr '" + net.container + "'\n" + - "\toption netmask '255.255.255.254'\n" + - "\toption gateway '" + gw_ip + "'\n" + - "\toption dns '" + gw_ip + "'\n"); + append_injail(name, render_section("lan", "eth0", proto, { + ipaddr: net.container, + netmask: "255.255.255.254", + gateway: gw_ip, + dns: gw_ip, + })); return 0; }; @@ -570,11 +612,10 @@ let backhaul_up = function(name, bh, m) { return 1; if (bh.address) - append_injail(name, "config interface 'backhaul'\n" + - "\toption device 'bh0'\n" + - "\toption proto 'static'\n" + - "\toption ipaddr '" + bh.address + "'\n" + - "\toption netmask '255.255.255.0'\n"); + append_injail(name, render_section("backhaul", "bh0", "static", { + ipaddr: bh.address, + netmask: "255.255.255.0", + })); return 0; }; @@ -620,6 +661,8 @@ let do_up = function(name, bundle) { attach = parse_attach(ann); bh = parse_backhaul(ann); + fs.unlink(injail_path(name)); + let roles = []; if (attach.kind == "bridged" || attach.kind == "routed") { push(roles, "h"); From ef6d6e62c946efc9497bf886d3c5f8d74e85ac85 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 18:50:01 +0100 Subject: [PATCH 61/82] uxc-net: add IPv6 addressing and a deterministic interface identifier The in-jail interface was IPv4 only; nothing in the contract could ask for an IPv6 address. Add org.openwrt.network.proto6, accepting 'dhcpv6', 'slaac', 'static' and 'none'. Where IPv6 is requested a second in-jail interface section is rendered on the moved device: 'dhcpv6' runs the full odhcp6c cycle, 'slaac' renders proto 'dhcpv6' with reqaddress 'none' and reqprefix 'no' so odhcp6c only processes router advertisements, and 'static' takes org.openwrt.network.address6 in address/prefix notation with an optional org.openwrt.network.gateway6. Add org.openwrt.network.ip6ifaceid, an address with a zero network part such as '::1234', rendered as option ip6ifaceid on the dhcpv6 section; /lib/netifd/proto/dhcpv6.sh hands it to odhcp6c through -i, giving the container a deterministic interface identifier within whatever prefix the link advertises. The default is 'none': no IPv6 section is rendered, preserving the existing behaviour for containers that do not ask for IPv6. Signed-off-by: Daniel Golle --- jail/uxc-net | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/jail/uxc-net b/jail/uxc-net index 34c543a..8d3e0cc 100644 --- a/jail/uxc-net +++ b/jail/uxc-net @@ -211,6 +211,53 @@ let bridged_static_opts = function(ann) { return opts; }; +let injail_proto6 = function(ann) { + let proto6 = ann["org.openwrt.network.proto6"]; + + if (proto6 == null || proto6 == "" || proto6 == "none") + return "none"; + if (proto6 == "dhcpv6" || proto6 == "slaac" || proto6 == "static") + return proto6; + warn(sprintf("uxc-net: unsupported org.openwrt.network.proto6 '%s'\n", proto6)); + return null; +}; + +let injail6_section = function(ann) { + let proto6 = injail_proto6(ann); + let ifaceid = ann["org.openwrt.network.ip6ifaceid"]; + let addr6 = ann["org.openwrt.network.address6"]; + let opts = {}; + + if (!proto6) + return null; + if (proto6 == "none") + return ""; + + if (proto6 == "static") { + if (addr6 == null || index(addr6, "/") < 0) { + warn("uxc-net: proto6 'static' needs org.openwrt.network.address6 as address/prefix\n"); + return null; + } + opts.ip6addr = addr6; + if (ann["org.openwrt.network.gateway6"]) + opts.ip6gw = ann["org.openwrt.network.gateway6"]; + return render_section("lan6", "eth0", "static", opts); + } + + if (proto6 == "slaac") { + opts.reqaddress = "none"; + opts.reqprefix = "no"; + } + if (ifaceid) { + if (!match(ifaceid, /^::/)) { + warn(sprintf("uxc-net: org.openwrt.network.ip6ifaceid '%s' must have a zero network part\n", ifaceid)); + return null; + } + opts.ip6ifaceid = ifaceid; + } + return render_section("lan6", "eth0", "dhcpv6", opts); +}; + let csv = function(val) { let list = [], i, parts; if (type(val) != "string") @@ -370,8 +417,9 @@ let bridged_up = function(name, ann, attach, m) { let net = attach.network; let vh = host_veth(name), vc = cont_veth(name); let proto = injail_proto(ann, "bridged"), opts = {}; + let section6 = injail6_section(ann); - if (!proto) + if (!proto || section6 == null) return 1; if (proto == "static") { opts = bridged_static_opts(ann); @@ -405,6 +453,8 @@ let bridged_up = function(name, ann, attach, m) { return 1; append_injail(name, render_section("lan", "eth0", proto, opts)); + if (section6 != "") + append_injail(name, section6); return 0; }; @@ -530,8 +580,9 @@ let routed_up = function(name, ann, m) { let gw_iface = gwif(name); let gw_ip = split(net.gw_cidr, "/")[0]; let proto = injail_proto(ann, "routed"); + let section6 = injail6_section(ann); - if (!proto) + if (!proto || section6 == null) return 1; if (call("network", "create_device", @@ -568,6 +619,8 @@ let routed_up = function(name, ann, m) { gateway: gw_ip, dns: gw_ip, })); + if (section6 != "") + append_injail(name, section6); return 0; }; From 85dce23e684aef2abbfb6e48b02613ab39258471 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 18:51:47 +0100 Subject: [PATCH 62/82] uxc-net: make routed containers a delegated-prefix downstream A routed container had no IPv6 path at all: the host gateway interface carried only the /31 IPv4 address, and nothing on the host answered router solicitations on the link, so a container asking for proto6 could never configure itself. Give the gateway interface created through network.add_dynamic an ip6assign of 64 and ip6ifaceid '::1'. netifd hands the whole blob to interface_alloc(), which parses every interface attribute, so netifd carves a /64 for the link out of the upstream delegated prefix and the host end takes the deterministic ::1 address within it; no netifd change is needed. Prefix assignment alone does not make the link work: router advertisements must actually be emitted on the gateway interface for the container to learn the prefix and its default route. That takes an odhcpd section, which is new host-side state: bring-up commits a per-container section named after the gateway interface to the persistent dhcp configuration, with ra and dhcpv6 in server mode, and reloads it through the config.change service event. Bring-down removes exactly that section again, following the lifecycle the per-container firewall sections already use. dnsmasq ignores the section because its dhcpv4 option defaults to disabled. fw_reload() becomes pkg_reload() so the firewall and dhcp reload paths share one helper. Signed-off-by: Daniel Golle --- jail/uxc-net | 49 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/jail/uxc-net b/jail/uxc-net index 8d3e0cc..161f211 100644 --- a/jail/uxc-net +++ b/jail/uxc-net @@ -44,6 +44,11 @@ let call = function(object, method, data) { return err ? -1 : 0; }; +let pkg_reload = function(pkg) { + ubus.call({ object: "service", method: "event", + data: { type: "config.change", data: { package: pkg } } }); +}; + let sidecar_path = function(name) { return "/tmp/run/uvol/.meta/uxc/" + name + ".annotations"; }; let state_dir = "/tmp/run/uvol/.meta/uxc/state"; @@ -484,11 +489,6 @@ let routed_subnet = function(name, ann) { }; }; -let fw_reload = function() { - ubus.call({ object: "service", method: "event", - data: { type: "config.change", data: { package: "firewall" } } }); -}; - let fw_uci_create = function(name, ann, net) { let czone = contzone(name); let cursor = uci.cursor(); @@ -549,7 +549,7 @@ let fw_uci_create = function(name, ann, net) { } cursor.commit("firewall"); - fw_reload(); + pkg_reload("firewall"); return 0; }; @@ -568,11 +568,42 @@ let fw_uci_remove = function(name) { cursor.delete("firewall", n); if (length(kill)) { cursor.commit("firewall"); - fw_reload(); + pkg_reload("firewall"); } return 0; }; +let ra_uci_create = function(name) { + let gw_iface = gwif(name); + let cursor = uci.cursor(); + + cursor.load("dhcp"); + if (cursor.get("dhcp", gw_iface)) + return 0; + + cursor.set("dhcp", gw_iface, "dhcp"); + cursor.set("dhcp", gw_iface, "interface", gw_iface); + cursor.set("dhcp", gw_iface, "ra", "server"); + cursor.set("dhcp", gw_iface, "dhcpv6", "server"); + cursor.commit("dhcp"); + pkg_reload("dhcp"); + return 0; +}; + +let ra_uci_remove = function(name) { + let gw_iface = gwif(name); + let cursor = uci.cursor(); + + cursor.load("dhcp"); + if (!cursor.get("dhcp", gw_iface)) + return 0; + + cursor.delete("dhcp", gw_iface); + cursor.commit("dhcp"); + pkg_reload("dhcp"); + return 0; +}; + let routed_up = function(name, ann, m) { let czone = contzone(name); let net = routed_subnet(name, ann); @@ -594,6 +625,8 @@ let routed_up = function(name, ann, m) { proto: "static", device: vh, ipaddr: [ net.gw_cidr ], + ip6assign: 64, + ip6ifaceid: "::1", force_link: true, persistent: true, zone: czone, @@ -612,6 +645,7 @@ let routed_up = function(name, ann, m) { return 1; fw_uci_create(name, ann, net); + ra_uci_create(name); append_injail(name, render_section("lan", "eth0", proto, { ipaddr: net.container, @@ -629,6 +663,7 @@ let routed_down = function(name) { let gw_iface = gwif(name); fw_uci_remove(name); + ra_uci_remove(name); call("network.interface." + accif(name), "remove", {}); call("network.interface." + gw_iface, "remove", {}); return 0; From 66781b2b3544ba552ed31b61897ec6471c4a52e4 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 18:53:37 +0100 Subject: [PATCH 63/82] uxc-net: remove the host configuration it creates ensure_network() commits persistent sections to network, dhcp and firewall when a bridged attachment names a network that does not exist, and bridged_down() never removed them, so a single container run left a network, a DHCP pool and a firewall zone in /etc/config forever. Removing the sections on bring-down is the right resolution rather than not committing them in the first place, because the DHCP pool can only exist as persistent configuration: dnsmasq reads UCI alone and offers no ubus path to hand it a pool, so an ephemerally created network would be one without addressing. The per-container firewall sections already follow this lifecycle for the same reason, stock fw4 ignoring zone data delivered over ubus: create at start, idempotent, delete on teardown. Record each auto-created network in a marker under the uxc state directory and name the created sections deterministically, the device section as _dev and the firewall zone section as . On bring-down of a bridged container the network is removed again when the marker exists and its bridge has no members left, deleting exactly the four sections creation wrote, and the marker with them. A network the operator defined is never touched, because it has no marker, and a network still carrying other containers keeps its configuration until the last member goes down. The marker lives on the uvol metadata volume, so a firmware re-image that wipes /etc/config but keeps the container registrations simply leads to the network being recreated on the next start. A marker can go stale if an auto-created network is never brought down and the operator later defines the same network by hand; the next last-member bring-down would then remove the hand-written sections. The window is narrow and the alternative is refcounting operator intent, which UCI cannot express. With this, bring-down restores /etc/config for bridged containers as well; while a container is up, its container-lifetime sections exist, exactly as the routed firewall and router advertisement sections do. Signed-off-by: Daniel Golle --- jail/uxc-net | 70 +++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 55 insertions(+), 15 deletions(-) diff --git a/jail/uxc-net b/jail/uxc-net index 161f211..6b5adb6 100644 --- a/jail/uxc-net +++ b/jail/uxc-net @@ -54,6 +54,8 @@ let sidecar_path = function(name) { return "/tmp/run/uvol/.meta/uxc/" + name + " let state_dir = "/tmp/run/uvol/.meta/uxc/state"; let macstore_dir = function(name) { return state_dir + "/" + name; }; let macstore_path = function(name) { return macstore_dir(name) + "/macaddrs"; }; +let autonet_dir = state_dir + "/autonets"; +let autonet_path = function(net) { return autonet_dir + "/" + net; }; let rand_mac = function() { let f = fs.open("/dev/urandom", "r"); @@ -160,6 +162,13 @@ let iface_exists = function(iface) { return !ubus.error(); }; +let bridge_members = function(br) { + let r = ubus.call({ object: "network.device", method: "status", data: { name: br } }); + if (ubus.error() || type(r) != "object" || type(r["bridge-members"]) != "array") + return -1; + return length(r["bridge-members"]); +}; + let append_injail = function(name, section) { let existed, f; @@ -387,7 +396,8 @@ let ensure_network = function(net) { cursor.load("firewall"); let br = "br-" + net; - let dev = cursor.add("network", "device"); + let dev = net + "_dev"; + cursor.set("network", dev, "device"); cursor.set("network", dev, "name", br); cursor.set("network", dev, "type", "bridge"); cursor.set("network", net, "interface"); @@ -405,19 +415,54 @@ let ensure_network = function(net) { cursor.set("dhcp", net, "dhcpv4", "server"); cursor.commit("dhcp"); - let zone = cursor.add("firewall", "zone"); - cursor.set("firewall", zone, "name", net); - cursor.set("firewall", zone, "network", net); - cursor.set("firewall", zone, "input", "ACCEPT"); - cursor.set("firewall", zone, "output", "ACCEPT"); - cursor.set("firewall", zone, "forward", "REJECT"); + cursor.set("firewall", net, "zone"); + cursor.set("firewall", net, "name", net); + cursor.set("firewall", net, "network", net); + cursor.set("firewall", net, "input", "ACCEPT"); + cursor.set("firewall", net, "output", "ACCEPT"); + cursor.set("firewall", net, "forward", "REJECT"); cursor.commit("firewall"); + fs.mkdir(state_dir, 0700); + fs.mkdir(autonet_dir, 0700); + let f = fs.open(autonet_path(net), "w"); + if (f) { + f.write(subnet + "\n"); + f.close(); + } + reload_and_wait(net, [ "network", "dhcp", "firewall" ]); return 1; }; +let autonet_remove = function(net) { + let cursor; + + if (!fs.stat(autonet_path(net))) + return; + if (bridge_members("br-" + net) != 0) + return; + + cursor = uci.cursor(); + cursor.load("network"); + cursor.load("dhcp"); + cursor.load("firewall"); + cursor.delete("network", net); + cursor.delete("network", net + "_dev"); + cursor.delete("dhcp", net); + cursor.delete("firewall", net); + cursor.commit("network"); + cursor.commit("dhcp"); + cursor.commit("firewall"); + + fs.unlink(autonet_path(net)); + + pkg_reload("network"); + pkg_reload("dhcp"); + pkg_reload("firewall"); +}; + let bridged_up = function(name, ann, attach, m) { let net = attach.network; let vh = host_veth(name), vc = cont_veth(name); @@ -468,8 +513,10 @@ let bridged_down = function(name, attach) { let vh = host_veth(name); call("network.interface." + accif(name), "remove", {}); - if (attach.network) + if (attach.network) { call("network.interface." + attach.network, "remove_device", { name: vh, "link-ext": false }); + autonet_remove(attach.network); + } return 0; }; @@ -708,13 +755,6 @@ let backhaul_up = function(name, bh, m) { return 0; }; -let bridge_members = function(br) { - let r = ubus.call({ object: "network.device", method: "status", data: { name: br } }); - if (ubus.error() || type(r) != "object" || type(r["bridge-members"]) != "array") - return -1; - return length(r["bridge-members"]); -}; - let backhaul_down = function(name, bh) { let br = bh_bridge(bh.id); let iface = bhseg(bh.id); From 9f0f087166064916ec287b73f14a6cb39df952df Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 18:57:01 +0100 Subject: [PATCH 64/82] jail: fail the container when uxc-net cannot configure its network run_uxc_net() reports whether the helper succeeded and the caller threw the answer away, so a container whose networking the helper refused, for instance a routed one asking for DHCP where the point-to-point link has no server, came up regardless with nothing but loopback inside its namespace. The diagnostic went to the log and the container looked healthy, which is the worst of both. Treat it as the setup failure it is. A network the runtime was asked for and could not build is not something to paper over, and a container that starts without it would have to be diagnosed from the inside. Signed-off-by: Daniel Golle --- jail/jail.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/jail/jail.c b/jail/jail.c index 975967e..4fb2a81 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -6821,8 +6821,11 @@ static void post_main(struct uloop_timeout *t) jail_chown_writable_surfaces(); } - if ((opts.namespace & CLONE_NEWNET) && opts.name && opts.ocibundle) - run_uxc_net("up"); + if ((opts.namespace & CLONE_NEWNET) && opts.name && opts.ocibundle && + run_uxc_net("up")) { + ERROR("uxc-net failed to configure the container network\n"); + free_and_exit(EXIT_FAILURE); + } if ((opts.namespace & CLONE_NEWNET) && opts.name) jail_network_attach(parent_ctx, opts.name, jail_process.pid); From db1872351e9823e4ee0f23957b50cee3d5843166 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 18:50:05 +0100 Subject: [PATCH 65/82] uxc: show container addresses and the host network namespace in list ujail's container state reply now carries an "org.openwrt.network" object describing the network namespace mode and the interfaces the container ended up with. Surface it in 'uxc list' so a glance at the table answers the most common question about a container: how is it reachable. The JSON output passes the object through verbatim, next to the annotations, so runc-style consumers see exactly what the jail reported. The table gains a NET column: "host" for a container sharing the host network namespace, "-" when there is nothing to show, otherwise the addresses. A container with a single interface prints a bare comma-separated address list, the common case staying compact; with several interfaces each address is prefixed with its interface name as "eth0=10.7.3.2/31", since a bare list from several interfaces cannot be told apart. The column is appended after OWNER rather than inserted: the package hook library /lib/functions/uxc.sh greps the table with '^$name[[:space:]].*[[:space:]]running', which an appended column leaves matching, and podman and conmon never parse this table. 'uxc state' needs no change as it prints the jail's blob verbatim. Signed-off-by: Daniel Golle --- uxc.c | 93 +++++++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 87 insertions(+), 6 deletions(-) diff --git a/uxc.c b/uxc.c index e86a173..a18f137 100644 --- a/uxc.c +++ b/uxc.c @@ -626,6 +626,7 @@ enum { STATE_PID, STATE_BUNDLE, STATE_ANNOTATIONS, + STATE_NETWORK, __STATE_MAX, }; @@ -636,6 +637,29 @@ static const struct blobmsg_policy state_policy[__STATE_MAX] = { [STATE_PID] = { .name = "pid", .type = BLOBMSG_TYPE_INT32 }, [STATE_BUNDLE] = { .name = "bundle", .type = BLOBMSG_TYPE_STRING }, [STATE_ANNOTATIONS] = { .name = "annotations", .type = BLOBMSG_TYPE_TABLE }, + [STATE_NETWORK] = { .name = "org.openwrt.network", .type = BLOBMSG_TYPE_TABLE }, +}; + +enum { + NET_NAMESPACE, + NET_INTERFACES, + __NET_MAX, +}; + +static const struct blobmsg_policy net_policy[__NET_MAX] = { + [NET_NAMESPACE] = { .name = "namespace", .type = BLOBMSG_TYPE_STRING }, + [NET_INTERFACES] = { .name = "interfaces", .type = BLOBMSG_TYPE_ARRAY }, +}; + +enum { + NET_IF_NAME, + NET_IF_ADDRESSES, + __NET_IF_MAX, +}; + +static const struct blobmsg_policy net_if_policy[__NET_IF_MAX] = { + [NET_IF_NAME] = { .name = "name", .type = BLOBMSG_TYPE_STRING }, + [NET_IF_ADDRESSES] = { .name = "addresses", .type = BLOBMSG_TYPE_ARRAY }, }; @@ -996,17 +1020,67 @@ static int uxc_state(char *name) return 0; } +static void netinfo_str(struct blob_attr *netinfo, char *out, size_t outlen) +{ + struct blob_attr *tn[__NET_MAX], *ti[__NET_IF_MAX]; + struct blob_attr *curif, *curaddr; + const char *ifname; + size_t len = 0; + int remif, remaddr; + int ifcount = 0; + + snprintf(out, outlen, "-"); + if (!netinfo) + return; + + blobmsg_parse(net_policy, __NET_MAX, tn, + blobmsg_data(netinfo), blobmsg_len(netinfo)); + if (tn[NET_NAMESPACE] && + !strcmp(blobmsg_get_string(tn[NET_NAMESPACE]), "host")) { + snprintf(out, outlen, "host"); + return; + } + if (!tn[NET_INTERFACES]) + return; + + blobmsg_for_each_attr(curif, tn[NET_INTERFACES], remif) + ifcount++; + + blobmsg_for_each_attr(curif, tn[NET_INTERFACES], remif) { + blobmsg_parse(net_if_policy, __NET_IF_MAX, ti, + blobmsg_data(curif), blobmsg_len(curif)); + if (!ti[NET_IF_ADDRESSES]) + continue; + + ifname = ti[NET_IF_NAME] ? + blobmsg_get_string(ti[NET_IF_NAME]) : "?"; + blobmsg_for_each_attr(curaddr, ti[NET_IF_ADDRESSES], remaddr) { + if (blobmsg_type(curaddr) != BLOBMSG_TYPE_STRING) + continue; + + len += snprintf(out + len, outlen - len, "%s%s%s%s", + len ? "," : "", + ifcount > 1 ? ifname : "", + ifcount > 1 ? "=" : "", + blobmsg_get_string(curaddr)); + if (len >= outlen) + len = outlen - 1; + } + } +} + static int uxc_list(void) { - struct blob_attr *cur, *tb[__CONF_MAX], *ts[__STATE_MAX]; + struct blob_attr *cur, *tb[__CONF_MAX], *ts[__STATE_MAX], *netinfo; int rem, pass; struct runtime_state *rsstate = NULL; char *name, *bundle, *ocistatus, *status, *created, *tmp; int container_pid; static struct blob_buf buf; void *arr, *obj, *ann; - size_t id_w = 2, pid_w = 3, status_w = 6, bundle_w = 6, created_w = 7; + size_t id_w = 2, pid_w = 3, status_w = 6, bundle_w = 6, created_w = 7, owner_w = 5; char pidstr[12]; + char netstr[512]; if (json_output) { blob_buf_init(&buf, 0); @@ -1015,10 +1089,11 @@ static int uxc_list(void) for (pass = json_output ? 1 : 0; pass < 2; pass++) { if (pass == 1 && !json_output) - printf("%-*s %-*s %-*s %-*s %-*s %s\n", + printf("%-*s %-*s %-*s %-*s %-*s %-*s %s\n", (int)id_w, "ID", (int)pid_w, "PID", (int)status_w, "STATUS", (int)bundle_w, "BUNDLE", - (int)created_w, "CREATED", "OWNER"); + (int)created_w, "CREATED", (int)owner_w, "OWNER", + "NET"); blobmsg_for_each_attr(cur, blob_data(conf.head), rem) { blobmsg_parse(conf_policy, __CONF_MAX, tb, @@ -1032,6 +1107,7 @@ static int uxc_list(void) ocistatus = NULL; container_pid = 0; created = "-"; + netinfo = NULL; rsstate = avl_find_element(&runtime, name, rsstate, avl); if (rsstate && rsstate->ocistate) { blobmsg_parse(state_policy, __STATE_MAX, ts, @@ -1043,6 +1119,7 @@ static int uxc_list(void) container_pid = blobmsg_get_u32(ts[STATE_PID]); if (ts[STATE_BUNDLE]) bundle = blobmsg_get_string(ts[STATE_BUNDLE]); + netinfo = ts[STATE_NETWORK]; } status = ocistatus?:(rsstate && rsstate->running)?"creating":(rsstate?"stopped":"uninitialized"); @@ -1074,13 +1151,17 @@ static int uxc_list(void) ann = blobmsg_open_table(&buf, "annotations"); blobmsg_close_table(&buf, ann); } + if (rsstate && rsstate->ocistate && ts[STATE_NETWORK]) + blobmsg_add_blob(&buf, ts[STATE_NETWORK]); blobmsg_add_string(&buf, "owner", "root"); blobmsg_close_table(&buf, obj); } else { - printf("%-*s %-*s %-*s %-*s %-*s %s\n", + netinfo_str(netinfo, netstr, sizeof(netstr)); + printf("%-*s %-*s %-*s %-*s %-*s %-*s %s\n", (int)id_w, name, (int)pid_w, pidstr, (int)status_w, status, (int)bundle_w, bundle, - (int)created_w, created, "root"); + (int)created_w, created, (int)owner_w, "root", + netstr); } } } From 7c05dc622aab6d81486134520e4ada50dc33aa15 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 18:50:49 +0100 Subject: [PATCH 66/82] uxc: accept --format json and --quiet like runc 'uxc list' only understood its own --json/-j spelling, while runc and crun both spell it '--format json' and both offer -q/--quiet. Accept all of them so tooling written against runc works unmodified: '--format table' and '--format json' select the output, an unknown format is rejected, and --json keeps working so nothing in-tree breaks. --quiet prints one container id per line and takes precedence over the format selection, matching runc's behaviour. Signed-off-by: Daniel Golle --- uxc.c | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/uxc.c b/uxc.c index a18f137..3804611 100644 --- a/uxc.c +++ b/uxc.c @@ -55,6 +55,7 @@ static bool verbose = false; static bool json_output = false; +static bool quiet_output = false; static int stdio_fds[STDIO_FDS_NUM] = { STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO }; static const char *confdir = UXC_ETC_CONFDIR; static struct ustream_fd cufd; @@ -123,7 +124,9 @@ static const struct option delete_opts[] = { }; static const struct option list_opts[] = { + {"format", required_argument, 0, 'f' }, {"json", no_argument, 0, 'j' }, + {"quiet", no_argument, 0, 'q' }, {0, 0, 0, 0 } }; @@ -377,7 +380,7 @@ static int usage(void) { printf("\t[--root ] [--rootless[=auto|true|false]]\n"); printf("\t[--systemd-cgroup] [--criu ]\n"); printf("commands:\n"); - printf("\tlist [--json]\t\t\t\tlist all configured containers (runc-compatible)\n"); + printf("\tlist [--json|--format json] [--quiet]\tlist all configured containers (runc-compatible)\n"); printf("\tattach \t\t\t\tattach to container console\n"); printf("\tcreate \t\t\t\t(re-)create \n"); printf("\t\t[--bundle ]\t\t\tOCI bundle at \n"); @@ -1082,6 +1085,18 @@ static int uxc_list(void) char pidstr[12]; char netstr[512]; + if (quiet_output) { + blobmsg_for_each_attr(cur, blob_data(conf.head), rem) { + blobmsg_parse(conf_policy, __CONF_MAX, tb, + blobmsg_data(cur), blobmsg_len(cur)); + if (!tb[CONF_NAME] || !tb[CONF_PATH]) + continue; + + printf("%s\n", blobmsg_get_string(tb[CONF_NAME])); + } + return 0; + } + if (json_output) { blob_buf_init(&buf, 0); arr = blobmsg_open_array(&buf, ""); @@ -2742,9 +2757,20 @@ int main(int argc, char **argv) opterr = 1; if (!strcmp(verb, "list")) { - while ((c = getopt_long(verb_argc, verb_argv, "j", list_opts, NULL)) != -1) { + while ((c = getopt_long(verb_argc, verb_argv, "f:jq", list_opts, NULL)) != -1) { switch (c) { + case 'f': + if (!strcmp(optarg, "json")) { + json_output = true; + } else if (!strcmp(optarg, "table")) { + json_output = false; + } else { + fprintf(stderr, "uxc: invalid format '%s'\n", optarg); + goto usage_out; + } + break; case 'j': json_output = true; break; + case 'q': quiet_output = true; break; default: goto usage_out; } } From 1091a16229e9dfbecf5d3faa9598bc18f99097c8 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 19:00:21 +0100 Subject: [PATCH 67/82] jail: report the container's creation time and rootfs in its state runc and crun both carry rootfs and created in the document their state operation returns, and tooling built against them expects to find them there. ujail reported neither, so uxc had nothing to print and showed a placeholder where every other runtime shows a timestamp. The runtime is the only honest source for both. It learns the resolved rootfs when it parses the bundle, and it knows the instant the container reached created state, which is the moment the OCI lifecycle calls its creation. Take the timestamp there rather than deriving it later from a process start time, which would answer a slightly different question and would oblige the caller to go reading /proc on the runtime's behalf. Both are added where the network object is added, after oci_state_fill(), so the document handed to OCI hooks on stdin keeps the shape the specification describes. Signed-off-by: Daniel Golle --- jail/jail.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/jail/jail.c b/jail/jail.c index 4fb2a81..e77594f 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -4898,6 +4898,31 @@ static const char *annotation_get(struct blob_attr *attrs, const char *key) return NULL; } +static struct timespec jail_created; + +static void oci_state_fill_runtime(struct blob_buf *b) +{ + char buf[40]; + struct tm tm; + size_t len; + + if (opts.extroot) + blobmsg_add_string(b, "rootfs", opts.extroot); + + if (!jail_created.tv_sec) + return; + + if (!gmtime_r(&jail_created.tv_sec, &tm)) + return; + + len = strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", &tm); + if (!len) + return; + + snprintf(buf + len, sizeof(buf) - len, ".%09ldZ", jail_created.tv_nsec); + blobmsg_add_string(b, "created", buf); +} + static void oci_state_fill_network(struct blob_buf *b) { struct blob_buf sidecar = { 0 }; @@ -5021,6 +5046,7 @@ static int handle_state(struct ubus_context *ctx, struct ubus_object *obj, { blob_buf_init(&bb, 0); oci_state_fill(&bb); + oci_state_fill_runtime(&bb); oci_state_fill_network(&bb); ubus_send_reply(ctx, req, bb.head); @@ -6995,6 +7021,7 @@ static void post_create_runtime(void) close(userns_pipe[3]); } + clock_gettime(CLOCK_REALTIME, &jail_created); jail_oci_state = OCI_STATE_CREATED; emit_instance_event("instance.ready"); From 1ee8da3608a283cad2ea9149e6d06ddece07f3e5 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 21 Aug 2026 19:01:18 +0100 Subject: [PATCH 68/82] uxc: show the creation time and rootfs the runtime reports The CREATED column was a hyphen for every container and the JSON carried neither rootfs nor created, both of which runc emits and tooling reads. The runtime now reports them, so take them from the state reply and print them. OWNER stays "root" deliberately. Containers here are created by procd, which runs as root, so the field is already truthful; inventing a per-container owner would mean uxc reading registration files, and uxc talks to procd and ujail over ubus and to nothing else. Signed-off-by: Daniel Golle --- uxc.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/uxc.c b/uxc.c index 3804611..0902c8f 100644 --- a/uxc.c +++ b/uxc.c @@ -630,6 +630,8 @@ enum { STATE_BUNDLE, STATE_ANNOTATIONS, STATE_NETWORK, + STATE_CREATED, + STATE_ROOTFS, __STATE_MAX, }; @@ -641,6 +643,8 @@ static const struct blobmsg_policy state_policy[__STATE_MAX] = { [STATE_BUNDLE] = { .name = "bundle", .type = BLOBMSG_TYPE_STRING }, [STATE_ANNOTATIONS] = { .name = "annotations", .type = BLOBMSG_TYPE_TABLE }, [STATE_NETWORK] = { .name = "org.openwrt.network", .type = BLOBMSG_TYPE_TABLE }, + [STATE_CREATED] = { .name = "created", .type = BLOBMSG_TYPE_STRING }, + [STATE_ROOTFS] = { .name = "rootfs", .type = BLOBMSG_TYPE_STRING }, }; enum { @@ -1077,7 +1081,7 @@ static int uxc_list(void) struct blob_attr *cur, *tb[__CONF_MAX], *ts[__STATE_MAX], *netinfo; int rem, pass; struct runtime_state *rsstate = NULL; - char *name, *bundle, *ocistatus, *status, *created, *tmp; + char *name, *bundle, *ocistatus, *status, *created, *rootfs, *tmp; int container_pid; static struct blob_buf buf; void *arr, *obj, *ann; @@ -1122,6 +1126,7 @@ static int uxc_list(void) ocistatus = NULL; container_pid = 0; created = "-"; + rootfs = NULL; netinfo = NULL; rsstate = avl_find_element(&runtime, name, rsstate, avl); if (rsstate && rsstate->ocistate) { @@ -1134,6 +1139,9 @@ static int uxc_list(void) container_pid = blobmsg_get_u32(ts[STATE_PID]); if (ts[STATE_BUNDLE]) bundle = blobmsg_get_string(ts[STATE_BUNDLE]); + if (ts[STATE_CREATED]) + created = blobmsg_get_string(ts[STATE_CREATED]); + rootfs = ts[STATE_ROOTFS] ? blobmsg_get_string(ts[STATE_ROOTFS]) : NULL; netinfo = ts[STATE_NETWORK]; } status = ocistatus?:(rsstate && rsstate->running)?"creating":(rsstate?"stopped":"uninitialized"); @@ -1168,6 +1176,12 @@ static int uxc_list(void) } if (rsstate && rsstate->ocistate && ts[STATE_NETWORK]) blobmsg_add_blob(&buf, ts[STATE_NETWORK]); + if (rootfs) + blobmsg_add_string(&buf, "rootfs", rootfs); + + if (strcmp(created, "-")) + blobmsg_add_string(&buf, "created", created); + blobmsg_add_string(&buf, "owner", "root"); blobmsg_close_table(&buf, obj); } else { From d22739f64cb1077d233a923d7de8183ab6735260 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Sat, 22 Aug 2026 22:23:28 +0100 Subject: [PATCH 69/82] uxc: purge orphaned state without forking rm Recursively remove the state directory with nftw() instead of spawning /bin/rm, whose exit status was discarded. uxc is meant to reach the system only through ubus, and this exec had nothing to do with volume management in the first place: it merely borrowed the uvol helper. Signed-off-by: Daniel Golle --- uxc.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/uxc.c b/uxc.c index 0902c8f..9f3cb39 100644 --- a/uxc.c +++ b/uxc.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -2432,15 +2433,24 @@ static bool uxc_registered(const char *name) return false; } +static int purge_cb(const char *path, const struct stat *sb, int typeflag, + struct FTW *ftwbuf) +{ + return remove(path); +} + static void reconcile_purge(const char *name, const char *statedir) { - char *rm[] = { "/bin/rm", "-rf", (char *)statedir, NULL }; char path[PATH_MAX]; snprintf(path, sizeof(path), "%s/settings/%s.json", UXC_VOL_CONFDIR, name); unlink(path); - run_uvol_argv(rm); + if (nftw(statedir, purge_cb, 16, FTW_DEPTH | FTW_PHYS)) { + fprintf(stderr, "uxc: reconcile: could not purge state for %s\n", name); + return; + } + fprintf(stderr, "uxc: reconcile: purged orphaned state for %s\n", name); } From 7a7f90c3256fa37ea9471e83f29a645c7ac4485f Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Sat, 22 Aug 2026 22:23:44 +0100 Subject: [PATCH 70/82] uxc: fail when the image volume cannot be activated The result of activating the container's image volume was discarded, so a volume left write-only by an interrupted upgrade, or one that vanished entirely, was only noticed later when ujail could not read the bundle. Report it where it happens and abort the creation. Fixes: d1fe47f6d789 ("uxc: provision and reap per-container volumes") Signed-off-by: Daniel Golle --- uxc.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/uxc.c b/uxc.c index 9f3cb39..8468bf6 100644 --- a/uxc.c +++ b/uxc.c @@ -1476,8 +1476,10 @@ static int uxc_create(char *name, bool immediately, const char *console_socket, path = blobmsg_get_string(tb[CONF_PATH]); imgvol = uvol_volume_name(path); - if (imgvol) - run_uvol("up", imgvol); + if (imgvol && run_uvol("up", imgvol)) { + fprintf(stderr, "uxc: failed to activate image volume %s\n", imgvol); + return -EIO; + } if (tb[CONF_PIDFILE]) pidfile = blobmsg_get_string(tb[CONF_PIDFILE]); From 3f85611c4debc71b60faf1728856b81cd25f70f8 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Sat, 22 Aug 2026 22:31:45 +0100 Subject: [PATCH 71/82] uxc: drive uvol over ubus where the plugin is present uvol now publishes its volume operations through rpcd, so use that instead of forking the CLI whenever the object is there, and keep the exec path for systems whose uvol predates it. The reply carries exactly the exit codes the callers already interpret, so create-versus-resize and "already larger, kept" keep working unchanged. Two properties of rpcd's exec plugins shape this. Its worker is killed once the exec timeout expires, and the caller then sees a reply-less failure rather than a timeout status, so treat any call that produced no reply as failure and never match on a particular status. The client timeout is derived from rpcd's own, read over ubus like the fstab lookup in uxc_boot() already is, so raising one raises the other. A create killed that way leaves the volume write-only or write-pending with the orphaned formatter still holding the locks, and retrying the identical create blocks behind it and then reclaims it. Boot gating moves to the readiness query: containers with data volumes or an overlay wait for the .meta volume, containers whose bundle lives on a uvol volume wait for the backend itself, which also stops a backend that is merely still coming up from being reported as an interrupted upgrade. Signed-off-by: Daniel Golle --- uxc.c | 247 +++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 236 insertions(+), 11 deletions(-) diff --git a/uxc.c b/uxc.c index 8468bf6..b4d327a 100644 --- a/uxc.c +++ b/uxc.c @@ -1239,6 +1239,8 @@ static int uvol_status(const char *vol); static const char *uvol_volume_name(const char *path); static int run_uvol(const char *action, const char *vol); static int provision_rw_uvol(const char *volname, const char *size); +static bool uvol_meta_pending(void); +static bool uvol_backend_pending(void); static int provision_data_volumes(const char *container, struct blob_attr *vols, struct blob_buf *req); @@ -2213,7 +2215,7 @@ static int uxc_boot(const char *mountpoint) if (checkvolumes(usettings->volumes)) continue; - if ((tb[CONF_DATA_VOLUMES] || tb[CONF_OVERLAY_SIZE]) && uvol_status(".meta")) + if ((tb[CONF_DATA_VOLUMES] || tb[CONF_OVERLAY_SIZE]) && uvol_meta_pending()) continue; name = strdup(blobmsg_get_string(tb[CONF_NAME])); @@ -2223,6 +2225,11 @@ static int uxc_boot(const char *mountpoint) } imgvol = uvol_volume_name(blobmsg_get_string(tb[CONF_PATH])); + if (imgvol && uvol_backend_pending()) { + free(name); + continue; + } + if (imgvol && uvol_status(imgvol)) { ERROR("uxc: %s image %s missing (interrupted upgrade?); run 'apk fix %s'\n", name, imgvol, name); @@ -2239,6 +2246,148 @@ static int uxc_boot(const char *mountpoint) return ret; } +#define UVOL_EXEC_TIMEOUT_DEFAULT 120 +#define UVOL_TIMEOUT_MARGIN_MS 10000 + +enum { + UVOL_CODE, + UVOL_READY, + UVOL_META, + __UVOL_MAX, +}; + +static const struct blobmsg_policy uvol_policy[__UVOL_MAX] = { + [UVOL_CODE] = { .name = "code", .type = BLOBMSG_TYPE_INT32 }, + [UVOL_READY] = { .name = "ready", .type = BLOBMSG_TYPE_BOOL }, + [UVOL_META] = { .name = "meta", .type = BLOBMSG_TYPE_BOOL }, +}; + +struct uvol_reply { + int code; + bool answered; + bool ready; + bool meta; +}; + +static void uvol_cb(struct ubus_request *req, int type, struct blob_attr *msg) +{ + struct blob_attr *tb[__UVOL_MAX]; + struct uvol_reply *reply = req->priv; + + if (!msg) + return; + + blobmsg_parse(uvol_policy, __UVOL_MAX, tb, blob_data(msg), blob_len(msg)); + + reply->answered = true; + + if (tb[UVOL_CODE]) + reply->code = blobmsg_get_u32(tb[UVOL_CODE]); + + if (tb[UVOL_READY]) + reply->ready = blobmsg_get_bool(tb[UVOL_READY]); + + if (tb[UVOL_META]) + reply->meta = blobmsg_get_bool(tb[UVOL_META]); +} + +static void uci_value_cb(struct ubus_request *req, int type, struct blob_attr *msg) +{ + static const struct blobmsg_policy pol = { + .name = "value", .type = BLOBMSG_TYPE_STRING + }; + struct blob_attr *tb; + + if (!msg) + return; + + blobmsg_parse(&pol, 1, &tb, blob_data(msg), blob_len(msg)); + if (tb) + *(int *)req->priv = atoi(blobmsg_get_string(tb)); +} + +static uint32_t uvol_ubus_id(void) +{ + static bool looked_up; + static uint32_t id; + + if (!looked_up) { + if (ubus_lookup_id(ctx, "uvol", &id)) + id = 0; + + looked_up = true; + } + + return id; +} + +static bool uvol_ubus_available(void) +{ + return uvol_ubus_id() != 0; +} + +static int uvol_timeout(void) +{ + static struct blob_buf req; + static int timeout_ms; + int secs = UVOL_EXEC_TIMEOUT_DEFAULT; + uint32_t id; + + if (timeout_ms) + return timeout_ms; + + if (!ubus_lookup_id(ctx, "uci", &id)) { + blob_buf_init(&req, 0); + blobmsg_add_string(&req, "config", "rpcd"); + blobmsg_add_string(&req, "section", "@rpcd[0]"); + blobmsg_add_string(&req, "option", "timeout"); + ubus_invoke(ctx, id, "get", req.head, uci_value_cb, &secs, 3000); + } + + if (secs < 1 || secs > 600) + secs = UVOL_EXEC_TIMEOUT_DEFAULT; + + timeout_ms = secs * 1000 + UVOL_TIMEOUT_MARGIN_MS; + + return timeout_ms; +} + +static int uvol_call(const char *method, struct blob_attr *args, + struct uvol_reply *reply) +{ + static struct blob_buf empty; + + memset(reply, 0, sizeof(*reply)); + + if (!args) { + blob_buf_init(&empty, 0); + args = empty.head; + } + + if (ubus_invoke(ctx, uvol_ubus_id(), method, args, uvol_cb, reply, + uvol_timeout())) + return -EIO; + + if (!reply->answered) + return -EIO; + + return 0; +} + +static int uvol_call_volume(const char *method, const char *vol) +{ + static struct blob_buf req; + struct uvol_reply reply; + + blob_buf_init(&req, 0); + blobmsg_add_string(&req, "name", vol); + + if (uvol_call(method, req.head, &reply)) + return -EIO; + + return reply.code; +} + static const char *uvol_volume_name(const char *path) { const char prefix[] = "/tmp/run/uvol/"; @@ -2278,31 +2427,94 @@ static int run_uvol(const char *action, const char *vol) { char *argv[] = { "/usr/sbin/uvol", (char *)action, (char *)vol, NULL }; + if (uvol_ubus_available()) + return uvol_call_volume(action, vol); + return run_uvol_argv(argv); } -static int run_uvol_create(const char *vol, const char *size, const char *type) +static int run_uvol_create(const char *vol, long long size, const char *mode) { + char sizebytes[32]; char *argv[] = { "/usr/sbin/uvol", "create", (char *)vol, - (char *)size, (char *)type, NULL }; + sizebytes, (char *)mode, NULL }; + static struct blob_buf req; + struct uvol_reply reply; - return run_uvol_argv(argv); + snprintf(sizebytes, sizeof(sizebytes), "%lld", size); + + if (!uvol_ubus_available()) + return run_uvol_argv(argv); + + blob_buf_init(&req, 0); + blobmsg_add_string(&req, "name", vol); + blobmsg_add_u64(&req, "size", size); + blobmsg_add_string(&req, "mode", mode); + + if (uvol_call("create", req.head, &reply)) + return -EIO; + + return reply.code; } -static int run_uvol_resize(const char *vol, const char *size) +static int run_uvol_resize(const char *vol, long long size) { - char *argv[] = { "/usr/sbin/uvol", "resize", (char *)vol, (char *)size, NULL }; + char sizebytes[32]; + char *argv[] = { "/usr/sbin/uvol", "resize", (char *)vol, sizebytes, NULL }; + static struct blob_buf req; + struct uvol_reply reply; - return run_uvol_argv(argv); + snprintf(sizebytes, sizeof(sizebytes), "%lld", size); + + if (!uvol_ubus_available()) + return run_uvol_argv(argv); + + blob_buf_init(&req, 0); + blobmsg_add_string(&req, "name", vol); + blobmsg_add_u64(&req, "size", size); + + if (uvol_call("resize", req.head, &reply)) + return -EIO; + + return reply.code; } static int uvol_status(const char *vol) { char *argv[] = { "/usr/sbin/uvol", "status", (char *)vol, NULL }; + if (uvol_ubus_available()) + return uvol_call_volume("status", vol); + return run_uvol_argv(argv); } +static bool uvol_meta_pending(void) +{ + struct uvol_reply reply; + + if (!uvol_ubus_available()) + return uvol_status(".meta") != 0; + + if (uvol_call("ready", NULL, &reply)) + return true; + + return !reply.meta; +} + +static bool uvol_backend_pending(void) +{ + struct uvol_reply reply; + + if (!uvol_ubus_available()) + return false; + + if (uvol_call("ready", NULL, &reply)) + return true; + + return !reply.ready; +} + static long long parse_size_bytes(const char *s) { char *end; @@ -2339,9 +2551,23 @@ static long long parse_size_bytes(const char *s) return v; } +static int create_rw_uvol(const char *volname, long long bytes) +{ + int ret, st; + + ret = run_uvol_create(volname, bytes, "rw"); + if (ret != -EIO) + return ret; + + st = uvol_status(volname); + if (st != 16 && st != 22) + return ret; + + return run_uvol_create(volname, bytes, "rw"); +} + static int provision_rw_uvol(const char *volname, const char *size) { - char sizebytes[32]; long long bytes; int st, rr; @@ -2350,16 +2576,15 @@ static int provision_rw_uvol(const char *volname, const char *size) fprintf(stderr, "uxc: invalid size '%s' for volume %s\n", size, volname); return -EINVAL; } - snprintf(sizebytes, sizeof(sizebytes), "%lld", bytes); st = uvol_status(volname); if (st == 2) { - if (run_uvol_create(volname, sizebytes, "rw")) { + if (create_rw_uvol(volname, bytes)) { fprintf(stderr, "uxc: failed to create volume %s\n", volname); return -EIO; } } else { - rr = run_uvol_resize(volname, sizebytes); + rr = run_uvol_resize(volname, bytes); if (rr == 22) fprintf(stderr, "uxc: volume %s larger than requested, kept\n", volname); else if (rr) { From 94688b4dc0470bb484a580f6f38cdaad2c74724e Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Sat, 22 Aug 2026 22:40:29 +0100 Subject: [PATCH 72/82] service: expose container data through get_data and set_data Containers live in their own avl tree, so neither data method could ever reach them: get_data walked the services tree alone and set_data looked up the name there alone, returning "not found" for every container. Consumers already fold procd data into their runtime configuration, fw4 reading type "firewall" and odhcpd type "dhcp", and a container is exactly the publisher that wants it, because the data is dropped when the instance goes away and the host configuration it describes goes with it. Walk both trees when dumping, and fall back to the container tree when attaching. Signed-off-by: Daniel Golle --- service/service.c | 48 +++++++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/service/service.c b/service/service.c index 3ed955f..95710cb 100644 --- a/service/service.c +++ b/service/service.c @@ -847,28 +847,14 @@ service_handle_validate(struct ubus_context *ctx, struct ubus_object *obj, return 0; } -static int -service_get_data(struct ubus_context *ctx, struct ubus_object *obj, - struct ubus_request_data *req, const char *method, - struct blob_attr *msg) +static void +service_dump_data(struct avl_tree *tree, const char *name, const char *instance, + const char *type) { struct service_instance *in; struct service *s; - struct blob_attr *tb[__DATA_MAX]; - const char *name = NULL; - const char *instance = NULL; - const char *type = NULL; - - blobmsg_parse(get_data_policy, __DATA_MAX, tb, blobmsg_data(msg), blobmsg_data_len(msg)); - if (tb[DATA_NAME]) - name = blobmsg_data(tb[DATA_NAME]); - if (tb[DATA_INSTANCE]) - instance = blobmsg_data(tb[DATA_INSTANCE]); - if (tb[DATA_TYPE]) - type = blobmsg_data(tb[DATA_TYPE]); - blob_buf_init(&b, 0); - avl_for_each_element(&services, s, avl) { + avl_for_each_element(tree, s, avl) { void *cs = NULL; void *ci = NULL; struct blobmsg_list_node *var; @@ -917,6 +903,29 @@ service_get_data(struct ubus_context *ctx, struct ubus_object *obj, if (cs) blobmsg_close_table(&b, cs); } +} + +static int +service_get_data(struct ubus_context *ctx, struct ubus_object *obj, + struct ubus_request_data *req, const char *method, + struct blob_attr *msg) +{ + struct blob_attr *tb[__DATA_MAX]; + const char *name = NULL; + const char *instance = NULL; + const char *type = NULL; + + blobmsg_parse(get_data_policy, __DATA_MAX, tb, blobmsg_data(msg), blobmsg_data_len(msg)); + if (tb[DATA_NAME]) + name = blobmsg_data(tb[DATA_NAME]); + if (tb[DATA_INSTANCE]) + instance = blobmsg_data(tb[DATA_INSTANCE]); + if (tb[DATA_TYPE]) + type = blobmsg_data(tb[DATA_TYPE]); + + blob_buf_init(&b, 0); + service_dump_data(&services, name, instance, type); + service_dump_data(&containers, name, instance, type); ubus_send_reply(ctx, req, b.head); return 0; @@ -942,6 +951,9 @@ service_handle_set_data(struct ubus_context *ctx, struct ubus_object *obj, name = blobmsg_get_string(tb[SET_DATA_NAME]); s = avl_find_element(&services, name, s, avl); + if (!s) + s = avl_find_element(&containers, name, s, avl); + if (!s) return UBUS_STATUS_NOT_FOUND; From 5f3bd0b66867afc9d7ff8bd9c11721f63a0be131 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Sat, 22 Aug 2026 23:55:02 +0100 Subject: [PATCH 73/82] uxc-net: publish the host configuration instead of writing UCI Every container brought a bridge, an interface, a firewall zone with its forwardings and rules, and a DHCP section into /etc/config, committed at ten sites across three packages. That is a flash write to the overlay on every start and stop, and whatever a power cut interrupts stays behind, describing a container that no longer exists. Stopping a container also left the deletions staged in /tmp/.uci rather than applied, so the next "Save & Apply" decided what the host config would be. Publish it as procd data instead. A container's zone, forwardings, redirects, rules and DHCP section go into its own instance data, which procd drops when the instance goes away, so the host configuration cannot outlive what it describes. fw4 already folds data of type "firewall" into its ruleset and odhcpd now does the same for "dhcp", and both are told to reload with the config.change event they already trigger on, because attaching data to an instance emits no event of its own. Networks created on demand for bridged containers are shared, so they cannot hang off one container's instance. They live in the data of a service of their own, and the published set is the state: the list is read back to add or drop a member, which is why no state file is needed and why nothing is lost when the volume holding it is not mounted. The zone a container gets is closed, so the services the host offers into it are opened explicitly, the way /etc/config/firewall opens them for wan rather than fw4 inferring anything: DNS always, ICMPv6 whenever the container has any IPv6 at all because neighbour discovery is ICMPv6 over IP and a static-IPv6 container behind a closed zone cannot resolve its gateway, and DHCPv6 only when the container asks for it. Creating the interface no longer waits for a reload to bring it up, so the thirty second wait for the interface object to appear is gone. Signed-off-by: Daniel Golle --- jail/uxc-net | 389 ++++++++++++++++++++++++++------------------------- 1 file changed, 197 insertions(+), 192 deletions(-) diff --git a/jail/uxc-net b/jail/uxc-net index 6b5adb6..828cb33 100644 --- a/jail/uxc-net +++ b/jail/uxc-net @@ -2,8 +2,6 @@ let fs = require("fs"); let ubus = require("ubus"); -let uci = require("uci"); -let uloop = require("uloop"); let fnv1a = function(s) { let h = 2166136261, i; @@ -49,13 +47,52 @@ let pkg_reload = function(pkg) { data: { type: "config.change", data: { package: pkg } } }); }; +let quiet_call = function(object, method, data) { + ubus.call({ object: object, method: method, data: data }); + return ubus.error() ? -1 : 0; +}; + +let autonet_service = "uxc-net"; + +let icmpv6_types = [ + "echo-request", "echo-reply", "destination-unreachable", + "packet-too-big", "time-exceeded", "bad-header", + "unknown-header-type", "router-solicitation", + "neighbour-solicitation", "router-advertisement", + "neighbour-advertisement", +]; + +let publish = function(name, fw, dh) { + let data = {}; + + if (length(fw)) + data.firewall = fw; + if (length(dh)) + data.dhcp = dh; + + if (call("service", "set_data", { name: name, instance: name, data: data })) + return 1; + + pkg_reload("firewall"); + pkg_reload("dhcp"); + + return 0; +}; + +let withdraw = function(name) { + quiet_call("service", "set_data", { name: name, instance: name, data: {} }); + + pkg_reload("firewall"); + pkg_reload("dhcp"); + + return 0; +}; + let sidecar_path = function(name) { return "/tmp/run/uvol/.meta/uxc/" + name + ".annotations"; }; let state_dir = "/tmp/run/uvol/.meta/uxc/state"; let macstore_dir = function(name) { return state_dir + "/" + name; }; let macstore_path = function(name) { return macstore_dir(name) + "/macaddrs"; }; -let autonet_dir = state_dir + "/autonets"; -let autonet_path = function(net) { return autonet_dir + "/" + net; }; let rand_mac = function() { let f = fs.open("/dev/urandom", "r"); @@ -343,124 +380,105 @@ let pick_subnet = function() { return null; }; -let reload_and_wait = function(iface, pkgs) { - let obj = "network.interface." + iface; - let conn = ubus.connect(); - let emit = function() { - for (let p in pkgs) - ubus.call({ object: "service", method: "event", - data: { type: "config.change", data: { package: p } } }); - }; +let autonet_list = function() { + let d = ubus.call({ object: "service", method: "get_data", + data: { name: autonet_service, type: "firewall" } }); + let nets = [], svc, k, v, sec; - if (!conn) { - emit(); - return; - } + if (ubus.error() || type(d) != "object") + return nets; - uloop.init(); - let ready = false; - let lst = conn.listener("ubus.object.add", function(ev, data) { - if (type(data) == "object" && data.path == obj) { - ready = true; - uloop.end(); - } - }); + svc = d[autonet_service]; + if (type(svc) != "object") + return nets; - emit(); + for (k, v in svc) { + if (type(v) != "object" || type(v.firewall) != "array") + continue; + for (sec in v.firewall) + if (sec.type == "zone" && sec.name) + push(nets, sec.name); + } - ubus.call({ object: obj, method: "status", data: {} }); - if (!ubus.error()) - ready = true; + return nets; +}; + +let autonet_publish = function(nets) { + let fw = [], dh = [], net; - if (!ready) { - let t = uloop.timer(30000, function() { uloop.end(); }); - uloop.run(); - t.cancel(); + for (net in nets) { + push(fw, { + type: "zone", + name: net, + network: [ net ], + input: "ACCEPT", + output: "ACCEPT", + forward: "REJECT", + }); + push(dh, { + type: "dhcp", + interface: net, + start: 100, + limit: 150, + leasetime: "12h", + dhcpv4: "server", + }); } - lst.remove(); - conn.disconnect(); + if (length(nets)) + call("service", "set", { name: autonet_service, + data: { firewall: fw, dhcp: dh } }); + else + quiet_call("service", "delete", { name: autonet_service }); + + pkg_reload("firewall"); + pkg_reload("dhcp"); }; let ensure_network = function(net) { + let br = "br-" + net; + let nets = autonet_list(); + let subnet; + if (iface_exists(net)) - return 0; + return (net in nets) ? 1 : 0; - let subnet = pick_subnet(); + subnet = pick_subnet(); if (!subnet) return 2; - let cursor = uci.cursor(); - cursor.load("network"); - cursor.load("dhcp"); - cursor.load("firewall"); + if (call("network", "create_device", { name: br, type: "bridge" })) + return 2; - let br = "br-" + net; - let dev = net + "_dev"; - cursor.set("network", dev, "device"); - cursor.set("network", dev, "name", br); - cursor.set("network", dev, "type", "bridge"); - cursor.set("network", net, "interface"); - cursor.set("network", net, "proto", "static"); - cursor.set("network", net, "device", br); - cursor.set("network", net, "ipaddr", subnet + ".1"); - cursor.set("network", net, "netmask", "255.255.255.0"); - cursor.commit("network"); - - cursor.set("dhcp", net, "dhcp"); - cursor.set("dhcp", net, "interface", net); - cursor.set("dhcp", net, "start", "100"); - cursor.set("dhcp", net, "limit", "150"); - cursor.set("dhcp", net, "leasetime", "12h"); - cursor.set("dhcp", net, "dhcpv4", "server"); - cursor.commit("dhcp"); - - cursor.set("firewall", net, "zone"); - cursor.set("firewall", net, "name", net); - cursor.set("firewall", net, "network", net); - cursor.set("firewall", net, "input", "ACCEPT"); - cursor.set("firewall", net, "output", "ACCEPT"); - cursor.set("firewall", net, "forward", "REJECT"); - cursor.commit("firewall"); - - fs.mkdir(state_dir, 0700); - fs.mkdir(autonet_dir, 0700); - let f = fs.open(autonet_path(net), "w"); - if (f) { - f.write(subnet + "\n"); - f.close(); + if (call("network", "add_dynamic", { + name: net, + proto: "static", + device: br, + ipaddr: [ subnet + ".1/24" ], + persistent: true, + })) { + quiet_call("network", "delete_device", { name: br }); + return 2; } - reload_and_wait(net, [ "network", "dhcp", "firewall" ]); + push(nets, net); + autonet_publish(nets); return 1; }; let autonet_remove = function(net) { - let cursor; + let nets = autonet_list(); - if (!fs.stat(autonet_path(net))) + if (!(net in nets)) return; - if (bridge_members("br-" + net) != 0) + if (bridge_members("br-" + net) > 0) return; - cursor = uci.cursor(); - cursor.load("network"); - cursor.load("dhcp"); - cursor.load("firewall"); - cursor.delete("network", net); - cursor.delete("network", net + "_dev"); - cursor.delete("dhcp", net); - cursor.delete("firewall", net); - cursor.commit("network"); - cursor.commit("dhcp"); - cursor.commit("firewall"); - - fs.unlink(autonet_path(net)); - - pkg_reload("network"); - pkg_reload("dhcp"); - pkg_reload("firewall"); + autonet_publish(filter(nets, function(n) { return n != net; })); + + call("network.interface." + net, "remove", {}); + quiet_call("network", "delete_device", { name: "br-" + net }); }; let bridged_up = function(name, ann, attach, m) { @@ -536,119 +554,108 @@ let routed_subnet = function(name, ann) { }; }; -let fw_uci_create = function(name, ann, net) { +let fw_specs = function(name, ann, net, proto6) { let czone = contzone(name); - let cursor = uci.cursor(); - let i, list, m, sec; - - cursor.load("firewall"); - if (cursor.get("firewall", czone)) - return 0; + let list = [], i, m, sub; + + push(list, { + type: "zone", + name: czone, + network: [ gwif(name) ], + input: "DROP", + output: "ACCEPT", + forward: "DROP", + }); - cursor.set("firewall", czone, "zone"); - cursor.set("firewall", czone, "name", czone); - cursor.set("firewall", czone, "network", gwif(name)); - cursor.set("firewall", czone, "input", "DROP"); - cursor.set("firewall", czone, "output", "ACCEPT"); - cursor.set("firewall", czone, "forward", "DROP"); - - list = csv(ann["org.openwrt.network.egress"]); - for (i = 0; i < length(list); i++) { - sec = czone + "f" + i; - cursor.set("firewall", sec, "forwarding"); - cursor.set("firewall", sec, "src", czone); - cursor.set("firewall", sec, "dest", zone_of(list[i])); - } + sub = csv(ann["org.openwrt.network.egress"]); + for (i = 0; i < length(sub); i++) + push(list, { + type: "forwarding", + src: czone, + dest: zone_of(sub[i]), + }); - list = csv(ann["org.openwrt.network.ingress"]); - for (i = 0; i < length(list); i++) { - m = match(list[i], /^([A-Za-z0-9_:]+):(tcp|udp)\/([0-9]+(-[0-9]+)?)$/); + sub = csv(ann["org.openwrt.network.ingress"]); + for (i = 0; i < length(sub); i++) { + m = match(sub[i], /^([A-Za-z0-9_:]+):(tcp|udp)\/([0-9]+(-[0-9]+)?)$/); if (!m) { - warn(sprintf("uxc-net: ignoring bad ingress '%s'\n", list[i])); + warn(sprintf("uxc-net: ignoring bad ingress '%s'\n", sub[i])); continue; } - sec = czone + "r" + i; - cursor.set("firewall", sec, "redirect"); - cursor.set("firewall", sec, "name", czone + "-in" + i); - cursor.set("firewall", sec, "src", zone_of(m[1])); - cursor.set("firewall", sec, "dest", czone); - cursor.set("firewall", sec, "proto", m[2]); - cursor.set("firewall", sec, "src_dport", m[3]); - cursor.set("firewall", sec, "dest_ip", net.container); - cursor.set("firewall", sec, "dest_port", m[3]); - cursor.set("firewall", sec, "target", "DNAT"); + push(list, { + type: "redirect", + name: czone + "-in" + i, + src: zone_of(m[1]), + dest: czone, + proto: m[2], + src_dport: m[3], + dest_ip: net.container, + dest_port: m[3], + target: "DNAT", + }); } - list = csv(ann["org.openwrt.network.host"]); - for (i = 0; i < length(list); i++) { - m = match(list[i], /^(tcp|udp)\/([0-9]+(-[0-9]+)?)$/); + sub = csv(ann["org.openwrt.network.host"]); + for (i = 0; i < length(sub); i++) { + m = match(sub[i], /^(tcp|udp)\/([0-9]+(-[0-9]+)?)$/); if (!m) { - warn(sprintf("uxc-net: ignoring bad host port '%s'\n", list[i])); + warn(sprintf("uxc-net: ignoring bad host port '%s'\n", sub[i])); continue; } - sec = czone + "h" + i; - cursor.set("firewall", sec, "rule"); - cursor.set("firewall", sec, "name", czone + "-host" + i); - cursor.set("firewall", sec, "src", czone); - cursor.set("firewall", sec, "proto", m[1]); - cursor.set("firewall", sec, "dest_port", m[2]); - cursor.set("firewall", sec, "target", "ACCEPT"); + push(list, { + type: "rule", + name: czone + "-host" + i, + src: czone, + proto: m[1], + dest_port: m[2], + target: "ACCEPT", + }); } - cursor.commit("firewall"); - pkg_reload("firewall"); - return 0; -}; + push(list, { + type: "rule", + name: czone + "-dns", + src: czone, + proto: [ "tcp", "udp" ], + dest_port: "53", + target: "ACCEPT", + }); -let fw_uci_remove = function(name) { - let czone = contzone(name); - let cursor = uci.cursor(); - let kill = [], t; - - cursor.load("firewall"); - for (t in [ "zone", "forwarding", "redirect", "rule" ]) - cursor.foreach("firewall", t, function(s) { - if (index(s[".name"], czone) == 0) - push(kill, s[".name"]); + if (proto6 != "none") { + push(list, { + type: "rule", + name: czone + "-icmpv6", + src: czone, + proto: "icmp", + family: "ipv6", + icmp_type: icmpv6_types, + limit: "1000/sec", + target: "ACCEPT", }); - for (let n in kill) - cursor.delete("firewall", n); - if (length(kill)) { - cursor.commit("firewall"); - pkg_reload("firewall"); } - return 0; -}; -let ra_uci_create = function(name) { - let gw_iface = gwif(name); - let cursor = uci.cursor(); - - cursor.load("dhcp"); - if (cursor.get("dhcp", gw_iface)) - return 0; + if (proto6 == "dhcpv6") { + push(list, { + type: "rule", + name: czone + "-dhcpv6", + src: czone, + proto: "udp", + family: "ipv6", + dest_port: "547", + target: "ACCEPT", + }); + } - cursor.set("dhcp", gw_iface, "dhcp"); - cursor.set("dhcp", gw_iface, "interface", gw_iface); - cursor.set("dhcp", gw_iface, "ra", "server"); - cursor.set("dhcp", gw_iface, "dhcpv6", "server"); - cursor.commit("dhcp"); - pkg_reload("dhcp"); - return 0; + return list; }; -let ra_uci_remove = function(name) { - let gw_iface = gwif(name); - let cursor = uci.cursor(); - - cursor.load("dhcp"); - if (!cursor.get("dhcp", gw_iface)) - return 0; - - cursor.delete("dhcp", gw_iface); - cursor.commit("dhcp"); - pkg_reload("dhcp"); - return 0; +let dhcp_specs = function(name) { + return [ { + type: "dhcp", + interface: gwif(name), + ra: "server", + dhcpv6: "server", + } ]; }; let routed_up = function(name, ann, m) { @@ -691,8 +698,7 @@ let routed_up = function(name, ann, m) { })) return 1; - fw_uci_create(name, ann, net); - ra_uci_create(name); + publish(name, fw_specs(name, ann, net, injail_proto6(ann)), dhcp_specs(name)); append_injail(name, render_section("lan", "eth0", proto, { ipaddr: net.container, @@ -709,8 +715,7 @@ let routed_up = function(name, ann, m) { let routed_down = function(name) { let gw_iface = gwif(name); - fw_uci_remove(name); - ra_uci_remove(name); + withdraw(name); call("network.interface." + accif(name), "remove", {}); call("network.interface." + gw_iface, "remove", {}); return 0; From 71061364c9c9716fd2cebefcdfcfe140d61c0fed Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Sun, 23 Aug 2026 16:12:29 +0100 Subject: [PATCH 74/82] uxc: refuse volume names outside a safe character set Volume names reach uvol, which builds shell command lines from them in its backends, so a name carrying a quote and a semicolon would run as root. uxc composes names from registration data, which comes from package Makefiles, stack templates and hand-written files rather than from uxc itself, so check them before either the ubus or the exec path is taken. Names are restricted to alphanumerics, dot, underscore and hyphen, which still admits the content-addressed image names and the internal .meta volume. Signed-off-by: Daniel Golle --- uxc.c | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/uxc.c b/uxc.c index b4d327a..9939980 100644 --- a/uxc.c +++ b/uxc.c @@ -2388,6 +2388,34 @@ static int uvol_call_volume(const char *method, const char *vol) return reply.code; } +static bool uvol_name_valid(const char *vol) +{ + size_t i; + + if (!vol || !*vol || strlen(vol) > 127) + return false; + + if (!isalnum((unsigned char)vol[0]) && vol[0] != '_' && vol[0] != '.') + return false; + + for (i = 0; vol[i]; i++) + if (!isalnum((unsigned char)vol[i]) && vol[i] != '.' && + vol[i] != '_' && vol[i] != '-') + return false; + + return true; +} + +static int uvol_name_check(const char *vol) +{ + if (uvol_name_valid(vol)) + return 0; + + fprintf(stderr, "uxc: refusing volume name '%s'\n", vol ? vol : ""); + + return -EINVAL; +} + static const char *uvol_volume_name(const char *path) { const char prefix[] = "/tmp/run/uvol/"; @@ -2399,6 +2427,9 @@ static const char *uvol_volume_name(const char *path) if (!path[plen] || strchr(path + plen, '/')) return NULL; + if (!uvol_name_valid(path + plen)) + return NULL; + return path + plen; } @@ -2427,6 +2458,9 @@ static int run_uvol(const char *action, const char *vol) { char *argv[] = { "/usr/sbin/uvol", (char *)action, (char *)vol, NULL }; + if (uvol_name_check(vol)) + return -EINVAL; + if (uvol_ubus_available()) return uvol_call_volume(action, vol); @@ -2441,6 +2475,9 @@ static int run_uvol_create(const char *vol, long long size, const char *mode) static struct blob_buf req; struct uvol_reply reply; + if (uvol_name_check(vol)) + return -EINVAL; + snprintf(sizebytes, sizeof(sizebytes), "%lld", size); if (!uvol_ubus_available()) @@ -2464,6 +2501,9 @@ static int run_uvol_resize(const char *vol, long long size) static struct blob_buf req; struct uvol_reply reply; + if (uvol_name_check(vol)) + return -EINVAL; + snprintf(sizebytes, sizeof(sizebytes), "%lld", size); if (!uvol_ubus_available()) @@ -2483,6 +2523,9 @@ static int uvol_status(const char *vol) { char *argv[] = { "/usr/sbin/uvol", "status", (char *)vol, NULL }; + if (uvol_name_check(vol)) + return -EINVAL; + if (uvol_ubus_available()) return uvol_call_volume("status", vol); From 8b66962cbf3ed41553e0c7f6bbffa83448bdeb2d Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 2 Sep 2026 00:11:29 +0100 Subject: [PATCH 75/82] uxc-net: publish host network as procd service data instead of driving netifd Describe the host side of a container's network as procd service data and let netifd materialise it on config load, rather than creating the objects one by one through the runtime ubus API (network create_device, network add_dynamic, network.interface.X add_device/remove_device/remove). Only the read-only queries (interface status, device status, interface dump, service get_data) remain. Everything a single container needs is attached to its own instance in one set_data call: the veth pair as a network-device entry, the jailed end (and the routed /31 gateway) as network-interface entries, the port membership in a user-owned UCI bridge as a bridge-port entry, plus the firewall and dhcp sections that were already published this way. The blob is assembled by a single function that always emits all five type keys, because set_data replaces the whole instance blob and a partial publish would silently withdraw the container's link. The lifecycle of these objects is therefore bound to the instance record: withdrawing the data on down, or deleting the instance, removes them on the next reload without any explicit teardown call. Bridges that uxc-net owns (on-demand br- autonets and bhr- backhaul segments) are shared between containers and live at service level on the uxc-net data service, since netifd replaces a bridge's ports list on every publish and exactly one publisher must hold the full union. The ports list of each owned bridge is the reference count: on every up and down the list is pruned to the veth names currently published by some instance, an owned bridge whose ports run empty is dropped together with its interface, zone and dhcp entries, and the service is deleted once nothing is left. The read-modify-write of the shared data, and the instance publish that follows it, are serialised with an exclusive flock so that concurrent container starts cannot lose an update or prune each other's pending port. Subnet selection for a new autonet also takes the published interfaces into account, not only netifd's runtime state. netifd re-reads procd data only during config load, so after each publish or withdrawal a synchronous network reload is issued before the firewall and dhcp config.change events. On up a failed reload is fatal, because the jailed interface and its veth must exist by the time ujail hands the network namespace over with netns_updown. The veth blob is byte-stable across republishes: MAC addresses come from the persisted store and are never regenerated, so a reload never restarts the pair and never destroys the container's eth0. The in-jail UCI rendering, annotation parsing and IPv6 handling are unchanged. The downstream-only persistent attribute is gone from all published interfaces. Signed-off-by: Daniel Golle --- jail/uxc-net | 598 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 365 insertions(+), 233 deletions(-) diff --git a/jail/uxc-net b/jail/uxc-net index 828cb33..2e73942 100644 --- a/jail/uxc-net +++ b/jail/uxc-net @@ -2,6 +2,7 @@ let fs = require("fs"); let ubus = require("ubus"); +let uci = require("uci"); let fnv1a = function(s) { let h = 2166136261, i; @@ -27,6 +28,7 @@ let contzone = function(name) { return "cz" + slug(name); }; let injail_dir = "/tmp/run/uxc-net"; let injail_path = function(name) { return injail_dir + "/" + name + ".network"; }; +let shared_lock_path = injail_dir + "/.shared.lock"; let loopback_section = "config interface 'loopback'\n" + "\toption device 'lo'\n" + @@ -52,6 +54,10 @@ let quiet_call = function(object, method, data) { return ubus.error() ? -1 : 0; }; +let network_reload = function() { + return call("network", "reload", {}); +}; + let autonet_service = "uxc-net"; let icmpv6_types = [ @@ -62,32 +68,6 @@ let icmpv6_types = [ "neighbour-advertisement", ]; -let publish = function(name, fw, dh) { - let data = {}; - - if (length(fw)) - data.firewall = fw; - if (length(dh)) - data.dhcp = dh; - - if (call("service", "set_data", { name: name, instance: name, data: data })) - return 1; - - pkg_reload("firewall"); - pkg_reload("dhcp"); - - return 0; -}; - -let withdraw = function(name) { - quiet_call("service", "set_data", { name: name, instance: name, data: {} }); - - pkg_reload("firewall"); - pkg_reload("dhcp"); - - return 0; -}; - let sidecar_path = function(name) { return "/tmp/run/uvol/.meta/uxc/" + name + ".annotations"; }; let state_dir = "/tmp/run/uvol/.meta/uxc/state"; @@ -149,6 +129,17 @@ let with_macs = function(dev, m, hostrole, peerrole) { return dev; }; +let veth_device = function(vc, m, hostrole, peerrole) { + return with_macs({ type: "veth", peer_name: vc }, m, hostrole, peerrole); +}; + +let jailed_interface = function(name, vc, jail_device, zone) { + let ifc = { proto: "none", device: vc, jail: name, jail_device: jail_device }; + if (zone) + ifc.zone = zone; + return ifc; +}; + let read_annotations = function(bundle, name) { let ann = {}, raw, side, cfg, s, k, v; @@ -194,16 +185,45 @@ let parse_backhaul = function(ann) { return { id: id, address: ann["org.openwrt.network.backhaul-address"] }; }; -let iface_exists = function(iface) { - ubus.call({ object: "network.interface." + iface, method: "status", data: {} }); - return !ubus.error(); +let iface_status = function(iface) { + let r = ubus.call({ object: "network.interface." + iface, method: "status", data: {} }); + if (ubus.error() || type(r) != "object") + return null; + return r; }; -let bridge_members = function(br) { - let r = ubus.call({ object: "network.device", method: "status", data: { name: br } }); - if (ubus.error() || type(r) != "object" || type(r["bridge-members"]) != "array") - return -1; - return length(r["bridge-members"]); +let uci_network_exists = function(net) { + let cursor = uci.cursor(); + + if (!cursor || !cursor.load("network")) + return false; + return cursor.get("network", net) != null; +}; + +let device_is_bridge = function(dev) { + let d = ubus.call({ object: "network.device", method: "status", data: { name: dev } }); + + if (ubus.error() || type(d) != "object") + return false; + return type(d["bridge-members"]) == "array"; +}; + +let bridge_of = function(net, status) { + let dev = status.device, m; + + if (type(dev) != "string") { + warn(sprintf("uxc-net: network '%s' has no device to attach to\n", net)); + return null; + } + if (device_is_bridge(dev)) + return { device: dev }; + + m = match(dev, /^(.+)\.([0-9]+)$/); + if (m && device_is_bridge(m[1])) + return { device: m[1], vlans: [ m[2] + ":*" ] }; + + warn(sprintf("uxc-net: device '%s' of network '%s' is not a bridge\n", dev, net)); + return null; }; @@ -335,30 +355,45 @@ let ip2int = function(s) { return (+p[0] * 16777216) + (+p[1] * 65536) + (+p[2] * 256) + (+p[3]); }; -let used_ranges = function() { +let range_add = function(ranges, ipint, plen) { + let size, start; + plen = +plen; + if (ipint == null || plen < 0 || plen > 32) + return; + size = (plen == 0) ? 4294967296 : (1 << (32 - plen)); + start = ipint - (ipint % size); + push(ranges, [ start, start + size - 1 ]); +}; + +let range_add_cidr = function(ranges, cidr) { + let p = split(cidr ?? "", "/"); + range_add(ranges, ip2int(p[0]), p[1] ?? 32); +}; + +let published_ranges = function(ranges, shared) { + let ifc, a; + for (ifc in values(shared.interfaces)) + for (a in ifc.ipaddr ?? []) + range_add_cidr(ranges, a); +}; + +let used_ranges = function(shared) { let ranges = []; - let add = function(ipint, plen) { - plen = +plen; - if (ipint == null || plen < 0 || plen > 32) - return; - let size = (plen == 0) ? 4294967296 : (1 << (32 - plen)); - let start = ipint - (ipint % size); - push(ranges, [ start, start + size - 1 ]); - }; let d = ubus.call({ object: "network.interface", method: "dump", data: {} }); let list = (type(d) == "object" && type(d.interface) == "array") ? d.interface : []; for (let intf in list) { for (let a in intf["ipv4-address"] ?? []) - add(ip2int(a.address), a.mask); + range_add(ranges, ip2int(a.address), a.mask); for (let r in intf.route ?? []) if (r.target != "0.0.0.0") - add(ip2int(r.target), r.mask); + range_add(ranges, ip2int(r.target), r.mask); } + published_ranges(ranges, shared); return ranges; }; -let pick_subnet = function() { - let used = used_ranges(); +let pick_subnet = function(shared) { + let used = used_ranges(shared); let avail = function(prefix) { let base = ip2int(prefix + ".0"); for (let r in used) @@ -380,30 +415,129 @@ let pick_subnet = function() { return null; }; -let autonet_list = function() { + +let shared_lock = function() { + let f; + + fs.mkdir(injail_dir, 0755); + f = fs.open(shared_lock_path, "w"); + if (!f || !f.lock("x")) { + warn("uxc-net: cannot lock the shared network state\n"); + return null; + } + return f; +}; + +let shared_unlock = function(f) { + if (!f) + return; + f.lock("u"); + f.close(); +}; + +let shared_read = function() { + let d = ubus.call({ object: "service", method: "get_data", data: { name: autonet_service } }); + let s = {}, cur; + + if (!ubus.error() && type(d) == "object") + cur = d[autonet_service]?.["*"]; + if (type(cur) == "object") + s = cur; + + return { + present: type(d?.[autonet_service]) == "object", + devices: type(s["network-device"]) == "object" ? s["network-device"] : {}, + interfaces: type(s["network-interface"]) == "object" ? s["network-interface"] : {}, + }; +}; + +let shared_empty = function(shared) { + return !length(shared.devices) && !length(shared.interfaces); +}; + +let collect_devices = function(inst, names) { + let dev; + + if (type(inst?.["network-device"]) != "object") + return; + for (dev in keys(inst["network-device"])) + names[dev] = true; +}; + +let published_devices = function() { let d = ubus.call({ object: "service", method: "get_data", - data: { name: autonet_service, type: "firewall" } }); - let nets = [], svc, k, v, sec; + data: { type: "network-device" } }); + let names = {}, svc_name, svc, inst_name, inst; if (ubus.error() || type(d) != "object") - return nets; + return names; + + for (svc_name, svc in d) { + if (type(svc) != "object") + continue; + for (inst_name, inst in svc) { + if (svc_name == autonet_service && inst_name == "*") + continue; + collect_devices(inst, names); + } + } - svc = d[autonet_service]; - if (type(svc) != "object") - return nets; + return names; +}; - for (k, v in svc) { - if (type(v) != "object" || type(v.firewall) != "array") +let shared_prune = function(shared, live) { + let dead = [], br, dev, ifc_name, ifc; + + for (br, dev in shared.devices) { + if (dev.type != "bridge") continue; - for (sec in v.firewall) - if (sec.type == "zone" && sec.name) - push(nets, sec.name); + dev.ports = filter(dev.ports ?? [], function(p) { return live[p] == true; }); + if (!length(dev.ports)) + push(dead, br); } + for (br in dead) + delete shared.devices[br]; + + dead = []; + for (ifc_name, ifc in shared.interfaces) + if (!shared.devices[ifc.device]) + push(dead, ifc_name); + for (ifc_name in dead) + delete shared.interfaces[ifc_name]; +}; + +let shared_bridge_ensure = function(shared, br, ipv6) { + let dev = shared.devices[br]; + + if (dev) + return dev; + dev = { type: "bridge" }; + if (ipv6 == false) + dev.ipv6 = false; + dev.bridge_empty = true; + dev.ports = []; + shared.devices[br] = dev; + return dev; +}; + +let shared_add_port = function(shared, br, port) { + let dev = shared.devices[br]; + + if (index(dev.ports, port) < 0) + push(dev.ports, port); + sort(dev.ports); +}; + +let autonet_nets = function(shared) { + let nets = [], n, ifc; - return nets; + for (n, ifc in shared.interfaces) + if (ifc.zone == n) + push(nets, n); + return sort(nets); }; -let autonet_publish = function(nets) { +let autonet_specs = function(nets) { let fw = [], dh = [], net; for (net in nets) { @@ -425,67 +559,99 @@ let autonet_publish = function(nets) { }); } - if (length(nets)) - call("service", "set", { name: autonet_service, - data: { firewall: fw, dhcp: dh } }); - else + return { firewall: fw, dhcp: dh }; +}; + +let shared_write = function(shared, before) { + let specs, data; + + if (shared_empty(shared)) { + if (!shared.present) + return 0; quiet_call("service", "delete", { name: autonet_service }); + return 1; + } - pkg_reload("firewall"); - pkg_reload("dhcp"); -}; + if (sprintf("%J", [ shared.devices, shared.interfaces ]) == before) + return 0; -let ensure_network = function(net) { - let br = "br-" + net; - let nets = autonet_list(); - let subnet; + specs = autonet_specs(autonet_nets(shared)); + data = { + "network-device": shared.devices, + "network-interface": shared.interfaces, + firewall: specs.firewall, + dhcp: specs.dhcp, + }; + if (call("service", "set", { name: autonet_service, data: data })) + return -1; + return 1; +}; - if (iface_exists(net)) - return (net in nets) ? 1 : 0; +let shared_snapshot = function(shared) { + if (shared_empty(shared)) + return null; + return sprintf("%J", [ shared.devices, shared.interfaces ]); +}; - subnet = pick_subnet(); - if (!subnet) - return 2; +let autonet_create = function(shared, net, vh) { + let br = "br-" + net; + let subnet = pick_subnet(shared); - if (call("network", "create_device", { name: br, type: "bridge" })) - return 2; + if (!subnet) { + warn(sprintf("uxc-net: no free subnet for network '%s'\n", net)); + return -1; + } - if (call("network", "add_dynamic", { - name: net, + shared_bridge_ensure(shared, br, null); + shared_add_port(shared, br, vh); + shared.interfaces[net] = { proto: "static", device: br, ipaddr: [ subnet + ".1/24" ], - persistent: true, - })) { - quiet_call("network", "delete_device", { name: br }); - return 2; - } - - push(nets, net); - autonet_publish(nets); + ip6assign: 64, + ip6ifaceid: "::1", + force_link: true, + zone: net, + }; + return 0; +}; - return 1; +let spec_new = function() { + return { devices: {}, interfaces: {}, bridge_ports: {}, firewall: [], dhcp: [] }; }; -let autonet_remove = function(net) { - let nets = autonet_list(); +let instance_publish = function(name, spec) { + return call("service", "set_data", { name: name, instance: name, data: { + "network-device": spec.devices, + "network-interface": spec.interfaces, + "bridge-port": spec.bridge_ports, + firewall: spec.firewall, + dhcp: spec.dhcp, + } }); +}; - if (!(net in nets)) - return; - if (bridge_members("br-" + net) > 0) - return; +let instance_withdraw = function(name) { + return quiet_call("service", "set_data", { name: name, instance: name, data: {} }); +}; - autonet_publish(filter(nets, function(n) { return n != net; })); +let instance_published = function(name) { + let d = ubus.call({ object: "service", method: "get_data", + data: { name: name, instance: name } }); + let inst; - call("network.interface." + net, "remove", {}); - quiet_call("network", "delete_device", { name: "br-" + net }); + if (ubus.error() || type(d) != "object") + return false; + inst = d[name]?.[name]; + return type(inst) == "object" && length(inst) > 0; }; -let bridged_up = function(name, ann, attach, m) { + +let bridged_up = function(name, ann, attach, m, spec, shared, owned) { let net = attach.network; let vh = host_veth(name), vc = cont_veth(name); let proto = injail_proto(ann, "bridged"), opts = {}; let section6 = injail6_section(ann); + let status, br; if (!proto || section6 == null) return 1; @@ -498,27 +664,25 @@ let bridged_up = function(name, ann, attach, m) { if (ann["org.openwrt.network.egress"] || ann["org.openwrt.network.ingress"]) warn("uxc-net: bridged takes no egress/ingress (L2 inherits the joined network's zone); ignoring\n"); - let created = ensure_network(net); - if (created == 2) - return 1; - - if (call("network", "create_device", - with_macs({ name: vh, type: "veth", peer_name: vc }, m, "h", "c"))) - return 1; - - if (call("network.interface." + net, "add_device", { name: vh, "link-ext": false })) - return 1; + if (shared.interfaces[net]) { + shared_add_port(shared, "br-" + net, vh); + } else if (index(owned, net) >= 0 || !uci_network_exists(net)) { + if (autonet_create(shared, net, vh)) + return 1; + } else { + status = iface_status(net); + if (!status) { + warn(sprintf("uxc-net: network '%s' is not known to netifd\n", net)); + return 1; + } + br = bridge_of(net, status); + if (!br) + return 1; + spec.bridge_ports[vh] = br; + } - if (call("network", "add_dynamic", { - name: accif(name), - proto: "none", - device: vc, - jail: name, - jail_device: "eth0", - zone: net, - persistent: true, - })) - return 1; + spec.devices[vh] = veth_device(vc, m, "h", "c"); + spec.interfaces[accif(name)] = jailed_interface(name, vc, "eth0", net); append_injail(name, render_section("lan", "eth0", proto, opts)); if (section6 != "") @@ -527,17 +691,6 @@ let bridged_up = function(name, ann, attach, m) { return 0; }; -let bridged_down = function(name, attach) { - let vh = host_veth(name); - - call("network.interface." + accif(name), "remove", {}); - if (attach.network) { - call("network.interface." + attach.network, "remove_device", { name: vh, "link-ext": false }); - autonet_remove(attach.network); - } - return 0; -}; - let routed_subnet = function(name, ann) { let h = 0, i, a, b, c; @@ -658,11 +811,10 @@ let dhcp_specs = function(name) { } ]; }; -let routed_up = function(name, ann, m) { +let routed_up = function(name, ann, m, spec) { let czone = contzone(name); let net = routed_subnet(name, ann); let vh = host_veth(name), vc = cont_veth(name); - let gw_iface = gwif(name); let gw_ip = split(net.gw_cidr, "/")[0]; let proto = injail_proto(ann, "routed"); let section6 = injail6_section(ann); @@ -670,35 +822,19 @@ let routed_up = function(name, ann, m) { if (!proto || section6 == null) return 1; - if (call("network", "create_device", - with_macs({ name: vh, type: "veth", peer_name: vc }, m, "h", "c"))) - return 1; - - if (call("network", "add_dynamic", { - name: gw_iface, + spec.devices[vh] = veth_device(vc, m, "h", "c"); + spec.interfaces[gwif(name)] = { proto: "static", device: vh, ipaddr: [ net.gw_cidr ], ip6assign: 64, ip6ifaceid: "::1", force_link: true, - persistent: true, - zone: czone, - })) - return 1; - - if (call("network", "add_dynamic", { - name: accif(name), - proto: "none", - device: vc, - jail: name, - jail_device: "eth0", zone: czone, - persistent: true, - })) - return 1; - - publish(name, fw_specs(name, ann, net, injail_proto6(ann)), dhcp_specs(name)); + }; + spec.interfaces[accif(name)] = jailed_interface(name, vc, "eth0", czone); + spec.firewall = fw_specs(name, ann, net, injail_proto6(ann)); + spec.dhcp = dhcp_specs(name); append_injail(name, render_section("lan", "eth0", proto, { ipaddr: net.container, @@ -712,44 +848,18 @@ let routed_up = function(name, ann, m) { return 0; }; -let routed_down = function(name) { - let gw_iface = gwif(name); - - withdraw(name); - call("network.interface." + accif(name), "remove", {}); - call("network.interface." + gw_iface, "remove", {}); - return 0; -}; - -let backhaul_up = function(name, bh, m) { +let backhaul_up = function(name, bh, m, spec, shared) { let br = bh_bridge(bh.id); let iface = bhseg(bh.id); let vh = bh_host_veth(name), vc = bh_cont_veth(name); - if (!iface_exists(iface)) { - if (call("network", "create_device", { name: br, type: "bridge", ipv6: false })) - return 1; - if (call("network", "add_dynamic", { - name: iface, proto: "none", device: br, persistent: true, - })) - return 1; - } - - if (call("network", "create_device", - with_macs({ name: vh, type: "veth", peer_name: vc }, m, "bh", "bc"))) - return 1; - if (call("network.interface." + iface, "add_device", { name: vh, "link-ext": false })) - return 1; + shared_bridge_ensure(shared, br, false); + shared_add_port(shared, br, vh); + if (!shared.interfaces[iface]) + shared.interfaces[iface] = { proto: "none", device: br }; - if (call("network", "add_dynamic", { - name: bhif(name), - proto: "none", - device: vc, - jail: name, - jail_device: "bh0", - persistent: true, - })) - return 1; + spec.devices[vh] = veth_device(vc, m, "bh", "bc"); + spec.interfaces[bhif(name)] = jailed_interface(name, vc, "bh0", null); if (bh.address) append_injail(name, render_section("backhaul", "bh0", "static", { @@ -760,32 +870,34 @@ let backhaul_up = function(name, bh, m) { return 0; }; -let backhaul_down = function(name, bh) { - let br = bh_bridge(bh.id); - let iface = bhseg(bh.id); - let vh = bh_host_veth(name); - call("network.interface." + iface, "remove_device", { name: vh, "link-ext": false }); - call("network.interface." + bhif(name), "remove", {}); +let shared_reconcile = function() { + let lock = shared_lock(), shared, before, changed; - if (bridge_members(br) == 0) { - call("network.interface." + iface, "remove", {}); - call("network", "delete_device", { name: br }); - } - - return 0; + if (!lock) + return 0; + shared = shared_read(); + before = shared_snapshot(shared); + shared_prune(shared, published_devices()); + changed = shared_write(shared, before); + shared_unlock(lock); + return changed; }; +let instance_rollback = function(name) { + instance_withdraw(name); + shared_reconcile(); +}; let do_up = function(name, bundle) { - let attach, bh; + let attach, bh, ann, roles, m, spec, shared, owned, before, lock, rc, written; if (!bundle) { warn("uxc-net: 'up' needs a bundle path\n"); return 1; } - let ann = read_annotations(bundle, name); + ann = read_annotations(bundle, name); if (ann == null) { warn("uxc-net: cannot read annotations from " + bundle + "/config.json\n"); return 1; @@ -796,7 +908,14 @@ let do_up = function(name, bundle) { fs.unlink(injail_path(name)); - let roles = []; + if (attach.kind == "unknown") { + warn(sprintf("uxc-net: attach '%s' not implemented\n", attach.raw ?? attach.kind)); + return 1; + } + if (attach.kind != "bridged" && attach.kind != "routed" && !bh) + return 0; + + roles = []; if (attach.kind == "bridged" || attach.kind == "routed") { push(roles, "h"); push(roles, "c"); @@ -805,50 +924,63 @@ let do_up = function(name, bundle) { push(roles, "bh"); push(roles, "bc"); } - let m = ensure_macs(name, roles); + m = ensure_macs(name, roles); + spec = spec_new(); - if (attach.kind == "bridged") { - if (bridged_up(name, ann, attach, m)) - return 1; - } else if (attach.kind == "routed") { - if (routed_up(name, ann, m)) - return 1; - } else if (attach.kind == "unknown") { - warn(sprintf("uxc-net: attach '%s' not implemented\n", attach.raw ?? attach.kind)); + lock = shared_lock(); + if (!lock) + return 1; + shared = shared_read(); + owned = autonet_nets(shared); + before = shared_snapshot(shared); + shared_prune(shared, published_devices()); + + rc = 0; + if (attach.kind == "bridged") + rc = bridged_up(name, ann, attach, m, spec, shared, owned); + else if (attach.kind == "routed") + rc = routed_up(name, ann, m, spec); + if (!rc && bh) + rc = backhaul_up(name, bh, m, spec, shared); + if (!rc && shared_write(shared, before) < 0) + rc = 1; + written = !rc; + if (!rc && instance_publish(name, spec)) + rc = 1; + shared_unlock(lock); + if (!rc && network_reload()) + rc = 1; + if (rc) { + if (written) + instance_rollback(name); return 1; } - if (bh) - if (backhaul_up(name, bh, m)) - return 1; + pkg_reload("firewall"); + pkg_reload("dhcp"); return 0; }; -let do_down = function(name, bundle) { - let ann = read_annotations(bundle, name); - if (ann == null) - ann = {}; +let do_down = function(name) { + let had, changed; - let configured = !!fs.stat(injail_path(name)); fs.unlink(injail_path(name)); - let attach = parse_attach(ann); - let bh = parse_backhaul(ann); + had = instance_published(name); + if (had) + instance_withdraw(name); - if (bh) - backhaul_down(name, bh); + changed = shared_reconcile(); - if (attach.kind == "none" || attach.kind == "host") + if (!had && changed <= 0) return 0; - if (attach.kind == "routed") - return routed_down(name); - if (attach.kind == "bridged") - return bridged_down(name, attach); - if (!configured) - return 0; - return routed_down(name); + network_reload(); + pkg_reload("firewall"); + pkg_reload("dhcp"); + + return 0; }; let name = ARGV[0]; @@ -863,7 +995,7 @@ if (!name || !action) { if (action == "up") exit(do_up(name, bundle)); else if (action == "down") - exit(do_down(name, bundle)); + exit(do_down(name)); warn(sprintf("uxc-net: unknown action '%s'\n", action)); exit(22); From 0aa2fc204f8a92bbacbba3cfdb42e71a6b95cd96 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 2 Sep 2026 08:51:24 +0100 Subject: [PATCH 76/82] jail: point the /dev/ptmx symlink at the relative pts/ptmx The OCI runtime-spec default device symlinks give /dev/ptmx the target "pts/ptmx", relative to /dev. ujail created it with an absolute target, so a bundle that validates the symlink target sees a mismatch. Use the relative target the spec and the conformance suite expect. Signed-off-by: Daniel Golle --- jail/jail.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jail/jail.c b/jail/jail.c index e77594f..7dd3c95 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -935,8 +935,8 @@ static int prepare_jail_dev(void) /* Dev symbolic links as defined in OCI spec */ snprintf(path, sizeof(path), "%s/ptmx", jail_dev); - if (symlink("/dev/pts/ptmx", path)) - WARNING("symlink() failed to create link to /dev/pts/ptmx"); + if (symlink("pts/ptmx", path)) + WARNING("symlink() failed to create link to pts/ptmx"); snprintf(path, sizeof(path), "%s/fd", jail_dev); if (symlink("/proc/self/fd", path)) From 4a6b6ea12d2bd21c25408a2df4d4d254ea4e95d7 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 2 Sep 2026 09:21:25 +0100 Subject: [PATCH 77/82] jail: apply OCI seccomp without forcing no-new-privs applyOCIlinuxseccomp() set PR_SET_NO_NEW_PRIVS unconditionally before installing the filter, so a bundle with process.noNewPrivileges left false still ended up with NoNewPrivs set once it carried a seccomp profile. The runtime-tools default configuration is exactly that, and runtimetest reads the flag back from /proc/self/status. The kernel accepts SECCOMP_SET_MODE_FILTER either under no-new-privs or from a task holding CAP_SYS_ADMIN in its user namespace, so the flag is not needed to install a filter, only the ordering matters. Follow runc: when noNewPrivileges is true, set the flag after the capability drop and install the filter last, so the filter need not permit the remaining setup syscalls; when it is false, install the filter in the container init before the capability drop, while the init still holds CAP_SYS_ADMIN in its own user namespace. The filter then has to permit the syscalls used up to execve (capset, prctl, setresuid, setgroups, umask, chdir, the landlock calls, execve), which is the same requirement runc imposes with that ordering. The ptrace injection path cannot serve the noNewPrivileges=false case at all: it acts after execve, when the bounding set of a typical bundle no longer contains CAP_SYS_ADMIN, which is why seccomp-inject sets the flag itself. Enforcing bundle profiles without no-new-privs therefore take the in-process route already used for SCMP_ACT_NOTIFY and flag-carrying profiles, where the linker-extended profile stays in place for the lifetime of the process instead of being narrowed at the entry point. The early install is confined to OCI bundles: a plain jail given a profile with -S also lands in opts.ociseccomp, and keeps the injection path whether or not procd passed -c, exactly as before. The CLOSE_RANGE_CLOEXEC sweep moves in front of the early install so that marking the inherited descriptors close-on-exec does not depend on the profile permitting close_range; the flag only takes effect at execve, and nothing opened between the two points is meant to survive it. Containers with noNewPrivileges=true are unaffected: they keep the flag, drop capabilities and get the filter last, whether injected or in-process. Fixes: ea7a790f210c ("jail: add support for running OCI bundle") Signed-off-by: Daniel Golle --- jail/jail.c | 23 +++++++++++++++++++---- jail/seccomp-oci.c | 5 ----- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/jail/jail.c b/jail/jail.c index 7dd3c95..0f9e238 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -1446,6 +1446,7 @@ static void notify_signal(int fd) } static bool jail_ptrace_seccomp(void); +static bool jail_inproc_seccomp(void); static void free_and_exit(int ret) { @@ -2938,6 +2939,7 @@ static void post_jail_fs(void) static void post_start_hook(void) { int pw_uid, pw_gid, gr_gid; + struct sock_fprog *seccomp_prog = opts.ociseccomp_linker ?: opts.ociseccomp; if (opts.scheduler.set && applyOCIprocessscheduler()) free_and_exit(EXIT_FAILURE); @@ -2945,6 +2947,12 @@ static void post_start_hook(void) if (opts.ioprio.set && applyOCIprocessiopriority()) free_and_exit(EXIT_FAILURE); + syscall(SYS_close_range, 3, ~0U, CLOSE_RANGE_CLOEXEC); + + if (seccomp_prog && jail_inproc_seccomp() && !opts.no_new_privs && + applyOCIlinuxseccomp(seccomp_prog, opts.name, opts.ocibundle)) + free_and_exit(EXIT_FAILURE); + /* * make sure setuid/setgid won't drop capabilities in case capabilities * have been specified explicitely. @@ -3076,13 +3084,12 @@ static void post_start_hook(void) free_and_exit(EXIT_FAILURE); } - if (opts.ociseccomp && seccomp_oci_needs_inproc() && - applyOCIlinuxseccomp(opts.ociseccomp_linker ?: opts.ociseccomp, opts.name, opts.ocibundle)) + if (seccomp_prog && jail_inproc_seccomp() && opts.no_new_privs && + applyOCIlinuxseccomp(seccomp_prog, opts.name, opts.ocibundle)) free_and_exit(EXIT_FAILURE); uloop_end(); free_opts(false); - syscall(SYS_close_range, 3, ~0U, CLOSE_RANGE_CLOEXEC); if (jail_ptrace_seccomp() && ptrace(PTRACE_TRACEME, 0, 0, 0)) { ERROR("PTRACE_TRACEME failed: %m\n"); exit(EXIT_FAILURE); @@ -7031,12 +7038,20 @@ static void post_create_runtime(void) pipe_send_start_container(NULL); } +static bool jail_inproc_seccomp(void) +{ + if (seccomp_oci_needs_inproc()) + return true; + + return opts.ocibundle && !opts.no_new_privs && opts.seccomp_mode == SECCOMP_MODE_ENFORCE; +} + static bool jail_ptrace_seccomp(void) { if (opts.seccomp_mode == SECCOMP_MODE_TRACE) return true; - return opts.ociseccomp && !seccomp_oci_needs_inproc(); + return opts.ociseccomp && !jail_inproc_seccomp(); } static void jail_seccomp_run(void) diff --git a/jail/seccomp-oci.c b/jail/seccomp-oci.c index 7d16103..c14c597 100644 --- a/jail/seccomp-oci.c +++ b/jail/seccomp-oci.c @@ -935,11 +935,6 @@ int applyOCIlinuxseccomp(struct sock_fprog *prog, const char *container_id, { int listener_fd = -1; - if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) { - ERROR("prctl(PR_SET_NO_NEW_PRIVS) failed: %m\n"); - goto errout; - } - if (seccomp_uses_notify) { if (!seccomp_listener_path) { ERROR("seccomp: SCMP_ACT_NOTIFY used without listenerPath\n"); From 7e164b2f06c5cf76cbf3ebb52bb2b5c408e520fe Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 2 Sep 2026 09:22:45 +0100 Subject: [PATCH 78/82] jail: honour linux.rootfsPropagation on the container root The field has been in the linux policy since OCI support arrived, but nothing consumed it: the value was parsed and dropped, and the root of the container kept whatever propagation the private remounts left it with. Map shared, slave, private and unbindable to their MS_* flags, all recursive, and apply the result to "/" as the last mount step once the rootfs is in place, from post_jail_fs(), which every path reaches after pivot_root() and, for a deferred user namespace, after the second unshare(CLONE_NEWNS) and the deferred masks. The MS_REC|MS_PRIVATE isolation applied earlier is untouched; it keeps the masks from leaking while the tree is built, and only the final propagation type of the container root changes, and only when the bundle asks for one. An empty string is treated as unset, as runc does; any other unknown value is rejected at parse time. The mount is skipped without a mount namespace, where "/" is the host root. Fixes: ea7a790f210c ("jail: add support for running OCI bundle") Signed-off-by: Daniel Golle --- jail/jail.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/jail/jail.c b/jail/jail.c index 0f9e238..717c737 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -223,6 +223,7 @@ static struct { int priority; } ioprio; unsigned long mdwe_flags; + unsigned long rootfs_propagation; struct landlock_config landlock; bool private_ubus; bool private_netifd; @@ -2920,6 +2921,12 @@ static void post_jail_fs(void) char buf[1]; ssize_t n; + if (opts.rootfs_propagation && (opts.namespace & CLONE_NEWNS) && + mount(NULL, "/", NULL, opts.rootfs_propagation, NULL)) { + ERROR("rootfsPropagation: %m\n"); + free_and_exit(EXIT_FAILURE); + } + do { n = read(pipes[2], buf, 1); } while (n < 0 && errno == EINTR); @@ -4233,6 +4240,27 @@ static int parseOCIlinuxpersonality(struct blob_attr *msg) return 0; } +static int parseOCIrootfspropagation(const char *mode) +{ + if (!*mode) + return 0; + + if (!strcmp(mode, "shared")) + opts.rootfs_propagation = MS_REC | MS_SHARED; + else if (!strcmp(mode, "slave")) + opts.rootfs_propagation = MS_REC | MS_SLAVE; + else if (!strcmp(mode, "private")) + opts.rootfs_propagation = MS_REC | MS_PRIVATE; + else if (!strcmp(mode, "unbindable")) + opts.rootfs_propagation = MS_REC | MS_UNBINDABLE; + else { + ERROR("unknown linux.rootfsPropagation %s\n", mode); + return EINVAL; + } + + return 0; +} + static int parseOCIlinux(struct blob_attr *msg) { struct blob_attr *tb[__OCI_LINUX_MAX]; @@ -4246,6 +4274,12 @@ static int parseOCIlinux(struct blob_attr *msg) blobmsg_parse(oci_linux_policy, __OCI_LINUX_MAX, tb, blobmsg_data(msg), blobmsg_len(msg)); + if (tb[OCI_LINUX_ROOTFSPROPAGATION]) { + res = parseOCIrootfspropagation(blobmsg_get_string(tb[OCI_LINUX_ROOTFSPROPAGATION])); + if (res) + return res; + } + if (tb[OCI_LINUX_PERSONALITY]) { res = parseOCIlinuxpersonality(tb[OCI_LINUX_PERSONALITY]); if (res) From e32d25a78b5b21cb44c7b88c589ee9cd1bd6b0f0 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 2 Sep 2026 09:23:47 +0100 Subject: [PATCH 79/82] jail: reject start of a container without a process The runtime spec makes process optional in config.json and requires start to fail when it is absent, while create is expected to succeed. parseOCI() refused the whole bundle with ENODATA instead, so such a container never reached the created state and start.t could not get to its assertion. Let create go ahead without a process section and have handle_start() refuse to run a container whose bundle defines none, logging the reason and answering UBUS_STATUS_INVALID_ARGUMENT like the other state rejections. A bundle without a process combined with -i, where ujail would start the container on its own right after create, is still rejected at parse time since there is nothing it could exec. uxc start now prints the ubus error of a refused start; it returned the status code silently before, and the operator saw nothing. Fixes: ea7a790f210c ("jail: add support for running OCI bundle") Signed-off-by: Daniel Golle --- jail/jail.c | 9 +++++++-- uxc.c | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/jail/jail.c b/jail/jail.c index 717c737..bfe7ea3 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -4540,12 +4540,12 @@ static int parseOCI(const char *jsonfile) if (tb[OCI_DOMAINNAME]) opts.domainname = strdup(blobmsg_get_string(tb[OCI_DOMAINNAME])); - if (!tb[OCI_PROCESS]) { + if (!tb[OCI_PROCESS] && opts.immediately) { res=ENODATA; goto errout; } - if ((res = parseOCIprocess(tb[OCI_PROCESS]))) + if (tb[OCI_PROCESS] && (res = parseOCIprocess(tb[OCI_PROCESS]))) goto errout; if (!tb[OCI_ROOT]) { @@ -4708,6 +4708,11 @@ static int handle_start(struct ubus_context *ctx, struct ubus_object *obj, if (jail_oci_state != OCI_STATE_CREATED) return UBUS_STATUS_INVALID_ARGUMENT; + if (!opts.jail_argv) { + ERROR("start refused: the bundle defines no process\n"); + return UBUS_STATUS_INVALID_ARGUMENT; + } + uloop_timeout_add(&start_container_timeout); return UBUS_STATUS_OK; diff --git a/uxc.c b/uxc.c index 9939980..c5de7ce 100644 --- a/uxc.c +++ b/uxc.c @@ -1676,6 +1676,7 @@ static int uxc_start(const char *name, bool console) ret = ubus_invoke(ctx, id, "start", NULL, NULL, NULL, 3000); if (ret) { + fprintf(stderr, "uxc: start %s: %s\n", name, ubus_strerror(ret)); uxc_wait_disarm(); return ret; } From 237ae2b81283e7ed88063dd4c682192fac0a4684 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 2 Sep 2026 09:26:43 +0100 Subject: [PATCH 80/82] jail: restart the container when netifd reappears The host side of a container network is dynamic: uxc-net publishes it as procd service data and netifd materialises the devices from that. A netifd restart drops those objects, and the container keeps running with its veth gone and nothing left to bring it back. Re-running uxc-net up is not enough, it re-plumbs the host end but never repeats the namespace hand-over done at create time by jail_network_attach() and the netns handling, so the container stays without its interface. Watch ubus.object.add for the network.interface object once the container is running, per container and only when it owns a network namespace, so the first appearance a jail sees is a genuine restart of netifd. On that event ujail stops its container the way uxc kill does, SIGTERM with the usual SIGKILL escalation, and lets the normal shutdown run: poststop withdraws the published network data through uxc-net down and runs the poststop hooks. Instead of exiting afterwards, the supervisor re-executes itself with its original arguments plus -i, so the fresh instance creates the container again, uxc-net up and the namespace hand-over included, and starts it right away. The pid and therefore procd's view of the instance do not change; procd has no restart operation for an instance and never respawns a container on its own, and uxc-managed instances created through create and start carry no -i, so a procd-side respawn would have stopped at the created state. The arguments are copied before option parsing, since getopt and the mount and namespace parsers split their operands in place, and the notify descriptor procd hands over with -a for the original create is left out of the copy handed to the new instance: its invoker returned long ago, and close-on-exec takes the descriptor with it. For the same reason the invoker is not signalled about an exit while the restart is under way. A restart is refused while a stop is already in flight, and a stop requested during the teardown wins over the pending restart, so a container being taken down by uxc kill or by procd is not revived. The handler is only registered once the container runs and the new instance registers its own only after it is up again, when the netifd object already exists, so a container cannot re-trigger itself. Signed-off-by: Daniel Golle --- jail/jail.c | 120 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 117 insertions(+), 3 deletions(-) diff --git a/jail/jail.c b/jail/jail.c index bfe7ea3..acae6dc 100644 --- a/jail/jail.c +++ b/jail/jail.c @@ -253,6 +253,9 @@ static long jail_clone3(struct clone_args *args) static int jail_process_pidfd = -1; static struct ubus_context *parent_ctx; +static bool jail_stop_requested; +static bool netifd_restart_pending; +static char **restart_argv; int console_fd; static int console_slave_fd = -1; @@ -1449,9 +1452,60 @@ static void notify_signal(int fd) static bool jail_ptrace_seccomp(void); static bool jail_inproc_seccomp(void); +static int restart_argv_save(int argc, char **argv) +{ + int i; + + restart_argv = calloc(argc + 1, sizeof(*restart_argv)); + if (!restart_argv) + return ENOMEM; + + for (i = 0; i < argc; i++) { + restart_argv[i] = strdup(argv[i]); + if (!restart_argv[i]) + return ENOMEM; + } + + return 0; +} + +static bool jail_restarting(void) +{ + return !exit_from_child && netifd_restart_pending && !jail_stop_requested; +} + +static void jail_restart_exec(void) +{ + char **argv; + int argc, n, i; + + for (argc = 0; restart_argv[argc]; argc++); + + argv = calloc(argc + 2, sizeof(*argv)); + if (!argv) + return; + + n = 0; + argv[n++] = restart_argv[0]; + argv[n++] = "-i"; + for (i = 1; i < argc; i++) { + if (!strcmp(restart_argv[i], "-a") && i + 1 < argc) { + i++; + continue; + } + argv[n++] = restart_argv[i]; + } + + INFO("restarting the container\n"); + syscall(SYS_close_range, 3, ~0U, CLOSE_RANGE_CLOEXEC); + execv("/proc/self/exe", argv); + ERROR("failed to re-execute for the restart: %m\n"); + free(argv); +} + static void free_and_exit(int ret) { - if (!exit_from_child) + if (!exit_from_child && !jail_restarting()) notify_signal(opts.notify_fd); if (!exit_from_child && opts.jail_network_started) { @@ -1478,6 +1532,9 @@ static void free_and_exit(int ret) free_opts(!exit_from_child); + if (jail_restarting()) + jail_restart_exec(); + exit(ret); } @@ -2200,6 +2257,9 @@ static void jail_process_timeout_cb(struct uloop_timeout *t) static void jail_handle_signal(int signo) { + if (signo == SIGTERM) + jail_stop_requested = true; + if (hook_running) { DEBUG("forwarding signal %d to the hook process\n", signo); kill(hook_process.pid, signo); @@ -5137,6 +5197,9 @@ container_handle_kill(struct ubus_context *ctx, struct ubus_object *obj, if (cur) all = blobmsg_get_bool(cur); + if (sig == SIGTERM || sig == SIGKILL) + jail_stop_requested = true; + if (jail_oci_state == OCI_STATE_CREATING) return UBUS_STATUS_NOT_FOUND; if (jail_oci_state == OCI_STATE_PAUSED && sig != SIGKILL && sig != 0) @@ -6068,6 +6131,13 @@ static struct ubus_object container_object = { .n_methods = ARRAY_SIZE(container_methods), }; +static void netifd_restart_cb(struct ubus_context *ctx, struct ubus_event_handler *ev, + const char *type, struct blob_attr *msg); + +static struct ubus_event_handler netifd_restart_handler = { + .cb = netifd_restart_cb, +}; + static void post_main(struct uloop_timeout *t); static struct uloop_timeout post_main_timeout = { .cb = post_main, @@ -6099,6 +6169,11 @@ int main(int argc, char **argv) return EXIT_FAILURE; } + if (restart_argv_save(argc, argv)) { + ERROR("out of memory\n"); + return EXIT_FAILURE; + } + /* those are filehandlers, so -1 indicates unused */ opts.setns.pid = -1; opts.setns.net = -1; @@ -6576,6 +6651,43 @@ static int run_uxc_net(const char *action) return (WIFEXITED(status) && WEXITSTATUS(status) == 0) ? 0 : -1; } +static void netifd_restart_cb(struct ubus_context *ctx, struct ubus_event_handler *ev, + const char *type, struct blob_attr *msg) +{ + static const struct blobmsg_policy pol = { + .name = "path", .type = BLOBMSG_TYPE_STRING + }; + struct blob_attr *tb; + + if (!msg) + return; + + blobmsg_parse(&pol, 1, &tb, blob_data(msg), blob_len(msg)); + if (!tb || strcmp(blobmsg_get_string(tb), "network.interface")) + return; + + if (!jail_running || jail_stop_requested || netifd_restart_pending) + return; + + INFO("netifd restarted, restarting the container\n"); + netifd_restart_pending = true; + if (jail_pidfd_send_signal(SIGTERM)) { + ERROR("cannot stop the container for the restart: %m\n"); + netifd_restart_pending = false; + return; + } + uloop_timeout_set(&jail_process_timeout, UXC_STOP_TIMEOUT * 1000); +} + +static void netifd_restart_watch(void) +{ + if (!(opts.namespace & CLONE_NEWNET) || !opts.ocibundle || !opts.name) + return; + + if (ubus_register_event_handler(parent_ctx, &netifd_restart_handler, "ubus.object.add")) + WARNING("cannot watch for netifd restarts\n"); +} + static void post_main(struct uloop_timeout *t) { int child_status; @@ -7321,10 +7433,12 @@ static void pipe_send_start_container(struct uloop_timeout *t) static void post_poststart(void) { - if (hook_chain_failed) + if (hook_chain_failed) { ERROR("poststart hook failed; stopping container\n"); - else + } else { + netifd_restart_watch(); uloop_run(); /* idle here while jail is running */ + } if (jail_running) { DEBUG("killing jail process\n"); From f3df40fae653df1c7c8163419cb75caa0c062d82 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 2 Sep 2026 09:27:06 +0100 Subject: [PATCH 81/82] uxc: let delete finish when the container is already gone uxc delete failed with EIO when procd answered NOT_FOUND to the instance removal, and delete -f aborted when the kill it issued first found no container process any more. Both happen when the container exits between the runtime state being read and the request being made, and both leave the configuration behind although the instance is gone, which is the state delete is meant to reach. Treat NOT_FOUND from the procd delete as done, skip the removal wait in that case since no instance is left to report it, and let a forced delete continue when its kill reports the container as already gone. uxc_kill now distinguishes that case as -ENOENT instead of folding it into -EIO; kill itself keeps failing for a stopped container, as the spec requires. Signed-off-by: Daniel Golle --- uxc.c | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/uxc.c b/uxc.c index c5de7ce..e3d4efe 100644 --- a/uxc.c +++ b/uxc.c @@ -1875,10 +1875,11 @@ static int uxc_kill(char *name, int signal, bool all) fprintf(stderr, "uxc: warning: cannot arm instance.* watcher\n"); } - if (ubus_invoke(ctx, id, "kill", req.head, NULL, NULL, 3000)) { + ret = ubus_invoke(ctx, id, "kill", req.head, NULL, NULL, 3000); + if (ret) { if (wait_stop) uxc_wait_disarm(); - return -EIO; + return (ret == UBUS_STATUS_NOT_FOUND) ? -ENOENT : -EIO; } if (wait_stop) { @@ -2786,16 +2787,16 @@ static int uxc_delete(char *name, bool force, bool volumes) rsstate = avl_find_element(&runtime, name, rsstate, avl); - if (rsstate && rsstate->running) { - if (force) { - ret = uxc_kill(name, SIGKILL, true); - if (ret) - goto errout; + if (rsstate && rsstate->running && !force) { + ret = -EWOULDBLOCK; + goto errout; + } - } else { - ret = -EWOULDBLOCK; + if (rsstate && rsstate->running) { + ret = uxc_kill(name, SIGKILL, true); + if (ret && ret != -ENOENT) goto errout; - } + ret = 0; } if (rsstate) { @@ -2828,7 +2829,8 @@ static int uxc_delete(char *name, bool force, bool volumes) fprintf(stderr, "uxc: warning: cannot arm instance.* watcher\n"); } - if (ubus_invoke(ctx, id, "delete", req.head, NULL, NULL, 3000)) { + ret = ubus_invoke(ctx, id, "delete", req.head, NULL, NULL, 3000); + if (ret && ret != UBUS_STATUS_NOT_FOUND) { blob_buf_free(&req); if (have_cont_obj) uxc_wait_disarm(); @@ -2836,12 +2838,13 @@ static int uxc_delete(char *name, bool force, bool volumes) goto errout; } - if (have_cont_obj) { - if (uxc_wait_run(&wait_state, 30000) == -ETIMEDOUT) - fprintf(stderr, "uxc: warning: timed out waiting for container.%s removal\n", - rsstate->container_name); + if (have_cont_obj && !ret && + uxc_wait_run(&wait_state, 30000) == -ETIMEDOUT) + fprintf(stderr, "uxc: warning: timed out waiting for container.%s removal\n", + rsstate->container_name); + if (have_cont_obj) uxc_wait_disarm(); - } + ret = 0; } usettings = avl_find_element(&settings, name, usettings, avl); From 6d44d6bb77e27ecbaa881976e2bb4d3cf8ccd253 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 2 Sep 2026 09:41:17 +0100 Subject: [PATCH 82/82] uxc-net: attach containers to multiple bridge VLANs A bridged container joined exactly one VLAN of a VLAN-filtering bridge: bridge_of() turned the br-X.N device of the attach network into a single "N:*" element on the bridge-port entry, so the host end of the veth was an untagged access port and the container could reach no other VLAN of that bridge. Add the annotation org.openwrt.network.vlans, a comma-separated list of [:t] entries that has a meaning only on top of bridged: where sits on a VLAN-filtering bridge. The attach network's own VLAN stays the untagged PVID member exactly as before; a bare adds the port as a further untagged member and :t adds it as a tagged trunk member. Without the annotation nothing changes, neither for a plain bridge nor for a single VLAN. On the host the bridge-port entry keeps its one "vlans" array, which now carries every requested element, for example ["20:*","30:t","40:t"]. netifd merges each element into the ports list of the UCI bridge-vlan section with that VID, so a single entry puts the veth into several VLANs with the right flag on each. VIDs must lie within 1 to 4094, identical entries are folded, and an entry is rejected when it is listed both tagged and untagged, when it names the attach network's own VLAN, or when the bridge has no such VLAN according to network.device status. The last check matters because netifd merges only into bridge-vlan sections that exist and a port of a VLAN-filtering bridge without any membership forwards nothing, so accepting the VID would leave a dead interface in the container. Networks created on demand are plain bridges owned by uxc-net and refuse the annotation. Inside the container a tagged VLAN is reachable only through an 802.1Q sub-interface, so for every :t the in-jail network file gains a device section of type 8021q named eth0. on top of eth0 and an interface 'vlan' on that device. The same netifd runs inside the jail and creates the device when the interface claims it, given the 8021q module on the host kernel. Untagged members need nothing inside: their frames are the untagged traffic on eth0. The VLAN interfaces default to proto none and switch to static when org.openwrt.network.vlan..address supplies an address/prefix. A container on a trunk typically uses its extra VLANs for plain L2 or for a service with a fixed address; a DHCP client on every VLAN is unusual and would compete for the default route that eth0 already provides. Teardown is unchanged: withdrawing the instance data removes the bridge-port entry with all of its memberships on the next reload, and the in-jail file is per container. Signed-off-by: Daniel Golle --- jail/uxc-net | 125 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 115 insertions(+), 10 deletions(-) diff --git a/jail/uxc-net b/jail/uxc-net index 2e73942..f8105d0 100644 --- a/jail/uxc-net +++ b/jail/uxc-net @@ -200,15 +200,46 @@ let uci_network_exists = function(net) { return cursor.get("network", net) != null; }; -let device_is_bridge = function(dev) { +let device_status = function(dev) { let d = ubus.call({ object: "network.device", method: "status", data: { name: dev } }); if (ubus.error() || type(d) != "object") - return false; - return type(d["bridge-members"]) == "array"; + return null; + return d; +}; + +let device_is_bridge = function(dev) { + return type(device_status(dev)?.["bridge-members"]) == "array"; }; -let bridge_of = function(net, status) { +let bridge_vlan_ids = function(br) { + return map(device_status(br)?.["bridge-vlans"] ?? [], function(v) { return v.id; }); +}; + +let bridge_port = function(br, pvid, vlans) { + let port = { device: br }, flags = [], have, v; + + if (pvid != null) + push(flags, pvid + ":*"); + if (length(vlans)) + have = bridge_vlan_ids(br); + for (v in vlans) { + if (v.vid == pvid) { + warn(sprintf("uxc-net: VLAN %d on '%s' is already the network's own VLAN\n", v.vid, br)); + return null; + } + if (index(have, v.vid) < 0) { + warn(sprintf("uxc-net: bridge '%s' has no VLAN %d\n", br, v.vid)); + return null; + } + push(flags, v.vid + (v.tagged ? ":t" : "")); + } + if (length(flags)) + port.vlans = flags; + return port; +}; + +let bridge_of = function(net, status, vlans) { let dev = status.device, m; if (type(dev) != "string") { @@ -216,11 +247,11 @@ let bridge_of = function(net, status) { return null; } if (device_is_bridge(dev)) - return { device: dev }; + return bridge_port(dev, null, vlans); m = match(dev, /^(.+)\.([0-9]+)$/); if (m && device_is_bridge(m[1])) - return { device: m[1], vlans: [ m[2] + ":*" ] }; + return bridge_port(m[1], +m[2], vlans); warn(sprintf("uxc-net: device '%s' of network '%s' is not a bridge\n", dev, net)); return null; @@ -329,6 +360,44 @@ let injail6_section = function(ann) { return render_section("lan6", "eth0", "dhcpv6", opts); }; +let render_vlan_device = function(parent, vid) { + return "config device\n" + + "\toption type '8021q'\n" + + "\toption name '" + parent + "." + vid + "'\n" + + "\toption ifname '" + parent + "'\n" + + "\toption vid '" + vid + "'\n"; +}; + +let injail_vlan_section = function(ann, vid) { + let addr = ann["org.openwrt.network.vlan." + vid + ".address"]; + let proto = "none", opts = {}; + + if (addr != null && addr != "") { + if (index(addr, "/") < 0) { + warn(sprintf("uxc-net: org.openwrt.network.vlan.%d.address must be address/prefix\n", vid)); + return null; + } + proto = "static"; + opts.ipaddr = addr; + } + return render_vlan_device("eth0", vid) + + render_section("vlan" + vid, "eth0." + vid, proto, opts); +}; + +let injail_vlan_sections = function(ann, vlans) { + let out = "", s, v; + + for (v in vlans) { + if (!v.tagged) + continue; + s = injail_vlan_section(ann, v.vid); + if (s == null) + return null; + out += s; + } + return out; +}; + let csv = function(val) { let list = [], i, parts; if (type(val) != "string") @@ -340,6 +409,30 @@ let csv = function(val) { return list; }; +let parse_vlans = function(ann) { + let list = [], vids = [], tok, m, vid, tagged, i; + + for (tok in csv(ann["org.openwrt.network.vlans"])) { + m = match(tok, /^([0-9]+)(:t)?$/); + vid = m ? +m[1] : 0; + if (vid < 1 || vid > 4094) { + warn(sprintf("uxc-net: bad org.openwrt.network.vlans entry '%s'\n", tok)); + return null; + } + tagged = m[2] != null; + i = index(vids, vid); + if (i >= 0 && list[i].tagged == tagged) + continue; + if (i >= 0) { + warn(sprintf("uxc-net: VLAN %d is listed both tagged and untagged in org.openwrt.network.vlans\n", vid)); + return null; + } + push(vids, vid); + push(list, { vid: vid, tagged: tagged }); + } + return list; +}; + let zone_of = function(token) { let m = match(token, /^(vpn|container):([A-Za-z0-9_.]+)$/); if (m) @@ -651,9 +744,13 @@ let bridged_up = function(name, ann, attach, m, spec, shared, owned) { let vh = host_veth(name), vc = cont_veth(name); let proto = injail_proto(ann, "bridged"), opts = {}; let section6 = injail6_section(ann); - let status, br; + let vlans = parse_vlans(ann); + let vlan_sections, on_demand, status, br; - if (!proto || section6 == null) + if (!proto || section6 == null || vlans == null) + return 1; + vlan_sections = injail_vlan_sections(ann, vlans); + if (vlan_sections == null) return 1; if (proto == "static") { opts = bridged_static_opts(ann); @@ -664,9 +761,15 @@ let bridged_up = function(name, ann, attach, m, spec, shared, owned) { if (ann["org.openwrt.network.egress"] || ann["org.openwrt.network.ingress"]) warn("uxc-net: bridged takes no egress/ingress (L2 inherits the joined network's zone); ignoring\n"); + on_demand = shared.interfaces[net] != null || index(owned, net) >= 0 || !uci_network_exists(net); + if (on_demand && length(vlans)) { + warn(sprintf("uxc-net: network '%s' is created on demand and has no VLANs to attach to\n", net)); + return 1; + } + if (shared.interfaces[net]) { shared_add_port(shared, "br-" + net, vh); - } else if (index(owned, net) >= 0 || !uci_network_exists(net)) { + } else if (on_demand) { if (autonet_create(shared, net, vh)) return 1; } else { @@ -675,7 +778,7 @@ let bridged_up = function(name, ann, attach, m, spec, shared, owned) { warn(sprintf("uxc-net: network '%s' is not known to netifd\n", net)); return 1; } - br = bridge_of(net, status); + br = bridge_of(net, status, vlans); if (!br) return 1; spec.bridge_ports[vh] = br; @@ -687,6 +790,8 @@ let bridged_up = function(name, ann, attach, m, spec, shared, owned) { append_injail(name, render_section("lan", "eth0", proto, opts)); if (section6 != "") append_injail(name, section6); + if (vlan_sections != "") + append_injail(name, vlan_sections); return 0; };