From 97f1d02abdf9a41f6f2666f2acf1949389ab6d22 Mon Sep 17 00:00:00 2001 From: Celrenheit Date: Tue, 28 Jul 2026 21:53:34 +0000 Subject: [PATCH] feat(linux): run the firecracker provider without sudo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Booting a sandbox on Linux took privilege twice per boot, and issue #9 was right that both asks were too large. Creating the bridge and TAPs shelled out to `sudo ip`, where a NOPASSWD entry hands over the host's entire network configuration to save one password prompt. Injecting the worktree loop-mounted the rootfs through `sudo clawk __loop-mount`, a root helper that passed its argv straight to syscall.Mount — the widest privilege in the tree, since anyone who could run it could mount any image anywhere. Neither is necessary. Networking: the kernel hands out CAP_NET_ADMIN for free inside a user namespace you own. The daemon forks an anchor into CLONE_NEWUSER|CLONE_NEWNET with a single-uid mapping (no /etc/subuid, no setuid helper — clawk wants capabilities over its own network, never over other users' files), builds br0 + tap0 + gv0 there with netlink, and passes the gvproxy-side TAP fd back over SCM_RIGHTS. gvproxy stays in the host namespace, where its egress sockets belong; only the VM's NIC moves, entered via nsenter because setns into a netns also demands CAP_SYS_ADMIN in the owning userns. Such a boot performs no privileged operation at all and leaves the host's interface list empty, and the VM loses its view of host networking — a real isolation gain. The anchor blocks on a pipe the daemon holds, so the namespace and every device in it dies with the VM: nothing to leak, nothing to clean up after a crash. Worktree: it becomes its own ext4 disk, built in userspace by the same writer that already builds every rootfs, and clawk-init mounts it from /dev/vdc via an additive Block/FSType transport on manifest mounts. The guest still sees /workspace/, so nothing downstream changes, and the loop-mount helper is deleted outright. Hosts that forbid unprivileged user namespaces (Ubuntu 24.04+ via AppArmor, or a zeroed sysctl) fall back to bridge mode. The CLI provisions those devices in the foreground, while it still has a terminal to authenticate on — the daemon never has one — so sudo prompts at most once per sandbox and never per boot. The mode is decided once and pinned for the daemon, with the reason travelling alongside it so the daemon log names the real cause. doctor reports which mode a host will use and the setting to change; CLAWK_NET_MODE pins either. Also here: one L2 segment and guest MAC per sandbox, so concurrent VMs sharing gvproxy's 192.168.127.2 stop colliding; every host device name scoped by uid, so two users' identically-named sandboxes cannot touch each other's devices; the in-guest binaries shipped prebuilt, so an installed clawk boots a sandbox with no Go toolchain present; the `vm ( kernel … )` override honoured instead of silently dropped; a Linux quick start; and CI publishing linux/amd64 and arm64 binaries. Verified on an x86_64 KVM host: rootless and bridge boots, snapshot/resume, worktree ownership, egress filtering, concurrent sandboxes, and a privilege matrix across four users — no sudo, NOPASSWD for ip only, and password sudo. A user with no sudo rights at all boots rootless with no host devices. --- .github/workflows/ci.yml | 21 + .github/workflows/release.yml | 77 ++- .gitignore | 5 + ARCHITECTURE.md | 5 +- CHANGELOG.md | 70 +++ Makefile | 24 +- README.md | 12 +- cmd/clawk/main.go | 7 +- docs/commands.md | 56 ++- docs/linux-quickstart.md | 249 +++++++++ go.mod | 2 + go.sum | 4 +- internal/agentembed/init_main.go.in | 36 +- internal/agentembed/prebuilt.go | 99 ++++ internal/agentembed/prebuilt/.gitignore | 4 + internal/agentembed/prebuilt/README.md | 14 + internal/cli/daemon_failure_test.go | 67 +++ internal/cli/doctor.go | 34 +- internal/cli/doctor_linux.go | 54 ++ internal/cli/fcd.go | 16 +- internal/cli/here.go | 50 ++ internal/guestbuild/cmd/gen-prebuilt/main.go | 47 ++ internal/guestbuild/guestbuild.go | 40 +- internal/guestbuild/guestbuild_test.go | 12 +- internal/guestbuild/prebuilt.go | 161 ++++++ internal/guestbuild/prebuilt_test.go | 94 ++++ internal/guestcfg/blockmount_test.go | 44 ++ internal/guestcfg/manifest.go | 11 + internal/sandbox/console.go | 12 +- internal/sandbox/fdpass_linux.go | 101 ++++ internal/sandbox/firecracker_linux.go | 379 +++++++++++--- internal/sandbox/firecracker_linux_test.go | 87 ++++ internal/sandbox/linux_shared.go | 255 +++++++--- internal/sandbox/loop_mount_linux.go | 133 ----- internal/sandbox/loop_mount_stub.go | 7 - internal/sandbox/netmode_linux.go | 135 +++++ internal/sandbox/netns_linux.go | 500 +++++++++++++++++++ internal/sandbox/netns_linux_test.go | 224 +++++++++ internal/sandbox/netns_stub.go | 15 + internal/sandbox/netstate_linux.go | 98 ++++ internal/sandbox/netstate_linux_test.go | 132 +++++ internal/sandbox/sudo_linux_test.go | 64 +++ machine/firecracker/firecracker_linux.go | 44 +- machine/firecracker/usermode_linux.go | 37 ++ machine/internal/ext4/tar2ext4.go | 20 +- machine/internal/ext4/tar2ext4_test.go | 63 ++- machine/internal/usermode/guestmac_test.go | 52 ++ machine/internal/usermode/usermode_unix.go | 43 +- machine/oci/dirdisk.go | 171 +++++++ machine/oci/dirdisk_test.go | 144 ++++++ machine/spec.go | 22 + machine/vz/vz_darwin.go | 1 + 52 files changed, 3728 insertions(+), 326 deletions(-) create mode 100644 docs/linux-quickstart.md create mode 100644 internal/agentembed/prebuilt.go create mode 100644 internal/agentembed/prebuilt/.gitignore create mode 100644 internal/agentembed/prebuilt/README.md create mode 100644 internal/cli/daemon_failure_test.go create mode 100644 internal/guestbuild/cmd/gen-prebuilt/main.go create mode 100644 internal/guestbuild/prebuilt.go create mode 100644 internal/guestbuild/prebuilt_test.go create mode 100644 internal/guestcfg/blockmount_test.go create mode 100644 internal/sandbox/fdpass_linux.go create mode 100644 internal/sandbox/firecracker_linux_test.go delete mode 100644 internal/sandbox/loop_mount_linux.go delete mode 100644 internal/sandbox/loop_mount_stub.go create mode 100644 internal/sandbox/netmode_linux.go create mode 100644 internal/sandbox/netns_linux.go create mode 100644 internal/sandbox/netns_linux_test.go create mode 100644 internal/sandbox/netns_stub.go create mode 100644 internal/sandbox/netstate_linux.go create mode 100644 internal/sandbox/netstate_linux_test.go create mode 100644 internal/sandbox/sudo_linux_test.go create mode 100644 machine/internal/usermode/guestmac_test.go create mode 100644 machine/oci/dirdisk.go create mode 100644 machine/oci/dirdisk_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de6fbb2..1a50fff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,27 @@ jobs: go vet ./... go test ./... + # The release path, which the tests above can't reach: a plain checkout + # embeds no guest binaries, so the prebuilt tests skip themselves. Run + # the generator, then assert a sandbox can be prepared with no Go + # toolchain — the whole point of shipping them prebuilt. + - name: prebuilt guest binaries — generate + use without a toolchain + run: | + make guestbin + go test ./internal/guestbuild/ -run 'TestBuildUsesPrebuilt|TestPrebuiltRejectsStaleSources' -v + + # Cross-build both release artifacts, so a tag can't be the first time we + # discover the Linux release doesn't compile. + - name: cross-build linux release artifacts + run: | + for arch in amd64 arm64; do + make clean-guestbin + make guestbin GUEST_ARCH="$arch" + CGO_ENABLED=0 GOOS=linux GOARCH="$arch" \ + go build -trimpath -o "/tmp/clawk-linux-$arch" ./cmd/clawk + ls -l "/tmp/clawk-linux-$arch" + done + macos: runs-on: macos-14 # Apple Silicon — required for the vz provider steps: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index aa96914..94e027f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,8 +1,15 @@ name: release # Cut a release by pushing a tag: `git tag v0.1.0 && git push origin v0.1.0`. -# Builds the macOS/arm64 binary, publishes it as a GitHub release, and (if -# HOMEBREW_TAP_TOKEN is set) bumps the Homebrew tap formula to match. +# Builds the macOS/arm64 and Linux (amd64 + arm64) binaries, publishes them as +# one GitHub release, and (if HOMEBREW_TAP_TOKEN is set) bumps the Homebrew tap +# formula to the macOS tarball. +# +# Every artifact runs `make guestbin` first, embedding the in-guest binaries so +# an installed clawk needs no Go toolchain to boot a sandbox. GUEST_ARCH is the +# artifact's OWN architecture: hardware virtualization can't cross +# architectures, so a linux/amd64 clawk only ever boots amd64 guests and needs +# exactly one set. on: push: tags: @@ -12,9 +19,69 @@ permissions: contents: write jobs: + # Create the release once, before anything uploads into it. + # + # The three artifact builds below (two Linux matrix legs plus macOS) stay + # deliberately independent of each other — a failure in one shouldn't + # withhold the others' binaries — but they cannot each CREATE the release: on + # a tag with no release yet they race to POST /releases, and the loser fails + # with 422 already_exists, dropping an architecture's tarball from a release + # that built fine. Doing it here also pins the generated notes to one + # deterministic run instead of whichever job happened to win. + create-release: + runs-on: ubuntu-latest + steps: + - uses: softprops/action-gh-release@v2 + with: + generate_release_notes: true + + # The Linux artifacts the firecracker provider needs. Pure Go — no cgo, no + # signing — so both architectures cross-compile from one runner. + linux: + needs: create-release + runs-on: ubuntu-latest + strategy: + matrix: + goarch: [amd64, arm64] + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: "1.26" + + - name: Prebuild guest binaries (linux/${{ matrix.goarch }}) + run: make guestbin GUEST_ARCH=${{ matrix.goarch }} + + - name: Build clawk (linux/${{ matrix.goarch }}) + env: + CGO_ENABLED: "0" + GOOS: linux + GOARCH: ${{ matrix.goarch }} + run: go build -trimpath -o clawk ./cmd/clawk + + - name: Package tarball + id: pkg + run: | + tarball="clawk-${GITHUB_REF_NAME}-linux-${{ matrix.goarch }}.tar.gz" + # No clawk.entitlements: codesigning is macOS-only. + tar czf "$tarball" clawk LICENSE + sha=$(sha256sum "$tarball" | awk '{print $1}') + echo "tarball=$tarball" >> "$GITHUB_OUTPUT" + { + echo "### clawk ${GITHUB_REF_NAME} (linux/${{ matrix.goarch }})" + echo "\`sha256: ${sha}\`" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload release asset + uses: softprops/action-gh-release@v2 + with: + files: ${{ steps.pkg.outputs.tarball }} + release: # Virtualization.framework is cgo/Objective-C: the binary MUST be built # on macOS arm64 — it cannot be cross-compiled from Linux. + needs: create-release runs-on: macos-14 env: TAP_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }} @@ -25,6 +92,9 @@ jobs: with: go-version: "1.26" + - name: Prebuild guest binaries (linux/arm64) + run: make guestbin GUEST_ARCH=arm64 + - name: Build clawk (darwin/arm64) run: go build -trimpath -o clawk ./cmd/clawk @@ -44,11 +114,10 @@ jobs: echo "\`sha256: ${sha}\`" } >> "$GITHUB_STEP_SUMMARY" - - name: Publish GitHub release + - name: Upload release asset uses: softprops/action-gh-release@v2 with: files: ${{ steps.pkg.outputs.tarball }} - generate_release_notes: true # Rewrites version/url/sha256 in the tap's Formula/clawk.rb and pushes. # Skipped automatically when HOMEBREW_TAP_TOKEN isn't configured — in diff --git a/.gitignore b/.gitignore index 565936b..7ec5eaa 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,8 @@ debug-logs/ # Local Claude Code settings and skills (personal, not shared) .claude/settings.local.json .claude/skills/ + +# Working notes: implementation plans, scratch write-ups. Kept out of the +# repo because they age into fiction the moment the code lands — the design +# rationale that matters belongs in the code and in docs/. +.plans/ diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 08298e4..9650f77 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -105,8 +105,9 @@ because it pins a vendored `gvisor-tap-vsock` fork; everything clawk-specific |--|----|-------------| | Host | macOS (Apple silicon) | Linux | | Hypervisor | Virtualization.framework (cgo, Code-Hex/vz) | `firecracker` + `/dev/kvm` | -| NIC | file-handle NIC speaking gvproxy's datagram protocol | host TAP, bridged to gvproxy by the frame pump | -| Worktree | virtio-fs live mount (`/home/agent/workspace`) | baked into the rootfs at create (`/workspace`); host edits don't propagate | +| NIC | file-handle NIC speaking gvproxy's datagram protocol | TAP, bridged to gvproxy by the frame pump | +| Network devices | none on the host | bridge + 2 TAPs, in a per-sandbox unprivileged user+netns (rootless, default) or on the host via sudo (fallback) | +| Worktree | virtio-fs live mount (`/home/agent/workspace`) | its own ext4 disk built in userspace, mounted at `/workspace`; host edits don't propagate | The split is `//go:build`-gated and total: a Linux checkout can't compile the vz code and vice versa. Most of clawk is shared; only the hypervisor glue is diff --git a/CHANGELOG.md b/CHANGELOG.md index ae9badc..6fbb27a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,35 @@ tagged. ### Added +- **Linux firecracker sandboxes boot without sudo.** Each sandbox's network + now lives in its own unprivileged user + network namespace, where clawk has + `CAP_NET_ADMIN` over its own bridge and TAPs without asking the host for + anything — so a boot performs no privileged operation, and no clawk + interfaces appear on the host at all. gvproxy stays in the host namespace + (that is where its egress sockets belong); only the VM's NIC moves, which + also means the VM no longer sees host networking. + + Hosts that forbid unprivileged user namespaces fall back to the previous + sudo-based bridge mode. `clawk doctor` reports which mode is active under + `host: network mode` and names the setting to change (on Ubuntu 24.04+, + AppArmor's `kernel.apparmor_restrict_unprivileged_userns`). + `CLAWK_NET_MODE=rootless|bridge` pins one. + +- **Release binaries no longer need a Go toolchain.** Booting a sandbox + compiled the three in-guest binaries on the host first, which made `go` a + hard prerequisite on every platform — `clawk doctor` failed without it — and + put a module download in the first-boot path. Release artifacts now embed + those binaries (`make guestbin`), so an installed clawk just unpacks them. + Building from source still compiles them on first boot, which is what a + contributor editing the guest sources wants; `doctor` reports the toolchain as + required only in that case. Costs ~7 MiB of binary size per artifact. + +- **Linux release binaries.** `linux/amd64` and `linux/arm64` tarballs are built + and published alongside the macOS one, so the firecracker provider no longer + requires building from source. Each artifact embeds the in-guest binaries for + its own architecture — guest arch always equals host arch, since hardware + virtualization can't cross architectures. + - **`env ( … )` gains aliases, defaults, and literals.** Entries now use shell / docker-compose parameter-expansion syntax: `NAME = ${HOST}` aliases a differently-named host variable, `${HOST:-default}` / `${HOST-default}` @@ -17,8 +46,49 @@ tagged. or quoted right-hand side (`EDITOR = vim`) sets a literal constant. Bare `NAME` passthrough is unchanged. +### Changed + +- **The worktree rides in on its own disk instead of being copied into the + rootfs.** Staging it used to loop-mount the rootfs as root — six privileged + operations per boot, plus a `sudo clawk __loop-mount` helper that mounted + whatever it was told. It is now built as a separate ext4 disk in userspace + and mounted by the guest from `/dev/vdc`. Same semantics (host edits still + don't propagate into a running guest); no privilege, and the root mount + helper is gone. + ### Fixed +- **Two firecracker sandboxes can run at once.** Every guest NIC used the same + hardcoded MAC and every TAP hung off one shared host bridge, so two guests + claimed one L2 identity on one segment — the second sandbox logged an IPv6 + duplicate-address error and the bridge's forwarding table decided which + guest received what. Guest MACs are now derived per VM and each sandbox gets + its own bridge (keyed on uid, so users don't share segments either). + + **Upgrading:** every host device name now carries the invoking uid, TAPs + included. Sandboxes created by an earlier clawk have differently-named + devices, so the first `clawk up` after upgrading reports the expected device + as missing — `clawk down && clawk up` re-provisions it. The old devices are + no longer named by anything and outlive `clawk destroy`; remove them with + `sudo ip link del clawk` (they are listed by `ip link | grep clawk`). +- **A worktree disk that fails to mount now fails the boot.** The guest logged + the error to its console and carried on, so a sandbox whose disk was + unreadable came up with an empty directory where the repo should be, + answered the agent, and reported success — an agent could then run a whole + session against nothing. Boots that would have been silently empty now stop + with `clawk-init: FATAL: mount /dev/vdc …` on the console. +- **First-boot failures on Linux say what went wrong.** The network setup ran + inside the detached VM daemon, which has no terminal — so on a host where + sudo needs a password it could never succeed, and its error went to a log + the CLI then deleted during rollback, leaving only "agent did not become + ready". Setup moved to the CLI (one prompt, in front of you), `clawk doctor` + gained a check for it, and boot failures now quote the daemon log. + ([#9](https://github.com/clawkwork/clawk/issues/9)) +- **A snapshot no longer risks a corrupted filesystem on restore.** `clawk up` + re-materialized the rootfs on every boot, including boots that were about to + restore a suspend state onto it — pairing a saved memory image with a disk + that had just been reset. Disks are left alone when a restorable state is + present. - **Forwarded `env ( … )` vars now reach the agent user.** The generated `/etc/profile.d/99-clawk-env.sh` was written `0600 root:root`, so the agent's login shells silently skipped it and the variables never arrived. diff --git a/Makefile b/Makefile index 4b1a404..ea56e95 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ BIN ?= ./bin/clawk ENTITLEMENTS := clawk.entitlements UNAME_S := $(shell uname -s) -.PHONY: install build sign test clean +.PHONY: install build sign test clean guestbin clean-guestbin install: go install ./cmd/clawk @@ -47,5 +47,25 @@ endif test: go test ./... -clean: +# Cross-compile the in-guest binaries into internal/agentembed/prebuilt/ so the +# next `go build` embeds them and the resulting clawk needs no Go toolchain to +# boot a sandbox. Release builds run this first; source builds skip it and +# compile the guest binaries on first boot instead. +# +# GUEST_ARCH must match the architecture the artifact will RUN on: hardware +# virtualization can't cross architectures, so an arm64 clawk only boots arm64 +# guests. Defaults to this machine's. +GUEST_ARCH ?= $(shell go env GOARCH) + +guestbin: + go run ./internal/guestbuild/cmd/gen-prebuilt -arch $(GUEST_ARCH) + +# Drop the embedded payload, restoring a plain source build. +clean-guestbin: + rm -f $(PREBUILT_DIR)/clawk-init $(PREBUILT_DIR)/clawk-pty-agent \ + $(PREBUILT_DIR)/clawk-time-sync $(PREBUILT_DIR)/manifest.json + +PREBUILT_DIR := internal/agentembed/prebuilt + +clean: clean-guestbin rm -rf ./bin diff --git a/README.md b/README.md index 10e154c..f86edec 100644 --- a/README.md +++ b/README.md @@ -122,9 +122,9 @@ not a sandboxed process." ## Install Requires macOS 14+ on Apple silicon. (Linux is supported via firecracker and -currently experimental; see -[VM providers](docs/commands.md#vm-providers) for the gaps. This README is -macOS-first.) +currently experimental — start with +**[docs/linux-quickstart.md](docs/linux-quickstart.md)**, which covers setup, +the workflow, and the gaps. This README is macOS-first.) ```sh brew install clawkwork/tap/clawk @@ -138,8 +138,10 @@ make install ``` Either way there's no extra host tooling: no Docker, no qemu, no sudo. The -hypervisor is Apple's Virtualization.framework, linked into the binary. First -run probes for anything missing and offers to fix it. +hypervisor is Apple's Virtualization.framework, linked into the binary, and the +release binaries carry the in-guest agent prebuilt — so a Go toolchain is only +needed if you build from source, in which case you have one. First run probes +for anything missing and offers to fix it. **Uninstall:** `clawk destroy` your sandboxes, `rm -rf ~/.clawk`, then remove the binary with `brew uninstall clawk` (or delete it from `$GOBIN` for a diff --git a/cmd/clawk/main.go b/cmd/clawk/main.go index 840d248..02ceacd 100644 --- a/cmd/clawk/main.go +++ b/cmd/clawk/main.go @@ -10,9 +10,10 @@ import ( ) func main() { - // Privileged mount/umount re-exec path — runs before cobra because these - // hidden subcommands exist only for `sudo $self __loop-mount/-unmount`. - sandbox.InitRootHelpers() + // Network-namespace helper re-execs (rootless mode) — before cobra, + // because these are the clawk binary acting as its own helper, not CLI + // verbs, and they must not touch the config store or flag parsing. + sandbox.InitNetNSHelpers() err := cli.Execute() if err == nil { return diff --git a/docs/commands.md b/docs/commands.md index 15d625a..b3f4e0d 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -131,7 +131,7 @@ without notice. | Provider | Host | Notes | |--------------------|-------|-----------------------------------------------------------------------| | `vz` (default) | macOS | Apple Virtualization.framework; no sudo. Live-mounts your worktree. | -| `firecracker` (experimental) | Linux | KVM microVM. Bakes the worktree into the rootfs (host edits don't propagate live), and skips host-file push, ssh-agent forwarding, and per-phase hooks today. | +| `firecracker` (experimental) | Linux | KVM microVM; no sudo on hosts that allow unprivileged user namespaces. Carries the worktree on its own disk (host edits don't propagate live), and skips host-file push, ssh-agent forwarding, and per-phase hooks today. | Pick one with `--provider`; the choice persists with the sandbox. Both run the same OCI rootfs, vsock agent, and egress allow-list — see @@ -139,6 +139,10 @@ same OCI rootfs, vsock agent, and egress allow-list — see ### Linux prerequisites (firecracker) +> New to clawk on Linux? **[linux-quickstart.md](linux-quickstart.md)** walks +> the whole path — setup, first run, the workflow, and what isn't there yet. +> This section is the reference detail. + The firecracker provider needs three things on the host; `clawk doctor` checks all of them: @@ -153,10 +157,46 @@ checks all of them: Without this, boots fail with firecracker's opaque `Permission denied (os error 13) ... /dev/kvm file's ACL`; `clawk up` now surfaces the kvm-group fix directly. -- **A Go toolchain** — used to cross-compile the tiny in-guest binaries on - first boot. Any `go` ≥ 1.21 works (older toolchains auto-download the one - the guest modules pin via `GOTOOLCHAIN=auto`). - -`clawk` shells out to `sudo` for the host bridge/TAP plumbing (`ip link`, -`ip tuntap`) and to stage the worktree into the rootfs at create — never to -run the VM itself, which firecracker does unprivileged against `/dev/kvm`. +- **`nsenter`** (from `util-linux`, already installed on mainstream distros) — + used by rootless mode to launch the VM inside its network namespace. Missing + it isn't fatal: clawk falls back to bridge mode and says so. + +A **Go toolchain** is not among them. A release binary carries the tiny +in-guest binaries prebuilt, and `clawk doctor` says so +(`host: go toolchain — not needed`). It's a prerequisite only for a clawk built +from source, which cross-compiles them on first boot; any `go` ≥ 1.21 works +there (older toolchains auto-download the one the guest modules pin via +`GOTOOLCHAIN=auto`). + +#### Privileges: rootless by default, sudo only as a fallback + +Creating a VM's network devices (a bridge and two TAPs) needs +`CAP_NET_ADMIN`. clawk gets it without sudo by running each sandbox's network +in its own **unprivileged user + network namespace**: inside a namespace you +own, you are root over your own network. Nothing else needs privilege — +firecracker itself runs unprivileged against `/dev/kvm`, and the worktree disk +is built in userspace. So on a host that allows unprivileged user namespaces, +a sandbox boots with **no privileged operation at all**, and no clawk +interfaces appear on the host. + +`clawk doctor` reports which mode is in use under `host: network mode`: + +- **`rootless`** — the default. Nothing to configure; the namespace (and every + device in it) is created and destroyed with the VM. +- **`bridge mode via sudo`** — the fallback for hosts that forbid unprivileged + user namespaces. Ubuntu 24.04+ is the common case: it blocks them via + AppArmor. Getting rootless mode there takes one root action, once: + `sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0` (persist it + in `/etc/sysctl.d/` if you want it to survive a reboot) — verified. The + narrower alternative is an AppArmor profile for the clawk binary granting + `userns,`, which keeps the restriction on for everything else; that route is + documented by Ubuntu but has not been tested here. In bridge mode clawk + creates host devices with + `sudo ip`: it prompts at most **once per sandbox** (never per boot — later + boots find the devices configured and touch no privilege), and it prompts in + the foreground rather than inside the background VM daemon, which has no + terminal to authenticate on. `NOPASSWD` for `ip` avoids the prompt entirely, + at the cost of granting network reconfiguration. + +Pin a mode with `CLAWK_NET_MODE=rootless|bridge` — useful to assert that no +sudo will ever be attempted (`rootless` fails loudly instead of falling back). diff --git a/docs/linux-quickstart.md b/docs/linux-quickstart.md new file mode 100644 index 0000000..5127ebe --- /dev/null +++ b/docs/linux-quickstart.md @@ -0,0 +1,249 @@ +# Linux quick start (firecracker) + +From nothing to an agent running in a microVM, on Linux, with an honest +account of what works today and what doesn't. + +The Linux provider is **experimental**. It boots the same guest stack as macOS +— OCI rootfs, `clawk-init` as PID 1, a vsock agent, gvproxy with the egress +allow-list — on firecracker instead of Apple's hypervisor. It is genuinely +usable for running an agent against a snapshot of your code. It is *not* yet a +replacement for the macOS workflow — read [§5 Limits](#5-limits) before you +plan a day around it. + +--- + +## 1. Prerequisites + +Three things, then `clawk doctor` tells you if you missed one. + +**The `firecracker` binary on `PATH`.** Grab a +[release](https://github.com/firecracker-microvm/firecracker/releases) (tested +against v1.12) and drop it in `/usr/local/bin`. + +**Read/write on `/dev/kvm`.** The one unavoidable root action: + +```sh +sudo usermod -aG kvm "$USER" # then log out and back in, or: newgrp kvm +``` + +**`util-linux`** — for `nsenter`, used to put the VM in its own network +namespace. Present on every mainstream distro already. + +No Go toolchain, no Docker, no qemu: the release binary carries the in-guest +agent prebuilt, and firecracker is a single static binary. + +### Install + +Grab the tarball for your architecture from the +[releases](https://github.com/clawkwork/clawk/releases) (there's no Homebrew tap +for Linux, and nothing to code-sign — that's a macOS requirement): + +```sh +tar xzf clawk-vX.Y.Z-linux-amd64.tar.gz # or -linux-arm64 +sudo install -m755 clawk /usr/local/bin/ +``` + +**From source** instead, if you're contributing — this one *does* need Go 1.26+, +and compiles the in-guest binaries on first boot rather than embedding them: + +```sh +git clone https://github.com/clawkwork/clawk && cd clawk +make install # → $GOBIN/clawk (or $GOPATH/bin/clawk) +``` + +--- + +## 2. Check the host + +```sh +clawk doctor # no arguments = host-level checks +``` + +On a host that can do rootless networking you want six OK lines — including +`host: go toolchain — not needed`, which is how a release binary reports that it +carries the in-guest agent. (A source build says the toolchain is required, +because it compiles that agent on first boot.) The one worth reading closely is +the last: + +``` +[OK] host: network mode — rootless (per-sandbox network namespace; no privileged operations) +``` + +That means **clawk will never ask for a password**. Creating a bridge and TAPs +needs `CAP_NET_ADMIN`, and clawk gets it by running each sandbox's network +inside an unprivileged user namespace it owns — no sudo, and no clawk +interfaces on your host at all. + +On **Ubuntu 24.04+** you'll see this instead — a `WARN`, not an `OK`, because +sudo will want a password at least once: + +``` +[WARN] host: network mode — bridge mode via sudo — rootless networking + unavailable: unprivileged user namespaces are blocked by AppArmor + (kernel.apparmor_restrict_unprivileged_userns=1); ... — sudo will + prompt once per sandbox, when it creates the devices +``` + +(With `NOPASSWD` for `ip`, the same host reports `OK` instead; with no terminal +to prompt on, `FAIL`.) + +Ubuntu blocks unprivileged user namespaces by default. Two choices: + +- **Get rootless mode** (one root action, once): + ```sh + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + # to persist across reboots: + echo 'kernel.apparmor_restrict_unprivileged_userns=0' | \ + sudo tee /etc/sysctl.d/99-clawk-userns.conf + ``` + (A narrower alternative is an AppArmor profile granting `userns,` for the + clawk binary only, which leaves the restriction on for everything else. + Ubuntu documents that route; it hasn't been tested here.) + +- **Stay in bridge mode.** It works fine. clawk creates host devices with + `sudo ip`, prompting **at most once per sandbox** — never per boot; later + boots find the devices already configured and touch no privilege. If the + prompt is the problem and you accept the trade, `NOPASSWD` for `ip` alone + removes it. + +Pin either mode with `CLAWK_NET_MODE=rootless|bridge`. Pinning `rootless` is +useful as an assertion: it fails loudly rather than silently falling back to +something that might prompt. + +--- + +## 3. Authenticate once + +Sandboxes get the Claude token from the host. Do this once: + +```sh +claude setup-token # on the HOST — generates a long-lived token +clawk auth set-token # paste it (interactive, echo off); or: set-token < token.txt +clawk auth status +``` + +Use this rather than relying on `~/.claude/.credentials.json`: that file holds +a *rotating* refresh token, and several sandboxes refreshing the same one +knock each other out. Details in [claude-auth.md](claude-auth.md). + +--- + +## 4. Run something + +```sh +cd ~/code/my-project +clawk +``` + +That creates a sandbox keyed to the directory, boots it (~7 s once the image +and kernel are cached; the first ever run downloads both), and attaches Claude +Code inside it. Leave the agent however you normally would — the VM keeps +running, and clawk prints how to get back in. + +Everyday verbs: + +```sh +clawk # re-attach to this directory's sandbox +clawk run shell # a plain login shell in the guest, instead of the agent +clawk status # what's running, memory, network policy +clawk list # every sandbox +clawk snapshot # freeze to disk (see §5 — this is how you keep work) +clawk resume # continue exactly where it left off +clawk destroy # throw it away +``` + +Networking and port forwards work as documented for macOS: + +```sh +clawk forward add 3000 # localhost:3000 on the host → guest:3000 + # (or 8080:80 to map across ports) +clawk network allow api.example.com +clawk network denials # what the guest tried to reach and was refused +``` + +The egress allow-list is enforced in gvproxy on the host, so the guest cannot +turn it off from the inside — that holds in rootless mode too (verified: a +blocked host is refused and logged, `acl: denied example.com`). + +--- + +## 5. Limits + +**Your worktree is copied in, not shared.** firecracker has no virtio-fs, and +the kernel clawk downloads for it has no 9p, so the worktree is built into its +own ext4 disk at boot and mounted at `/workspace/`. Consequences: + +- Host edits made while the VM runs **do not** appear in the guest. +- Guest edits **do not** appear on the host. + +**`down` + `up` discards in-guest changes. `snapshot` + `resume` keeps them.** +This is the most important thing to know. Measured, not theorised: + +| You run | In-guest changes | +|---|---| +| detach / re-attach | **kept** (the VM never stopped) | +| `clawk snapshot` → `clawk resume` | **kept** | +| `clawk down` → `clawk up` | **lost** — the disk is rebuilt from your host worktree | +| `clawk destroy` | gone, obviously | + +So on Linux today: snapshot when you step away, and don't `down` a sandbox +holding work you care about. + +**Nothing carries the guest's commits back to the host.** There's no ssh-agent +forwarding on Linux, so the agent can't `git push` with your host keys, and no +sync back to your worktree — so `clawk pr` would open a PR on a branch that +never received the work. `clawk run shell` gets you inside to read or salvage +results, and a guest can push over HTTPS with a token if your allow-list +permits the host; but neither is a workflow yet. Treat Linux sandboxes as +places whose *output you read* — analysis, review, exploration, test runs. + +**Agent history lives in the VM, not on the host.** On macOS, Claude's +`projects/` and `memory/` are host directories mounted into the guest, so a +conversation survives `clawk destroy`. firecracker mounts only the worktree +disk, so on Linux that history is inside the disposable disk and follows the +same table above — the "conversation memory persists across destroys" promise +in `clawk --help` is macOS-only today. + +**No idle stop.** The daemon can't observe client sessions on firecracker, so a +sandbox you forget about keeps its RAM until you `snapshot`, `down` or +`destroy` it. On macOS it would park itself after 30 minutes. + +**Also not wired up on Linux yet:** the `files ( … )` host-file push, per-phase +`on up` hooks, and toolchain cache shares. `clawk debug vshell` doesn't work +either (it looks for the macOS socket layout) — use `clawk run shell`. + +Most of the above has the same fix: mount the worktree live over 9p-over-vsock, +as macOS does over virtio-fs. firecracker itself has no virtio-fs device (its +device model is deliberately minimal — net, block, vsock, balloon, rng) and the +firecracker-CI kernel it boots has neither transport. But clawk *already* +publishes a Kata-based guest kernel with 9p for both architectures +(`vmlinux-amd64` / `vmlinux-arm64`, see `images/guest-kernel/`), and +`internal/ninep` already serves 9p over vsock for macOS — so this is wiring the +firecracker path to what exists, not new infrastructure. Until then, the disk +semantics above are what you get. + +--- + +## 6. When something breaks + +| Symptom | Meaning | +|---|---| +| `host: /dev/kvm — missing` | no KVM: enable virtualization in BIOS, or you're in a VM without nested virt | +| `host: /dev/kvm — permission denied` | not in the `kvm` group yet, or you didn't start a new login session | +| `network mode … no terminal to prompt on` | bridge mode, password-sudo, and a non-interactive shell — run it from a terminal, or enable rootless, or `NOPASSWD` for `ip` | +| `rootless unavailable: nsenter is not on PATH` | install `util-linux` | +| `rootless unavailable: /dev/net/tun is not usable` | unusual perms on the device; most distros ship it world-accessible | +| `agent did not become ready` | the guest didn't boot. The error quotes both the daemon log and the guest console — read those first | +| `host: legacy bridge` warning | a leftover `clawkbr0` from an older clawk; `sudo ip link del clawkbr0` | +| `host network device … is missing` | bridge mode, and something removed the devices under a running sandbox: `clawk down && clawk up` | + +Everything a sandbox owns lives under `~/.clawk/namespaces//vms//`: +`fcd.log` (host side — the VM daemon, gvproxy, the allow-list) and +`console.log` (guest side — kernel and `clawk-init`). Those two files answer +most questions. + +--- + +Provider differences in one table: [commands.md](commands.md#vm-providers). +Network policy: [networking.md](networking.md). Images and kernels: +[images.md](images.md). diff --git a/go.mod b/go.mod index 04c0e98..0c78411 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/prashantgupta24/mac-sleep-notifier v1.0.1 github.com/spf13/cobra v1.10.2 github.com/stretchr/testify v1.11.1 + github.com/vishvananda/netlink v1.3.1 golang.org/x/term v0.43.0 ) @@ -56,6 +57,7 @@ require ( github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/vbatts/tar-split v0.11.3 // indirect + github.com/vishvananda/netns v0.0.5 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect golang.org/x/exp v0.0.0-20231219180239-dc181d75b848 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 8ebf8f0..47974e2 100644 --- a/go.sum +++ b/go.sum @@ -38,8 +38,6 @@ github.com/charmbracelet/x/windows v0.2.2 h1:IofanmuvaxnKHuV04sC0eBy/smG6kIKrWG2 github.com/charmbracelet/x/windows v0.2.2/go.mod h1:/8XtdKZzedat74NQFn0NGlGL4soHB0YQZrETF96h75k= github.com/clawkwork/gvisor-tap-vsock v0.8.9-1 h1:bbxRTiyD2PhRdGgUO78Y4iEFatw4rdasHTdoVjgjxcw= github.com/clawkwork/gvisor-tap-vsock v0.8.9-1/go.mod h1:jXUIFs9TJIaiq+3hMo86pCkK/RTtXCv/rytP2KfWNDE= -github.com/clawkwork/p9 v0.0.0-20260708181804-a3a49aec6c3c h1:8dFQAWzO+f8YV9CYbRAlkzuoqWGGQouI0GRFvwj16iQ= -github.com/clawkwork/p9 v0.0.0-20260708181804-a3a49aec6c3c/go.mod h1:LoNwfBWP+QlCkjS1GFNylCthRIk/TkMZd6ICTbC+hrI= github.com/clawkwork/p9 v0.0.0-20260708203442-03a4a1385461 h1:GS2MHUMN4XAbiUThZsxMvGYL/GzN4TbLYoVJSfsBPoU= github.com/clawkwork/p9 v0.0.0-20260708203442-03a4a1385461/go.mod h1:LoNwfBWP+QlCkjS1GFNylCthRIk/TkMZd6ICTbC+hrI= github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSEFgwIwO+UVM8= @@ -204,6 +202,8 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= diff --git a/internal/agentembed/init_main.go.in b/internal/agentembed/init_main.go.in index dc98895..64c360b 100644 --- a/internal/agentembed/init_main.go.in +++ b/internal/agentembed/init_main.go.in @@ -82,6 +82,11 @@ type mountSpec struct { // host's kern.maxfiles. Tag is still the virtio-fs fallback. Kept in // lock-step with internal/guestcfg.Mount. NinePVSockPort uint32 `json:"ninep_port,omitempty"` + // Block, when set, mounts a local block device (FSType, default ext4) + // rather than any host-shared filesystem — how firecracker carries the + // worktree, having neither virtio-fs nor 9p. Takes precedence over both. + Block string `json:"block,omitempty"` + FSType string `json:"fstype,omitempty"` } type fileSpec struct { @@ -139,9 +144,21 @@ func main() { } for _, mt := range m.Mounts { - if err := mountShare(mt, m.User); err != nil { - logf("mount %s -> %s: %v", mt.Tag, mt.Path, err) + err := mountShare(mt, m.User) + if err == nil { + continue } + // A block share IS the workspace — firecracker carries the worktree on + // its own disk, with no host transport to fall back to — so a failed + // mount here must stop the boot. Logging and carrying on brings the VM + // up with an empty directory where the repo should be, the agent + // answers, `clawk up` reports success, and a whole session runs against + // nothing. Host-shared mounts are the opposite case: a toolchain cache + // that didn't mount makes the sandbox slow, not wrong. + if mt.Block != "" { + fatalf("mount %s on %s: %v", mt.Block, mt.Path, err) + } + logf("mount %s -> %s: %v", mt.Tag, mt.Path, err) } for _, f := range m.Files { @@ -647,6 +664,21 @@ func mountShare(mt mountSpec, u *manifestUser) error { flags |= unix.MS_RDONLY } + // A local block device needs no host transport at all, so it is tried + // first and never falls back: if the disk the host built for us won't + // mount, silently mounting something else would hide the workspace. + if mt.Block != "" { + fstype := mt.FSType + if fstype == "" { + fstype = "ext4" + } + if err := unix.Mount(mt.Block, mt.Path, fstype, flags, ""); err != nil { + return fmt.Errorf("mount %s (%s) on %s: %w", mt.Block, fstype, mt.Path, err) + } + logf("mounted %s (%s) on %s", mt.Block, fstype, mt.Path) + return nil + } + if mt.NinePVSockPort > 0 { if err := mountNineP(mt, flags); err == nil { logf("mounted 9p %s on %s (vsock port %d)", mt.Tag, mt.Path, mt.NinePVSockPort) diff --git a/internal/agentembed/prebuilt.go b/internal/agentembed/prebuilt.go new file mode 100644 index 0000000..45f4730 --- /dev/null +++ b/internal/agentembed/prebuilt.go @@ -0,0 +1,99 @@ +package agentembed + +import ( + "crypto/sha256" + "embed" + "encoding/hex" + "encoding/json" + "io/fs" +) + +// Prebuilt guest binaries, compiled at release time and embedded so a user +// never needs a Go toolchain to boot a sandbox. +// +// The directory is EMPTY in git (just .gitkeep and a README) — the payload is +// generated by `make guestbin` and gitignored. That keeps ~7 MiB of binaries +// out of the history and, more importantly, keeps them from going stale: a +// source install builds the guest binaries from the embedded .in sources as it +// always has, so a contributor editing init_main.go.in gets their edit rather +// than a committed artifact from three months ago. +// +// So there are two shapes of clawk binary: +// +// - release artifact — `make guestbin` ran first, the binaries are embedded, +// and nothing needs `go` at runtime; +// - source build (`make install`, `go build`) — nothing embedded, and +// guestbuild compiles from source on first boot, exactly as before. +// +// all: so the embed pattern still matches when the dir holds only dotfiles; +// without it `go:embed prebuilt` would fail to compile on a clean checkout. +// +//go:embed all:prebuilt +var prebuiltFS embed.FS + +// PrebuiltDir is where `make guestbin` writes, relative to this package. +const PrebuiltDir = "prebuilt" + +// PrebuiltManifestName identifies the metadata `make guestbin` writes beside +// the binaries. +const PrebuiltManifestName = "manifest.json" + +// PrebuiltManifest describes what the embedded binaries are, so a consumer can +// refuse a set that doesn't match what it needs. +type PrebuiltManifest struct { + // Arch is the GOARCH the binaries were built for. Guest arch always + // equals host arch (hardware virtualization can't cross architectures), + // but recording it means a mismatch is caught rather than assumed. + Arch string `json:"arch"` + + // SourcesSHA256 is SourcesHash() at build time. It is the one thing worth + // verifying: it catches an artifact whose embedded binaries were built + // from different .in sources than the ones embedded beside them, which is + // otherwise invisible and would ship a guest that silently predates the + // source tree. + SourcesSHA256 string `json:"sources_sha256"` + + // GoVersion built them, for diagnostics. + GoVersion string `json:"go_version,omitempty"` +} + +// SourcesHash digests every embedded guest source, in a fixed order. Both the +// generator and the consumer compute it the same way, so it can be compared +// across a build boundary. +func SourcesHash() string { + h := sha256.New() + for _, s := range []struct { + name string + data []byte + }{ + {"clawk-init/main.go", InitMainGo}, + {"clawk-init/go.mod", InitGoMod}, + {"clawk-pty-agent/main.go", AgentMainGo}, + {"clawk-pty-agent/go.mod", AgentGoMod}, + {"clawk-time-sync/main.go", TimeSyncMainGo}, + {"clawk-time-sync/go.mod", TimeSyncGoMod}, + } { + h.Write([]byte(s.name)) + h.Write(s.data) + } + return hex.EncodeToString(h.Sum(nil)) +} + +// Prebuilt returns the embedded binaries and their manifest. +// +// ok is false for an ordinary source build, where the directory holds no +// payload — that is the normal case for contributors, not an error. +func Prebuilt() (files fs.FS, m PrebuiltManifest, ok bool) { + sub, err := fs.Sub(prebuiltFS, PrebuiltDir) + if err != nil { + return nil, PrebuiltManifest{}, false + } + data, err := fs.ReadFile(sub, PrebuiltManifestName) + if err != nil { + return nil, PrebuiltManifest{}, false + } + if err := json.Unmarshal(data, &m); err != nil { + return nil, PrebuiltManifest{}, false + } + return sub, m, true +} diff --git a/internal/agentembed/prebuilt/.gitignore b/internal/agentembed/prebuilt/.gitignore new file mode 100644 index 0000000..f62350e --- /dev/null +++ b/internal/agentembed/prebuilt/.gitignore @@ -0,0 +1,4 @@ +# Generated by `make guestbin`; see README.md. +* +!.gitignore +!README.md diff --git a/internal/agentembed/prebuilt/README.md b/internal/agentembed/prebuilt/README.md new file mode 100644 index 0000000..9189ad4 --- /dev/null +++ b/internal/agentembed/prebuilt/README.md @@ -0,0 +1,14 @@ +# Prebuilt guest binaries (generated — not committed) + +`make guestbin` writes `clawk-init`, `clawk-pty-agent`, `clawk-time-sync` and +`manifest.json` here, and `go:embed` bakes them into the clawk binary so a +release needs no Go toolchain at runtime. + +Everything except this file and `.gitignore` is generated and ignored by git: +the binaries are ~7 MiB per architecture, and committing them would both bloat +the history and let them drift behind the `.go.in` sources they are built from. + +A plain `go build` / `make install` leaves this directory empty, and +`internal/guestbuild` then compiles the guest binaries from the embedded +sources on first boot — the long-standing behaviour, and the one contributors +want while editing those sources. diff --git a/internal/cli/daemon_failure_test.go b/internal/cli/daemon_failure_test.go new file mode 100644 index 0000000..783b0c5 --- /dev/null +++ b/internal/cli/daemon_failure_test.go @@ -0,0 +1,67 @@ +package cli + +import ( + "os" + "path/filepath" + "testing" + + "github.com/clawkwork/clawk/internal/config" + "github.com/stretchr/testify/require" +) + +// reportDaemonFailure is the last chance to say WHY a first boot failed, because +// the VM dir (and the daemon log in it) is about to be deleted. It walks both +// providers' logs, and must not let an uninformative one stop the walk — a +// daemon log that exists but holds nothing printable is a partially flushed +// file, not an answer. +func TestReportDaemonFailureFallsThroughToTheOtherLog(t *testing.T) { + report := func(t *testing.T, logs map[string]string) string { + t.Helper() + prev := store + store = config.NewStoreAt(t.TempDir()) + t.Cleanup(func() { store = prev }) + + sb := &config.Sandbox{Name: "boot-fail"} + vmDir := store.VMDir(sb.Name) + require.NoError(t, os.MkdirAll(vmDir, 0o755)) + for name, body := range logs { + require.NoError(t, os.WriteFile(filepath.Join(vmDir, name), []byte(body), 0o644)) + } + + r, w, err := os.Pipe() + require.NoError(t, err) + orig := os.Stderr + os.Stderr = w + defer func() { os.Stderr = orig }() + + reportDaemonFailure(sb) + require.NoError(t, w.Close()) + buf := make([]byte, 8192) + n, _ := r.Read(buf) + return string(buf[:n]) + } + + t.Run("blank fcd.log must not hide vzd.log", func(t *testing.T) { + out := report(t, map[string]string{ + // Non-empty (so the len==0 skip doesn't fire) but with no printable + // line: exactly what a boot killed right after openDaemonLog leaves. + "fcd.log": "\n \n\t\n", + "vzd.log": "2026/07/28 10:00:00 FATAL: bridge: sudo ip: a password is required\n", + }) + require.Contains(t, out, "FATAL: bridge: sudo ip: a password is required", + "the other daemon's verdict is the only diagnosis there is; got %q", out) + }) + + t.Run("a log with a verdict wins and stops the walk", func(t *testing.T) { + out := report(t, map[string]string{ + "fcd.log": "2026/07/28 10:00:00 FATAL: firecracker refused the drive\n", + "vzd.log": "2026/07/28 10:00:00 FATAL: should not be reached\n", + }) + require.Contains(t, out, "firecracker refused the drive") + require.NotContains(t, out, "should not be reached") + }) + + t.Run("no logs at all stays silent", func(t *testing.T) { + require.Empty(t, report(t, nil)) + }) +} diff --git a/internal/cli/doctor.go b/internal/cli/doctor.go index 8d62a32..42cbee7 100644 --- a/internal/cli/doctor.go +++ b/internal/cli/doctor.go @@ -14,6 +14,7 @@ import ( "time" "github.com/clawkwork/clawk/internal/config" + "github.com/clawkwork/clawk/internal/guestbuild" "github.com/clawkwork/clawk/internal/sandbox" "github.com/clawkwork/clawk/internal/vsockclient" "github.com/clawkwork/clawk/machine/oci" @@ -111,6 +112,31 @@ func fail(name, detail, fix string) doctorCheck { return doctorCheck{Name: name, Status: statusFail, Detail: detail, FixHint: fix} } +// goToolchainCheck reports on the Go toolchain, which is a prerequisite only +// for a clawk built from source. +// +// A release binary carries the in-guest binaries prebuilt, so it needs no +// toolchain and no module download to boot a sandbox — reporting a missing +// `go` as a failure there would send people installing something they don't +// need. A source build compiles them on first boot, and then `go` really is +// required. +func goToolchainCheck() doctorCheck { + const name = "host: go toolchain" + _, err := exec.LookPath("go") + switch { + case err == nil: + return ok(name, "on PATH") + case guestbuild.Prebuilt(runtime.GOARCH): + return ok(name, "not needed — this build ships the guest agent prebuilt") + default: + return fail(name, + "not found on PATH — this clawk was built from source, so it compiles the "+ + "guest init/agent on first boot", + "install from https://go.dev/dl or brew install go — or use a release binary, "+ + "which ships them prebuilt") + } +} + // runDoctorChecks dispatches the right set of checks based on whether // a sandbox name resolved. Empty name → host-only. func runDoctorChecks(name string) []doctorCheck { @@ -149,13 +175,7 @@ func hostChecks() []doctorCheck { Detail: fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH), }) - if _, err := exec.LookPath("go"); err != nil { - results = append(results, fail("host: go toolchain", - "not found on PATH (builds the guest init/agent for image-based sandboxes)", - "install from https://go.dev/dl or brew install go")) - } else { - results = append(results, ok("host: go toolchain", "on PATH")) - } + results = append(results, goToolchainCheck()) root := clawkRoot() if _, err := os.Stat(root); err != nil { diff --git a/internal/cli/doctor_linux.go b/internal/cli/doctor_linux.go index ef21d3a..fb8e30b 100644 --- a/internal/cli/doctor_linux.go +++ b/internal/cli/doctor_linux.go @@ -7,6 +7,8 @@ import ( "fmt" "os" "os/exec" + + "github.com/clawkwork/clawk/internal/sandbox" ) // platformHostChecks probes the Linux/firecracker host prerequisites that @@ -44,5 +46,57 @@ func platformHostChecks() []doctorCheck { "ensure the invoking user can read/write /dev/kvm")) } + results = append(results, hostNetSetupCheck()) + if name, stale := sandbox.StaleLegacyBridge(); stale { + results = append(results, warn("host: legacy bridge", + name+" is present with nothing attached — leftover from a clawk that shared one bridge across sandboxes", + "harmless; remove it with: sudo ip link del "+name)) + } + return results } + +// hostNetSetupCheck reports how clawk will obtain the CAP_NET_ADMIN it needs +// to create a sandbox's bridge and TAPs — the host prerequisite that used to +// be invisible until a boot failed (issue #9: doctor said 5 ok / 0 fail on a +// host that could not boot at all, because nothing checked this). +// +// Rootless mode needs nothing from the host and is the happy answer. Otherwise +// we fall back to sudo, and how bad that is depends: device creation happens +// once per sandbox, not per boot (see ensureTAP), so a host where sudo prompts +// is usable — it just needs a terminal at create time. Only a host that can +// neither authenticate nor prompt is broken. +func hostNetSetupCheck() doctorCheck { + const name = "host: network mode" + mode, why := sandbox.SelectNetMode() + if mode == sandbox.NetModeRootless { + // A pin is taken at face value everywhere else — but doctor exists to + // tell the truth, so verify a pinned rootless really works rather than + // reporting OK on a host where every boot will fail. + if pinned, isPinned := sandbox.NetModePinned(); isPinned && pinned == sandbox.NetModeRootless { + if avail, reason := sandbox.NetNSAvailable(); !avail { + return fail(name, + "pinned to rootless by CLAWK_NET_MODE, but this host cannot do it: "+reason, + "fix the cause above, or unset CLAWK_NET_MODE to fall back to bridge mode") + } + } + return ok(name, "rootless (per-sandbox network namespace; no privileged operations)") + } + + detail := "bridge mode via sudo — " + why + if sandbox.SudoIPWorksUnprompted() { + return ok(name, detail+" (passwordless sudo for ip, so nothing will prompt)") + } + if sandbox.StdinIsTerminal() { + return warn(name, + detail+" — sudo will prompt once per sandbox, when it creates the devices", + "fine if you're happy to type it; to avoid it entirely, enable unprivileged "+ + "user namespaces (see the reason above), or grant NOPASSWD for ip only: "+ + "' ALL=(ALL) NOPASSWD: /usr/sbin/ip'") + } + return fail(name, + detail+" — and sudo needs a password with no terminal to prompt on, so "+ + "'clawk up' cannot create its network devices", + "run 'clawk up' from a terminal, enable unprivileged user namespaces, or "+ + "grant NOPASSWD for ip: ' ALL=(ALL) NOPASSWD: /usr/sbin/ip'") +} diff --git a/internal/cli/fcd.go b/internal/cli/fcd.go index a28069a..22ae496 100644 --- a/internal/cli/fcd.go +++ b/internal/cli/fcd.go @@ -97,13 +97,23 @@ func runFcd(_ *cobra.Command, args []string) (retErr error) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - // The provider sets up the host bridge/TAPs and returns the spec, whose - // UserMode net carries the egress filter; the backend brings up gvproxy. + // The provider wires up the sandbox's network — a private namespace it + // creates here (rootless) or the host devices the CLI pre-created (bridge) + // — and returns the spec, whose UserMode net carries the egress filter; + // the backend brings up gvproxy. prov := sandbox.NewFirecrackerProvider(store) - spec, err := prov.DaemonSpec(sb, allow) + if mode, why := prov.NetModeForLog(); why != "" { + logger.Printf("network mode: %s (%s)", mode, why) + } else { + logger.Printf("network mode: %s", mode) + } + spec, netCleanup, err := prov.DaemonSpec(sb, allow) if err != nil { return fmt.Errorf("building spec: %w", err) } + // Releases the network namespace (rootless) once the VM is gone; a no-op in + // bridge mode, whose devices outlive the daemon by design. + defer netCleanup() logger.Printf("spec: vcpu=%d mem=%dMiB forwards=%d", spec.VCPU, spec.MemoryMiB, countForwards(spec)) diff --git a/internal/cli/here.go b/internal/cli/here.go index cfea81c..26f56fc 100644 --- a/internal/cli/here.go +++ b/internal/cli/here.go @@ -27,6 +27,11 @@ import ( // remove. Call only for a sandbox created in this same invocation; best-effort, // so cleanup errors are warned rather than masking the boot error. func rollbackFailedCreate(sb *config.Sandbox) { + // Destroy removes the VM dir, and the daemon log inside it is usually the + // only record of WHY the boot failed (the CLI's own error is often just a + // vsock timeout). Salvage its verdict before deleting it, or a first-boot + // failure is undiagnosable without re-running by hand. + reportDaemonFailure(sb) if provider, err := providerFor(sb); err == nil { if err := provider.Destroy(sb); err != nil { fmt.Fprintf(os.Stderr, "warning: rolling back VM for %q: %v\n", sb.DisplayName(), err) @@ -37,6 +42,51 @@ func rollbackFailedCreate(sb *config.Sandbox) { } } +// reportDaemonFailure prints the VM daemon's own verdict to stderr, for a +// sandbox whose VM dir is about to be deleted. It prefers the FATAL line the +// daemon logs on its way out (see the deferred logger in __vzd/__fcd) and +// falls back to the tail, so the user sees "bridge: sudo … a password is +// required" rather than only the CLI's downstream "agent did not become +// ready". Silent when there's no log — a failure before the daemon started. +func reportDaemonFailure(sb *config.Sandbox) { + vmDir := store.VMDir(sb.Name) + for _, name := range []string{"fcd.log", "vzd.log"} { + path := filepath.Join(vmDir, name) + data, err := os.ReadFile(path) + if err != nil || len(data) == 0 { + continue + } + var fatal string + var lines []string + for _, l := range strings.Split(string(data), "\n") { + if strings.TrimSpace(l) == "" { + continue + } + lines = append(lines, l) + if strings.Contains(l, "FATAL:") { + fatal = l + } + } + switch { + case fatal != "": + fmt.Fprintf(os.Stderr, "daemon log (%s): %s\n", path, fatal) + case len(lines) > 0: + if len(lines) > 5 { + lines = lines[len(lines)-5:] + } + fmt.Fprintf(os.Stderr, "daemon log (%s), last %d lines:\n%s\n", + path, len(lines), strings.Join(lines, "\n")) + default: + // Present but with nothing printable in it — a partially flushed + // log, or a boot killed just after the file was created. Returning + // here would consume the whole function and leave the other + // daemon's log, which may hold the actual FATAL line, unread. + continue + } + return + } +} + // hereCWDAndName resolves the cwd-vm sandbox name derived from the // current working directory. The returned cwd is canonical (absolute, // symlinks resolved) because it becomes the record's Anchor and InPlace diff --git a/internal/guestbuild/cmd/gen-prebuilt/main.go b/internal/guestbuild/cmd/gen-prebuilt/main.go new file mode 100644 index 0000000..72c70c4 --- /dev/null +++ b/internal/guestbuild/cmd/gen-prebuilt/main.go @@ -0,0 +1,47 @@ +// gen-prebuilt cross-compiles the in-guest binaries and writes them, with a +// manifest, into internal/agentembed/prebuilt/ for go:embed to pick up. Run by +// `make guestbin` before building a release artifact, so the shipped clawk +// needs no Go toolchain to boot a sandbox. +// +// The guest architecture must match the artifact's host architecture — +// hardware virtualization cannot cross architectures, so an arm64 clawk only +// ever boots arm64 guests. It defaults to the host's GOARCH; -arch overrides +// it when cross-building a release for another platform. +package main + +import ( + "context" + "flag" + "fmt" + "os" + "path/filepath" + "runtime" + + "github.com/clawkwork/clawk/internal/agentembed" + "github.com/clawkwork/clawk/internal/guestbuild" +) + +func main() { + arch := flag.String("arch", runtime.GOARCH, "guest GOARCH to build for (must match the target host's)") + out := flag.String("out", defaultOut(), "directory to write the binaries and manifest into") + flag.Parse() + + if err := guestbuild.GeneratePrebuilt(context.Background(), *out, *arch); err != nil { + fmt.Fprintf(os.Stderr, "gen-prebuilt: %v\n", err) + os.Exit(1) + } + fmt.Printf(" guest binaries for linux/%s → %s (sources %.12s)\n", + *arch, *out, agentembed.SourcesHash()) +} + +// defaultOut locates the embed directory from this file's own path, so the +// target works from any working directory. +func defaultOut() string { + _, self, _, ok := runtime.Caller(0) + if !ok { + return filepath.Join("internal", "agentembed", agentembed.PrebuiltDir) + } + // .../internal/guestbuild/cmd/gen-prebuilt/main.go → .../internal/agentembed/prebuilt + root := filepath.Dir(filepath.Dir(filepath.Dir(filepath.Dir(self)))) + return filepath.Join(root, "agentembed", agentembed.PrebuiltDir) +} diff --git a/internal/guestbuild/guestbuild.go b/internal/guestbuild/guestbuild.go index 71cda4a..38e3d57 100644 --- a/internal/guestbuild/guestbuild.go +++ b/internal/guestbuild/guestbuild.go @@ -7,9 +7,11 @@ // static binaries are baked into the rootfs by machine/oci's inject // support. // -// Results are cached under /guestbin//, keyed by the -// source content, target arch and Go toolchain version, so the ~seconds -// build cost is paid once per clawk version. +// Release binaries carry these prebuilt (see internal/agentembed/prebuilt.go), +// so an installed clawk needs no Go toolchain at all. Building from source +// leaves them out and they are compiled here on first use instead, cached +// under /guestbin// keyed by source content, target arch and +// toolchain version — so the ~seconds cost is paid once per clawk version. package guestbuild import ( @@ -50,10 +52,13 @@ func modules() []module { } } -// Build compiles the guest binaries for linux/, reusing a previous -// build when sources, arch and toolchain are unchanged. Requires `go` on -// PATH; the first build also needs network access for the guest modules' -// dependencies (cached by the host's module cache afterwards). +// Build returns the guest binaries for linux/. +// +// A release binary embeds them (see internal/agentembed/prebuilt.go), so this +// only unpacks and returns — no Go toolchain, no network. A source build has +// nothing embedded and compiles them from the embedded sources instead, +// caching the result; that path needs `go` on PATH, and network the first time +// for the guest modules' dependencies. func Build(ctx context.Context, cacheDir, arch string) (Binaries, error) { if cacheDir == "" { return Binaries{}, fmt.Errorf("guestbuild: cacheDir is required") @@ -61,10 +66,29 @@ func Build(ctx context.Context, cacheDir, arch string) (Binaries, error) { if arch == "" { return Binaries{}, fmt.Errorf("guestbuild: arch is required") } + if bins, ok := fromPrebuilt(cacheDir, arch); ok { + return bins, nil + } + return buildFromSource(ctx, cacheDir, arch) +} + +// Prebuilt reports whether this binary carries guest binaries for arch, i.e. +// whether a sandbox can boot with no Go toolchain present. `clawk doctor` asks, +// so it can tell a hard prerequisite from an optional one. +func Prebuilt(arch string) bool { + _, m, ok := agentembed.Prebuilt() + return ok && m.Arch == arch && m.SourcesSHA256 == agentembed.SourcesHash() +} + +// buildFromSource compiles the guest binaries, reusing a previous build when +// sources, arch and toolchain are unchanged. +func buildFromSource(ctx context.Context, cacheDir, arch string) (Binaries, error) { goBin, err := exec.LookPath("go") if err != nil { return Binaries{}, fmt.Errorf( - "guestbuild: `go` not found — the Go toolchain is required to build the guest agent (install from https://go.dev/dl or brew install go)") + "guestbuild: `go` not found — this clawk was built from source, so it needs the Go " + + "toolchain to compile the guest agent (install from https://go.dev/dl or brew install go). " + + "Release binaries ship the guest agent prebuilt and need no toolchain") } key, err := cacheKey(goBin, arch) diff --git a/internal/guestbuild/guestbuild_test.go b/internal/guestbuild/guestbuild_test.go index 6c4664b..9ddefbe 100644 --- a/internal/guestbuild/guestbuild_test.go +++ b/internal/guestbuild/guestbuild_test.go @@ -15,6 +15,10 @@ import ( // the first time for the guest modules' deps) and exercises the cache. // This is the host-side equivalent of agentembed's compile test, plus // the init module that only this package builds. +// +// It drives buildFromSource rather than Build, because it is specifically +// about the compile path: Build short-circuits to the embedded binaries when +// the artifact ships them, which is what TestBuildUsesPrebuiltWithoutGo covers. func TestBuild(t *testing.T) { if _, err := exec.LookPath("go"); err != nil { t.Skip("`go` not on PATH") @@ -24,8 +28,8 @@ func TestBuild(t *testing.T) { } cache := t.TempDir() - bins, err := Build(context.Background(), cache, runtime.GOARCH) - require.NoError(t, err, "Build") + bins, err := buildFromSource(context.Background(), cache, runtime.GOARCH) + require.NoError(t, err, "buildFromSource") require.False(t, bins.Cached, "fresh build reported Cached") for _, p := range []string{bins.Init, bins.Agent, bins.TimeSync} { fi, err := os.Stat(p) @@ -40,8 +44,8 @@ func TestBuild(t *testing.T) { // cache hit must not rewrite the files. before, err := os.Stat(bins.Init) require.NoError(t, err) - again, err := Build(context.Background(), cache, runtime.GOARCH) - require.NoError(t, err, "cached Build") + again, err := buildFromSource(context.Background(), cache, runtime.GOARCH) + require.NoError(t, err, "cached buildFromSource") require.Equal(t, bins.Init, again.Init, "cache key changed: %s != %s", again.Init, bins.Init) require.True(t, again.Cached, "second build did not report Cached") after, err := os.Stat(bins.Init) diff --git a/internal/guestbuild/prebuilt.go b/internal/guestbuild/prebuilt.go new file mode 100644 index 0000000..d34a64a --- /dev/null +++ b/internal/guestbuild/prebuilt.go @@ -0,0 +1,161 @@ +package guestbuild + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "io/fs" + "os" + "os/exec" + "path/filepath" + + "github.com/clawkwork/clawk/internal/agentembed" +) + +// fromPrebuilt materializes the guest binaries embedded at release time, +// returning ok=false when this build has none (a source build) or has a set +// that doesn't fit the request. +// +// Extraction, rather than using them in place: callers hand these paths to +// machine/oci as inject sources, which opens them as files — an embed.FS entry +// has no path on disk. They land in the same cache layout a source build uses, +// keyed by the manifest's source hash so a prebuilt clawk and a source-built +// one can share a cache dir without colliding. +func fromPrebuilt(cacheDir, arch string) (Binaries, bool) { + files, m, ok := agentembed.Prebuilt() + if !ok { + return Binaries{}, false + } + if m.Arch != arch { + // Not an error: an aarch64 host asking for amd64 guest binaries is a + // caller bug, but falling through to a source build is the honest + // response, and hardware virtualization makes it unreachable anyway. + return Binaries{}, false + } + if want := agentembed.SourcesHash(); m.SourcesSHA256 != want { + // The embedded binaries were built from different sources than the + // ones embedded beside them — a broken release, and exactly the + // failure that would otherwise ship a stale guest silently. Say so and + // let the source build take over. + fmt.Fprintf(os.Stderr, + "warning: ignoring embedded guest binaries: built from sources %.12s, this build embeds %.12s\n", + m.SourcesSHA256, want) + return Binaries{}, false + } + + outDir := filepath.Join(cacheDir, "guestbin", fmt.Sprintf("prebuilt-%s-%.16s", arch, m.SourcesSHA256)) + bins := Binaries{ + Init: filepath.Join(outDir, "clawk-init"), + Agent: filepath.Join(outDir, "clawk-pty-agent"), + TimeSync: filepath.Join(outDir, "clawk-time-sync"), + } + if allExist(bins.Init, bins.Agent, bins.TimeSync) { + bins.Cached = true + return bins, true + } + if err := extractAll(files, outDir); err != nil { + // A cache we can't write is not fatal while `go` might still be + // available; warn and let Build fall through. + fmt.Fprintf(os.Stderr, "warning: unpacking embedded guest binaries: %v\n", err) + return Binaries{}, false + } + bins.Cached = true + return bins, true +} + +// extractAll writes the three binaries into outDir via a temp dir + rename, so +// an interrupted extraction never leaves a partial set that allExist accepts. +func extractAll(files fs.FS, outDir string) error { + if err := os.MkdirAll(filepath.Dir(outDir), 0o755); err != nil { + return err + } + tmpDir, err := os.MkdirTemp(filepath.Dir(outDir), "unpack-*") + if err != nil { + return err + } + defer os.RemoveAll(tmpDir) + + for _, m := range modules() { + src, err := files.Open(m.name) + if err != nil { + return fmt.Errorf("%s: %w", m.name, err) + } + err = writeExecutable(filepath.Join(tmpDir, m.name), src) + src.Close() + if err != nil { + return fmt.Errorf("%s: %w", m.name, err) + } + } + + if err := os.Rename(tmpDir, outDir); err != nil { + // A concurrent extraction may have won; its result is as good as ours. + if _, statErr := os.Stat(outDir); statErr == nil { + return nil + } + return err + } + return nil +} + +func writeExecutable(path string, r io.Reader) error { + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o755) + if err != nil { + return err + } + if _, err := io.Copy(f, r); err != nil { + f.Close() + return err + } + return f.Close() +} + +// GeneratePrebuilt builds the guest binaries for arch and writes them, plus a +// manifest, into destDir for `go:embed` to pick up. It is the implementation +// behind `make guestbin`; nothing at runtime calls it. +// +// It deliberately reuses Build, so the binaries a release embeds are produced +// by the same code path a source build uses — one compiler invocation, one set +// of flags, no second recipe to drift. +func GeneratePrebuilt(ctx context.Context, destDir, arch string) error { + cache, err := os.MkdirTemp("", "clawk-guestbin-*") + if err != nil { + return err + } + defer os.RemoveAll(cache) + + bins, err := buildFromSource(ctx, cache, arch) + if err != nil { + return err + } + if err := os.MkdirAll(destDir, 0o755); err != nil { + return err + } + for name, src := range map[string]string{ + "clawk-init": bins.Init, + "clawk-pty-agent": bins.Agent, + "clawk-time-sync": bins.TimeSync, + } { + data, err := os.ReadFile(src) + if err != nil { + return err + } + if err := os.WriteFile(filepath.Join(destDir, name), data, 0o755); err != nil { + return err + } + } + + m := agentembed.PrebuiltManifest{ + Arch: arch, + SourcesSHA256: agentembed.SourcesHash(), + } + if out, err := exec.Command("go", "env", "GOVERSION").Output(); err == nil { + m.GoVersion = string(bytes.TrimSpace(out)) + } + data, err := json.MarshalIndent(m, "", " ") + if err != nil { + return err + } + return os.WriteFile(filepath.Join(destDir, agentembed.PrebuiltManifestName), append(data, '\n'), 0o644) +} diff --git a/internal/guestbuild/prebuilt_test.go b/internal/guestbuild/prebuilt_test.go new file mode 100644 index 0000000..0e57274 --- /dev/null +++ b/internal/guestbuild/prebuilt_test.go @@ -0,0 +1,94 @@ +package guestbuild + +import ( + "context" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/clawkwork/clawk/internal/agentembed" + "github.com/stretchr/testify/require" +) + +// TestBuildUsesPrebuiltWithoutGo is the point of the whole mechanism: when the +// binary carries prebuilt guest binaries, a sandbox can be prepared with no Go +// toolchain on PATH at all. +// +// Skips on a source build (nothing embedded), which is the normal state of a +// contributor's checkout — run `make guestbin` to exercise it, as CI and the +// release workflow do. +func TestBuildUsesPrebuiltWithoutGo(t *testing.T) { + if !Prebuilt(runtime.GOARCH) { + t.Skip("no embedded guest binaries in this build; run `make guestbin` first") + } + + // An empty PATH is the strongest form of the claim: not "go is elsewhere", + // but "no go exists". + t.Setenv("PATH", "") + + cache := t.TempDir() + bins, err := Build(context.Background(), cache, runtime.GOARCH) + require.NoError(t, err, "a prebuilt clawk must not need a Go toolchain") + + for _, p := range []string{bins.Init, bins.Agent, bins.TimeSync} { + st, err := os.Stat(p) + require.NoError(t, err) + require.Greater(t, st.Size(), int64(500<<10), "%s looks hollow", p) + require.NotZero(t, st.Mode()&0o111, "%s must be executable", p) + } + + t.Run("second call reuses the extraction", func(t *testing.T) { + again, err := Build(context.Background(), cache, runtime.GOARCH) + require.NoError(t, err) + require.True(t, again.Cached) + require.Equal(t, bins.Init, again.Init) + }) + + t.Run("a foreign arch falls through instead of lying", func(t *testing.T) { + // The embedded set is for one arch. Asking for another must not hand + // back binaries of the wrong architecture; with no `go` to fall back + // to, that means an error. + other := "amd64" + if runtime.GOARCH == "amd64" { + other = "arm64" + } + _, err := Build(context.Background(), t.TempDir(), other) + require.Error(t, err) + require.Contains(t, err.Error(), "`go` not found") + }) +} + +// TestPrebuiltRejectsStaleSources pins the integrity check: binaries built from +// different sources than the ones embedded beside them must be ignored, because +// shipping them would silently run a guest older than the source tree. +func TestPrebuiltRejectsStaleSources(t *testing.T) { + if !Prebuilt(runtime.GOARCH) { + t.Skip("no embedded guest binaries in this build") + } + _, m, ok := agentembed.Prebuilt() + require.True(t, ok) + require.Equal(t, agentembed.SourcesHash(), m.SourcesSHA256, + "this build's embedded binaries and sources disagree — `make guestbin` is stale") +} + +// TestGeneratePrebuilt covers the generator `make guestbin` runs. +func TestGeneratePrebuilt(t *testing.T) { + if testing.Short() { + t.Skip("compiles three guest binaries") + } + dest := t.TempDir() + require.NoError(t, GeneratePrebuilt(context.Background(), dest, runtime.GOARCH)) + + for _, name := range []string{"clawk-init", "clawk-pty-agent", "clawk-time-sync"} { + st, err := os.Stat(filepath.Join(dest, name)) + require.NoError(t, err) + require.Greater(t, st.Size(), int64(500<<10)) + require.NotZero(t, st.Mode()&0o111, "%s must be executable", name) + } + data, err := os.ReadFile(filepath.Join(dest, agentembed.PrebuiltManifestName)) + require.NoError(t, err) + require.Contains(t, string(data), agentembed.SourcesHash(), + "the manifest must record the sources it was built from") + require.Contains(t, string(data), runtime.GOARCH) +} diff --git a/internal/guestcfg/blockmount_test.go b/internal/guestcfg/blockmount_test.go new file mode 100644 index 0000000..fa65db0 --- /dev/null +++ b/internal/guestcfg/blockmount_test.go @@ -0,0 +1,44 @@ +package guestcfg + +import ( + "encoding/json" + "strings" + "testing" +) + +// TestMountBlockAdditive verifies Block/FSType are a clean additive change, +// the same contract TestMountNinePVSockPortAdditive holds for 9p: they +// round-trip, and empty values stay off the wire so a manifest without a +// block mount is byte-identical to the pre-change JSON — no Version bump, +// no forced sandbox recreation. +func TestMountBlockAdditive(t *testing.T) { + m := Manifest{ + Version: Version, + Mounts: []Mount{ + {Path: "/workspace/proj", Block: "/dev/vdc", FSType: "ext4"}, + {Tag: "claude_agents", Path: "/home/agent/.claude/agents"}, // virtio-fs only + }, + } + b, err := json.Marshal(m) + if err != nil { + t.Fatalf("marshal: %v", err) + } + js := string(b) + if strings.Contains(js, `"block":""`) || strings.Contains(js, `"fstype":""`) { + t.Errorf("empty block fields not omitted (old inits would see new keys): %s", js) + } + if !strings.Contains(js, `"block":"/dev/vdc"`) || !strings.Contains(js, `"fstype":"ext4"`) { + t.Errorf("block mount missing from wire: %s", js) + } + + var got Manifest + if err := json.Unmarshal(b, &got); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if got.Mounts[0].Block != "/dev/vdc" || got.Mounts[0].FSType != "ext4" { + t.Errorf("block mount = %+v, want /dev/vdc ext4", got.Mounts[0]) + } + if got.Mounts[1].Block != "" { + t.Errorf("share mount picked up a block device: %+v", got.Mounts[1]) + } +} diff --git a/internal/guestcfg/manifest.go b/internal/guestcfg/manifest.go index 110e3cd..bc59f27 100644 --- a/internal/guestcfg/manifest.go +++ b/internal/guestcfg/manifest.go @@ -85,11 +85,22 @@ type User struct { // the virtio-fs share, and a 9p-capable init falls back to it if the 9p mount // fails (e.g. a kernel without CONFIG_NET_9P_FD). This keeps the change // additive — no Version bump, no forced sandbox recreation. +// +// Block selects a third transport, for hosts with no file-sharing transport +// at all: firecracker has neither virtio-fs nor (with the firecracker-CI +// kernel) 9p, so its worktree rides in on its own virtio-blk disk, built on +// the host in userspace (machine/oci.WriteDirDisk). Block is the in-guest +// device path and takes precedence over both other transports; Tag is +// irrelevant to it. Additive like NinePVSockPort — an older clawk-init +// ignores it, which is safe because only the firecracker provider sets it +// and its guest binaries are rebuilt from the same source tree. type Mount struct { Tag string `json:"tag"` Path string `json:"path"` ReadOnly bool `json:"ro,omitempty"` NinePVSockPort uint32 `json:"ninep_port,omitempty"` + Block string `json:"block,omitempty"` // e.g. "/dev/vdc" + FSType string `json:"fstype,omitempty"` // defaults to ext4 } // File is one file written into the guest at boot. Content is raw bytes diff --git a/internal/sandbox/console.go b/internal/sandbox/console.go index 5acc07a..96ed403 100644 --- a/internal/sandbox/console.go +++ b/internal/sandbox/console.go @@ -15,6 +15,14 @@ import ( // time (kernel panics, clawk-init errors, missing init binaries); making // the CLI surface it directly turns "go read this file" into an answer. func ConsoleTail(path string, n int) string { + return LogTail(path, n, "last guest console output") +} + +// LogTail is ConsoleTail for any log file, with a caller-chosen label. The +// host-side daemon log needs the same treatment as the guest console: it +// holds the real cause of most boot failures, and on a first `clawk` in a +// directory the rollback deletes it moments later. +func LogTail(path string, n int, label string) string { data, err := os.ReadFile(path) if err != nil || len(data) == 0 { return "" @@ -31,6 +39,6 @@ func ConsoleTail(path string, n int) string { if len(lines) > n { lines = lines[len(lines)-n:] } - return fmt.Sprintf("\n--- last guest console output (%s) ---\n%s\n---", - path, strings.Join(lines, "\n")) + return fmt.Sprintf("\n--- %s (%s) ---\n%s\n---", + label, path, strings.Join(lines, "\n")) } diff --git a/internal/sandbox/fdpass_linux.go b/internal/sandbox/fdpass_linux.go new file mode 100644 index 0000000..d5d1ee7 --- /dev/null +++ b/internal/sandbox/fdpass_linux.go @@ -0,0 +1,101 @@ +//go:build linux + +// SCM_RIGHTS fd passing between the daemon and its namespace anchor. +// +// A file descriptor sent this way is the same open file in the receiver, with +// none of the sender's namespace attached to it — which is the whole trick +// behind rootless mode: the anchor creates the TAP where it has +// CAP_NET_ADMIN, and the daemon reads and writes frames on it from the host +// network namespace, where gvproxy lives. + +package sandbox + +import ( + "errors" + "fmt" + "net" + "os" + "time" + + "golang.org/x/sys/unix" +) + +// handshakeFD is where the anchor finds its end of the socketpair: exec'd +// children see cmd.ExtraFiles[0] as fd 3. +const handshakeFD = 3 + +// handshakePair returns the two ends of a unix stream socket: the parent's as +// a *net.UnixConn (so it has deadlines and ReadMsgUnix), the child's as an +// *os.File to hand to exec.Cmd.ExtraFiles. +func handshakePair() (*net.UnixConn, *os.File, error) { + fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0) + if err != nil { + return nil, nil, fmt.Errorf("socketpair: %w", err) + } + parentFile := os.NewFile(uintptr(fds[0]), "netns-handshake") + // FileConn dups the fd and registers the copy with the runtime poller, + // which is what makes read deadlines work; our original is then surplus. + conn, err := net.FileConn(parentFile) + _ = parentFile.Close() + if err != nil { + _ = unix.Close(fds[1]) + return nil, nil, fmt.Errorf("wrapping handshake socket: %w", err) + } + uconn, ok := conn.(*net.UnixConn) + if !ok { + _ = conn.Close() + _ = unix.Close(fds[1]) + return nil, nil, fmt.Errorf("handshake socket is %T, want *net.UnixConn", conn) + } + return uconn, os.NewFile(uintptr(fds[1]), "netns-handshake-child"), nil +} + +// sendFD sends one file descriptor plus a one-byte status over the raw socket +// fd. The byte matters: SCM_RIGHTS needs at least one byte of ordinary +// payload to ride along, and it doubles as the anchor's "topology is up" +// signal. +func sendFD(sockFD, fd int, status byte) error { + if err := unix.Sendmsg(sockFD, []byte{status}, unix.UnixRights(fd), nil, 0); err != nil { + return fmt.Errorf("sendmsg: %w", err) + } + return nil +} + +// recvFD waits up to timeout for one descriptor sent by sendFD. +// +// The timeout is what turns "the anchor wedged before sending anything" into a +// prompt error instead of a hang; an anchor that died outright closes the +// socket, which surfaces as EOF. +func recvFD(conn *net.UnixConn, timeout time.Duration) (int, error) { + if err := conn.SetReadDeadline(time.Now().Add(timeout)); err != nil { + return -1, fmt.Errorf("setting handshake deadline: %w", err) + } + buf := make([]byte, 1) + oob := make([]byte, unix.CmsgSpace(4)) // exactly one fd + n, oobn, _, _, err := conn.ReadMsgUnix(buf, oob) + if err != nil { + return -1, fmt.Errorf("reading the namespace handshake: %w", err) + } + if n == 0 && oobn == 0 { + return -1, errors.New("the namespace helper exited without handing over its tap") + } + + scms, err := unix.ParseSocketControlMessage(oob[:oobn]) + if err != nil { + return -1, fmt.Errorf("parsing control message: %w", err) + } + for _, scm := range scms { + fds, err := unix.ParseUnixRights(&scm) + if err != nil { + continue + } + if len(fds) > 0 { + // Close any extras rather than leak them; we asked for one. + for _, extra := range fds[1:] { + _ = unix.Close(extra) + } + return fds[0], nil + } + } + return -1, errors.New("the namespace handshake carried no file descriptor") +} diff --git a/internal/sandbox/firecracker_linux.go b/internal/sandbox/firecracker_linux.go index d5d0a8f..3d1770e 100644 --- a/internal/sandbox/firecracker_linux.go +++ b/internal/sandbox/firecracker_linux.go @@ -12,22 +12,29 @@ // gvproxy can't drive firecracker's TAP directly, so the daemon bridges // the two with a frame pump (fcnet_linux.go). // +// Each sandbox gets its own L2 bridge and its own guest MAC, so the shared +// guest IP (every gvproxy hands out 192.168.127.2) is confined to one segment +// per VM — concurrent sandboxes no longer collide. +// // Known limitations: -// - One shared bridge / one fixed guest IP, so two firecracker sandboxes -// at once collide. A per-sandbox /30 is the next step. -// - No virtio-fs, so the phase worktree is baked into the rootfs at -// Create time rather than live-mounted; host edits don't propagate. +// - The worktree rides in on its own disk built at Create time rather than +// live-mounted, so host edits don't propagate into a running guest. The +// default (firecracker-CI) kernel has no filesystem transport at all — no +// 9p, no FUSE, no virtio-fs. clawk's own published kernel has all three +// and boots here fine (`vm ( kernel … )`), so the missing piece is a host +// server wired to one of them, not the kernel. virtio-fs is the exception: +// firecracker ships no such device, whatever the guest supports. package sandbox import ( "context" "errors" "fmt" + "io/fs" "os" "os/exec" "path/filepath" "runtime" - "strings" "syscall" "time" @@ -37,6 +44,7 @@ import ( "github.com/clawkwork/clawk/internal/netfilter" "github.com/clawkwork/clawk/internal/vsockclient" "github.com/clawkwork/clawk/machine" + "github.com/clawkwork/clawk/machine/kernel" "github.com/clawkwork/clawk/machine/oci" // Register the firecracker backend. @@ -51,6 +59,19 @@ const ( fcVSockCID = 3 // fcAgentPort is the vsock port clawk-pty-agent listens on in the guest. fcAgentPort = 1024 + // fcWorktreeDevice is where the worktree disk lands in the guest. Drives + // are attached in spec order after the rootfs (machine/firecracker puts + // rootfs first, then Spec.Disks), so vda=rootfs, vdb=guestcfg, vdc=this. + fcWorktreeDevice = "/dev/vdc" + // worktreeDiskSlack is the free space added to a worktree disk beyond the + // tree itself, for whatever the agent builds in there. Sparse, so unused + // capacity costs nothing on the host. + worktreeDiskSlack = 4 << 30 + // kernelResolveTimeout bounds resolveKernel's network work: an S3 listing, + // plus a multi-megabyte download on a cold cache. Generous for that, but + // finite — DaemonSpec runs it in a detached daemon with no terminal and no + // supervisor, where an unreachable host would otherwise hang for good. + kernelResolveTimeout = 10 * time.Minute ) // FirecrackerProvider implements the Provider + agent interfaces using the @@ -83,8 +104,8 @@ func (f *FirecrackerProvider) vsockPath(sb *config.Sandbox) string { func (f *FirecrackerProvider) GuestWorkspaceRoot() string { return "/workspace" } // Create stages everything the VM boots from: the firecracker-CI kernel, -// the cross-compiled guest binaries, the OCI rootfs (with the worktree -// baked in), and the clawk-init manifest config disk. +// the cross-compiled guest binaries, the OCI rootfs, the worktree disk, and +// the clawk-init manifest config disk. func (f *FirecrackerProvider) Create(sb *config.Sandbox) error { if _, err := exec.LookPath("firecracker"); err != nil { return errors.New("firecracker binary not on PATH") @@ -99,24 +120,33 @@ func (f *FirecrackerProvider) Create(sb *config.Sandbox) error { if err := os.MkdirAll(vmDir, 0o755); err != nil { return fmt.Errorf("creating vm dir: %w", err) } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), kernelResolveTimeout) defer cancel() - if _, err := ciEnsureKernel(ctx); err != nil { + if _, err := f.resolveKernel(ctx, sb); err != nil { return fmt.Errorf("kernel: %w", err) } bins, err := guestbuild.Build(ctx, f.store.CacheDir(), runtime.GOARCH) if err != nil { return fmt.Errorf("guest binaries: %w", err) } - // Materialize the OCI rootfs (build cache + clone) into a per-VM disk - // we can bake the worktree into — firecracker has no virtio-fs. + // A boot that will restore a suspend state must keep the exact disks the + // memory image was saved against: re-materializing the rootfs (a clone + // that truncates its destination) or rebuilding the worktree disk under a + // restore corrupts the guest filesystem — the invariant machine.Suspendable + // documents and restoreOrStart relies on. Everything above this line is + // content-addressed and safe to re-run. rootfs := filepath.Join(vmDir, "rootfs.raw") + if f.hasRestorableState(sb, vmDir, rootfs) { + sb.GuestIP = guestIP + return nil + } + // Materialize the OCI rootfs (build cache + clone) into a per-VM disk. if _, err := oci.Materialize(ctx, OCIRootFS(sb, f.store.CacheDir(), bins), rootfs); err != nil { return fmt.Errorf("materializing rootfs: %w", err) } - if err := f.bakeWorktree(rootfs, sb.Phases[0].Worktree); err != nil { - return fmt.Errorf("baking worktree: %w", err) + if err := f.writeWorktreeDisk(sb); err != nil { + return fmt.Errorf("worktree disk: %w", err) } if err := guestcfg.WriteDisk(f.manifest(sb), filepath.Join(vmDir, "guestcfg.img")); err != nil { return fmt.Errorf("guest config disk: %w", err) @@ -125,12 +155,36 @@ func (f *FirecrackerProvider) Create(sb *config.Sandbox) error { return nil } +// hasRestorableState reports whether this VM dir holds a suspend state that +// the next boot will restore onto the disks already present — in which case +// Create must not touch them. A state whose rootfs is gone can never restore +// (restoreOrStart discards it loudly), so it doesn't count. +// +// EVERY disk buildSpec attaches has to be there, not just the rootfs: skipping +// the build leaves whatever is on disk as the whole disk set. A state saved by +// a clawk that baked the worktree into the rootfs — no worktree.img at all — +// would otherwise survive this check, fail the suspend fingerprint (the disk +// count changed), and cold-boot against a drive that does not exist, which +// firecracker refuses outright. Rebuilding instead costs only the snapshot, +// which restoreOrStart was going to discard in that case anyway. +func (f *FirecrackerProvider) hasRestorableState(sb *config.Sandbox, vmDir, rootfs string) bool { + if !machine.SuspendStateExists(filepath.Join(vmDir, "suspend")) { + return false + } + for _, disk := range []string{rootfs, f.worktreeDiskPath(sb), filepath.Join(vmDir, "guestcfg.img")} { + if _, err := os.Stat(disk); err != nil { + return false + } + } + return true +} + // manifest is the clawk-init boot manifest for firecracker. The guest is // configured statically with gvproxy's addresses (no DHCP client in arbitrary // images): gateway + DNS are gvproxy, so DNS answers flow through gvproxy's // resolver and feed the allow-list's domain matcher. Same values as the vz -// manifest (OCIGuestManifest). No virtio-fs mounts — the worktree is baked -// into the rootfs. +// manifest (OCIGuestManifest). No virtio-fs share — the worktree arrives on +// its own disk (see writeWorktreeDisk), mounted from /dev/vdc. func (f *FirecrackerProvider) manifest(sb *config.Sandbox) guestcfg.Manifest { return guestcfg.Manifest{ Hostname: sb.Name, @@ -141,25 +195,90 @@ func (f *FirecrackerProvider) manifest(sb *config.Sandbox) guestcfg.Manifest { DNS: []string{gvproxyGateway}, MTU: gvproxyMTU, }, + Mounts: []guestcfg.Mount{{ + Path: f.guestWorktreePath(sb), + Block: fcWorktreeDevice, + FSType: "ext4", + }}, Services: []guestcfg.Service{{Name: "pty-agent", Path: guestcfg.AgentPath}}, } } -// bakeWorktree copies the host worktree into the rootfs at the guest -// workspace path. Done at Create time because firecracker can't live-mount. -func (f *FirecrackerProvider) bakeWorktree(rootfs, worktreeSrc string) error { - rel := strings.TrimPrefix(f.GuestWorkspaceRoot(), "/") - return mountedRootfs(rootfs, func(mnt string) error { - target := filepath.Join(mnt, rel, filepath.Base(worktreeSrc)) - if err := runSudo("mkdir", "-p", target); err != nil { +// guestWorktreePath is where the worktree disk is mounted in the guest — +// the same /workspace/ layout the bake produced, so nothing downstream +// (sessions, runners, cwd inference) sees a change. +func (f *FirecrackerProvider) guestWorktreePath(sb *config.Sandbox) string { + return filepath.Join(f.GuestWorkspaceRoot(), filepath.Base(sb.Phases[0].Worktree)) +} + +// writeWorktreeDisk builds the phase worktree into its own ext4 disk, in +// userspace. +// +// Firecracker has no virtio-fs and the firecracker-CI kernel has no 9p, so +// the worktree has to be inside a block device. It used to be copied into the +// rootfs through a loop mount, which cost six privileged operations per boot +// (losetup, mount, mkdir, cp, umount, losetup -d) and required a root helper +// that mounted whatever it was told. A separate disk needs none of that: the +// same ext4 writer that builds the rootfs writes it as this user. +// +// One disk per VM, rebuilt on each Create — same lifetime and semantics as +// the bake it replaces (host edits still don't propagate into a running +// guest; a live worktree needs a 9p-capable guest kernel). +func (f *FirecrackerProvider) writeWorktreeDisk(sb *config.Sandbox) error { + // Resolved once, here, so the walk that sizes the disk and the walk that + // fills it see the same tree: both lstat their root, so a worktree path + // whose last component is a symlink would size to bare slack. + src, err := filepath.EvalSymlinks(sb.Phases[0].Worktree) + if err != nil { + return fmt.Errorf("resolving worktree %s: %w", sb.Phases[0].Worktree, err) + } + size, err := worktreeDiskSize(src) + if err != nil { + return err + } + return oci.WriteDirDisk(src, f.worktreeDiskPath(sb), size) +} + +func (f *FirecrackerProvider) worktreeDiskPath(sb *config.Sandbox) string { + return filepath.Join(f.vmDir(sb), "worktree.img") +} + +// worktreeDiskSize picks the worktree disk's capacity: the tree's own size +// plus headroom for what the agent does inside it (build outputs, node_modules, +// a second checkout). Doubling covers small repos poorly, so take whichever of +// "twice the tree" and "the tree plus a flat slack" is larger. The padding is +// sparse — it costs no physical disk until the guest writes to it — so being +// generous here is close to free, and running out mid-session is not. +// +// TODO: make this configurable (clawk.mod `disk`) once anyone hits the ceiling. +func worktreeDiskSize(dir string) (int64, error) { + var used int64 + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + // A file that vanished mid-walk shouldn't fail sizing (the disk + // build tolerates the same) — but the worktree root going missing + // means there is nothing to boot with, and must not size to slack. + if os.IsNotExist(err) && path != dir { + return nil + } return err } - // Trailing /. copies contents into target, not the dir itself. - if err := runSudo("cp", "-a", strings.TrimRight(worktreeSrc, "/")+"/.", target); err != nil { - return fmt.Errorf("copying worktree: %w", err) + info, err := d.Info() + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + if info.Mode().IsRegular() { + used += info.Size() } return nil }) + if err != nil { + return 0, fmt.Errorf("sizing worktree %s: %w", dir, err) + } + return max(2*used, used+worktreeDiskSlack), nil } // Start spawns the detached __fcd daemon — which owns gvproxy, the frame @@ -185,6 +304,19 @@ func (f *FirecrackerProvider) Start(sb *config.Sandbox) error { sb.GuestIP = guestIP return nil // already running } + // Decide the network mode once, here, and hand the decision to the daemon + // below: if each probed independently they could disagree — the CLI + // skipping the host devices because rootless looked available, the daemon + // then falling back to bridge mode and finding none — and the resulting + // error would tell the user to run the command they just ran. + mode, why := SelectNetMode() + // In bridge mode, create the host devices here, in the foreground, while we + // still have the user's terminal to authenticate on. The daemon never can. + if mode == NetModeBridge { + if err := f.ensureBridgeHostNet(sb); err != nil { + return fmt.Errorf("%w (%s)", err, why) + } + } self, err := os.Executable() if err != nil { return fmt.Errorf("locating clawk binary: %w", err) @@ -194,6 +326,11 @@ func (f *FirecrackerProvider) Start(sb *config.Sandbox) error { cmd := exec.Command(self, "__fcd", sb.Name) cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true} cmd.Stdin, cmd.Stdout, cmd.Stderr = nil, nil, nil + // Pin the mode we just decided (and provisioned for), so the daemon can't + // probe its way to a different answer — and pass the reason with it, or the + // daemon reports our own handoff as if the user had pinned it, and the real + // cause never reaches fcd.log. + cmd.Env = append(os.Environ(), NetModeHandoff(mode, why)...) if err := cmd.Start(); err != nil { return fmt.Errorf("spawning fcd: %w", err) } @@ -207,8 +344,15 @@ func (f *FirecrackerProvider) Start(sb *config.Sandbox) error { ctx, cancel := context.WithTimeout(context.Background(), 150*time.Second) defer cancel() if err := vsockclient.Ping(ctx, f.vsockPath(sb), fcAgentPort, 120*time.Second); err != nil { - return fmt.Errorf("agent did not become ready: %w%s", - err, ConsoleTail(filepath.Join(vmDir, "console.log"), 20)) + // Both logs: the console shows guest-side failures (kernel panic, + // clawk-init), fcd.log host-side ones (spec build, gvproxy, the + // hypervisor refusing to start). A daemon that died before the VM + // existed leaves an empty console and the real cause in fcd.log, + // where the CLI's own error otherwise reads as a bare vsock timeout. + return fmt.Errorf("agent did not become ready: %w%s%s", + err, + LogTail(filepath.Join(vmDir, "fcd.log"), 10, "clawk daemon log"), + ConsoleTail(filepath.Join(vmDir, "console.log"), 20)) } return nil } @@ -216,23 +360,47 @@ func (f *FirecrackerProvider) Start(sb *config.Sandbox) error { // FCStateDir exposes the machine-library state dir to the __fcd daemon. func (f *FirecrackerProvider) FCStateDir(sb *config.Sandbox) string { return f.fcStateDir(sb) } -// DaemonSpec sets up the host network plumbing (IP-less bridge + the guest's -// TAP + the daemon-owned gvproxy TAP) and returns the machine.Spec the __fcd -// daemon boots. It runs in the daemon process; the returned spec carries a -// UserMode net in TAP-bridge mode so the firecracker backend brings up -// gvproxy (with allow as the egress filter) bridged to the guest's NIC. -func (f *FirecrackerProvider) DaemonSpec(sb *config.Sandbox, allow *netfilter.AllowList) (machine.Spec, error) { - if err := ensureLinuxBridge(); err != nil { - return machine.Spec{}, fmt.Errorf("bridge: %w", err) +// ensureBridgeHostNet provisions bridge mode's host-side network plumbing for +// sb: the IP-less bridge, the guest's TAP, and the daemon-owned gvproxy TAP. +// (Rootless mode has no host devices at all — see netmode_linux.go.) +// +// It runs in the CLI, from Start, before __fcd is spawned, because creating +// those devices needs CAP_NET_ADMIN and the CLI is the only process that can +// obtain it interactively. The daemon cannot: it is Setsid-detached, and +// sudo's timestamp records are tty-scoped, so a password the user typed a +// second earlier is invisible there and every `sudo -n` fails with "a +// password is required" — forever, no matter how recently they +// authenticated. Provisioning here means a password-sudo host prompts once, +// in front of the user, instead of failing inside a daemon whose log the +// CLI never shows. See issue #9. +// +// Idempotent and privilege-free once the devices exist (see ensureTAP). +func (f *FirecrackerProvider) ensureBridgeHostNet(sb *config.Sandbox) error { + bridge := bridgeDevice(sb.Name) + if err := ensureLinuxBridge(bridge); err != nil { + return fmt.Errorf("bridge: %w", err) } - fcTap := tapDevice(sb.Name) - if err := ensureTAP(fcTap); err != nil { - return machine.Spec{}, fmt.Errorf("guest tap: %w", err) + if err := ensureTAP(tapDevice(sb.Name), bridge); err != nil { + return fmt.Errorf("guest tap: %w", err) } - gvTap := gvTapDevice(sb.Name) - if err := ensureTAP(gvTap); err != nil { - return machine.Spec{}, fmt.Errorf("gvproxy tap: %w", err) + if err := ensureTAP(gvTapDevice(sb.Name), bridge); err != nil { + return fmt.Errorf("gvproxy tap: %w", err) } + return nil +} + +// DaemonSpec returns the machine.Spec the __fcd daemon boots, plus a cleanup +// to run when the daemon exits. It runs in the daemon process; the returned +// spec carries a UserMode net in TAP-bridge mode so the firecracker backend +// brings up gvproxy (with allow as the egress filter) bridged to the guest's +// NIC. +// +// Rootless mode builds that bridge inside a fresh user+network namespace here +// and now — it needs no privilege, so the daemon can do it despite having no +// terminal. Bridge mode instead uses host devices that ensureBridgeHostNet created +// in the CLI, and only verifies them, because creating them needs a sudo the +// daemon could never authenticate (see ensureBridgeHostNet). +func (f *FirecrackerProvider) DaemonSpec(sb *config.Sandbox, allow *netfilter.AllowList) (machine.Spec, func(), error) { forwards := make([]machine.PortForward, 0, len(sb.Forwards)) for _, fwd := range sb.Forwards { forwards = append(forwards, machine.PortForward{ @@ -240,20 +408,94 @@ func (f *FirecrackerProvider) DaemonSpec(sb *config.Sandbox, allow *netfilter.Al Proto: machine.ProtoTCP, }) } - spec := f.buildSpec(sb) - spec.Net = []machine.Net{machine.UserMode{ - Filter: allow, - Forwards: forwards, - GuestTAP: fcTap, - HostTAP: gvTap, - }} - return spec, nil + net := machine.UserMode{Filter: allow, Forwards: forwards} + + mode, why := SelectNetMode() + var cleanup = func() {} + var ns *sandboxNetNS // rootless mode only + switch mode { + case NetModeRootless: + self, err := os.Executable() + if err != nil { + return machine.Spec{}, nil, fmt.Errorf("locating clawk binary: %w", err) + } + ns, err = startSandboxNetNS(self, os.Stderr) + if err != nil { + return machine.Spec{}, nil, fmt.Errorf("rootless network: %w", err) + } + // ns still owns the TAP fd, so cleanup closes it on any failure + // between here and the return below. Ownership moves to the backend + // only once the spec is really going to it (TakeHostTAP, at the end). + cleanup = func() { _ = ns.Close() } + net.GuestTAP = nsGuestTAP + net.NetNSExec = ns.ExecPrefix + default: + fcTap, gvTap := tapDevice(sb.Name), gvTapDevice(sb.Name) + for _, dev := range []string{bridgeDevice(sb.Name), fcTap, gvTap} { + if !linkExists(dev) { + return machine.Spec{}, nil, fmt.Errorf( + "host network device %s for sandbox %q is missing (bridge mode: %s) — "+ + "it is provisioned by 'clawk up %s'; if something removed it, "+ + "run 'clawk down %s && clawk up %s'", + dev, sb.Name, why, sb.Name, sb.Name, sb.Name) + } + } + net.GuestTAP, net.HostTAP = fcTap, gvTap + } + + // Bounded, and not with context.Background(): resolveKernel revalidates a + // URL override on every call and can download the CI kernel, so an + // unreachable host (allow-list refusal, captive portal, dead mirror) would + // wedge the detached daemon here forever — holding the network namespace it + // just created, while the CLI reports only its own vsock timeout and nothing + // ever reaps the daemon. + kernelCtx, cancelKernel := context.WithTimeout(context.Background(), kernelResolveTimeout) + defer cancelKernel() + kernelPath, err := f.resolveKernel(kernelCtx, sb) + if err != nil { + cleanup() + return machine.Spec{}, nil, fmt.Errorf("kernel: %w", err) + } + spec := f.buildSpec(sb, kernelPath) + if ns != nil { + // Handed over here and nowhere earlier: machine.UserMode documents that + // HostTAPFile's ownership travels with the spec to the backend, so this + // is the point where cleanup must stop accounting for the fd — and + // every failure before it is a path where cleanup still has to close it. + net.HostTAPFile = ns.TakeHostTAP() + } + spec.Net = []machine.Net{net} + return spec, cleanup, nil +} + +// NetModeForLog reports the mode the daemon will use and why it isn't +// rootless, for the daemon's startup log line. +func (f *FirecrackerProvider) NetModeForLog() (NetMode, string) { return SelectNetMode() } + +// resolveKernel returns the vmlinux to direct-boot: the sandbox's override +// (clawk.mod `vm ( kernel … )`, or --kernel) when set, else the firecracker-CI +// kernel. +// +// The override used to be ignored here, which was worse than it sounds. The CI +// kernel is deliberately minimal — no 9p, no FUSE, no virtio-fs, so no +// filesystem transport of any kind — and the flag is precisely how a user says +// "boot something that has one". Silently dropping it made that impossible and +// looked like the kernel was at fault. +func (f *FirecrackerProvider) resolveKernel(ctx context.Context, sb *config.Sandbox) (string, error) { + if sb.Kernel == "" { + return ciEnsureKernel(ctx) + } + return kernel.Fetch(ctx, kernel.Options{ + CacheDir: f.store.CacheDir(), + Arch: runtime.GOARCH, + Override: sb.Kernel, + }) } -// buildSpec assembles the resource/boot/rootfs parts of the machine.Spec. -// The Net entry is filled in by DaemonSpec once the TAPs exist. -func (f *FirecrackerProvider) buildSpec(sb *config.Sandbox) machine.Spec { - kernel, _ := ciEnsureKernel(context.Background()) +// buildSpec assembles the resource/boot/rootfs parts of the machine.Spec, given +// the kernel DaemonSpec resolved. The Net entry is filled in by DaemonSpec once +// the TAPs exist. +func (f *FirecrackerProvider) buildSpec(sb *config.Sandbox, kernelPath string) machine.Spec { vcpu := uint(1) if sb.CPU > 0 { vcpu = sb.CPU @@ -272,15 +514,21 @@ func (f *FirecrackerProvider) buildSpec(sb *config.Sandbox) machine.Spec { MemoryMiB: memMiB, MemoryMaxMiB: memMaxMiB, Boot: machine.DirectKernel{ - Vmlinux: kernel, + Vmlinux: kernelPath, // Firecracker's serial is ttyS0 (no virtio-console/hvc0). // clawk-init reads the manifest on /dev/vdb and configures the // network from it, so there's no ip= cmdline. Cmdline: "console=ttyS0 reboot=k panic=1 pci=off root=/dev/vda rw " + "init=" + guestcfg.InitPath + " clawk.cfg=/dev/vdb", }, - RootFS: machine.RawDisk{Path: filepath.Join(f.vmDir(sb), "rootfs.raw")}, - Disks: []machine.Disk{{Path: filepath.Join(f.vmDir(sb), "guestcfg.img"), ReadOnly: true}}, + RootFS: machine.RawDisk{Path: filepath.Join(f.vmDir(sb), "rootfs.raw")}, + // Order is the guest's device order: vdb=guestcfg, vdc=worktree + // (fcWorktreeDevice). Appending anything here shifts later devices, + // so keep new disks after these two. + Disks: []machine.Disk{ + {Path: filepath.Join(f.vmDir(sb), "guestcfg.img"), ReadOnly: true}, + {Path: f.worktreeDiskPath(sb)}, + }, VSockCID: fcVSockCID, Serial: machine.Serial{LogPath: filepath.Join(f.vmDir(sb), "console.log")}, } @@ -301,10 +549,25 @@ func (f *FirecrackerProvider) Stop(sb *config.Sandbox) error { return stopByPIDFile(filepath.Join(f.vmDir(sb), "fc.pid"), 25*time.Second) } +// Destroy stops the VM and removes its host devices and state. +// +// Device teardown is best-effort and deliberately non-interactive +// (runSudoQuiet): a destroy that can't reach sudo should still delete the VM +// dir, not stop to ask for a password on the way out. It is also driven by +// what actually exists rather than by the current mode — a rootless sandbox +// has no host devices to remove (its namespace took them with it), while a +// sandbox created back when the host used bridge mode still does, and those +// must not be leaked just because rootless works now. func (f *FirecrackerProvider) Destroy(sb *config.Sandbox) error { _ = f.Stop(sb) - _ = runSudo("ip", "link", "del", tapDevice(sb.Name)) - _ = runSudo("ip", "link", "del", gvTapDevice(sb.Name)) + // TAPs first, then the bridge they were enslaved to — it belongs to this + // sandbox alone, so leaving it behind would leak one interface per + // destroyed sandbox. + for _, dev := range []string{tapDevice(sb.Name), gvTapDevice(sb.Name), bridgeDevice(sb.Name)} { + if linkExists(dev) { + _ = runSudoQuiet("ip", "link", "del", dev) + } + } return os.RemoveAll(f.vmDir(sb)) } diff --git a/internal/sandbox/firecracker_linux_test.go b/internal/sandbox/firecracker_linux_test.go new file mode 100644 index 0000000..ada44bc --- /dev/null +++ b/internal/sandbox/firecracker_linux_test.go @@ -0,0 +1,87 @@ +//go:build linux + +package sandbox + +import ( + "os" + "path/filepath" + "testing" + + "github.com/clawkwork/clawk/internal/config" + "github.com/stretchr/testify/require" +) + +// hasRestorableState gates whether Create skips building the disks. It must +// answer "no" unless EVERY disk buildSpec attaches is already there: skipping +// the build leaves whatever is on disk as the whole disk set, so a suspend +// state next to an incomplete one means a cold boot against a drive that +// doesn't exist, which firecracker refuses outright. +func TestHasRestorableState(t *testing.T) { + root := t.TempDir() + f := &FirecrackerProvider{store: config.NewStoreAt(root)} + sb := &config.Sandbox{Name: "proj"} + vmDir := f.vmDir(sb) + rootfs := filepath.Join(vmDir, "rootfs.raw") + + touch := func(path string) { + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte("x"), 0o644)) + } + + require.False(t, f.hasRestorableState(sb, vmDir, rootfs), "no suspend state at all") + + touch(filepath.Join(vmDir, "suspend", "snapshot.state")) + require.False(t, f.hasRestorableState(sb, vmDir, rootfs), "state but no rootfs") + + touch(rootfs) + touch(filepath.Join(vmDir, "guestcfg.img")) + require.False(t, f.hasRestorableState(sb, vmDir, rootfs), + "a state saved before the worktree had its own disk must not be trusted") + + touch(f.worktreeDiskPath(sb)) + require.True(t, f.hasRestorableState(sb, vmDir, rootfs), "the full disk set is present") + + require.NoError(t, os.Remove(filepath.Join(vmDir, "guestcfg.img"))) + require.False(t, f.hasRestorableState(sb, vmDir, rootfs), "guestcfg.img is attached too") +} + +func TestWorktreeDiskSize(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "sub"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "a"), make([]byte, 1024), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "sub", "b"), make([]byte, 2048), 0o644)) + // Symlinks and dirs must not be counted as content. + require.NoError(t, os.Symlink("a", filepath.Join(dir, "link"))) + + size, err := worktreeDiskSize(dir) + require.NoError(t, err) + // 3 KiB of content is dwarfed by the slack floor, which is the point: + // small repos still get room for the agent to build in. + require.Equal(t, int64(3072+worktreeDiskSlack), size) + + t.Run("large trees scale past the floor", func(t *testing.T) { + // A tree bigger than the slack doubles instead of adding a constant. + big := t.TempDir() + f, err := os.Create(filepath.Join(big, "big")) + require.NoError(t, err) + // Sparse: Truncate reserves no blocks, but Size() reports it, which is + // what the sizing walk reads. + require.NoError(t, f.Truncate(worktreeDiskSlack+1)) + require.NoError(t, f.Close()) + + size, err := worktreeDiskSize(big) + require.NoError(t, err) + require.Equal(t, int64(2*(worktreeDiskSlack+1)), size) + }) + + t.Run("empty tree still gets headroom", func(t *testing.T) { + size, err := worktreeDiskSize(t.TempDir()) + require.NoError(t, err) + require.Equal(t, int64(worktreeDiskSlack), size) + }) + + t.Run("missing tree is an error", func(t *testing.T) { + _, err := worktreeDiskSize(filepath.Join(dir, "absent")) + require.Error(t, err) + }) +} diff --git a/internal/sandbox/linux_shared.go b/internal/sandbox/linux_shared.go index ccf532b..9e5b5f1 100644 --- a/internal/sandbox/linux_shared.go +++ b/internal/sandbox/linux_shared.go @@ -7,6 +7,7 @@ package sandbox import ( + "bytes" "context" "crypto/sha256" "encoding/hex" @@ -22,18 +23,51 @@ import ( "sort" "strconv" "strings" + "sync" "syscall" "time" + + "golang.org/x/term" ) -// linuxBridge is the shared L2 bridge firecracker TAPs attach to. It -// deliberately carries NO host IP: gvproxy owns L3 (gateway, DHCP, NAT, -// DNS) entirely in userspace — same as the vz provider — so the bridge is -// just a dumb switch joining each VM's TAP to the daemon-owned TAP that -// feeds gvproxy. Because no real interface ever claims gvproxy's -// 192.168.127.1/24, this is safe even when clawk runs nested inside a vz -// VM that already uses that subnet on its own eth0. See fcnet_linux.go. -const linuxBridge = "clawkbr0" +// legacyLinuxBridge is the single bridge every sandbox shared before bridges +// became per-sandbox. Kept only so doctor can point at a stale leftover. +const legacyLinuxBridge = "clawkbr0" + +// bridgeDevice returns the name of the L2 bridge a sandbox's TAPs attach to. +// +// The bridge deliberately carries NO host IP: gvproxy owns L3 (gateway, DHCP, +// NAT, DNS) entirely in userspace — same as the vz provider — so the bridge is +// just a dumb switch joining the VM's TAP to the daemon-owned TAP that feeds +// gvproxy. Because no real interface ever claims gvproxy's 192.168.127.1/24, +// this is safe even when clawk runs nested inside a vz VM that already uses +// that subnet on its own eth0. +// +// One bridge PER SANDBOX, not one shared: every guest gets the same +// 192.168.127.2 from its own gvproxy, so a shared bridge put two guests with +// one IP (and, before per-VM MACs, one MAC) on a single L2 segment — frames +// went to whichever guest the forwarding table had seen last. Separate +// bridges make each sandbox its own L2 domain, which is what vz gives every +// VM for free. +// +// IFNAMSIZ is 16 (15 usable): "clawkbr" + 8 hex chars is 15. +func bridgeDevice(sbName string) string { return "clawkbr" + deviceHash(sbName) } + +// deviceHash is the suffix every host network device name for a sandbox +// carries: 8 hex chars of sha256(uid + name). +// +// The uid is in the hash, and must be in EVERY device's — a name that collides +// across users is worse than it sounds. Two users with an identically-named +// sandbox would land on one TAP: the second `clawk up` finds the device +// already there, skips creation, and re-enslaves the first user's LIVE TAP to +// its own bridge (killing that VM's network) — then firecracker cannot open it +// anyway, because the device belongs to the other uid. `clawk destroy` by +// either user would delete the other's devices too. Keeping the hash in one +// place is what stops a new device kind from reintroducing that. +func deviceHash(sbName string) string { + sum := sha256.Sum256([]byte(strconv.Itoa(os.Getuid()) + ":" + sbName)) + return hex.EncodeToString(sum[:4]) +} // firecracker-ci kernel catalog. The bucket exposes versioned // `vmlinux-X.Y.Z` objects; we pick the newest by component-wise version @@ -60,28 +94,23 @@ func ciArch() string { } // tapDevice returns a deterministic TAP name for a sandbox. IFNAMSIZ is 16 -// (15 usable); "clawk" + 8 hex chars of sha256(name) is 13. -func tapDevice(sbName string) string { - sum := sha256.Sum256([]byte(sbName)) - return "clawk" + hex.EncodeToString(sum[:4]) -} - -// linkExists reports whether a network link with the given name is present. -// `ip link show` exits 0 when found, 1 otherwise — we rely on the exit code -// rather than parsing output. -func linkExists(name string) bool { - return exec.Command("ip", "link", "show", name).Run() == nil -} +// (15 usable); "clawk" + 8 hex chars (see deviceHash) is 13. +func tapDevice(sbName string) string { return "clawk" + deviceHash(sbName) } -// ensureLinuxBridge is idempotent: creates the shared (IP-less) bridge and -// brings it up. No address is assigned — see linuxBridge. -func ensureLinuxBridge() error { - if !linkExists(linuxBridge) { - if err := runSudo("ip", "link", "add", "name", linuxBridge, "type", "bridge"); err != nil { +// ensureLinuxBridge is idempotent: creates the sandbox's (IP-less) bridge and +// brings it up. No address is assigned — see bridgeDevice. Each step is +// skipped when sysfs already reports it done (netstate_linux.go), so +// re-booting an existing sandbox performs no privileged work. +func ensureLinuxBridge(bridge string) error { + if !linkExists(bridge) { + if err := runSudo("ip", "link", "add", "name", bridge, "type", "bridge"); err != nil { return err } } - return runSudo("ip", "link", "set", linuxBridge, "up") + if linkIsUp(bridge) { + return nil + } + return runSudo("ip", "link", "set", bridge, "up") } // gvTapDevice returns the deterministic name of the daemon-owned TAP that @@ -92,32 +121,162 @@ func gvTapDevice(sbName string) string { return tapDevice(sbName) + "g" } // ensureTAP creates the TAP device if missing, enslaves it to the bridge, // and brings it up. The device is owned by the current uid so the (non- // root) daemon and firecracker can open its fd without CAP_NET_ADMIN. -// Safe to call on an already-configured device. -func ensureTAP(tap string) error { +// Safe to call on an already-configured device, and free of privileged +// calls when it already is one. +func ensureTAP(tap, bridge string) error { if !linkExists(tap) { uid := strconv.Itoa(os.Getuid()) if err := runSudo("ip", "tuntap", "add", "dev", tap, "mode", "tap", "user", uid); err != nil { return err } } - if err := runSudo("ip", "link", "set", tap, "master", linuxBridge); err != nil { - return err + if linkMaster(tap) != bridge { + if err := runSudo("ip", "link", "set", tap, "master", bridge); err != nil { + return err + } + } + if linkIsUp(tap) { + return nil } return runSudo("ip", "link", "set", tap, "up") } -// runSudo runs `sudo -n ` and wraps the error with the command -// output for diagnosis. The -n flag means "never prompt" — fail fast instead -// of hanging waiting for a password. +// sudoAuthError is a `sudo -n` failure sudo attributed to a missing +// password rather than to the command itself. runSudo retries these +// interactively when it has a terminal; every other failure is final. +type sudoAuthError struct{ err error } + +func (e *sudoAuthError) Error() string { return e.err.Error() } +func (e *sudoAuthError) Unwrap() error { return e.err } + +// runSudo runs a privileged command, preferring the non-interactive path. +// +// `sudo -n` first: on a host with passwordless sudo (or a live timestamp +// record for this terminal) it succeeds silently, and that is both the +// common case and the only path available to a caller with no terminal. +// When it fails purely for want of a password and we DO have a terminal, +// retry without -n so sudo can ask — one prompt in the foreground beats a +// hard failure, which is what issue #9 hit. Teardown paths that must never +// prompt call runSudoQuiet directly. func runSudo(cmd string, args ...string) error { - full := append([]string{"-n", cmd}, args...) - out, err := exec.Command("sudo", full...).CombinedOutput() - if err != nil { - return fmt.Errorf("sudo %s %s: %w (%s)", cmd, strings.Join(args, " "), err, strings.TrimSpace(string(out))) + err := runSudoQuiet(cmd, args...) + var authErr *sudoAuthError + if err == nil || !errors.As(err, &authErr) { + return err + } + if !StdinIsTerminal() { + return fmt.Errorf("%w — and no terminal to prompt on: "+ + "run this from a terminal, or grant passwordless sudo for ip", err) + } + noteSudoPrompt() + full := append([]string{cmd}, args...) + c := exec.Command("sudo", full...) + // Prompt and any command output go to stderr: stdout belongs to the + // CLI's own (sometimes parsed) output. + c.Stdin, c.Stdout, c.Stderr = os.Stdin, os.Stderr, os.Stderr + if runErr := c.Run(); runErr != nil { + return fmt.Errorf("sudo %s %s: %w", cmd, strings.Join(args, " "), runErr) } return nil } +// runSudoQuiet runs `sudo -n ` and never prompts, wrapping +// the error with the command output for diagnosis. Failures sudo blames on +// a missing password come back as *sudoAuthError so runSudo can decide +// whether a prompt is possible. +func runSudoQuiet(cmd string, args ...string) error { + full := append([]string{"-n", cmd}, args...) + c := exec.Command("sudo", full...) + c.Env = cLocaleEnv() + out, err := c.CombinedOutput() + if err == nil { + return nil + } + wrapped := fmt.Errorf("sudo %s %s: %w (%s)", cmd, strings.Join(args, " "), err, strings.TrimSpace(string(out))) + if sudoOutputMentionsPassword(out) { + return &sudoAuthError{err: wrapped} + } + return wrapped +} + +// cLocaleEnv is our environment with LC_ALL=C, so sudo's own diagnostics come +// back in English and sudoOutputMentionsPassword can match them. sudo emits +// those from the front-end process, before env_reset touches what the command +// sees, so this only affects sudo's messages — not the command's behavior. +func cLocaleEnv() []string { + env := os.Environ() + out := make([]string, 0, len(env)+1) + for _, kv := range env { + if strings.HasPrefix(kv, "LC_ALL=") || strings.HasPrefix(kv, "LANG=") { + continue + } + out = append(out, kv) + } + return append(out, "LC_ALL=C") +} + +// sudoOutputMentionsPassword reports whether a failed `sudo -n` attempt was +// sudo declining to authenticate rather than the command itself failing. +// +// Matching sudo's own message is the whole test, and reliable because we force +// LC_ALL=C (see cLocaleEnv). Anything sudo doesn't blame on authentication +// belongs to the command, which matters because runSudo RETRIES what this +// returns true for: a misclassified "RTNETLINK answers: File exists" re-runs a +// privileged command that already ran. +// +// There is deliberately no "can sudo run anything at all?" fallback probe. It +// looks like a safety net and is the opposite: on a sudoers that grants +// NOPASSWD for `ip` alone — the setup clawk documents — `sudo -n true` fails on +// every host, including ones where `sudo ip` runs freely, so every ordinary +// command failure came back as "needs a password". +// +// Deliberately NOT `sudo -n -v` either, which looks like the obvious probe and +// is wrong for the mirror-image reason: -v validates the credential cache, and +// a host with NOPASSWD has no cache to validate, so sudo tries to authenticate +// and -n makes that fail — "a password is required" on a host where every +// command in fact runs freely. +func sudoOutputMentionsPassword(out []byte) bool { + lower := bytes.ToLower(out) + // "no tty present and no askpass program specified" is the same refusal + // worded without the word "password" (older sudo, and sudo built without + // the -n message); the ones that do say it cover the rest. + return bytes.Contains(lower, []byte("password")) || + bytes.Contains(lower, []byte("askpass")) +} + +// SudoIPWorksUnprompted reports whether `sudo ip` runs without asking for a +// password — the one privileged thing bridge mode needs. +// +// It probes with `ip -V`, the actual binary under the actual sudoers rules: +// side-effect free, and accurate for a sudoers that permits only `ip` (where +// probing `true` or `-v` would wrongly report that a password is needed). +func SudoIPWorksUnprompted() bool { + c := exec.Command("sudo", "-n", "ip", "-V") + c.Env = cLocaleEnv() + return c.Run() == nil +} + +// StdinIsTerminal reports whether we can prompt the user for a password. +// Exported because doctor's verdict on bridge mode turns on the same question. +func StdinIsTerminal() bool { return term.IsTerminal(int(os.Stdin.Fd())) } + +// sudoPromptOnce keeps the explanation to one line per process, however +// many devices the run has to create. +var sudoPromptOnce sync.Once + +// noteSudoPrompt explains, once, why a VM boot is asking for a password — +// otherwise a bare "[sudo] password for …" in the middle of `clawk up` +// looks like the agent escalating, which is exactly the fear issue #9 +// raised. +func noteSudoPrompt() { + sudoPromptOnce.Do(func() { + fmt.Fprintln(os.Stderr, + "clawk needs sudo once per sandbox to create its network devices "+ + "(ip tuntap / ip link); the VM itself runs unprivileged. "+ + "Run 'clawk doctor' for the zero-sudo setup.") + }) +} + // ciCacheDir is the user-local cache for the downloaded kernel. func ciCacheDir() string { home, _ := os.UserHomeDir() @@ -239,28 +398,6 @@ func downloadAsset(ctx context.Context, url, dst string) error { return nil } -// mountedRootfs loop-mounts an ext4 rootfs, invokes fn with the mountpoint -// inside it, and unmounts on return. Uses loop_mount_linux.go's self-exec -// helpers instead of `sudo mount`/`sudo umount` because util-linux mount(8) -// and umount(8) crash with SIGILL on some aarch64 nested-virt kernels. -func mountedRootfs(rootfs string, fn func(mnt string) error) (retErr error) { - mnt, err := os.MkdirTemp("", "clawk-rootfs-") - if err != nil { - return err - } - defer os.RemoveAll(mnt) - loop, err := loopMountExt4(rootfs, mnt) - if err != nil { - return err - } - defer func() { - if err := loopUnmountExt4(mnt, loop); err != nil && retErr == nil { - retErr = err - } - }() - return fn(mnt) -} - // checkKVMAccess verifies the invoking user can open /dev/kvm read/write — // the one host prerequisite firecracker can't degrade around. It returns an // actionable error (naming the kvm-group fix) instead of letting the failure diff --git a/internal/sandbox/loop_mount_linux.go b/internal/sandbox/loop_mount_linux.go deleted file mode 100644 index 281b65c..0000000 --- a/internal/sandbox/loop_mount_linux.go +++ /dev/null @@ -1,133 +0,0 @@ -//go:build linux - -// Loop-mount plumbing for rootfs injection. We do NOT call `sudo mount -o loop` -// because util-linux mount(8) and umount(8) crash with SIGILL/SIGSEGV on some -// aarch64 nested-virt kernels (the mount(2) syscall itself succeeds, but the -// binaries die in post-syscall housekeeping). Instead we re-exec the clawk -// binary under sudo and invoke syscall.Mount / syscall.Unmount directly. -// -// The hidden subcommands MountHelperCmd / UnmountHelperCmd are dispatched by -// InitRootHelpers, which must be called at the top of main before any CLI -// parsing so sudo re-exec paths don't run cobra. - -package sandbox - -import ( - "fmt" - "os" - "os/exec" - "strings" - "syscall" -) - -// Hidden subcommand names. Prefixed with "__" so they don't collide with -// user-facing cobra commands and are obviously private. -const ( - mountHelperCmd = "__loop-mount" - unmountHelperCmd = "__loop-unmount" -) - -// InitRootHelpers handles privileged mount/umount re-exec before cobra runs. -// Returns only if os.Args does not match a helper command; otherwise it -// exits the process. -func InitRootHelpers() { - if len(os.Args) < 2 { - return - } - switch os.Args[1] { - case mountHelperCmd: - // args: __loop-mount - if len(os.Args) != 5 { - fmt.Fprintf(os.Stderr, "usage: clawk %s \n", mountHelperCmd) - os.Exit(2) - } - if err := syscall.Mount(os.Args[2], os.Args[3], os.Args[4], 0, ""); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - os.Exit(0) - case unmountHelperCmd: - // args: __loop-unmount - if len(os.Args) != 3 { - fmt.Fprintf(os.Stderr, "usage: clawk %s \n", unmountHelperCmd) - os.Exit(2) - } - // MNT_DETACH so a lingering reference (e.g. a half-torn-down util-linux - // mount(8) from this kernel's SIGILL bug) can't pin the mountpoint. - if err := syscall.Unmount(os.Args[2], 0x00000002 /* MNT_DETACH */); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - os.Exit(0) - } -} - -// loopMountExt4 attaches backing to a free loop device and mounts it as ext4 -// at mnt. Returns the loop device path so the caller can detach it after -// unmounting. -func loopMountExt4(backing, mnt string) (string, error) { - loop, err := losetupFind(backing) - if err != nil { - return "", err - } - if err := selfMount(loop, mnt, "ext4"); err != nil { - _ = losetupDetach(loop) - return "", err - } - return loop, nil -} - -// loopUnmountExt4 unmounts mnt and detaches the loop device. Both steps run -// even if the first fails, so the caller isn't left with half-torn-down -// kernel state; the first error encountered is returned. -func loopUnmountExt4(mnt, loop string) error { - umErr := selfUnmount(mnt) - loErr := losetupDetach(loop) - switch { - case umErr != nil: - return umErr - case loErr != nil: - return loErr - } - return nil -} - -func selfMount(src, dst, fstype string) error { - self, err := os.Executable() - if err != nil { - return fmt.Errorf("resolving self exe: %w", err) - } - out, err := exec.Command("sudo", "-n", self, mountHelperCmd, src, dst, fstype).CombinedOutput() - if err != nil { - return fmt.Errorf("%s %s %s: %w (%s)", mountHelperCmd, src, dst, err, strings.TrimSpace(string(out))) - } - return nil -} - -func selfUnmount(dst string) error { - self, err := os.Executable() - if err != nil { - return fmt.Errorf("resolving self exe: %w", err) - } - out, err := exec.Command("sudo", "-n", self, unmountHelperCmd, dst).CombinedOutput() - if err != nil { - return fmt.Errorf("%s %s: %w (%s)", unmountHelperCmd, dst, err, strings.TrimSpace(string(out))) - } - return nil -} - -func losetupFind(backing string) (string, error) { - out, err := exec.Command("sudo", "-n", "losetup", "--find", "--show", backing).CombinedOutput() - if err != nil { - return "", fmt.Errorf("losetup --find %s: %w (%s)", backing, err, strings.TrimSpace(string(out))) - } - return strings.TrimSpace(string(out)), nil -} - -func losetupDetach(loop string) error { - out, err := exec.Command("sudo", "-n", "losetup", "-d", loop).CombinedOutput() - if err != nil { - return fmt.Errorf("losetup -d %s: %w (%s)", loop, err, strings.TrimSpace(string(out))) - } - return nil -} diff --git a/internal/sandbox/loop_mount_stub.go b/internal/sandbox/loop_mount_stub.go deleted file mode 100644 index de89bef..0000000 --- a/internal/sandbox/loop_mount_stub.go +++ /dev/null @@ -1,7 +0,0 @@ -//go:build !linux - -package sandbox - -// InitRootHelpers is a no-op on non-linux — the hidden mount/umount helpers -// only exist for the Linux-only VM providers. -func InitRootHelpers() {} diff --git a/internal/sandbox/netmode_linux.go b/internal/sandbox/netmode_linux.go new file mode 100644 index 0000000..24803f2 --- /dev/null +++ b/internal/sandbox/netmode_linux.go @@ -0,0 +1,135 @@ +//go:build linux + +// How the firecracker provider obtains the CAP_NET_ADMIN it needs to create a +// sandbox's network devices. Two modes, picked per boot: +// +// - NetModeRootless: an unprivileged user+network namespace per sandbox +// (netns_linux.go). No privileged operation, ever, and each VM gets its own +// L2 domain. The default wherever the kernel allows it. +// +// - NetModeBridge: host devices created through sudo, one prompt per sandbox +// at most (linux_shared.go). The fallback for hosts that forbid +// unprivileged user namespaces — Ubuntu 24.04+ does by default, via +// AppArmor — and the only mode before rootless existed. +// +// The choice is made by trying: whether an unprivileged user namespace works +// depends on kernel config, sysctls, LSM policy and container nesting at once, +// and Ubuntu's AppArmor restriction in particular is invisible to every check +// short of attempting the clone. CLAWK_NET_MODE overrides, for operators who +// want one or the other pinned. + +package sandbox + +import ( + "fmt" + "os" + "strings" + "sync" +) + +// NetMode is how a sandbox's host-side network devices come into being. +type NetMode string + +const ( + NetModeRootless NetMode = "rootless" + NetModeBridge NetMode = "bridge" +) + +// netModeEnv pins the mode when set to a known value. +const netModeEnv = "CLAWK_NET_MODE" + +// netModeWhyEnv carries the reason behind a mode the CLI already decided. +// +// It exists because Start pins its decision in the daemon's environment (see +// FirecrackerProvider.Start) so the daemon can't probe its way to a different +// answer — which makes every inherited decision indistinguishable from an +// operator's own pin. Without the reason travelling too, the daemon log +// reported "pinned by CLAWK_NET_MODE=bridge" for a variable the user never set, +// and the real cause (AppArmor, a missing nsenter) appeared nowhere in it. +const netModeWhyEnv = "CLAWK_NET_MODE_WHY" + +// NetModePinned reports the CLAWK_NET_MODE override, if a valid one is set. +// A pin is honored as-is: someone who pinned rootless does not want a silent +// fallback to a mode that may ask for their password. +func NetModePinned() (NetMode, bool) { + switch strings.ToLower(strings.TrimSpace(os.Getenv(netModeEnv))) { + case string(NetModeRootless): + return NetModeRootless, true + case string(NetModeBridge): + return NetModeBridge, true + case "": + return "", false + default: + // A typo is not a mode; say so rather than silently picking something. + fmt.Fprintf(os.Stderr, "warning: ignoring unknown %s=%q (want rootless or bridge)\n", + netModeEnv, os.Getenv(netModeEnv)) + return "", false + } +} + +// SelectNetMode reports the mode to use and, when it isn't rootless, a +// complete phrase saying why — "pinned by CLAWK_NET_MODE=bridge" for a +// deliberate override, "rootless networking unavailable: …" for a host that +// cannot do it. +// +// Callers render the phrase verbatim rather than prefixing their own reason. +// The two facts are different — one is the operator's own configuration, the +// other a host capability — and reporting a pin as "rootless unavailable" +// handed users fixes for a restriction that wasn't there. +func SelectNetMode() (NetMode, string) { + if pinned, ok := NetModePinned(); ok { + if pinned == NetModeBridge { + return NetModeBridge, pinnedBridgeWhy() + } + return pinned, "" + } + if ok, why := NetNSAvailable(); !ok { + return NetModeBridge, "rootless networking unavailable: " + why + } + return NetModeRootless, "" +} + +// pinnedBridgeWhy explains a bridge pin: the reason the CLI passed down when +// this process inherited its decision (netModeWhyEnv), otherwise the pin +// itself, which is all there is to say when an operator set it by hand. +func pinnedBridgeWhy() string { + if why := strings.TrimSpace(os.Getenv(netModeWhyEnv)); why != "" { + return why + } + return "pinned by " + netModeEnv + "=bridge" +} + +// NetModeHandoff is the environment a child process needs to inherit this +// process's network-mode decision verbatim — the mode so it cannot probe its +// way to a different answer, and the reason so it reports the truth about why. +func NetModeHandoff(mode NetMode, why string) []string { + env := []string{netModeEnv + "=" + string(mode)} + if why != "" { + env = append(env, netModeWhyEnv+"="+why) + } + return env +} + +// netNSProbe caches the namespace probe: forking a namespace costs a few +// milliseconds, several call sites ask, and a host's capability cannot change +// under a running process. +var netNSProbe = sync.OnceValues(probeNetNS) + +// NetNSAvailable reports whether this host can create the unprivileged +// user+network namespace rootless mode needs, and if not, why — phrased as the +// thing a user would have to change. +func NetNSAvailable() (bool, string) { return netNSProbe() } + +func probeNetNS() (bool, string) { + self, err := os.Executable() + if err != nil { + return false, fmt.Sprintf("cannot locate the clawk binary: %v", err) + } + ns, err := startSandboxNetNS(self, nil) // quiet: the reason is returned, not logged + if err != nil { + return false, netNSHint(err) + } + // Close covers the TAP fd too — nobody took ownership of it here. + _ = ns.Close() + return true, "" +} diff --git a/internal/sandbox/netns_linux.go b/internal/sandbox/netns_linux.go new file mode 100644 index 0000000..1f3053c --- /dev/null +++ b/internal/sandbox/netns_linux.go @@ -0,0 +1,500 @@ +//go:build linux + +// Rootless per-sandbox networking: one unprivileged user+network namespace +// per VM, owning the bridge and the two TAPs. +// +// Creating a network device needs CAP_NET_ADMIN. Linux hands that out for +// free inside a user namespace you own — so instead of asking sudo for `ip`, +// clawk forks a child into CLONE_NEWUSER|CLONE_NEWNET, where it is root over +// its own private network, and builds the topology there with netlink. The +// result on a working host is a sandbox that boots with no privileged +// operation at all, and a host whose interface list stays empty. +// +// host netns (the __fcd daemon): sandbox netns (the anchor): +// gvproxy ── pump ── gv0 fd ────────────► gv0 ─┐ +// (SCM_RIGHTS) ├── br0 +// firecracker (via nsenter) ───────────► tap0 ─┘ +// +// gvproxy stays in the host namespace on purpose: its whole job is egress +// through host sockets. Only the VM's NIC moves. A TAP fd is namespace- +// agnostic once open, so the pump reaches into the sandbox's L2 with nothing +// more than the fd the anchor sends it — while the hypervisor, which can only +// name a TAP, is launched inside the namespace (see nsenterExecPrefix). +// +// Why an anchor process rather than a named namespace: an unnamed netns lives +// exactly as long as something references it, so the anchor holding it open +// (and dying with the daemon, whose pipe it blocks on) makes teardown +// automatic — no `ip netns del`, no leaked interfaces, nothing to clean up +// after a crash. It also means the namespace is unreachable to anything that +// doesn't already have the daemon's fds. +// +// This is the same shape rootlesskit/slirp4netns give rootless podman. + +package sandbox + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync" + "syscall" + "time" + "unsafe" + + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" +) + +// netnsAnchorCmd is the hidden subcommand for the anchor, dispatched before +// cobra sees os.Args. Prefixed with "__" so it can't collide with a +// user-facing verb. +const netnsAnchorCmd = "__netns-init" + +// Device names inside a sandbox's namespace. Fixed, because the namespace +// holds exactly one VM — no hashing needed (contrast tapDevice, which shares +// the host's flat namespace in bridge mode). +const ( + nsBridge = "br0" + nsGuestTAP = "tap0" + nsHostTAP = "gv0" +) + +// anchorReadyByte is the handshake the anchor writes once the topology is up, +// alongside the gv0 fd. +const anchorReadyByte = 'R' + +// anchorStartTimeout bounds how long we wait for that handshake. Generous: +// the work is a handful of netlink calls, so anything approaching this means +// the child is wedged, not slow. +const anchorStartTimeout = 15 * time.Second + +// InitNetNSHelpers dispatches the two hidden namespace subcommands before +// cobra runs, and exits the process when it handles one. Returns normally +// when os.Args is an ordinary invocation. +// +// Must be called at the very top of main: these paths are re-execs of the +// clawk binary as a helper, not CLI commands, and must never touch the CLI's +// config store or flag parsing. +func InitNetNSHelpers() { + if len(os.Args) < 2 { + return + } + if os.Args[1] == netnsAnchorCmd { + runNetNSAnchor() + } +} + +// runNetNSAnchor is the child side of the namespace: it builds the topology, +// hands the host-side TAP fd back over the inherited socket (fd 3), and then +// blocks on stdin so the namespace outlives this call for as long as the +// parent holds the pipe. Never returns. +func runNetNSAnchor() { + if err := buildNetNSTopology(); err != nil { + fmt.Fprintf(os.Stderr, "netns anchor: %v\n", err) + os.Exit(1) + } + tapFD, err := openTAPFD(nsHostTAP) + if err != nil { + fmt.Fprintf(os.Stderr, "netns anchor: opening %s: %v\n", nsHostTAP, err) + os.Exit(1) + } + if err := sendFD(handshakeFD, tapFD, anchorReadyByte); err != nil { + fmt.Fprintf(os.Stderr, "netns anchor: handing over %s: %v\n", nsHostTAP, err) + os.Exit(1) + } + // The parent owns the fd now; ours would keep the TAP attached. + _ = unix.Close(tapFD) + // Hold the namespace open until the parent's pipe closes (daemon exit), + // then die and let the kernel reclaim everything in it. + _, _ = io.Copy(io.Discard, os.Stdin) + os.Exit(0) +} + +// buildNetNSTopology creates the bridge and both TAPs inside the current +// (fresh, empty) network namespace and brings everything up. All netlink, no +// iproute2: the anchor is root here, so no privilege is requested from the +// host at any point. +func buildNetNSTopology() error { + if lo, err := netlink.LinkByName("lo"); err == nil { + // Not fatal — nothing clawk does needs guest-side loopback on the + // host end — but a down lo confuses anyone debugging in here. + _ = netlink.LinkSetUp(lo) + } + + br := &netlink.Bridge{LinkAttrs: netlink.LinkAttrs{Name: nsBridge}} + if err := netlink.LinkAdd(br); err != nil { + return fmt.Errorf("creating bridge %s: %w", nsBridge, err) + } + + // tap0 is created persistent and then released: firecracker attaches to + // it by name, and a TAP that still has an owning fd rejects a second + // TUNSETIFF with EBUSY. Persistence keeps the device alive in the + // namespace with no fd held — exactly what `ip tuntap add` does. + for _, name := range []string{nsGuestTAP, nsHostTAP} { + if err := addPersistentTAP(name); err != nil { + return err + } + link, err := netlink.LinkByName(name) + if err != nil { + return fmt.Errorf("looking up %s: %w", name, err) + } + if err := netlink.LinkSetMaster(link, br); err != nil { + return fmt.Errorf("enslaving %s to %s: %w", name, nsBridge, err) + } + if err := netlink.LinkSetUp(link); err != nil { + return fmt.Errorf("bringing up %s: %w", name, err) + } + } + if err := netlink.LinkSetUp(br); err != nil { + return fmt.Errorf("bringing up %s: %w", nsBridge, err) + } + return nil +} + +// addPersistentTAP creates a persistent TAP device by name and drops its fd, +// leaving the device in place with nothing attached. +func addPersistentTAP(name string) error { + fd, err := openTAPFD(name) + if err != nil { + return fmt.Errorf("creating tap %s: %w", name, err) + } + defer unix.Close(fd) + if _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, uintptr(fd), + uintptr(tunSetPersist), 1); errno != 0 { + return fmt.Errorf("TUNSETPERSIST %s: %w", name, errno) + } + return nil +} + +// TUN/TAP ioctls and flags from , spelled out locally like +// the backend's openTAP does rather than pulling in a dependency for three +// constants. +const ( + tunSetIface = 0x400454ca // TUNSETIFF + tunSetPersist = 0x400454cb // TUNSETPERSIST + iffTAP = 0x0002 // IFF_TAP: ethernet frames + iffNoPI = 0x1000 // IFF_NO_PI: no packet-info prefix +) + +// openTAPFD creates (or attaches to) a TAP device by name and returns its raw +// fd in nonblocking mode. Nonblocking before any os.File wraps it: that is +// what registers the fd with Go's runtime poller, and machine.UserMode's +// HostTAPFile contract requires it (a blocking fd there cannot be shut down). +func openTAPFD(name string) (int, error) { + if len(name) >= 16 { // IFNAMSIZ + return -1, fmt.Errorf("tap name %q too long", name) + } + fd, err := unix.Open("/dev/net/tun", unix.O_RDWR, 0) + if err != nil { + return -1, fmt.Errorf("open /dev/net/tun: %w", err) + } + var ifr [40]byte // struct ifreq + copy(ifr[:15], name) + flags := uint16(iffTAP | iffNoPI) + ifr[16], ifr[17] = byte(flags), byte(flags>>8) + // uintptr(unsafe.Pointer(…)) written inline, in the call's own argument + // list, and NOT behind a helper: only that syntactic form gets the + // compiler's syscall special case, which keeps ifr alive and un-moved for + // the duration of the call. Hidden behind a function the conversion is just + // arithmetic on an address the compiler no longer sees as a reference, so a + // stack move would leave the kernel writing into memory that moved — and + // go vet's unsafeptr check goes quiet about it too. + if _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, uintptr(fd), + uintptr(tunSetIface), uintptr(unsafe.Pointer(&ifr[0]))); errno != 0 { + unix.Close(fd) + return -1, fmt.Errorf("TUNSETIFF %q: %w", name, errno) + } + if err := unix.SetNonblock(fd, true); err != nil { + unix.Close(fd) + return -1, fmt.Errorf("tap %q set nonblock: %w", name, err) + } + return fd, nil +} + +// nsenterExecPrefix returns the argv prefix that runs a command inside the +// anchor's namespaces, given the anchor's pid. +// +// It joins the USER namespace as well as the network one, and that is not +// optional: setns(2) into a netns demands CAP_SYS_ADMIN both in the caller's +// own user namespace and in the one owning the target netns. We are +// unprivileged in the initial user namespace, so joining the anchor's userns +// first — where our euid, as its owner, holds every capability — is what makes +// the netns join legal. `nsenter --net` alone fails with EPERM. +// +// And it is nsenter rather than a few lines of Go, because joining a user +// namespace is forbidden to multithreaded processes, which every Go program is +// (the runtime spins up threads before main). nsenter is single-threaded C +// written for exactly this; podman, docker and kubernetes all shell out to it +// for the same reason. +func nsenterExecPrefix(nsenter string, pid int) []string { + proc := filepath.Join("/proc", strconv.Itoa(pid), "ns") + return []string{ + nsenter, + "--user=" + filepath.Join(proc, "user"), + "--net=" + filepath.Join(proc, "net"), + // Keep our real uid/gid instead of nsenter's default setuid(0): the + // mapping already presents us as root inside, and /dev/kvm access is + // checked against the real (unmapped) ids, which must not change. + "--preserve-credentials", + "--", + } +} + +// lookNsenter resolves nsenter, or explains its absence in the terms the +// rootless-availability check reports. +func lookNsenter() (string, error) { + path, err := exec.LookPath("nsenter") + if err != nil { + return "", fmt.Errorf("nsenter is not on PATH (it ships with util-linux) — "+ + "rootless mode needs it to put the VM in its network namespace: %w", err) + } + return path, nil +} + +// sandboxNetNS is a running namespace: the anchor process plus the host-side +// TAP fd it produced. +type sandboxNetNS struct { + // HostTAP is the gvproxy-side TAP, already nonblocking. This struct owns + // it until a caller takes it (TakeHostTAP); Close closes whatever is left. + HostTAP *os.File + // ExecPrefix re-execs a command inside the namespace. + ExecPrefix []string + + anchor *exec.Cmd + stdin *os.File // closing it tells the anchor to exit +} + +// TakeHostTAP hands the gvproxy-side TAP fd to the caller and forgets it, so +// Close stops accounting for it. Call it exactly where ownership really moves: +// machine.UserMode.HostTAPFile transfers the fd to the backend, which closes +// it on shutdown, and Close must not race that. +// +// Everything that does NOT take the fd — the availability probe, every failure +// between the handshake and a spec the backend accepts — relies on Close to +// close it, since nothing else will. +func (n *sandboxNetNS) TakeHostTAP() *os.File { + f := n.HostTAP + n.HostTAP = nil + return f +} + +// Close tears the namespace down: closing the anchor's stdin ends it, and the +// kernel reclaims the namespace (and every device in it) once the last +// reference — the anchor, plus any VM still running in it — is gone. The TAP fd +// goes too, unless a caller took ownership of it (TakeHostTAP). +func (n *sandboxNetNS) Close() error { + if n.HostTAP != nil { + _ = n.HostTAP.Close() + n.HostTAP = nil + } + if n.stdin != nil { + _ = n.stdin.Close() + n.stdin = nil + } + if n.anchor == nil { + return nil + } + cmd := n.anchor + n.anchor = nil + // cmd.Wait (not Process.Wait) so the goroutine copying the anchor's stderr + // is joined and its pipe closed. The anchor exits as soon as the stdin + // close above lands, so this returns promptly — but it must not be able to + // hang a daemon shutdown, hence the kill after a grace period. + done := make(chan struct{}) + go func() { + _ = cmd.Wait() + close(done) + }() + select { + case <-done: + case <-time.After(anchorExitGrace): + _ = cmd.Process.Kill() + <-done + } + return nil +} + +// anchorExitGrace bounds how long Close waits for the anchor to notice its +// stdin closed. It has nothing to do but return from a read, so this only +// fires on a wedge. A var so tests can shorten it. +var anchorExitGrace = 2 * time.Second + +// startSandboxNetNS forks the anchor into a fresh user+network namespace, +// waits for its topology, and returns a handle to it. +// +// The single-uid mapping (our uid ↔ 0 inside) is what makes this unprivileged +// and is also why it needs no /etc/subuid entry, unlike container runtimes: +// clawk only wants capabilities over its own network, never over other users' +// files. +// diagOut is where the anchor's stderr is echoed; nil discards it (used when +// merely probing, where the failure reason is returned rather than logged). +func startSandboxNetNS(self string, diagOut io.Writer) (_ *sandboxNetNS, err error) { + // Resolved before the fork: a namespace we cannot put the VM into is worse + // than no namespace, and this is the cheapest of the two failures. + nsenter, err := lookNsenter() + if err != nil { + return nil, fmt.Errorf("%w: %w", errNetNSUnavailable, err) + } + parentSock, childSock, err := handshakePair() + if err != nil { + return nil, err + } + defer parentSock.Close() + + stdinR, stdinW, err := os.Pipe() + if err != nil { + childSock.Close() + return nil, fmt.Errorf("netns lifetime pipe: %w", err) + } + defer stdinR.Close() + + // The anchor's stderr is where its failures are described; keep a copy for + // the error we return (the fallback path logs it) while still letting it + // flow to the caller's log, if it wants one. + diag := &teeBuffer{out: diagOut, max: 4 << 10} + + cmd := exec.Command(self, netnsAnchorCmd) + cmd.Stdin = stdinR + cmd.Stdout = nil + cmd.Stderr = diag + cmd.ExtraFiles = []*os.File{childSock} // becomes fd 3 in the child + cmd.SysProcAttr = &syscall.SysProcAttr{ + Cloneflags: syscall.CLONE_NEWUSER | syscall.CLONE_NEWNET, + UidMappings: []syscall.SysProcIDMap{ + {ContainerID: 0, HostID: os.Getuid(), Size: 1}, + }, + GidMappings: []syscall.SysProcIDMap{ + {ContainerID: 0, HostID: os.Getgid(), Size: 1}, + }, + // setgroups must stay denied for an unprivileged mapping; the kernel + // refuses to write gid_map otherwise. + GidMappingsEnableSetgroups: false, + } + startErr := cmd.Start() + // Our copy of the child's socket must go NOW, whether or not the start + // worked: while we hold it, the socket never reaches EOF, so an anchor that + // died on its first syscall would cost us the full handshake timeout + // instead of failing instantly. + childSock.Close() + if startErr != nil { + stdinW.Close() + return nil, fmt.Errorf("%w: %w", errNetNSUnavailable, startErr) + } + // One teardown, called from at most one place: the recvFD path below has to + // reap before it can quote the anchor's stderr, and the deferred guard has + // to cover every other failure. Two copies of the same three calls made the + // second Wait return "Wait was already called" and the second Kill ESRCH — + // swallowed, but it left no readable rule about who owns the child. + var teardownOnce sync.Once + teardown := func() { + teardownOnce.Do(func() { + stdinW.Close() + _ = cmd.Process.Kill() + _ = cmd.Wait() // reap, and flush the stderr copier + }) + } + defer func() { + if err != nil { + teardown() + } + }() + + tapFD, err := recvFD(parentSock, anchorStartTimeout) + if err != nil { + // Reap first so the anchor's own diagnosis (which is far more specific + // than "handshake failed") is in the buffer before we quote it. + teardown() + if d := diag.String(); d != "" { + return nil, fmt.Errorf("%w: %s", errNetNSUnavailable, d) + } + return nil, fmt.Errorf("%w: %w", errNetNSUnavailable, err) + } + // Nonblocking is set by the anchor before the fd travels, but SCM_RIGHTS + // carries the file, not the fd flags of the sender's descriptor — so set + // it here too, on the raw fd, before os.NewFile decides about the poller. + if err := unix.SetNonblock(tapFD, true); err != nil { + unix.Close(tapFD) + return nil, fmt.Errorf("received tap fd set nonblock: %w", err) + } + + return &sandboxNetNS{ + HostTAP: os.NewFile(uintptr(tapFD), "netns:"+nsHostTAP), + ExecPrefix: nsenterExecPrefix(nsenter, cmd.Process.Pid), + anchor: cmd, + stdin: stdinW, + }, nil +} + +// teeBuffer forwards writes to out while retaining the first max bytes, so a +// child's stderr can both reach the log and be quoted in an error. +type teeBuffer struct { + out io.Writer + max int + + mu sync.Mutex + buf bytes.Buffer +} + +func (t *teeBuffer) Write(p []byte) (int, error) { + t.mu.Lock() + if room := t.max - t.buf.Len(); room > 0 { + t.buf.Write(p[:min(len(p), room)]) + } + t.mu.Unlock() + if t.out != nil { + return t.out.Write(p) + } + return len(p), nil +} + +func (t *teeBuffer) String() string { + t.mu.Lock() + defer t.mu.Unlock() + return strings.TrimSpace(t.buf.String()) +} + +// errNetNSUnavailable marks a host that cannot do rootless networking, as +// opposed to one where it broke. Callers fall back to bridge mode. +var errNetNSUnavailable = errors.New("rootless networking unavailable") + +// netNSHint turns a namespace-creation failure into one actionable sentence, +// naming the setting to change rather than echoing errno. The text lands in +// `clawk doctor` and the daemon log, so it stays short: the raw error is +// already in the log line beside it. +func netNSHint(err error) string { + // Strip our own framing so the hint doesn't repeat itself. + msg := strings.TrimPrefix(err.Error(), errNetNSUnavailable.Error()+": ") + msg = strings.TrimPrefix(msg, "netns anchor: ") + + permission := strings.Contains(msg, "operation not permitted") || + strings.Contains(msg, "permission denied") + switch { + case strings.Contains(msg, "/dev/net/tun"): + return "/dev/net/tun is not usable by this user (rootless mode needs it; " + + "most distros ship it world-accessible)" + case permission && sysctlIs("/proc/sys/kernel/apparmor_restrict_unprivileged_userns", "1"): + // Ubuntu 24.04+ denies unprivileged user namespaces this way. + return "unprivileged user namespaces are blocked by AppArmor " + + "(kernel.apparmor_restrict_unprivileged_userns=1); allow clawk with an " + + "AppArmor profile granting 'userns,', or set that sysctl to 0" + case permission && sysctlIs("/proc/sys/user/max_user_namespaces", "0"): + return "unprivileged user namespaces are disabled " + + "(user.max_user_namespaces=0); raise it to enable rootless mode" + case permission: + return "the kernel refused an unprivileged user namespace: " + msg + default: + return msg + } +} + +// sysctlIs reports whether a /proc sysctl file holds exactly want. +func sysctlIs(path, want string) bool { + v, err := os.ReadFile(path) + return err == nil && strings.TrimSpace(string(v)) == want +} diff --git a/internal/sandbox/netns_linux_test.go b/internal/sandbox/netns_linux_test.go new file mode 100644 index 0000000..70ee443 --- /dev/null +++ b/internal/sandbox/netns_linux_test.go @@ -0,0 +1,224 @@ +//go:build linux + +package sandbox + +import ( + "bytes" + "errors" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// buildClawk compiles the real binary: the namespace helpers are re-execs of +// it, and os.Executable() under `go test` is the test binary, which has no +// InitNetNSHelpers dispatch. +func buildClawk(t *testing.T) string { + t.Helper() + if _, err := exec.LookPath("go"); err != nil { + t.Skip("`go` not on PATH") + } + bin := filepath.Join(t.TempDir(), "clawk") + out, err := exec.Command("go", "build", "-o", bin, "github.com/clawkwork/clawk/cmd/clawk").CombinedOutput() + require.NoErrorf(t, err, "building clawk:\n%s", out) + return bin +} + +// TestSandboxNetNS exercises the whole rootless path on a host that allows it: +// the unprivileged user+network namespace, the netlink topology inside it, the +// TAP fd crossing back over SCM_RIGHTS, and the exec trampoline. +// +// Skips (rather than fails) where the host forbids it — an unprivileged +// user namespace can be denied by AppArmor (Ubuntu 24.04+) or a sysctl, and +// /dev/net/tun is not always accessible to non-root. Those hosts fall back to +// bridge mode in production, which is the behavior asserted below. +func TestSandboxNetNS(t *testing.T) { + ns, err := startSandboxNetNS(buildClawk(t), nil) + if err != nil { + require.ErrorIs(t, err, errNetNSUnavailable, + "a host that can't do rootless must say so in a way the provider can fall back on") + t.Skipf("rootless networking unavailable here: %v", err) + } + t.Cleanup(func() { + _ = ns.HostTAP.Close() + _ = ns.Close() + }) + + t.Run("tap fd is poller-owned", func(t *testing.T) { + // machine.UserMode.HostTAPFile requires nonblocking, or the frame pump + // can't be shut down. The backend rejects a blocking fd, so this is + // the contract both sides agree on. + rc, err := ns.HostTAP.SyscallConn() + require.NoError(t, err) + var flags int + require.NoError(t, rc.Control(func(fd uintptr) { + r, _, _ := syscall.Syscall(syscall.SYS_FCNTL, fd, syscall.F_GETFL, 0) + flags = int(r) + })) + require.NotZero(t, flags&syscall.O_NONBLOCK, "tap fd must be nonblocking") + }) + + t.Run("topology exists inside the namespace", func(t *testing.T) { + out := inNetNS(t, ns, "sh", "-c", + "cat /sys/class/net/"+nsBridge+"/flags; "+ + "basename $(readlink /sys/class/net/"+nsGuestTAP+"/master); "+ + "basename $(readlink /sys/class/net/"+nsHostTAP+"/master)") + lines := strings.Fields(out) + require.Len(t, lines, 3, "expected bridge flags + two masters, got %q", out) + + flags, err := parseHexFlags(lines[0]) + require.NoError(t, err) + require.NotZero(t, flags&iffUp, "bridge must be up (flags=%s)", lines[0]) + require.Equal(t, nsBridge, lines[1], "guest tap must be enslaved to the bridge") + require.Equal(t, nsBridge, lines[2], "gvproxy tap must be enslaved to the bridge") + }) + + t.Run("host network is untouched", func(t *testing.T) { + // The point of the namespace: no clawk interfaces on the host at all, + // so nothing to leak and nothing to clean up. + for _, dev := range []string{nsBridge, nsGuestTAP, nsHostTAP} { + require.False(t, linkExists(dev), + "%s must not exist in the host namespace", dev) + } + }) + + t.Run("namespace is not the host's", func(t *testing.T) { + inside := strings.TrimSpace(inNetNS(t, ns, "readlink", "/proc/self/ns/net")) + outside, err := os.Readlink("/proc/self/ns/net") + require.NoError(t, err) + require.NotEqual(t, outside, inside, "trampoline did not change namespace") + }) +} + +// TestSandboxNetNSFailsFast pins the behavior that made the first version of +// this code take 15s to report a dead anchor: the parent must not hold the +// child's socket end, or a helper that dies instantly costs a full handshake +// timeout instead of surfacing its error. +func TestSandboxNetNSFailsFast(t *testing.T) { + // A binary that exits immediately stands in for an anchor that dies on its + // first syscall. + fake := filepath.Join(t.TempDir(), "false") + require.NoError(t, os.WriteFile(fake, []byte("#!/bin/sh\nexit 3\n"), 0o755)) + + done := make(chan error, 1) + go func() { + _, err := startSandboxNetNS(fake, nil) + done <- err + }() + select { + case err := <-done: + require.Error(t, err) + require.ErrorIs(t, err, errNetNSUnavailable) + case <-time.After(anchorStartTimeout - time.Second): + t.Fatalf("a dead anchor should fail immediately, not wait out the %s handshake timeout", + anchorStartTimeout) + } +} + +// TestSandboxNetNSClose covers teardown without needing a real namespace: the +// anchor is any process that reads stdin, since that is the whole contract — +// closing the pipe ends it, and the namespace dies with it. +func TestSandboxNetNSClose(t *testing.T) { + t.Run("reaps an anchor that exits on stdin close", func(t *testing.T) { + cmd, stdin := fakeAnchor(t, "cat") // exits at EOF + ns := &sandboxNetNS{anchor: cmd, stdin: stdin} + + start := time.Now() + require.NoError(t, ns.Close()) + require.Less(t, time.Since(start), anchorExitGrace, + "should exit on stdin close, not by hitting the grace timeout") + // Reaped, not merely signalled: no zombie left behind. + require.Error(t, cmd.Process.Signal(syscall.Signal(0))) + }) + + t.Run("kills an anchor that ignores stdin", func(t *testing.T) { + // A wedged helper must never hang a daemon shutdown. + old := anchorExitGrace + anchorExitGrace = 50 * time.Millisecond + t.Cleanup(func() { anchorExitGrace = old }) + + cmd, stdin := fakeAnchor(t, "sleep", "60") + ns := &sandboxNetNS{anchor: cmd, stdin: stdin} + + done := make(chan struct{}) + go func() { _ = ns.Close(); close(done) }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Close hung on an anchor that ignores stdin") + } + require.Error(t, cmd.Process.Signal(syscall.Signal(0))) + }) + + t.Run("is idempotent", func(t *testing.T) { + cmd, stdin := fakeAnchor(t, "cat") + ns := &sandboxNetNS{anchor: cmd, stdin: stdin} + require.NoError(t, ns.Close()) + require.NoError(t, ns.Close()) + }) +} + +// fakeAnchor starts a stand-in anchor process wired like the real one: stdin is +// a pipe the caller holds, stderr is a teeBuffer. +func fakeAnchor(t *testing.T, argv ...string) (*exec.Cmd, *os.File) { + t.Helper() + if _, err := exec.LookPath(argv[0]); err != nil { + t.Skipf("%s not available", argv[0]) + } + r, w, err := os.Pipe() + require.NoError(t, err) + cmd := exec.Command(argv[0], argv[1:]...) + cmd.Stdin = r + cmd.Stderr = &teeBuffer{max: 64} + require.NoError(t, cmd.Start()) + require.NoError(t, r.Close()) // the child holds it now + return cmd, w +} + +func TestNetNSHint(t *testing.T) { + t.Run("permission errors name the likely cause", func(t *testing.T) { + hint := netNSHint(errors.New("fork/exec: operation not permitted")) + require.NotEmpty(t, hint) + // Whichever branch this host takes, the hint has to name something the + // user can act on rather than echo errno. + require.Regexp(t, `apparmor|max_user_namespaces|refused`, hint) + }) + t.Run("other errors pass through", func(t *testing.T) { + require.Contains(t, netNSHint(errors.New("no such file or directory")), + "no such file or directory") + }) +} + +func TestTeeBuffer(t *testing.T) { + var out bytes.Buffer + tb := &teeBuffer{out: &out, max: 8} + n, err := tb.Write([]byte("hello ")) + require.NoError(t, err) + require.Equal(t, 6, n) + _, err = tb.Write([]byte("world\n")) + require.NoError(t, err) + + require.Equal(t, "hello world", strings.TrimSpace(out.String()), "must forward everything") + require.Equal(t, "hello wo", tb.String(), "must retain only the first max bytes") +} + +// inNetNS runs a command inside the namespace via the exec trampoline and +// returns its combined output. +func inNetNS(t *testing.T, ns *sandboxNetNS, argv ...string) string { + t.Helper() + full := append(append([]string{}, ns.ExecPrefix...), argv...) + out, err := exec.Command(full[0], full[1:]...).CombinedOutput() + require.NoErrorf(t, err, "%v:\n%s", full, out) + return string(out) +} + +func parseHexFlags(s string) (uint64, error) { + return strconv.ParseUint(strings.TrimSpace(s), 0, 64) +} diff --git a/internal/sandbox/netns_stub.go b/internal/sandbox/netns_stub.go new file mode 100644 index 0000000..4f03861 --- /dev/null +++ b/internal/sandbox/netns_stub.go @@ -0,0 +1,15 @@ +//go:build !linux + +package sandbox + +// InitNetNSHelpers is a no-op off Linux. The hidden subcommands it dispatches +// are re-execs of the clawk binary into a network namespace, which only the +// firecracker provider needs (see netns_linux.go); every other platform has no +// such helper to dispatch. +// +// It exists because main calls it unconditionally, before cobra — the helper +// paths must not touch flag parsing or the config store, so they cannot be +// hidden behind a build-tagged provider. Deleting the previous half of this +// pair (InitRootHelpers' !linux stub, removed with the loop mount) is what +// broke the macOS build once; keep the two definitions in lock-step. +func InitNetNSHelpers() {} diff --git a/internal/sandbox/netstate_linux.go b/internal/sandbox/netstate_linux.go new file mode 100644 index 0000000..6c116fd --- /dev/null +++ b/internal/sandbox/netstate_linux.go @@ -0,0 +1,98 @@ +//go:build linux + +// Host network-state readers, consulted before any privileged `ip` call. +// +// Every device the firecracker provider needs is created once and then +// reused for the life of the sandbox, so a boot that finds its bridge and +// TAPs already configured must perform no privileged work at all. That is +// the difference between one sudo prompt per sandbox and one per boot on a +// host where sudo needs a password — and the daemon, which cannot prompt at +// all (see ensureBridgeHostNet), only ever reads this state. +// +// State comes from sysfs rather than `ip link show`: no subprocess, no +// iproute2 on PATH, and the two facts the callers act on — administratively +// up, and which bridge a device is enslaved to — are plain files. + +package sandbox + +import ( + "bytes" + "os" + "path/filepath" + "strconv" +) + +// sysClassNet holds one subdirectory (symlink) per network link. +const sysClassNet = "/sys/class/net" + +// iffUp is IFF_UP from — the administrative up flag, i.e. what +// `ip link set up` sets. Deliberately not operstate: a bridge with no +// carrier (every clawk bridge before a VM attaches) reports operstate=down +// while administratively up, so keying off operstate would re-run the +// privileged call on every single boot. +const iffUp = 0x1 + +// linkExistsIn reports whether a link named name is present under root. +func linkExistsIn(root, name string) bool { + st, err := os.Stat(filepath.Join(root, name)) + return err == nil && st.IsDir() +} + +// linkIsUpIn reports whether name has IFF_UP set. A missing or unparsable +// flags file reports false, so the caller falls back to issuing the same +// `ip link set up` it would have issued before this check existed. +func linkIsUpIn(root, name string) bool { + data, err := os.ReadFile(filepath.Join(root, name, "flags")) + if err != nil { + return false + } + // One 0x-prefixed hex mask, e.g. "0x1003\n". + txt := string(bytes.TrimSpace(data)) + flags, err := strconv.ParseUint(txt, 0, 64) + if err != nil { + return false + } + return flags&iffUp != 0 +} + +// linkMasterIn returns the name of the bridge name is enslaved to, or "" if +// it has none — the master symlink is absent on an unenslaved device. +func linkMasterIn(root, name string) string { + dst, err := os.Readlink(filepath.Join(root, name, "master")) + if err != nil { + return "" + } + return filepath.Base(dst) +} + +// linkExists reports whether a network link with the given name is present. +func linkExists(name string) bool { return linkExistsIn(sysClassNet, name) } + +// linkIsUp reports whether the named link is administratively up. +func linkIsUp(name string) bool { return linkIsUpIn(sysClassNet, name) } + +// linkMaster returns the bridge the named link is enslaved to, or "". +func linkMaster(name string) string { return linkMasterIn(sysClassNet, name) } + +// bridgeMemberCountIn returns how many links are enslaved to a bridge, or -1 +// when it isn't a bridge (or doesn't exist). A bridge lists its members as +// symlinks under brif/. +func bridgeMemberCountIn(root, name string) int { + ents, err := os.ReadDir(filepath.Join(root, name, "brif")) + if err != nil { + return -1 + } + return len(ents) +} + +// StaleLegacyBridge reports whether the pre-per-sandbox shared bridge is +// still on this host with nothing attached — dead weight from an older clawk +// that doctor can offer to clean up. A bridge that still has members is left +// alone: something (a sandbox from the older clawk, or an unrelated tool that +// happened to pick the name) is using it. +func StaleLegacyBridge() (name string, stale bool) { + if !linkExists(legacyLinuxBridge) { + return legacyLinuxBridge, false + } + return legacyLinuxBridge, bridgeMemberCountIn(sysClassNet, legacyLinuxBridge) == 0 +} diff --git a/internal/sandbox/netstate_linux_test.go b/internal/sandbox/netstate_linux_test.go new file mode 100644 index 0000000..8f0ed97 --- /dev/null +++ b/internal/sandbox/netstate_linux_test.go @@ -0,0 +1,132 @@ +//go:build linux + +package sandbox + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// fakeLink materializes one sysfs-shaped link directory: //flags +// (and a master symlink when master != ""). +func fakeLink(t *testing.T, root, name, flags, master string) { + t.Helper() + dir := filepath.Join(root, name) + require.NoError(t, os.MkdirAll(dir, 0o755)) + if flags != "" { + require.NoError(t, os.WriteFile(filepath.Join(dir, "flags"), []byte(flags), 0o644)) + } + if master != "" { + // sysfs points at ../../; only the base name is read. + require.NoError(t, os.Symlink(filepath.Join("..", master), filepath.Join(dir, "master"))) + } +} + +func TestLinkExistsIn(t *testing.T) { + root := t.TempDir() + fakeLink(t, root, "clawkbr0", "0x1003", "") + require.NoError(t, os.WriteFile(filepath.Join(root, "notadir"), nil, 0o644)) + + require.True(t, linkExistsIn(root, "clawkbr0")) + require.False(t, linkExistsIn(root, "clawkbr1")) + require.False(t, linkExistsIn(root, "notadir"), "a plain file is not a link") +} + +func TestLinkIsUpIn(t *testing.T) { + root := t.TempDir() + // A bridge with no carrier: administratively up (IFF_UP set) even though + // operstate would read "down". This is the case that must NOT re-run the + // privileged `ip link set up` on every boot. + fakeLink(t, root, "carrierless", "0x1003", "") + fakeLink(t, root, "down", "0x1002", "") + fakeLink(t, root, "lo", "0x9", "") + fakeLink(t, root, "noflags", "", "") + fakeLink(t, root, "garbage", "not-a-number", "") + + require.True(t, linkIsUpIn(root, "carrierless")) + require.True(t, linkIsUpIn(root, "lo")) + require.False(t, linkIsUpIn(root, "down")) + require.False(t, linkIsUpIn(root, "noflags"), "unreadable flags must fail open (re-issue the call)") + require.False(t, linkIsUpIn(root, "garbage")) + require.False(t, linkIsUpIn(root, "absent")) +} + +func TestLinkMasterIn(t *testing.T) { + root := t.TempDir() + fakeLink(t, root, "enslaved", "0x1003", "clawkbr0") + fakeLink(t, root, "free", "0x1003", "") + + require.Equal(t, "clawkbr0", linkMasterIn(root, "enslaved")) + require.Equal(t, "", linkMasterIn(root, "free")) + require.Equal(t, "", linkMasterIn(root, "absent")) +} + +func TestBridgeMemberCountIn(t *testing.T) { + root := t.TempDir() + fakeLink(t, root, "empty-bridge", "0x1003", "") + require.NoError(t, os.MkdirAll(filepath.Join(root, "empty-bridge", "brif"), 0o755)) + fakeLink(t, root, "busy-bridge", "0x1003", "") + require.NoError(t, os.MkdirAll(filepath.Join(root, "busy-bridge", "brif"), 0o755)) + require.NoError(t, os.Symlink("../../tap0", filepath.Join(root, "busy-bridge", "brif", "tap0"))) + fakeLink(t, root, "not-a-bridge", "0x1003", "") + + require.Equal(t, 0, bridgeMemberCountIn(root, "empty-bridge")) + require.Equal(t, 1, bridgeMemberCountIn(root, "busy-bridge")) + require.Equal(t, -1, bridgeMemberCountIn(root, "not-a-bridge"), "no brif → not a bridge") + require.Equal(t, -1, bridgeMemberCountIn(root, "absent")) +} + +func TestBridgeDevice(t *testing.T) { + a, b := bridgeDevice("proj-a"), bridgeDevice("proj-b") + require.NotEqual(t, a, b, "distinct sandboxes need distinct L2 segments") + require.Equal(t, a, bridgeDevice("proj-a"), "must be stable across boots") + require.NotEqual(t, legacyLinuxBridge, a) + for _, name := range []string{"a", "proj-a", "a-very-long-sandbox-name-indeed"} { + dev := bridgeDevice(name) + require.LessOrEqual(t, len(dev), 15, "IFNAMSIZ-1 exceeded: %q", dev) + require.True(t, strings.HasPrefix(dev, "clawkbr"), dev) + } + // The bridge and the TAPs on it must never be the same device name. + require.NotEqual(t, bridgeDevice("x"), tapDevice("x")) + require.NotEqual(t, bridgeDevice("x"), gvTapDevice("x")) +} + +// EVERY host device name has to carry the uid, not just the bridge. A TAP name +// that collides across users is worse than a clash: the second user's `clawk +// up` finds the device present, skips creation, and re-enslaves the first +// user's live TAP to its own bridge — killing that VM's network — and either +// user's `clawk destroy` deletes the other's devices. +func TestDeviceNamesCarryTheUID(t *testing.T) { + sum := sha256.Sum256([]byte(strconv.Itoa(os.Getuid()) + ":proj-a")) + want := hex.EncodeToString(sum[:4]) + + require.Equal(t, "clawkbr"+want, bridgeDevice("proj-a")) + require.Equal(t, "clawk"+want, tapDevice("proj-a")) + require.Equal(t, "clawk"+want+"g", gvTapDevice("proj-a")) + + for _, name := range []string{"a", "proj-a", "a-very-long-sandbox-name-indeed"} { + for _, dev := range []string{tapDevice(name), gvTapDevice(name)} { + require.LessOrEqual(t, len(dev), 15, "IFNAMSIZ-1 exceeded: %q", dev) + } + } +} + +// The real sysfs must parse with the same readers — the fixtures above only +// prove the parsing, not that the format guess is right. Every Linux host has +// lo, and lo is always administratively up. +func TestLinkReadersAgainstRealSysfs(t *testing.T) { + if _, err := os.Stat(filepath.Join(sysClassNet, "lo")); err != nil { + t.Skip("no /sys/class/net/lo") + } + require.True(t, linkExists("lo")) + require.True(t, linkIsUp("lo"), "lo is always up") + require.Equal(t, "", linkMaster("lo"), "lo has no master") + require.False(t, linkExists("clawk-nonexistent-device")) +} diff --git a/internal/sandbox/sudo_linux_test.go b/internal/sandbox/sudo_linux_test.go new file mode 100644 index 0000000..e79c8e1 --- /dev/null +++ b/internal/sandbox/sudo_linux_test.go @@ -0,0 +1,64 @@ +//go:build linux + +package sandbox + +import ( + "os" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSudoOutputMentionsPassword(t *testing.T) { + // What sudo -n actually prints when it declines to authenticate. We force + // LC_ALL=C so this stays the message we get (see cLocaleEnv). + require.True(t, sudoOutputMentionsPassword([]byte("sudo: a password is required"))) + require.True(t, sudoOutputMentionsPassword([]byte("[sudo] Password for deploy:"))) + // Command failures must NOT be classified as auth problems — a retry would + // re-run a command that already ran. + require.False(t, sudoOutputMentionsPassword([]byte("RTNETLINK answers: File exists"))) + // "no terminal to read the password" is also an auth failure: sudo wants a + // password and cannot ask, which is precisely the retry-on-a-tty case. + require.True(t, sudoOutputMentionsPassword([]byte( + "sudo: a terminal is required to read the password; either use the -S option"))) + // A policy rejection is not an auth failure; prompting cannot fix it. + require.False(t, sudoOutputMentionsPassword([]byte( + "Sorry, user deploy is not allowed to execute '/usr/sbin/ip' as root"))) + // The same refusal worded without the word "password", which older sudo + // emits. This used to be caught by a `sudo -n true` fallback probe — and + // that probe also caught every ordinary command failure on a sudoers that + // permits `ip` alone, which is why matching the message is the whole test + // now (see sudoOutputMentionsPassword). + require.True(t, sudoOutputMentionsPassword([]byte( + "sudo: no tty present and no askpass program specified"))) + // A host that grants NOPASSWD for ip only: `sudo -n true` fails there on + // every host, so nothing may treat a command's own failure as an auth + // problem — runSudo would re-run a privileged command that already ran. + require.False(t, sudoOutputMentionsPassword([]byte( + "Cannot find device \"clawkbr1234abcd\""))) +} + +func TestCLocaleEnv(t *testing.T) { + t.Setenv("LC_ALL", "fr_FR.UTF-8") + t.Setenv("LANG", "fr_FR.UTF-8") + t.Setenv("CLAWK_KEEP_ME", "yes") + + env := cLocaleEnv() + var lcAll, lang []string + var kept bool + for _, kv := range env { + switch { + case strings.HasPrefix(kv, "LC_ALL="): + lcAll = append(lcAll, kv) + case strings.HasPrefix(kv, "LANG="): + lang = append(lang, kv) + case kv == "CLAWK_KEEP_ME=yes": + kept = true + } + } + require.Equal(t, []string{"LC_ALL=C"}, lcAll, "exactly one LC_ALL, forced to C") + require.Empty(t, lang, "a localized LANG would still translate sudo's message") + require.True(t, kept, "the rest of the environment must survive") + require.Len(t, env, len(os.Environ())-1, "LANG dropped, LC_ALL replaced") +} diff --git a/machine/firecracker/firecracker_linux.go b/machine/firecracker/firecracker_linux.go index e8c674d..a29848b 100644 --- a/machine/firecracker/firecracker_linux.go +++ b/machine/firecracker/firecracker_linux.go @@ -77,10 +77,13 @@ func checkSpec(spec machine.Spec) error { // ok case machine.UserMode: // gvproxy bridges to firecracker only in TAP-bridge mode: the VM - // boots on GuestTAP and gvproxy is pumped to HostTAP. The fd-NIC - // mode (both empty) has no firecracker analogue. - if nn.GuestTAP == "" || nn.HostTAP == "" { - return fmt.Errorf("%w: firecracker UserMode requires GuestTAP and HostTAP (TAP-bridge mode)", machine.ErrUnsupportedSpec) + // boots on GuestTAP and gvproxy is pumped to the host-side TAP, + // given either by name (HostTAP) or as an already-open fd + // (HostTAPFile, for a TAP in another network namespace — which we + // could not open by name from here). The fd-NIC mode (nothing set) + // has no firecracker analogue. + if nn.GuestTAP == "" || (nn.HostTAP == "" && nn.HostTAPFile == nil) { + return fmt.Errorf("%w: firecracker UserMode requires GuestTAP plus HostTAP or HostTAPFile (TAP-bridge mode)", machine.ErrUnsupportedSpec) } case machine.Unixgram: return fmt.Errorf("%w: firecracker does not support %T net", machine.ErrUnsupportedSpec, n) @@ -218,14 +221,23 @@ func (v *vm) startUserMode() error { SockPath: filepath.Join(v.stateDir, "gvproxy.sock"), Forwards: um.Forwards, Filter: um.Filter, + ID: v.spec.ID, }) if err != nil { return fmt.Errorf("firecracker: gvproxy: %w", err) } - tap, err := openTAP(um.HostTAP) - if err != nil { + // A caller that created the TAP in another network namespace hands us its + // fd instead of a name — we could not open it by name from here. + tap := um.HostTAPFile + if tap == nil { + tap, err = openTAP(um.HostTAP) + if err != nil { + stack.Close() + return fmt.Errorf("firecracker: open host tap %q: %w", um.HostTAP, err) + } + } else if err := requireNonblockingTAP(tap); err != nil { stack.Close() - return fmt.Errorf("firecracker: open host tap %q: %w", um.HostTAP, err) + return fmt.Errorf("firecracker: host tap fd: %w", err) } ctx, cancel := context.WithCancel(context.Background()) if err := startPump(ctx, tap, stack.VMSocket); err != nil { @@ -241,6 +253,17 @@ func (v *vm) startUserMode() error { return nil } +// netNSExecPrefix returns the argv prefix that puts the hypervisor in the +// VM's network namespace, or nil to spawn it in ours. See +// machine.UserMode.NetNSExec. +func (v *vm) netNSExecPrefix() []string { + um, ok := v.userModeNet() + if !ok || len(um.NetNSExec) == 0 { + return nil + } + return append([]string(nil), um.NetNSExec...) +} + // stopUserMode tears down the gvproxy stack and frame pump. Idempotent. func (v *vm) stopUserMode() { if v.umCancel != nil { @@ -543,7 +566,12 @@ func (v *vm) spawn(_ context.Context) error { defer consoleFile.Close() } - cmd := exec.Command(Name, "--api-sock", apiSock) + // A UserMode net may ask for the VM to live in another network namespace + // (the guest's TAP is there, and only there). The prefix re-execs us into + // it; gvproxy deliberately stays behind in this one, since its egress + // sockets belong to the host's network. + argv := append(v.netNSExecPrefix(), Name, "--api-sock", apiSock) + cmd := exec.Command(argv[0], argv[1:]...) cmd.Stdout = consoleFile cmd.Stderr = logFile cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} diff --git a/machine/firecracker/usermode_linux.go b/machine/firecracker/usermode_linux.go index 4750bde..22eb981 100644 --- a/machine/firecracker/usermode_linux.go +++ b/machine/firecracker/usermode_linux.go @@ -66,6 +66,43 @@ func openTAP(name string) (*os.File, error) { return os.NewFile(uintptr(fd), "/dev/net/tun:"+name), nil } +// requireNonblockingTAP verifies a TAP fd handed in from elsewhere (a +// namespace-owning helper, over SCM_RIGHTS) is already nonblocking, as +// openTAP's own fds are. +// +// It checks rather than fixes, because this cannot be fixed from here: +// os.NewFile decides whether to register the fd with Go's runtime poller +// based on its blocking state at that moment, and a File wrapped around a +// blocking fd never joins the poller — so Close could not interrupt the +// pump's in-flight Read, and flipping O_NONBLOCK behind the File's back +// would instead turn that Read into a permanent EAGAIN. The sender has to +// SetNonblock on the raw fd before wrapping it; this catches the mistake +// with a message that says so, instead of a pump that won't shut down. +// +// SyscallConn (not Fd) because Fd itself removes the file from the poller. +func requireNonblockingTAP(f *os.File) error { + rc, err := f.SyscallConn() + if err != nil { + return fmt.Errorf("tap fd: %w", err) + } + var flags int + var errno syscall.Errno + if cerr := rc.Control(func(fd uintptr) { + r, _, e := syscall.Syscall(syscall.SYS_FCNTL, fd, syscall.F_GETFL, 0) + flags, errno = int(r), e + }); cerr != nil { + return fmt.Errorf("tap fd: %w", cerr) + } + if errno != 0 { + return fmt.Errorf("tap fd F_GETFL: %w", errno) + } + if flags&syscall.O_NONBLOCK == 0 { + return fmt.Errorf("tap fd is in blocking mode: set O_NONBLOCK on the raw fd " + + "before wrapping it with os.NewFile, or the frame pump cannot be shut down") + } + return nil +} + // frameConn is the message-oriented half of *os.File / *net.UnixConn: each // Read yields exactly one frame/datagram and each Write sends exactly one. type frameConn interface { diff --git a/machine/internal/ext4/tar2ext4.go b/machine/internal/ext4/tar2ext4.go index 7a877dd..6084682 100644 --- a/machine/internal/ext4/tar2ext4.go +++ b/machine/internal/ext4/tar2ext4.go @@ -82,7 +82,18 @@ func Convert(r io.Reader, w io.ReadWriteSeeker, opts ...ConvertOption) error { } name, ok := normalize(hdr.Name) if !ok { - continue + // The one entry normalize rejects that still carries information + // is the root's own ("/", ".", "./"): its mode and ownership + // belong to the image root, which the writer otherwise leaves at + // 0755 root:root. That default makes a mounted tree's root + // unwritable to any non-root guest user even though every file + // inside it has the right owner, so honor it. Only a directory + // entry can describe the root; anything else naming it is + // nonsense (a hardlink to the root, notably, is not a thing). + if hdr.Typeflag != tar.TypeDir || !namesRoot(hdr.Name) { + continue + } + name = "" // how compactext4 spells the root } if strings.HasPrefix(path.Base(name), ".wh.") { return fmt.Errorf("ext4: whiteout entry %q: input must be a flattened tar", hdr.Name) @@ -187,3 +198,10 @@ func normalize(name string) (string, bool) { } return name, true } + +// namesRoot reports whether a tar entry path refers to the archive root itself +// rather than to something inside it. Note it does NOT accept "..": that +// escapes the root, and path.Clean("/"+name) would flatten the difference. +func namesRoot(name string) bool { + return path.Clean(strings.TrimPrefix(name, "/")) == "." +} diff --git a/machine/internal/ext4/tar2ext4_test.go b/machine/internal/ext4/tar2ext4_test.go index 19dac4a..fdd7c9e 100644 --- a/machine/internal/ext4/tar2ext4_test.go +++ b/machine/internal/ext4/tar2ext4_test.go @@ -104,7 +104,9 @@ func TestConvert(t *testing.T) { image := convertToImage(t, buildTar(t, []tarEntry{ // PAX global header noise, as emitted by BuildKit. {name: "pax_global_header", typeflag: tar.TypeXGlobalHeader}, - // Root entry — refers to the rootfs itself, must be skipped. + // Root entry — configures the image root rather than creating an + // entry (see TestConvertRootEntry). 0755 root:root is what OCI + // layers carry here, i.e. the writer's own default. {name: "./", typeflag: tar.TypeDir, mode: 0o755}, // Out-of-order: child before its parent's own entry; the parent // entry arrives later with a tighter mode and must win. @@ -185,6 +187,65 @@ func TestConvertRejectsWhiteout(t *testing.T) { require.Contains(t, err.Error(), "whiteout", "Convert with whiteout entry: err = %v, want whiteout rejection", err) } +// The tar entry naming the archive root ("./", "/", ".") carries the root +// directory's own mode and ownership. The writer creates its root inode as +// 0755 root:root, so ignoring that entry — as this converter used to — left +// every mounted tree with a root the owner of its contents could not write in. +// +// This path is shared with every OCI rootfs build, not just clawk's worktree +// disks, so the cases that must NOT be mistaken for the root matter as much as +// the one that must: ".." escapes the archive, and only a directory can +// describe a directory. +func TestConvertRootEntry(t *testing.T) { + rootStat := func(t *testing.T, entries []tarEntry) string { + t.Helper() + image := convertToImage(t, buildTar(t, entries), Writable(), TotalSize(8<<20)) + e2fsckClean(t, image) + return debugfsStat(t, image, "<2>") // inode 2 is the ext4 root + } + + t.Run("root entry sets owner and mode", func(t *testing.T) { + out := rootStat(t, []tarEntry{ + {name: "./", typeflag: tar.TypeDir, mode: 0o750, uid: 1000, gid: 1000}, + {name: "./README.md", typeflag: tar.TypeReg, mode: 0o644, uid: 1000, gid: 1000, body: "hi\n"}, + }) + for _, want := range []string{"Type: directory", "Mode: 0750", "User: 1000", "Group: 1000"} { + require.Contains(t, out, want, "stat <2>:\n%s", out) + } + }) + + t.Run("children survive the root being reconfigured", func(t *testing.T) { + image := convertToImage(t, buildTar(t, []tarEntry{ + {name: "./", typeflag: tar.TypeDir, mode: 0o750, uid: 1000, gid: 1000}, + {name: "etc/hostname", typeflag: tar.TypeReg, mode: 0o644, body: "box\n"}, + }), Writable(), TotalSize(8<<20)) + e2fsckClean(t, image) + ls := debugfsCmd(t, image, "ls -l /") + // lost+found is created before the root entry is read; reusing the + // root inode must not drop the directory's existing entries. + require.Contains(t, ls, "lost+found", "ls /:\n%s", ls) + require.Contains(t, ls, "etc", "ls /:\n%s", ls) + }) + + t.Run("dotdot is an escape, not the root", func(t *testing.T) { + out := rootStat(t, []tarEntry{ + {name: "..", typeflag: tar.TypeDir, mode: 0o700, uid: 4242, gid: 4242}, + {name: "etc/hostname", typeflag: tar.TypeReg, mode: 0o644, body: "box\n"}, + }) + require.Contains(t, out, "Mode: 0755", "'..' must not reconfigure the root:\n%s", out) + require.NotContains(t, out, "User: 4242", "'..' must not reconfigure the root:\n%s", out) + }) + + t.Run("a non-directory naming the root is ignored", func(t *testing.T) { + out := rootStat(t, []tarEntry{ + {name: ".", typeflag: tar.TypeReg, mode: 0o600, uid: 4242, gid: 4242, body: "nonsense"}, + {name: "etc/hostname", typeflag: tar.TypeReg, mode: 0o644, body: "box\n"}, + }) + require.Contains(t, out, "Type: directory", "the root must stay a directory:\n%s", out) + require.Contains(t, out, "Mode: 0755", "stat <2>:\n%s", out) + }) +} + // nopSeeker adapts a buffer for Convert's signature in error-path tests // that never reach a real seek. type nopSeeker struct{ *bytes.Buffer } diff --git a/machine/internal/usermode/guestmac_test.go b/machine/internal/usermode/guestmac_test.go new file mode 100644 index 0000000..2590cab --- /dev/null +++ b/machine/internal/usermode/guestmac_test.go @@ -0,0 +1,52 @@ +//go:build darwin || linux + +package usermode + +import ( + "net" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGuestMAC(t *testing.T) { + t.Run("deterministic in id", func(t *testing.T) { + // Stability is load-bearing: gvproxy's static DHCP lease keys on this + // MAC, so a value that drifted between boots would move the guest to a + // dynamic IP and break every port forward. + require.Equal(t, guestMAC("proj-a"), guestMAC("proj-a")) + }) + + t.Run("distinct per id", func(t *testing.T) { + seen := map[string]string{} + for _, id := range []string{"a", "b", "proj", "proj-2", "INFRA-123", "test-proj"} { + mac := guestMAC(id) + if prev, dup := seen[mac]; dup { + t.Fatalf("ids %q and %q collide on %s", prev, id, mac) + } + seen[mac] = id + } + }) + + t.Run("well-formed, locally administered unicast", func(t *testing.T) { + for _, id := range []string{"", "a", "some-long-sandbox-name"} { + hw, err := net.ParseMAC(guestMAC(id)) + require.NoErrorf(t, err, "id %q produced %q", id, guestMAC(id)) + require.Len(t, hw, 6) + require.EqualValues(t, 0, hw[0]&0x1, "multicast bit must be clear (id %q)", id) + require.EqualValues(t, 0x2, hw[0]&0x2, "locally-administered bit must be set (id %q)", id) + } + }) + + t.Run("empty id falls back", func(t *testing.T) { + require.Equal(t, fallbackGuestMAC, guestMAC("")) + }) +} + +// The NIC MAC the backend configures and gvproxy's static lease must agree, or +// the guest gets a dynamic address on a different IP. +func TestGvproxyConfigLeaseMatchesGuestMAC(t *testing.T) { + cfg := Config{ID: "proj-a", SockPath: "/tmp/x"} + gv := buildGvproxyConfig(cfg) + require.Equal(t, guestMAC(cfg.ID), gv.DHCPStaticLeases[defaultGuestIP]) +} diff --git a/machine/internal/usermode/usermode_unix.go b/machine/internal/usermode/usermode_unix.go index 798fbd4..7a525c8 100644 --- a/machine/internal/usermode/usermode_unix.go +++ b/machine/internal/usermode/usermode_unix.go @@ -29,17 +29,18 @@ import ( "github.com/containers/gvisor-tap-vsock/pkg/virtualnetwork" ) -// Default addresses gvproxy assigns. The MACs are fixed so the DHCP static -// lease on the gvproxy side always matches the NIC MAC we configure on the -// hypervisor side — a MAC drift produces a dynamic lease on a different IP, -// which silently breaks every port forward. +// Default addresses gvproxy assigns. The gateway MAC is fixed; the guest MAC +// is derived per VM (see guestMAC). Whatever the value, gvproxy's DHCP static +// lease and the NIC MAC the hypervisor configures must agree — a MAC drift +// produces a dynamic lease on a different IP, which silently breaks every +// port forward. const ( defaultMTU = 1500 defaultSubnet = "192.168.127.0/24" defaultGatewayIP = "192.168.127.1" defaultGatewayMAC = "5a:94:ef:e4:0c:ee" defaultGuestIP = "192.168.127.2" - defaultGuestMAC = "5a:94:ef:e4:0c:dd" + fallbackGuestMAC = "5a:94:ef:e4:0c:dd" // macOS ships unix-socket buffers in the single-digit KB by default. // Fine for DHCP-sized frames, silently drops fragmented TCP once the @@ -70,6 +71,11 @@ type Config struct { // into the gvproxy Configuration (not a process global), so multiple // Stacks can run concurrently with independent filters. Filter machine.Filter + + // ID identifies the VM this stack serves; it seeds the guest MAC so + // concurrently running VMs don't share an L2 identity. Empty is + // tolerated (fallbackGuestMAC) but only safe for one VM at a time. + ID string } // Stack is a running UserMode network. A backend obtains one via Start, @@ -125,7 +131,7 @@ func Start(cfg Config) (_ *Stack, err error) { return &Stack{ VMSocket: vmSocket, GuestIP: defaultGuestIP, - GuestMAC: defaultGuestMAC, + GuestMAC: guestMAC(cfg.ID), host: host, vn: vn, cfg: cfg, @@ -353,6 +359,27 @@ func bumpBuffers(c *net.UnixConn, bytes int) error { return setErr } +// guestMAC derives a stable, VM-unique MAC from id. +// +// Every VM used to get the same hardcoded address. On vz that is invisible +// (each VM has its own private L2 segment), but on firecracker all guests +// share one host bridge: two sandboxes at once claimed the same MAC and the +// same IPv6 link-local, flapping the bridge's forwarding table and delivering +// frames to the wrong guest ("duplicate address detected" in the console). +// +// Deterministic in id, so a MAC survives reboots and suspend/restore — the +// gvproxy static lease keys on it, and a drifting MAC would move the guest to +// a dynamic IP and silently break every port forward. 0x5a keeps the +// locally-administered bit set and the multicast bit clear. +func guestMAC(id string) string { + if id == "" { + return fallbackGuestMAC + } + sum := sha256.Sum256([]byte("clawk-guest-mac:" + id)) + return fmt.Sprintf("5a:%02x:%02x:%02x:%02x:%02x", + sum[0], sum[1], sum[2], sum[3], sum[4]) +} + func buildGvproxyConfig(cfg Config) *types.Configuration { forwards := make(map[string]string, len(cfg.Forwards)) for _, f := range cfg.Forwards { @@ -365,7 +392,9 @@ func buildGvproxyConfig(cfg Config) *types.Configuration { GatewayIP: defaultGatewayIP, GatewayMacAddress: defaultGatewayMAC, DHCPStaticLeases: map[string]string{ - defaultGuestIP: defaultGuestMAC, + // Must match Stack.GuestMAC — the backend configures that on the + // NIC, and a mismatch downgrades the guest to a dynamic lease. + defaultGuestIP: guestMAC(cfg.ID), }, Forwards: forwards, NAT: map[string]string{ diff --git a/machine/oci/dirdisk.go b/machine/oci/dirdisk.go new file mode 100644 index 0000000..d2f905e --- /dev/null +++ b/machine/oci/dirdisk.go @@ -0,0 +1,171 @@ +// Userspace directory → ext4 disk image. The firecracker provider carries a +// sandbox's worktree on its own virtio-blk disk built through here, because +// the alternative — loop-mounting the rootfs and copying files in — needs +// CAP_SYS_ADMIN on the host for every boot, which is the privilege clawk +// exists to avoid handing out. +// +// The same ext4 writer already builds every OCI rootfs (writeDisk in oci.go); +// this is that path with a filesystem walk in front of it instead of a +// flattened image. + +package oci + +import ( + "archive/tar" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/clawkwork/clawk/machine/internal/ext4" +) + +// WriteDirDisk builds a writable ext4 image at dst holding the tree rooted +// at srcDir, padded to sizeBytes so the guest has room to allocate (the +// padding is a sparse hole; it costs no physical disk until written). The +// tree lands at the image root: srcDir/foo is /foo in the image. +// +// Ownership, mode bits and symlinks are preserved as-is — including srcDir's +// own, so the mounted root belongs to whoever owns the source rather than to +// root — which keeps the semantics of the `cp -a` this replaced. Sockets and +// devices are skipped: a worktree has none, and neither survives a copy +// meaningfully. Hardlinks are written as independent regular files: git's +// object store has none (objects are distinct files), and duplicating is safer +// than emitting a link whose target the walk hasn't reached yet. +// +// The image is built under a temporary name and renamed into place, so an +// interrupted build never leaves a plausible-looking partial disk behind. +func WriteDirDisk(srcDir, dst string, sizeBytes int64) error { + // Resolved before anything looks at it, because the walk below lstats its + // root: a srcDir whose last component is a symlink to a directory passes + // the IsDir check (os.Stat follows) and then yields exactly one walk entry + // — the root — producing a well-formed EMPTY image with no error anywhere. + // Errors keep naming the path the caller passed, not the resolved one. + given := srcDir + srcDir, err := filepath.EvalSymlinks(given) + if err != nil { + return fmt.Errorf("oci: dir disk source %q: %w", given, err) + } + info, err := os.Stat(srcDir) + if err != nil { + return fmt.Errorf("oci: dir disk source %q: %w", given, err) + } + if !info.IsDir() { + return fmt.Errorf("oci: dir disk source %q is not a directory", given) + } + + pr, pw := io.Pipe() + go func() { + tw := tar.NewWriter(pw) + err := tarDir(tw, srcDir) + if err == nil { + err = tw.Close() + } + pw.CloseWithError(err) + }() + // Closing the read end unblocks the walk goroutine if the conversion + // bails before draining the stream. + defer pr.Close() + + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + return fmt.Errorf("oci: preparing dir disk dir: %w", err) + } + tmp := dst + ".tmp" + f, err := os.Create(tmp) + if err != nil { + return fmt.Errorf("oci: creating dir disk: %w", err) + } + defer func() { + f.Close() + os.Remove(tmp) + }() + + if err := ext4.Convert(pr, f, ext4.Writable(), ext4.TotalSize(sizeBytes)); err != nil { + return fmt.Errorf("oci: converting %s to ext4: %w", srcDir, err) + } + if err := f.Close(); err != nil { + return fmt.Errorf("oci: closing dir disk: %w", err) + } + if err := os.Rename(tmp, dst); err != nil { + return fmt.Errorf("oci: promoting dir disk: %w", err) + } + return nil +} + +// tarDir walks root and writes every entry to tw with paths relative to +// root. Walk order is lexical (fs.WalkDir), so parents always precede +// children — though the converter tolerates any order. +// +// The root's own entry ("./") is emitted like any other: the converter reads it +// for the image root's owner and mode, which it would otherwise leave at +// 0755 root:root — a workspace whose root the guest user cannot write in. +func tarDir(tw *tar.Writer, root string) error { + return filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + info, err := d.Info() + if err != nil { + // A file that vanished mid-walk (a build artifact, an editor's + // temp file) must not fail the whole disk. + if os.IsNotExist(err) { + return nil + } + return err + } + switch { + case info.Mode().IsRegular(), d.IsDir(), info.Mode()&os.ModeSymlink != 0: + default: + return nil // sockets, devices, fifos + } + + var link string + if info.Mode()&os.ModeSymlink != 0 { + if link, err = os.Readlink(path); err != nil { + return err + } + } + hdr, err := tar.FileInfoHeader(info, link) + if err != nil { + return err + } + // Slash-separated, no leading slash — same shape appendExtra uses. + // (Uid/Gid need no help: tar.FileInfoHeader fills them from the + // syscall.Stat_t on every unix.) + hdr.Name = filepath.ToSlash(rel) + if d.IsDir() && !strings.HasSuffix(hdr.Name, "/") { + hdr.Name += "/" + } + if err := tw.WriteHeader(hdr); err != nil { + return fmt.Errorf("oci: writing %s: %w", hdr.Name, err) + } + if !info.Mode().IsRegular() { + return nil + } + src, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + defer src.Close() + // CopyN, not Copy: the header committed to info.Size() bytes, and a + // file being appended to while we walk would otherwise desync the + // stream. Short files are padded by the zero writer below. + n, err := io.CopyN(tw, src, info.Size()) + if err == io.EOF && n < info.Size() { + _, err = tw.Write(make([]byte, info.Size()-n)) + } + if err != nil && err != io.EOF { + return fmt.Errorf("oci: copying %s: %w", hdr.Name, err) + } + return nil + }) +} diff --git a/machine/oci/dirdisk_test.go b/machine/oci/dirdisk_test.go new file mode 100644 index 0000000..00e5d0f --- /dev/null +++ b/machine/oci/dirdisk_test.go @@ -0,0 +1,144 @@ +package oci + +import ( + "os" + "os/exec" + "path/filepath" + "strconv" + "testing" + + "github.com/stretchr/testify/require" +) + +// The image is validated with debugfs (see debugfsRun in +// integration_test.go): read-only, unprivileged, no loop mount — the same +// property that makes WriteDirDisk worth having. +func TestWriteDirDisk(t *testing.T) { + requireTool(t, "debugfs") + + src := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(src, "sub", "deep"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(src, "README.md"), []byte("hello worktree\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(src, "sub", "deep", "nested.txt"), []byte("deep\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(src, "build.sh"), []byte("#!/bin/sh\nexit 0\n"), 0o755)) + require.NoError(t, os.Symlink("README.md", filepath.Join(src, "link-to-readme"))) + // An empty dir must survive: a git worktree has plenty. + require.NoError(t, os.MkdirAll(filepath.Join(src, "empty"), 0o755)) + + disk := filepath.Join(t.TempDir(), "worktree.img") + require.NoError(t, WriteDirDisk(src, disk, 32<<20)) + + st, err := os.Stat(disk) + require.NoError(t, err) + require.GreaterOrEqual(t, st.Size(), int64(32<<20), "image should be padded to the requested size") + + t.Run("content and layout", func(t *testing.T) { + require.Contains(t, debugfsRun(t, disk, "cat README.md"), "hello worktree") + require.Contains(t, debugfsRun(t, disk, "cat sub/deep/nested.txt"), "deep") + root := debugfsRun(t, disk, "ls -l /") + for _, want := range []string{"README.md", "build.sh", "sub", "empty", "link-to-readme"} { + require.Contains(t, root, want) + } + }) + + t.Run("modes and ownership survive", func(t *testing.T) { + // stat prints e.g. "Mode: 0755" and "User: 1000 Group: 1000". + require.Contains(t, debugfsRun(t, disk, "stat build.sh"), "0755") + require.Contains(t, debugfsRun(t, disk, "stat sub/deep/nested.txt"), "0600") + // debugfs column-pads: "User: 501 Group: 20". + out := debugfsRun(t, disk, "stat README.md") + require.Regexp(t, + `User:\s+`+strconv.Itoa(os.Getuid())+`\s+Group:\s+`+strconv.Itoa(os.Getgid()), + out, "host uid/gid must be preserved (cp -a parity)") + }) + + // The regression that motivated emitting the root's own tar entry: the ext4 + // writer creates its root inode as 0755 root:root, so without it the guest + // user owns every file in the tree but cannot create anything AT the root — + // no `git init`, no node_modules, no new top-level file — while `ls -l` + // looks entirely normal. + t.Run("root directory is not root-owned", func(t *testing.T) { + require.NoError(t, os.Chmod(src, 0o750)) + disk := filepath.Join(t.TempDir(), "root-owner.img") + require.NoError(t, WriteDirDisk(src, disk, 8<<20)) + + out := debugfsRun(t, disk, "stat <2>") // inode 2 is the ext4 root + require.Regexp(t, + `User:\s+`+strconv.Itoa(os.Getuid())+`\s+Group:\s+`+strconv.Itoa(os.Getgid()), + out, "the image root must belong to the source's owner, not root:\n%s", out) + require.Contains(t, out, "0750", "the source root's mode must survive too:\n%s", out) + }) + + t.Run("symlink is a symlink", func(t *testing.T) { + out := debugfsRun(t, disk, "stat link-to-readme") + require.Contains(t, out, "symlink") + require.Contains(t, out, "README.md", "link target should be stored:\n%s", out) + }) + + t.Run("filesystem is consistent and writable", func(t *testing.T) { + requireTool(t, "e2fsck") + // -fn: force a full check, answer no to every repair prompt. Exit + // codes: 0 clean, 4 = errors left uncorrected (we answered no). + out, err := exec.Command("e2fsck", "-fn", disk).CombinedOutput() + if err != nil { + var code int + if ee, ok := err.(*exec.ExitError); ok { + code = ee.ExitCode() + } + require.Failf(t, "e2fsck reported problems", "exit=%d\n%s", code, out) + } + // A read-only-compat image would make the guest's rw mount fail. + require.NotContains(t, debugfsRun(t, disk, "features"), "read-only") + }) +} + +func TestWriteDirDiskRejectsNonDir(t *testing.T) { + dir := t.TempDir() + file := filepath.Join(dir, "f") + require.NoError(t, os.WriteFile(file, []byte("x"), 0o644)) + + err := WriteDirDisk(file, filepath.Join(dir, "out.img"), 1<<20) + require.Error(t, err) + require.Contains(t, err.Error(), "not a directory") + + err = WriteDirDisk(filepath.Join(dir, "absent"), filepath.Join(dir, "out.img"), 1<<20) + require.Error(t, err) +} + +// TestWriteDirDiskFollowsSymlinkedSource covers the quietest way this can go +// wrong: os.Stat follows symlinks and filepath.WalkDir does not, so a srcDir +// whose last component is a link to a directory used to pass validation and +// then yield a single walk entry — a well-formed, EMPTY image, no error. +func TestWriteDirDiskFollowsSymlinkedSource(t *testing.T) { + requireTool(t, "debugfs") + + real := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(real, "README.md"), []byte("hi\n"), 0o644)) + link := filepath.Join(t.TempDir(), "worktree") + require.NoError(t, os.Symlink(real, link)) + + disk := filepath.Join(t.TempDir(), "out.img") + require.NoError(t, WriteDirDisk(link, disk, 4<<20)) + require.Contains(t, debugfsRun(t, disk, "cat README.md"), "hi", + "a symlinked source must be walked through, not silently produce an empty image") +} + +// TestWriteDirDiskNoPartialOnFailure proves the .tmp + rename discipline: +// a failed build leaves no file at the destination path. +func TestWriteDirDiskNoPartialOnFailure(t *testing.T) { + dir := t.TempDir() + dst := filepath.Join(dir, "sub", "out.img") + // Size 0 with content still converts, so force failure differently: an + // unwritable parent. + require.NoError(t, os.MkdirAll(filepath.Join(dir, "sub"), 0o500)) + t.Cleanup(func() { _ = os.Chmod(filepath.Join(dir, "sub"), 0o700) }) + if os.Getuid() == 0 { + t.Skip("root ignores directory permissions") + } + src := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(src, "a"), []byte("a"), 0o644)) + + require.Error(t, WriteDirDisk(src, dst, 1<<20)) + _, err := os.Stat(dst) + require.True(t, os.IsNotExist(err), "no partial image should remain") +} diff --git a/machine/spec.go b/machine/spec.go index bbc5894..944bbd1 100644 --- a/machine/spec.go +++ b/machine/spec.go @@ -3,6 +3,7 @@ package machine import ( "fmt" "net" + "os" ) // Spec describes a VM. It is the input to Backend.New. @@ -198,6 +199,27 @@ type UserMode struct { // ignored by backends that attach the fd NIC. GuestTAP string HostTAP string + + // HostTAPFile, when set, is an already-open fd for the gvproxy-side TAP, + // and HostTAP is ignored. It exists so the TAP can live in a network + // namespace the backend is not in: creating a TAP needs CAP_NET_ADMIN, + // but a TAP fd is namespace-agnostic once open, so the caller creates it + // wherever it has that capability (for clawk, an unprivileged user + // namespace) and passes the fd here. Ownership transfers to the backend. + // + // The raw fd MUST be put in nonblocking mode before it is wrapped in the + // os.File, so Go's poller owns it and closing the file interrupts an + // in-flight read; backends reject a blocking fd rather than deadlock on + // shutdown. + HostTAPFile *os.File + + // NetNSExec, when set, is an argv prefix that re-execs its arguments + // inside the VM's network namespace — e.g. {"/proc/self/exe", + // "__in-netns", "/proc/1234/ns/net", "--"}. The backend prepends it when + // spawning the hypervisor, so the VM's NIC lives in that namespace while + // gvproxy keeps the backend's own (which is where its egress sockets have + // to be). Empty leaves the hypervisor in the current namespace. + NetNSExec []string } func (UserMode) isNet() {} diff --git a/machine/vz/vz_darwin.go b/machine/vz/vz_darwin.go index f540670..128b697 100644 --- a/machine/vz/vz_darwin.go +++ b/machine/vz/vz_darwin.go @@ -856,6 +856,7 @@ func (v *vm) build() (_ *codevz.VirtualMachineConfiguration, _ *codevz.VirtualMa SockPath: sockPath, Forwards: nn.Forwards, Filter: nn.Filter, + ID: v.spec.ID, }) if err != nil { return nil, nil, fmt.Errorf("vz: usermode stack: %w", err)