From 5daa85eb474b5077b09884bf1f51a82cfccb71d0 Mon Sep 17 00:00:00 2001 From: Josh Zhang Date: Sun, 20 Sep 2026 00:51:47 +0000 Subject: [PATCH 1/7] feat(protocol): the bootstrap token and boot_config words, and nothing that sets them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 1 of docs/design/2026-09-20-microvm-bootstrap-token-and-vsock.md section 8: the additive wire vocabulary a microVM session's configuration needs, with no producer and no consumer yet. Nothing in this change sets any of it, so every existing message is byte-identical — which is what makes "controld rolls before runners" true for the three steps that follow. protocol/runner gains two omitempty fields on Spec (BootstrapToken, the single-use capability that replaces an environment's decrypted secret values for a microVM session; SecretNames, the names those values will arrive under), the two method words the exchange rides (fetch_session_secrets, mint_session_bootstrap), their protocol version, CapabilityMicrovmV1, and BootConfig — the first control frame a microVM host sends its guest. The schema lives beside Spec rather than in internal/relay because every field of it is a field of the create it came from (the note's open question 4). internal/relay gains ControlEvent.Cold on a suspending notice ("this is not a freeze — flush, unmount, forget every delivered secret") and the boot_config kind. Frame is untouched. The tests are the compatibility claim rather than a restatement of the struct: the golden create with no bootstrap is byte-for-byte the one TestPublicRunnerWireShapes already pins, a Docker create still carries its env values verbatim, a warm suspend notice still has no cold key, and an older decoder reads a cold notice as the plain suspending it has always answered. Co-Authored-By: Claude Fable 5.1 --- internal/relay/bootconfig_test.go | 88 +++++++++++++++++ internal/relay/frame.go | 26 +++++ protocol/runner/bootstrap_test.go | 157 ++++++++++++++++++++++++++++++ protocol/runner/messages.go | 133 +++++++++++++++++++++++++ 4 files changed, 404 insertions(+) create mode 100644 internal/relay/bootconfig_test.go create mode 100644 protocol/runner/bootstrap_test.go diff --git a/internal/relay/bootconfig_test.go b/internal/relay/bootconfig_test.go new file mode 100644 index 0000000..d606810 --- /dev/null +++ b/internal/relay/bootconfig_test.go @@ -0,0 +1,88 @@ +package relay + +import ( + "bytes" + "encoding/json" + "strings" + "testing" +) + +// TestColdIsAdditiveOnTheSuspendNotice is the compatibility promise for the +// one field the cold-suspend handshake adds. runnerd runs on the host and +// sessiond ships inside the session image, and a session keeps the sessiond +// it booted with for life — so the suspend notice a sandbox already reads +// must be byte-identical after this, and a sandbox that never learns the +// field must read a plain "suspending". +func TestColdIsAdditiveOnTheSuspendNotice(t *testing.T) { + warm, err := json.Marshal(ControlEvent{Kind: KindSuspending, ID: 4}) + if err != nil { + t.Fatal(err) + } + if string(warm) != `{"kind":"suspending","id":4}` { + t.Fatalf("a warm suspend notice = %s", warm) + } + if strings.Contains(string(warm), `"cold"`) { + t.Fatalf("a warm suspend notice leaked a cold key: %s", warm) + } + + cold, err := json.Marshal(ControlEvent{Kind: KindSuspending, ID: 5, Cold: true}) + if err != nil { + t.Fatal(err) + } + if string(cold) != `{"kind":"suspending","id":5,"cold":true}` { + t.Fatalf("a cold suspend notice = %s", cold) + } + + // An older sandbox decodes it as the plain notice it has always answered: + // unknown JSON keys are dropped, so it quiesces its execs and acks, and + // the host-side unmount happens without its help. + var old struct { + Kind string `json:"kind"` + ID uint64 `json:"id,omitempty"` + } + if err := json.Unmarshal(cold, &old); err != nil { + t.Fatalf("an old build could not decode a cold notice at all: %v", err) + } + if old.Kind != KindSuspending || old.ID != 5 { + t.Fatalf("an old build decoded a cold notice as %+v", old) + } + + var back ControlEvent + if err := json.Unmarshal(cold, &back); err != nil { + t.Fatal(err) + } + if !back.Cold || back.ID != 5 { + t.Fatalf("cold round trip mangled: %+v", back) + } +} + +// TestBootConfigKindRidesAnOrdinaryControlFrame pins that the boot +// configuration needs no new frame type and no new envelope: it is a +// FrameControl with AttachID 0 carrying a ControlEvent whose payload relay +// does not read, exactly like every request and response already on this +// channel. +func TestBootConfigKindRidesAnOrdinaryControlFrame(t *testing.T) { + if KindBootConfig != "boot_config" { + t.Fatalf("boot config kind = %q", KindBootConfig) + } + ev, err := json.Marshal(ControlEvent{Kind: KindBootConfig, + Payload: json.RawMessage(`{"protocol":1,"session_id":"sess_example"}`)}) + if err != nil { + t.Fatal(err) + } + const wantEvent = `{"kind":"boot_config","payload":{"protocol":1,"session_id":"sess_example"}}` + if string(ev) != wantEvent { + t.Fatalf("a boot config event = %s\nwant %s", ev, wantEvent) + } + raw, err := Encode(Frame{Type: FrameControl, Payload: ev}) + if err != nil { + t.Fatal(err) + } + var f Frame + if err := json.Unmarshal(raw, &f); err != nil { + t.Fatal(err) + } + if f.Type != FrameControl || f.AttachID != 0 || !bytes.Equal(f.Payload, ev) { + t.Fatalf("boot config frame round trip mangled: %+v", f) + } +} diff --git a/internal/relay/frame.go b/internal/relay/frame.go index b02f734..e540840 100644 --- a/internal/relay/frame.go +++ b/internal/relay/frame.go @@ -143,6 +143,20 @@ const ( // // See docs/design/exec-idle-stop.md. KindExecCount = "exec_count" + // KindBootConfig is the FIRST control frame a microVM host sends down the + // vsock channel the guest has just opened: the session's whole + // configuration, carried as a runner.BootConfig in Payload. + // + // It travels downward only, and only on that path. A Docker session is + // configured through its container's environment block before sessiond + // starts and never sees this kind at all; a sessiond that predates it + // logs one unknown frame and carries on, which is the honest outcome for + // a guest that was configured some other way. + // + // Its schema is runner.BootConfig, in protocol/runner beside Spec, and + // not here: every field of it is a field of the create this session came + // from, and relay interprets no payload it carries. + KindBootConfig = "boot_config" ) type ControlEvent struct { @@ -199,6 +213,18 @@ type ControlEvent struct { // does not speak this event. Live int `json:"live,omitempty"` Seq uint64 `json:"seq,omitempty"` + // Cold qualifies a KindSuspending: this is not a freeze, it is the end of + // this VM. A cold-suspended microVM session is TERMINATED — ADR-0003 §2.2 + // keeps no memory image anywhere — so the sandbox is being asked to flush + // what it has, unmount the agent home, and forget every secret the + // bootstrap exchange delivered, before it answers KindSuspendReady. + // + // It is omitempty, and false is both the zero value and today's whole + // meaning: a warm pause carries no `cold`, so the suspend notice a Plan 6 + // sandbox has always read is byte-identical. A sessiond that predates the + // field reads a plain "suspending", quiesces its execs, and answers; the + // host-side unmount still happens, the guest simply did not help. + Cold bool `json:"cold,omitempty"` } func Encode(f Frame) ([]byte, error) { return json.Marshal(f) } diff --git a/protocol/runner/bootstrap_test.go b/protocol/runner/bootstrap_test.go new file mode 100644 index 0000000..3d1418f --- /dev/null +++ b/protocol/runner/bootstrap_test.go @@ -0,0 +1,157 @@ +package runner_test + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/tokencanopy/rainier/protocol/runner" +) + +// TestBootstrapFieldsAreAdditive is the compatibility promise of the microVM +// bootstrap design, at the hop where it is made: controld rolls before +// runners, so every create a control plane that KNOWS about the token +// dispatches to a runner that does not must be byte-identical to the one that +// runner already reads. +// +// The golden create is TestPublicRunnerWireShapes'. It is repeated here +// literally rather than referenced, because the point of a golden is that a +// change has to edit the bytes and say so. +func TestBootstrapFieldsAreAdditive(t *testing.T) { + create := runner.ToRunner{ + Type: "create", ReqID: 7, Session: "sess_example", + Spec: &runner.Spec{Image: "example.invalid/agent@sha256:0000", Cmd: []string{"bash"}}, + } + got, err := json.Marshal(create) + if err != nil { + t.Fatal(err) + } + const want = `{"type":"create","req_id":7,"session":"sess_example",` + + `"spec":{"image":"example.invalid/agent@sha256:0000","cmd":["bash"]}}` + if string(got) != want { + t.Fatalf("a create with no bootstrap = %s\nwant %s", got, want) + } + for _, tag := range []string{"bootstrap_token", "secret_names"} { + if strings.Contains(string(got), `"`+tag+`"`) { + t.Fatalf("a create with no bootstrap leaked %q: %s", tag, got) + } + } + + // And a Docker create carrying secret values keeps carrying them: the + // withholding is keyed on the capability, not on the field existing. + docker, err := json.Marshal(runner.Spec{Image: "img", Env: map[string]string{"TOKEN": "value_example"}}) + if err != nil { + t.Fatal(err) + } + if string(docker) != `{"image":"img","env":{"TOKEN":"value_example"}}` { + t.Fatalf("a docker create = %s", docker) + } +} + +// TestBootstrapSpecRoundTrip pins the two tags a microVM create adds and +// proves they survive the hop — an omitempty field that never carried its +// value would fence nobody. +func TestBootstrapSpecRoundTrip(t *testing.T) { + in := runner.ToRunner{Type: "create", ReqID: 21, Session: "sess_example", Spec: &runner.Spec{ + Image: "img", + BootstrapToken: "dG9rZW5fZXhhbXBsZQ", + SecretNames: []string{"DEPLOY_KEY", "NPM_TOKEN"}, + }} + b, err := json.Marshal(in) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `"bootstrap_token":"dG9rZW5fZXhhbXBsZQ"`, + `"secret_names":["DEPLOY_KEY","NPM_TOKEN"]`, + } { + if !strings.Contains(string(b), want) { + t.Fatalf("bootstrap tags wrong on the wire: %s", b) + } + } + // A microVM create carries the NAMES and no values at all. + if strings.Contains(string(b), `"env"`) { + t.Fatalf("a withheld create still carried an env block: %s", b) + } + var out runner.ToRunner + if err := json.Unmarshal(b, &out); err != nil { + t.Fatal(err) + } + if out.Spec == nil || out.Spec.BootstrapToken != in.Spec.BootstrapToken || + len(out.Spec.SecretNames) != 2 || out.Spec.SecretNames[1] != "NPM_TOKEN" { + t.Fatalf("round trip mangled the bootstrap: %+v", out.Spec) + } +} + +// TestBootstrapVocabulary pins the three wire words. They are read by a +// control plane, a runner and a sandbox that are three separately released +// binaries, so a typo in one of them would be a silently unanswered method +// rather than a build failure. +func TestBootstrapVocabulary(t *testing.T) { + if runner.MethodFetchSessionSecrets != "fetch_session_secrets" { + t.Fatalf("fetch method = %q", runner.MethodFetchSessionSecrets) + } + if runner.MethodMintSessionBootstrap != "mint_session_bootstrap" { + t.Fatalf("mint method = %q", runner.MethodMintSessionBootstrap) + } + if runner.CapabilityMicrovmV1 != "microvm.v1" { + t.Fatalf("microvm capability = %q", runner.CapabilityMicrovmV1) + } + if runner.SessionBootstrapProtocolVersion != 1 { + t.Fatalf("bootstrap protocol = %d", runner.SessionBootstrapProtocolVersion) + } +} + +// TestBootConfigWireShape pins the bytes a host sends a guest as its first +// control frame. sessiond ships inside the session image and a session keeps +// the one it booted with for life, so these tags are a contract between +// builds that may be months apart. +func TestBootConfigWireShape(t *testing.T) { + minimal, err := json.Marshal(runner.BootConfig{ + Protocol: runner.SessionBootstrapProtocolVersion, + SessionID: "sess_example", + Cmd: []string{"bash"}, + }) + if err != nil { + t.Fatal(err) + } + const wantMinimal = `{"protocol":1,"session_id":"sess_example","cmd":["bash"]}` + if string(minimal) != wantMinimal { + t.Fatalf("a minimal boot config = %s\nwant %s", minimal, wantMinimal) + } + + full := runner.BootConfig{ + Protocol: runner.SessionBootstrapProtocolVersion, + SessionID: "sess_example", + Cmd: []string{"bash", "-l"}, + ProxyURL: "http://sess_example@proxy.invalid:3128", + NoProxy: "127.0.0.1,localhost", + EgressAllow: []string{"registry.npmjs.org"}, + Setup: "apt-get install -y jq\n", + SetupTimeoutSec: 900, + Init: "make dev\n", + InitTimeoutSec: 300, + Repos: []runner.RepoSpec{{Owner: "acme", Name: "app", BaseBranch: "main", + SessionBranch: "rainier/work", Dir: "app"}}, + GitAuthorName: "example", + GitAuthorEmail: "42+example@users.noreply.github.com", + Env: map[string]string{"CLAUDE_CONFIG_DIR": "/rainier/agents/claude"}, + SecretNames: []string{"DEPLOY_KEY"}, + BootstrapToken: "dG9rZW5fZXhhbXBsZQ", + } + b, err := json.Marshal(full) + if err != nil { + t.Fatal(err) + } + var back runner.BootConfig + if err := json.Unmarshal(b, &back); err != nil { + t.Fatal(err) + } + if back.SessionID != full.SessionID || back.NoProxy != full.NoProxy || + back.SetupTimeoutSec != 900 || back.InitTimeoutSec != 300 || + len(back.Repos) != 1 || back.Repos[0].Dir != "app" || + back.Env["CLAUDE_CONFIG_DIR"] != "/rainier/agents/claude" || + len(back.SecretNames) != 1 || back.BootstrapToken != full.BootstrapToken { + t.Fatalf("boot config round trip mangled: %+v", back) + } +} diff --git a/protocol/runner/messages.go b/protocol/runner/messages.go index 7fabe24..888def2 100644 --- a/protocol/runner/messages.go +++ b/protocol/runner/messages.go @@ -55,6 +55,54 @@ const ( MethodRevokeAgentCredentials = "revoke_agent_credentials" ) +// The two methods a microVM session's bootstrap token rides, added by +// docs/design/2026-09-20-microvm-bootstrap-token-and-vsock.md §3. +// +// They ride the shapes that already exist — up as a FromRunner "session_req", +// down as a ToRunner "session_rpc" — so neither RPCEnvelope nor either +// direction's envelope changes. Only the switch that answers them grows. +// +// Neither puts a value anywhere a forwarder reads: runnerd is documented as +// passing RPCEnvelope.Payload through without parsing it, and the control +// plane's answer names a count in its log and never a name's value. +const ( + // MethodFetchSessionSecrets is sandbox → control plane, once per boot: + // {"protocol": 1, "token": ""} → {"env": {"NAME": "value"}}. + // Refused as the usual {"error": sentence} on ok:false when the token is + // spent, expired, or fenced by a placement generation that has moved. + // + // It buys exactly the environment's decrypted secret_refs and nothing + // else: a github credential and a coding agent's login set already have + // their own methods, unchanged by this one. + MethodFetchSessionSecrets = "fetch_session_secrets" + // MethodMintSessionBootstrap is runner → control plane, on a cold resume: + // {"protocol": 1} → {"token": "", "expires_in_sec": 120}. It + // carries no session id: the id is FromRunner.Session, which the + // placement guard has already checked, which is what keeps a runner + // holding session A from minting for session B. + MethodMintSessionBootstrap = "mint_session_bootstrap" +) + +// SessionBootstrapProtocolVersion is the independently negotiated version of +// the two methods above, carried as "protocol" in both request bodies for the +// same reason AgentCredentialProtocolVersion is: the sandbox ships inside the +// session image and keeps the sessiond it booted with for life, so the two +// ends of this exchange are routinely different builds and a mismatch must be +// an answer rather than a misparse. +const SessionBootstrapProtocolVersion = 1 + +// CapabilityMicrovmV1 is the capability token a runnerd built with the +// microVM driver announces. It is a fact about the BUILD and the configured +// driver, not a claim an operator makes, so runnerd appends it rather than +// waiting to be told — the same rule CapabilityExecV1 follows. +// +// It is also the fence the control plane keys withholding on: createSpec +// leaves an environment's decrypted secret values out of Spec.Env, and mints +// a bootstrap token in their place, exactly for a placement whose runner +// announced this. A runner that does not announce it is dispatched today's +// Spec, byte for byte. +const CapabilityMicrovmV1 = "microvm.v1" + // HomeMount is the agent home a create mounts into a sandbox: one writable // volume per (creator, workspace), landing at Path, inside which each coding // agent gets its own subdirectory. It is what makes "log in once" true across @@ -314,6 +362,13 @@ type Spec struct { GitAuthorEmail string `json:"git_author_email,omitempty"` // Env is injected into the container's environment. Values are secrets // as often as not, so this field is never logged verbatim. + // + // With one exception, which is the whole of the microVM bootstrap design: + // for a placement whose runner announced CapabilityMicrovmV1 the control + // plane leaves an environment's decrypted secret_refs OUT of this map and + // sets BootstrapToken and SecretNames instead, so what remains is + // configuration — agent-home paths and the agent manifest. Either the + // values or the token, never both. Env map[string]string `json:"env,omitempty"` // Home is the agent home this session mounts: the (creator, workspace) // volume every coding agent keeps its own configuration and credential @@ -323,6 +378,84 @@ type Spec struct { // does not know the field mounts nothing and the session's agents simply // ask for a login, which is the truthful state, not a failure. Home *HomeMount `json:"home,omitempty"` + // BootstrapToken is the single-use capability a microVM session exchanges + // for its environment's decrypted secrets, which for such a session are + // deliberately absent from Env. Minted per create and per cold resume, + // fenced by the placement generation, never written to host disk. Absent + // on every Docker create and from every older control plane, which is why + // Env keeps its meaning. + BootstrapToken string `json:"bootstrap_token,omitempty"` + // SecretNames are the NAMES the token will deliver — a name is not a + // value, as the launch-material resolver already says — so the guest can + // tell "no secrets declared" from "declared and never arrived". + SecretNames []string `json:"secret_names,omitempty"` +} + +// BootConfig is the first control frame a microVM session's host sends down +// the vsock channel, and the whole of what the guest is configured with. +// +// It lives here, beside Spec, rather than in internal/relay, because it is +// the same vocabulary a create resolved: every field below is a field of Spec +// or a value derived from one, and the two drifting apart would be a session +// configured with something the control plane never dispatched. relay carries +// it as an opaque payload on a FrameControl of kind relay.KindBootConfig and +// interprets none of it. +// +// It is NOT a runner-plane message: it never travels between runnerd and +// controld. It travels between a runner and the sandbox it booted, over the +// one channel that exists before the guest's network does. +// +// Nothing in it is a secret. That is the point of the whole design: Env below +// is the non-secret configuration block (agent-home paths and the agent +// manifest — paths and names), SecretNames are names, and BootstrapToken is a +// capability the control plane can refuse, not a credential. A microVM host +// writes no part of this to disk, and the values SecretNames names arrive in +// the guest over the token exchange, from the control plane, never from here. +type BootConfig struct { + // Protocol is SessionBootstrapProtocolVersion. A guest that reads a + // version it does not speak fails its boot chain rather than booting on a + // configuration it has half understood. + Protocol int `json:"protocol"` + // SessionID is the session this guest IS. On the WebSocket path the guest + // asserts its id in a query parameter and runnerd believes it; over vsock + // the listening socket is inside one VM's own directory, so the id is the + // host's statement and not the guest's claim. + SessionID string `json:"session_id"` + // Cmd is the agent argv, ProxyURL the egress proxy every outbound request + // must route through (already carrying the session's identity as URL + // userinfo), and EgressAllow the allowlist the host enforces and the + // guest is told about. + Cmd []string `json:"cmd,omitempty"` + ProxyURL string `json:"proxy_url,omitempty"` + NoProxy string `json:"no_proxy,omitempty"` + EgressAllow []string `json:"egress_allow,omitempty"` + // The boot chain, in the shape Spec carries it rather than base64 in an + // environment block: a JSON string holds a multi-line script directly, and + // the encoding only ever existed because `docker run -e K=V` carries one + // line per variable. + Setup string `json:"setup,omitempty"` + SetupTimeoutSec int `json:"setup_timeout_sec,omitempty"` + Init string `json:"init,omitempty"` + InitTimeoutSec int `json:"init_timeout_sec,omitempty"` + Repos []RepoSpec `json:"repos,omitempty"` + GitAuthorName string `json:"git_author_name,omitempty"` + GitAuthorEmail string `json:"git_author_email,omitempty"` + // Env is the session's NON-SECRET configuration environment: for a + // microVM session that is exactly the agent-home path variables and the + // agent manifest (RAINIER_AGENTS_B64), which are paths and names. An + // environment's decrypted secret_refs are never in it — the control plane + // withholds them from Spec.Env for a runner announcing + // CapabilityMicrovmV1, and a host that finds values here with no token + // refuses the create rather than delivering them. + Env map[string]string `json:"env,omitempty"` + // SecretNames and BootstrapToken are the exchange: which names are + // expected, and the one-shot capability that fetches their values. + // SecretNames non-empty with an empty token is a boot that must FAIL — + // the guest was promised credentials it cannot get — and that refusal is + // the guest's, not the host's, because only the guest knows whether it + // has already been given them. + SecretNames []string `json:"secret_names,omitempty"` + BootstrapToken string `json:"bootstrap_token,omitempty"` } // Attach is the dial_attach block of a ToRunner: controld tells the runner From 6208bef9a7eac11a78741771739056177982dfe0 Mon Sep 17 00:00:00 2001 From: Josh Zhang Date: Sun, 20 Sep 2026 01:08:30 +0000 Subject: [PATCH 2/7] =?UTF-8?q?feat(control):=20mint,=20withhold,=20and=20?= =?UTF-8?q?answer=20=E2=80=94=20the=20bootstrap=20token's=20control=20plan?= =?UTF-8?q?e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 2 of the microVM bootstrap design's rollout: controld can now mint a session bootstrap token and answer the exchange, and nothing asks yet, because no runner announces microvm.v1. This is the step that makes "controld rolls before runners" true. createSpec keys the one new decision on the placement's runner's announced capabilities, which now travel with the placement rather than being re-read at dispatch. For a runner announcing microvm.v1 it leaves material.Environment's values out of Spec.Env, sets SecretNames to what it withheld, and mints a 32-byte crypto/rand base64url token whose sha256 is stored against the session with the create's placement generation and a 120-second expiry. Every other runner is dispatched exactly the Spec it always was — asserted as strictly as the microVM row, because that is the compatibility floor. Three rules that are easy to get subtly wrong, each with a test that states why: the mint is unconditional on microvm.v1 rather than conditional on there being secrets (a session with a creator has a non-empty Spec.Env either way, and the driver's refusal keys on values-without-a-token, so "mint only when needed" would refuse every scratch session on a microVM runner); a secret_ref spelled like an agent-home variable is dropped from SecretNames exactly as it is already dropped from Spec.Env, or the reservation would be defeated one hop later inside the guest; and a mint the store will not record fails the create, because the alternative is dispatching the secrets after all. The store seam is control.SessionBootstrapStore: a port of two methods, with the spend as ONE atomic step, because single use enforced by a read followed by a write is not single use. Both stores implement it — the in-memory one under its mutex, Postgres as a predicated UPDATE with a second read only to say which of the four refusals applies — and controlapp/repotest gains three cases so the two are pinned against the same contract rather than against each other. internal/controld grows the two arms. fetch_session_secrets spends first and resolves the environment's secrets afterwards through the same launchMaterial the scheduler dispatches with, so there is one decryption path and not two; mint_session_bootstrap is the cold-resume half and takes no session id, because the id is the one the placement guard already checked. Neither is gated on the capability: a runner that did not announce microvm.v1 was handed the values in Spec.Env already, so a second, weaker fence in front of the real one could only ever disagree with it. The refusals are four distinguishable sentences that name a condition and never a token, a value, or another session, and the log discipline is a test rather than a claim: one case captures this package's whole log output across a mint, a successful fetch and a replay, and greps it for the fixture secret and for every token that crossed the wire — then proves the grep ran over real output by requiring the session id to be in it. Co-Authored-By: Claude Fable 5.1 --- control/bootstrap.go | 91 ++++ controlapp/agents_test.go | 6 +- controlapp/bootstrap.go | 117 +++++ controlapp/bootstrap_stub_external_test.go | 24 ++ controlapp/bootstrap_stub_test.go | 83 ++++ controlapp/bootstrap_test.go | 269 ++++++++++++ controlapp/fleet.go | 17 +- controlapp/fleet_external_test.go | 1 + controlapp/fleet_test.go | 9 +- controlapp/repotest/bootstrap.go | 155 +++++++ controlapp/repotest/repotest.go | 11 +- controlapp/scheduler.go | 110 ++++- controlapp/scheduler_test.go | 10 +- controlapp/uow_test.go | 1 + internal/controld/controld.go | 5 + internal/controld/memstore.go | 69 +++ internal/controld/memstore_test.go | 1 + internal/controld/pgstore/bootstrap.go | 112 +++++ .../migrations/0015_session_bootstrap.sql | 30 ++ internal/controld/pgstore/pgstore_test.go | 1 + internal/controld/srpc.go | 4 + internal/controld/srpc_bootstrap.go | 230 ++++++++++ internal/controld/srpc_bootstrap_test.go | 404 ++++++++++++++++++ internal/controld/store.go | 6 + 24 files changed, 1749 insertions(+), 17 deletions(-) create mode 100644 control/bootstrap.go create mode 100644 controlapp/bootstrap.go create mode 100644 controlapp/bootstrap_stub_external_test.go create mode 100644 controlapp/bootstrap_stub_test.go create mode 100644 controlapp/bootstrap_test.go create mode 100644 controlapp/repotest/bootstrap.go create mode 100644 internal/controld/pgstore/bootstrap.go create mode 100644 internal/controld/pgstore/migrations/0015_session_bootstrap.sql create mode 100644 internal/controld/srpc_bootstrap.go create mode 100644 internal/controld/srpc_bootstrap_test.go diff --git a/control/bootstrap.go b/control/bootstrap.go new file mode 100644 index 0000000..613edb7 --- /dev/null +++ b/control/bootstrap.go @@ -0,0 +1,91 @@ +package control + +import ( + "context" + "errors" + "time" +) + +// SessionBootstrap is the control plane's whole memory of one minted session +// bootstrap token: the hash of the token, the placement it was minted under, +// and when it stops being one. The token itself is put on the wire once and +// never stored — a store that held it would be a store that could hand an +// environment's secrets to anyone who could read a row. +// +// There is no "consumed" field here because a caller never writes one: the +// spend is the store's own atomic step (ConsumeSessionBootstrap), and a +// record a caller could mark spent is a record two callers could both spend. +type SessionBootstrap struct { + // Hash is the hex-encoded SHA-256 of the token. Hex rather than raw + // bytes so a durable store keeps one comparable column and no encoding + // decision of its own — the same shape the bearer-token hash already has. + Hash string + // PlacementGeneration is the generation the sandbox this token was minted + // for belongs to. It is the fence: a session re-placed onto another + // runner has moved past it, and the token minted for the sandbox that no + // longer exists stops being an answer. + PlacementGeneration uint64 + // ExpiresAt is when the token stops being one, whatever else is true of + // it. Absolute rather than a TTL because the store is the authority on + // "now" for every replica reading the row. + ExpiresAt time.Time +} + +// The four ways a bootstrap exchange is refused. They are four sentinels +// rather than one because each names a different fact about the fleet, and a +// single "no" would make a routine cold resume (fenced) and a replay attempt +// (spent) indistinguishable in an operator's log — while the sandbox still +// gets the same closed answer either way. +// +// None of them, and nothing wrapping them, may carry a token or a secret +// value: a refusal names a condition and a session, never a credential. +var ( + // ErrBootstrapUnknown reports that the session has no minted token whose + // hash matches the one presented. It is also the answer for ANOTHER + // session's token, because the lookup is keyed by the session the + // placement guard read and never by anything in the request. + ErrBootstrapUnknown = errors.New("control: bootstrap token unknown") + + // ErrBootstrapSpent reports a token that matched and has already been + // exchanged. Single-use is the design's decision: a second boot inside + // one VM asks for a second mint rather than replaying the first token. + ErrBootstrapSpent = errors.New("control: bootstrap token already used") + + // ErrBootstrapExpired reports a token whose ExpiresAt has passed. + ErrBootstrapExpired = errors.New("control: bootstrap token expired") + + // ErrBootstrapFenced reports a token whose PlacementGeneration is not the + // row's current one: the session has been placed again since, so the + // sandbox this token was minted for is not the sandbox asking. + ErrBootstrapFenced = errors.New("control: bootstrap token superseded by a newer placement") +) + +// SessionBootstrapStore is the persistence behind the one-shot token a +// microVM session exchanges for its environment's decrypted secrets. It is +// its own port, sized to two methods, for the reason every other port here is +// sized to its job: minting and spending a capability has nothing to do with +// a session's lifecycle, and a repository that could do both would put "hand +// out this workspace's secrets" one method away from "list sessions". +type SessionBootstrapStore interface { + // PutSessionBootstrap records b as id's ONLY acceptable token, replacing + // whatever was recorded before — a fresh mint invalidates its + // predecessor, which is what makes a cold resume's new token the only one + // that works. ErrInvalid on an empty workspace, session, or hash. + PutSessionBootstrap(ctx context.Context, ws WorkspaceID, id SessionID, b SessionBootstrap) error + + // ConsumeSessionBootstrap spends id's token, atomically with respect to + // every other caller, and reports which of the four refusals applies when + // it cannot. gen is the session row's CURRENT placement generation, read + // by the caller under the same guard that authorized the request, and now + // is the caller's clock. + // + // The atomicity is the whole contract: two exchanges racing on one token + // must produce exactly one success, or single-use means nothing. An + // implementation that reads the record and then writes it back is not + // this method, whatever its tests say on an idle machine. + // + // It returns nothing but an error on purpose. The VALUES the token buys + // are not the store's to know — they are resolved above it, from the + // environment, by the one component that holds the secrets key. + ConsumeSessionBootstrap(ctx context.Context, ws WorkspaceID, id SessionID, hash string, gen uint64, now time.Time) error +} diff --git a/controlapp/agents_test.go b/controlapp/agents_test.go index 558f03a..55b09ec 100644 --- a/controlapp/agents_test.go +++ b/controlapp/agents_test.go @@ -146,7 +146,7 @@ func TestCreateSpecCarriesTheHome(t *testing.T) { row := control.Session{ID: "sess_example", WorkspaceID: "ws_example", CreatorID: "user_example", Spec: control.PortableSpec{Image: "registry.example.invalid/base@sha256:0000"}} - spec, fail := fx.service.createSpec(fleetCtx, row, nil) + spec, fail := fx.service.createSpec(fleetCtx, row, nil, nil, 0) if fail != "" { t.Fatalf("createSpec failed: %s", fail) } @@ -216,7 +216,7 @@ func TestCreateSpecCarriesTheHome(t *testing.T) { // a login that cannot happen. anon := row anon.CreatorID = "" - spec, fail = fx.service.createSpec(fleetCtx, anon, nil) + spec, fail = fx.service.createSpec(fleetCtx, anon, nil, nil, 0) if fail != "" { t.Fatalf("createSpec failed: %s", fail) } @@ -247,7 +247,7 @@ func TestCreateSpecCarriesTheHome(t *testing.T) { agentsEnvVar: "untrusted-manifest", "USER_VALUE": "preserved", }}}) - spec, fail = fx.service.createSpec(fleetCtx, row, nil) + spec, fail = fx.service.createSpec(fleetCtx, row, nil, nil, 0) if fail != "" { t.Fatalf("createSpec failed: %s", fail) } diff --git a/controlapp/bootstrap.go b/controlapp/bootstrap.go new file mode 100644 index 0000000..ef673c0 --- /dev/null +++ b/controlapp/bootstrap.go @@ -0,0 +1,117 @@ +package controlapp + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "errors" + "time" + + "github.com/tokencanopy/rainier/control" +) + +// The session bootstrap token: what a microVM session presents, once, to be +// given the environment secrets its Spec.Env deliberately does not carry. +// See docs/design/2026-09-20-microvm-bootstrap-token-and-vsock.md §3. +// +// It is opaque to every hop that carries it — not a JWT, carrying no claims — +// because the control plane holds the state. What a hop can do with it is +// therefore exactly nothing except hand it on. + +// SessionBootstrapTTL is how long a minted token lives. It must cover a base +// microVM claim, a guest boot and one round trip; the note's open question 1 +// is whether 120 seconds is the right number, and until Phase 1 measures it +// this constant is the one place to change it. +const SessionBootstrapTTL = 120 * time.Second + +// sessionBootstrapTokenBytes is the token's entropy: 32 bytes from +// crypto/rand. Base64url-encoded it is 43 characters with no padding, and it +// is never truncated, prefixed or decorated — a token with a readable prefix +// is a token somebody grep's a log for. +const sessionBootstrapTokenBytes = 32 + +// HashSessionBootstrapToken is the one-way function the store keeps a token +// under: hex-encoded SHA-256, the same shape the bearer-token hash already +// has. It is exported because the mint and the exchange are answered in two +// different packages and must agree byte for byte; a second spelling would be +// a token that can never be redeemed. +func HashSessionBootstrapToken(token string) string { + sum := sha256.Sum256([]byte(token)) + return hex.EncodeToString(sum[:]) +} + +// SessionBootstrapMinter mints and records one session's bootstrap token. +// +// It is a value rather than a service because it holds no policy: the TTL is +// a constant above it, the fence is the caller's placement generation, and +// the only decision it makes is that the plaintext leaves here and is stored +// nowhere. Two callers share it — the scheduler, at create, and the host's +// answer to a runner's mint_session_bootstrap on a cold resume — and they +// share it precisely so a resume's token cannot end up minted differently +// from a create's. +type SessionBootstrapMinter struct { + Store control.SessionBootstrapStore + Clock control.Clock +} + +// Mint returns a fresh token for id and records its hash against gen with an +// expiry SessionBootstrapTTL from now, replacing any token id already had. +// +// The plaintext is returned and never retained: it exists in this process +// only long enough to be copied into the message that carries it, which is +// the same rule the git credential mint follows. +// +// A store that cannot record the hash is a hard failure, not a token minted +// anyway. The alternative — handing a sandbox a capability nothing can ever +// verify — is a create that boots and then cannot get its secrets, reported +// as a healthy session. +func (m SessionBootstrapMinter) Mint(ctx context.Context, ws control.WorkspaceID, id control.SessionID, gen uint64) (string, error) { + if m.Store == nil || m.Clock == nil { + return "", errors.New("controlapp: this control plane cannot mint a session bootstrap token") + } + b := make([]byte, sessionBootstrapTokenBytes) + if _, err := rand.Read(b); err != nil { + // A broken entropy source has no safe fallback: every caller needs an + // unguessable capability, and a predictable one is worse than none. + return "", errors.New("controlapp: the session bootstrap token could not be minted") + } + token := base64.RawURLEncoding.EncodeToString(b) + if err := m.Store.PutSessionBootstrap(ctx, ws, id, control.SessionBootstrap{ + Hash: HashSessionBootstrapToken(token), + PlacementGeneration: gen, + ExpiresAt: m.Clock.Now().Add(SessionBootstrapTTL), + }); err != nil { + return "", err + } + return token, nil +} + +// SessionBootstrapRefusal renders one of control's four bootstrap sentinels +// as the sentence a sandbox is given, and reports whether err was one of +// them at all. +// +// The four sentences are deliberately different from each other and +// deliberately identical in what they withhold: none names a token, a secret, +// a name, or another session. They say which condition holds so that a person +// reading a failed boot can tell "this token has already been used" from +// "this session was placed again while it was booting" — two facts with +// completely different remedies — and nothing more. +// +// Anything that is not one of the four is NOT rendered here: the caller +// answers with its own flat sentence, because an error nobody wrote to be +// shown may quote a row, a column, or a value. +func SessionBootstrapRefusal(err error) (string, bool) { + switch { + case errors.Is(err, control.ErrBootstrapUnknown): + return "this session has no bootstrap token matching the one presented", true + case errors.Is(err, control.ErrBootstrapSpent): + return "this session's bootstrap token has already been exchanged", true + case errors.Is(err, control.ErrBootstrapExpired): + return "this session's bootstrap token has expired", true + case errors.Is(err, control.ErrBootstrapFenced): + return "this session has been placed again since its bootstrap token was minted", true + } + return "", false +} diff --git a/controlapp/bootstrap_stub_external_test.go b/controlapp/bootstrap_stub_external_test.go new file mode 100644 index 0000000..059dbb6 --- /dev/null +++ b/controlapp/bootstrap_stub_external_test.go @@ -0,0 +1,24 @@ +package controlapp_test + +import ( + "context" + "time" + + "github.com/tokencanopy/rainier/control" +) + +// extBootstrapStub is the external mirror of bootstrapStub: controlapp_test +// is a separate package, exactly as a Rainier Cloud module would be, so it +// supplies its own bootstrap store rather than borrowing the internal one — +// which is itself the proof that the port is satisfiable from outside. +var _ control.SessionBootstrapStore = extBootstrapStub{} + +type extBootstrapStub struct{} + +func (extBootstrapStub) PutSessionBootstrap(context.Context, control.WorkspaceID, control.SessionID, control.SessionBootstrap) error { + return nil +} + +func (extBootstrapStub) ConsumeSessionBootstrap(context.Context, control.WorkspaceID, control.SessionID, string, uint64, time.Time) error { + return control.ErrBootstrapUnknown +} diff --git a/controlapp/bootstrap_stub_test.go b/controlapp/bootstrap_stub_test.go new file mode 100644 index 0000000..186605c --- /dev/null +++ b/controlapp/bootstrap_stub_test.go @@ -0,0 +1,83 @@ +package controlapp + +import ( + "context" + "sync" + "time" + + "github.com/tokencanopy/rainier/control" +) + +// bootstrapStub is control.SessionBootstrapStore for every fixture in this +// package: it records what a mint wrote and answers a consume with the same +// four sentinels a real store does. +// +// It is a real (if small) implementation rather than a recorder that always +// says yes, because single use is a rule about the STORE — a fixture that +// accepted every token would let a change that spends one twice pass every +// test here that is not specifically about the exchange. +var _ control.SessionBootstrapStore = (*bootstrapStub)(nil) + +type bootstrapStub struct { + mu sync.Mutex + rows map[control.SessionID]*stubBootstrapRow + // putErr, when set, fails every mint. It is how a test drives the one + // path createSpec must fail closed on. + putErr error +} + +type stubBootstrapRow struct { + hash string + gen uint64 + expiresAt time.Time + spent bool +} + +func newBootstrapStub() *bootstrapStub { + return &bootstrapStub{rows: map[control.SessionID]*stubBootstrapRow{}} +} + +func (b *bootstrapStub) PutSessionBootstrap(_ context.Context, ws control.WorkspaceID, id control.SessionID, rec control.SessionBootstrap) error { + if ws == "" || id == "" || rec.Hash == "" { + return control.ErrInvalid + } + b.mu.Lock() + defer b.mu.Unlock() + if b.putErr != nil { + return b.putErr + } + b.rows[id] = &stubBootstrapRow{hash: rec.Hash, gen: rec.PlacementGeneration, expiresAt: rec.ExpiresAt} + return nil +} + +func (b *bootstrapStub) ConsumeSessionBootstrap(_ context.Context, ws control.WorkspaceID, id control.SessionID, hash string, gen uint64, now time.Time) error { + if ws == "" || id == "" || hash == "" { + return control.ErrInvalid + } + b.mu.Lock() + defer b.mu.Unlock() + row, ok := b.rows[id] + switch { + case !ok || row.hash != hash: + return control.ErrBootstrapUnknown + case row.gen != gen: + return control.ErrBootstrapFenced + case !now.Before(row.expiresAt): + return control.ErrBootstrapExpired + case row.spent: + return control.ErrBootstrapSpent + } + row.spent = true + return nil +} + +// minted returns the record held for id, if any. +func (b *bootstrapStub) minted(id control.SessionID) (stubBootstrapRow, bool) { + b.mu.Lock() + defer b.mu.Unlock() + row, ok := b.rows[id] + if !ok { + return stubBootstrapRow{}, false + } + return *row, true +} diff --git a/controlapp/bootstrap_test.go b/controlapp/bootstrap_test.go new file mode 100644 index 0000000..aa7a5eb --- /dev/null +++ b/controlapp/bootstrap_test.go @@ -0,0 +1,269 @@ +package controlapp + +import ( + "context" + "encoding/base64" + "errors" + "slices" + "strings" + "testing" + "time" + + "github.com/tokencanopy/rainier/control" + "github.com/tokencanopy/rainier/protocol/runner" +) + +// bootstrapRow is the session every case here dispatches: a creator (so the +// agent-home block runs and Spec.Env is non-empty even with nothing secret in +// it), an environment, and an image. +func bootstrapRow() control.Session { + return control.Session{ + ID: "sess_example", WorkspaceID: "ws_example", CreatorID: "user_example", + EnvironmentID: "env_example", + Spec: control.PortableSpec{Image: "registry.example.invalid/base@sha256:0000"}, + } +} + +// TestCreateSpecWithholdsOnlyForMicrovm is the whole of §3's "either the +// secrets or the token, never both", over the matrix of capabilities a +// placement's runner can announce. +// +// The Docker rows are the compatibility floor and are asserted as strictly as +// the microVM one: a runner that does not announce microvm.v1 — which is +// every runner in the fleet today — must be dispatched the values in +// Spec.Env, with no token and no names, byte for byte what it has always +// received. +func TestCreateSpecWithholdsOnlyForMicrovm(t *testing.T) { + for _, tc := range []struct { + name string + caps []string + withheld bool + }{ + {"a runner announcing nothing", nil, false}, + {"a runner announcing exec.v1", []string{runner.CapabilityExecV1}, false}, + {"a runner announcing microvm.v1", []string{runner.CapabilityMicrovmV1}, true}, + {"a runner announcing both", []string{runner.CapabilityExecV1, runner.CapabilityMicrovmV1}, true}, + {"a runner announcing something that merely looks like it", []string{"microvm"}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + fx := newFleetFixture(t) + fx.resolver.material = LaunchMaterial{Environment: map[string]string{ + "DEPLOY_KEY": "value_must_not_travel", + "NPM_TOKEN": "value_must_not_travel_either", + }} + + spec, fail := fx.service.createSpec(fleetCtx, bootstrapRow(), nil, tc.caps, 7) + if fail != "" { + t.Fatalf("createSpec failed: %s", fail) + } + + if !tc.withheld { + if spec.BootstrapToken != "" || len(spec.SecretNames) != 0 { + t.Fatalf("a runner that announced no microvm.v1 was sent a token or names: %+v", spec) + } + if spec.Env["DEPLOY_KEY"] != "value_must_not_travel" || + spec.Env["NPM_TOKEN"] != "value_must_not_travel_either" { + t.Fatalf("the docker path lost its secret values: %v", slices.Sorted(mapKeys(spec.Env))) + } + if _, minted := fx.bootstraps.minted("sess_example"); minted { + t.Fatal("a docker create minted a bootstrap token") + } + return + } + + // Withheld: the names travel, the values do not, and a token does. + if !slices.Equal(spec.SecretNames, []string{"DEPLOY_KEY", "NPM_TOKEN"}) { + t.Fatalf("secret names = %v, want both, sorted", spec.SecretNames) + } + for _, name := range spec.SecretNames { + if v, present := spec.Env[name]; present { + t.Fatalf("a withheld create carried %s = %q in Spec.Env", name, v) + } + } + if spec.BootstrapToken == "" { + t.Fatal("a withheld create carries no bootstrap token; its guest can never get its secrets") + } + // The agent home is configuration and stays, which is what makes + // the boot config a useful message at all. + for k, v := range AgentsEnv(AgentProviders()) { + if spec.Env[k] != v { + t.Fatalf("a withheld create lost the agent-home key %s: %v", + k, slices.Sorted(mapKeys(spec.Env))) + } + } + + // The token is 32 bytes of base64url, and what was RECORDED is its + // hash against this placement — never the token. + raw, err := base64.RawURLEncoding.DecodeString(spec.BootstrapToken) + if err != nil || len(raw) != 32 { + t.Fatalf("token decodes to %d bytes (%v), want 32 base64url bytes", len(raw), err) + } + rec, minted := fx.bootstraps.minted("sess_example") + if !minted { + t.Fatal("nothing was recorded for the token that was handed out") + } + if rec.hash != HashSessionBootstrapToken(spec.BootstrapToken) { + t.Fatal("the recorded hash is not this token's") + } + if rec.hash == spec.BootstrapToken || strings.Contains(rec.hash, spec.BootstrapToken) { + t.Fatal("the plaintext token reached the store") + } + if rec.gen != 7 { + t.Fatalf("recorded placement generation = %d, want the create's 7", rec.gen) + } + want := fx.clock.Now().Add(SessionBootstrapTTL) + if !rec.expiresAt.Equal(want) { + t.Fatalf("expiry = %s, want %s (%s from the mint)", rec.expiresAt, want, SessionBootstrapTTL) + } + }) + } +} + +// TestCreateSpecMintsEvenWithNoSecrets pins the rule that makes the driver's +// own refusal safe: a microVM create ALWAYS carries a token, even when the +// environment declares no secret_refs at all. +// +// The alternative — mint only when there is something to fetch — looks +// tidier and breaks the compatibility row it is supposed to serve. A session +// with a creator and no secrets still has a non-empty Spec.Env (the agent +// manifest), and the microVM driver refuses a create with values and no +// token; so "mint only when needed" would refuse every scratch session on a +// microVM runner. +func TestCreateSpecMintsEvenWithNoSecrets(t *testing.T) { + fx := newFleetFixture(t) + fx.resolver.material = LaunchMaterial{} + + spec, fail := fx.service.createSpec(fleetCtx, bootstrapRow(), nil, []string{runner.CapabilityMicrovmV1}, 3) + if fail != "" { + t.Fatalf("createSpec failed: %s", fail) + } + if spec.BootstrapToken == "" { + t.Fatal("a microVM create with no secrets carries no token, so its driver will refuse it") + } + if len(spec.SecretNames) != 0 { + t.Fatalf("secret names = %v, want none declared", spec.SecretNames) + } + if len(spec.Env) == 0 { + t.Fatal("the create lost the agent manifest, which is configuration and must stay") + } +} + +// TestCreateSpecFailsClosedWhenTheTokenCannotBeRecorded is the fail-closed +// half. A store that will not record the hash leaves nothing able to verify +// the token, so the alternatives are refusing the create or dispatching the +// secrets after all — and the second is the exposure this design removes. +func TestCreateSpecFailsClosedWhenTheTokenCannotBeRecorded(t *testing.T) { + fx := newFleetFixture(t) + fx.resolver.material = LaunchMaterial{Environment: map[string]string{"DEPLOY_KEY": "value_must_not_travel"}} + fx.bootstraps.putErr = errors.New("the store is down") + + spec, fail := fx.service.createSpec(fleetCtx, bootstrapRow(), nil, []string{runner.CapabilityMicrovmV1}, 1) + if fail == "" { + t.Fatalf("createSpec succeeded with an unrecordable token: %+v", spec) + } + if spec != nil { + t.Fatalf("a failed create still produced a spec: %+v", spec) + } + if strings.Contains(fail, "value_must_not_travel") || strings.Contains(fail, "DEPLOY_KEY") { + t.Fatalf("the failure reason quotes the material: %q", fail) + } +} + +// TestWithholdableNamesRespectsTheAgentHomeReservation pins that a +// secret_ref spelled like an agent-home variable is dropped on BOTH paths. +// +// On the Docker path the reservation already wins — AgentsEnv's keys are +// launch invariants a workspace's configuration may not replace — and a +// microVM guest that was handed the same name as a name to fetch would apply +// the workspace's value over its own credential-custody path after boot, +// which is the reservation defeated one hop later. +func TestWithholdableNamesRespectsTheAgentHomeReservation(t *testing.T) { + fx := newFleetFixture(t) + reserved := AgentsEnv(AgentProviders()) + var anyReserved string + for k := range reserved { + if anyReserved == "" || k < anyReserved { + anyReserved = k + } + } + fx.resolver.material = LaunchMaterial{Environment: map[string]string{ + anyReserved: "value_must_not_travel", + "DEPLOY_KEY": "value_must_not_travel_either", + }} + + spec, fail := fx.service.createSpec(fleetCtx, bootstrapRow(), nil, []string{runner.CapabilityMicrovmV1}, 1) + if fail != "" { + t.Fatalf("createSpec failed: %s", fail) + } + if slices.Contains(spec.SecretNames, anyReserved) { + t.Fatalf("the reserved key %q was promised to the guest as a secret name: %v", anyReserved, spec.SecretNames) + } + if !slices.Contains(spec.SecretNames, "DEPLOY_KEY") { + t.Fatalf("the ordinary secret name was dropped too: %v", spec.SecretNames) + } +} + +// TestSessionBootstrapMinterRecordsAndNeverRepeats pins the two properties of +// the mint that no store can supply: the token is fresh every time, and what +// leaves the minter is the only copy. +func TestSessionBootstrapMinterRecordsAndNeverRepeats(t *testing.T) { + store := newBootstrapStub() + clock := &fleetFakeClock{now: time.Unix(1_700_000_000, 0)} + m := SessionBootstrapMinter{Store: store, Clock: clock} + + seen := map[string]struct{}{} + for i := 0; i < 64; i++ { + tok, err := m.Mint(context.Background(), "ws_example", "sess_example", uint64(i)) + if err != nil { + t.Fatalf("mint %d: %v", i, err) + } + if _, dup := seen[tok]; dup { + t.Fatalf("mint %d repeated a token", i) + } + seen[tok] = struct{}{} + } + + // And a minter with no store refuses rather than handing out a + // capability nothing can ever check. + if _, err := (SessionBootstrapMinter{Clock: clock}).Mint(context.Background(), "ws_example", "sess_example", 1); err == nil { + t.Fatal("a minter with no store handed out a token") + } +} + +// TestSessionBootstrapRefusalNamesTheConditionAndNothingElse pins §15.1 at +// the one place these sentences are written: four distinguishable answers, +// none of which carries a token, a value, or another session's id. +func TestSessionBootstrapRefusalNamesTheConditionAndNothingElse(t *testing.T) { + seen := map[string]struct{}{} + for _, err := range []error{ + control.ErrBootstrapUnknown, control.ErrBootstrapSpent, + control.ErrBootstrapExpired, control.ErrBootstrapFenced, + } { + sentence, ok := SessionBootstrapRefusal(err) + if !ok || sentence == "" { + t.Fatalf("%v produced no sentence", err) + } + if _, dup := seen[sentence]; dup { + t.Fatalf("%v repeats another refusal's sentence: %q", err, sentence) + } + seen[sentence] = struct{}{} + } + if _, ok := SessionBootstrapRefusal(errors.New("the store is down")); ok { + t.Fatal("an error nobody wrote to be shown was rendered as a refusal") + } + if _, ok := SessionBootstrapRefusal(nil); ok { + t.Fatal("a nil error was rendered as a refusal") + } +} + +// mapKeys is slices.Sorted's input for a map whose keys a failure message +// names. It exists so a failure prints key names and never values. +func mapKeys(m map[string]string) func(func(string) bool) { + return func(yield func(string) bool) { + for k := range m { + if !yield(k) { + return + } + } + } +} diff --git a/controlapp/fleet.go b/controlapp/fleet.go index 773504a..0f4eed4 100644 --- a/controlapp/fleet.go +++ b/controlapp/fleet.go @@ -52,6 +52,15 @@ type FleetOptions struct { // elsewhere; nothing in this task asks it yet. Checkpoints control.CheckpointLocator + // Bootstraps records the single-use token a microVM session exchanges for + // its environment's secrets. It is required rather than optional, like + // every other port here, and for a sharper reason than symmetry: a + // dispatch that could not mint must REFUSE, and a service with no store + // behind it would have to choose between refusing every microVM create + // and quietly falling back to putting the secrets in Spec.Env — which is + // the exposure this whole design removes. + Bootstraps control.SessionBootstrapStore + // DefaultEgress replaces the built-in developer egress baseline // (DefaultDeveloperEgressHosts) that every dispatched session's allowlist // is unioned with. It is a POINTER because "leave it alone" and "make it @@ -96,6 +105,11 @@ type FleetService struct { uow control.UnitOfWork checkpoints control.CheckpointLocator + // bootstraps mints and records a microVM session's bootstrap token + // (FleetOptions.Bootstraps), composed once here over the same clock every + // other decision in this service reads. + bootstraps SessionBootstrapMinter + // defaultEgress is the host's developer egress baseline, resolved once at // construction (FleetOptions.DefaultEgress). Held as a plain slice because // by this point "unset" has already been answered. @@ -126,7 +140,7 @@ func NewFleetService(opts FleetOptions) (*FleetService, error) { if opts.LaunchMaterial == nil { return nil, control.ErrInvalid } - if opts.UnitOfWork == nil || opts.Checkpoints == nil { + if opts.UnitOfWork == nil || opts.Checkpoints == nil || opts.Bootstraps == nil { return nil, control.ErrInvalid } return &FleetService{ @@ -145,6 +159,7 @@ func NewFleetService(opts FleetOptions) (*FleetService, error) { defaultInitTimeout: opts.DefaultInitTimeoutSec, uow: opts.UnitOfWork, checkpoints: opts.Checkpoints, + bootstraps: SessionBootstrapMinter{Store: opts.Bootstraps, Clock: opts.Clock}, defaultEgress: resolveDefaultEgress(opts.DefaultEgress), wake: make(chan control.PoolID, 64), known: make(map[control.PoolID]struct{}), diff --git a/controlapp/fleet_external_test.go b/controlapp/fleet_external_test.go index 3884358..5e3f27f 100644 --- a/controlapp/fleet_external_test.go +++ b/controlapp/fleet_external_test.go @@ -31,6 +31,7 @@ func fleetExternalOptions() controlapp.FleetOptions { LaunchMaterial: fleetExtResolver{}, UnitOfWork: extDirectUOW{}, Checkpoints: extLocatorStub{}, + Bootstraps: extBootstrapStub{}, } } diff --git a/controlapp/fleet_test.go b/controlapp/fleet_test.go index 0f66881..3a29e0e 100644 --- a/controlapp/fleet_test.go +++ b/controlapp/fleet_test.go @@ -567,6 +567,9 @@ type fleetFixture struct { // checkpoints is the fixture's locator. A test sets its answer BEFORE // starting the scheduler loop, which is the only reader. checkpoints *locatorStub + // bootstraps is the fixture's session bootstrap store: what a microVM + // create minted, and the four refusals an exchange can meet. + bootstraps *bootstrapStub } // fleetTestNoBaseline switches the developer egress baseline OFF for the @@ -602,6 +605,7 @@ func newFleetFixtureWith(t *testing.T, resolver LaunchMaterialResolver, defaultE clock := &fleetFakeClock{now: time.Unix(1_700_000_000, 0)} ids := &fleetFakeIDs{} ckpts := &locatorStub{} + boots := newBootstrapStub() r := resolver if r == nil { r = &fleetFakeResolver{} @@ -624,6 +628,7 @@ func newFleetFixtureWith(t *testing.T, resolver LaunchMaterialResolver, defaultE LaunchMaterial: r, UnitOfWork: directUOW{}, Checkpoints: ckpts, + Bootstraps: boots, DefaultEgress: defaultEgress, }) if err != nil { @@ -632,7 +637,7 @@ func newFleetFixtureWith(t *testing.T, resolver LaunchMaterialResolver, defaultE return &fleetFixture{ service: svc, auth: auth, sessions: sessions, envs: envs, fleet: fleet, pools: pools, transport: transport, events: events, clock: clock, ids: ids, st: st, - resolver: fr, checkpoints: ckpts, + resolver: fr, checkpoints: ckpts, bootstraps: boots, } } @@ -695,6 +700,7 @@ func TestNewFleetServiceRequiresEveryPort(t *testing.T) { LaunchMaterial: &fleetFakeResolver{}, UnitOfWork: directUOW{}, Checkpoints: locatorStub{}, + Bootstraps: newBootstrapStub(), } } if _, err := NewFleetService(base()); err != nil { @@ -713,6 +719,7 @@ func TestNewFleetServiceRequiresEveryPort(t *testing.T) { "launch material": func(o *FleetOptions) { o.LaunchMaterial = nil }, "unit of work": func(o *FleetOptions) { o.UnitOfWork = nil }, "checkpoints": func(o *FleetOptions) { o.Checkpoints = nil }, + "bootstraps": func(o *FleetOptions) { o.Bootstraps = nil }, } { o := base() zero(&o) diff --git a/controlapp/repotest/bootstrap.go b/controlapp/repotest/bootstrap.go new file mode 100644 index 0000000..a67472e --- /dev/null +++ b/controlapp/repotest/bootstrap.go @@ -0,0 +1,155 @@ +package repotest + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/tokencanopy/rainier/control" +) + +// The bootstrap-token half of the contract. It is short because the port is +// two methods, and it is here rather than in either store's own tests because +// it is the one place where "the in-memory store and the Postgres store agree" +// is checked rather than assumed — and because a hosted cell's store has to +// pass exactly the same cases. +// +// Every case creates its session first: a durable store may (and this repo's +// does) key the token to the session row, so a token minted against a session +// that does not exist is not a case the contract has an opinion about. + +// bootstrapSession is the session every case below mints against, placed on a +// runner at a known generation so the fence has something to fence. +func bootstrapSession(t *testing.T, s Stores, id control.SessionID) control.Session { + t.Helper() + return mustCreate(t, s, Alpha, control.Session{ + ID: id, CreatorID: "act_a", State: control.StateQueued, PoolID: PoolA, + }) +} + +// caseSessionBootstrap (B1) is the whole of single-use and the whole of the +// fence, in the five outcomes the design note's test plan names: one success +// and four refusals, each distinguishable from the others. +func caseSessionBootstrap(t *testing.T, s Stores) { + ctx := context.Background() + bootstrapSession(t, s, "sess_boot") + bootstrapSession(t, s, "sess_other") + + now := baseTime() + const hash = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + rec := control.SessionBootstrap{Hash: hash, PlacementGeneration: 4, ExpiresAt: now.Add(120 * time.Second)} + if err := s.Bootstraps.PutSessionBootstrap(ctx, Alpha, "sess_boot", rec); err != nil { + t.Fatalf("put: %v", err) + } + + // A token nobody minted — including, and especially, ANOTHER session's — + // is unknown. The lookup is keyed by the session, so "sess_other presents + // sess_boot's token" and "sess_boot presents a token out of thin air" are + // deliberately the same answer: neither caller learns anything about the + // token that does exist. + if err := s.Bootstraps.ConsumeSessionBootstrap(ctx, Alpha, "sess_other", hash, 4, now); !errors.Is(err, control.ErrBootstrapUnknown) { + t.Fatalf("another session's token: err = %v, want ErrBootstrapUnknown", err) + } + const otherHash = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + if err := s.Bootstraps.ConsumeSessionBootstrap(ctx, Alpha, "sess_boot", otherHash, 4, now); !errors.Is(err, control.ErrBootstrapUnknown) { + t.Fatalf("a token that was never minted: err = %v, want ErrBootstrapUnknown", err) + } + + // A placement that has moved past the one the token was minted under. + if err := s.Bootstraps.ConsumeSessionBootstrap(ctx, Alpha, "sess_boot", hash, 5, now); !errors.Is(err, control.ErrBootstrapFenced) { + t.Fatalf("a superseded generation: err = %v, want ErrBootstrapFenced", err) + } + + // Expiry is checked at the boundary: at exactly ExpiresAt the token is + // already gone, so a 120-second TTL is 120 seconds and not 121. + if err := s.Bootstraps.ConsumeSessionBootstrap(ctx, Alpha, "sess_boot", hash, 4, rec.ExpiresAt); !errors.Is(err, control.ErrBootstrapExpired) { + t.Fatalf("at the expiry instant: err = %v, want ErrBootstrapExpired", err) + } + + // The one success, and then the replay. + if err := s.Bootstraps.ConsumeSessionBootstrap(ctx, Alpha, "sess_boot", hash, 4, now); err != nil { + t.Fatalf("a fresh token: err = %v, want success", err) + } + if err := s.Bootstraps.ConsumeSessionBootstrap(ctx, Alpha, "sess_boot", hash, 4, now); !errors.Is(err, control.ErrBootstrapSpent) { + t.Fatalf("a replayed token: err = %v, want ErrBootstrapSpent", err) + } + + // And the token is not the other workspace's to spend, whatever it holds. + if err := s.Bootstraps.ConsumeSessionBootstrap(ctx, Beta, "sess_boot", hash, 4, now); !errors.Is(err, control.ErrBootstrapUnknown) { + t.Fatalf("another workspace: err = %v, want ErrBootstrapUnknown", err) + } +} + +// caseSessionBootstrapRemint (B2) is what makes a cold resume safe: the token +// runnerd fetches before it boots a new VM is the only one that works, and the +// one the previous boot was handed stops being an answer — including when that +// one was never spent. +func caseSessionBootstrapRemint(t *testing.T, s Stores) { + ctx := context.Background() + bootstrapSession(t, s, "sess_remint") + now := baseTime() + const first = "1111111111111111111111111111111111111111111111111111111111111111" + const second = "2222222222222222222222222222222222222222222222222222222222222222" + + if err := s.Bootstraps.PutSessionBootstrap(ctx, Alpha, "sess_remint", control.SessionBootstrap{ + Hash: first, PlacementGeneration: 1, ExpiresAt: now.Add(time.Minute)}); err != nil { + t.Fatalf("first put: %v", err) + } + if err := s.Bootstraps.PutSessionBootstrap(ctx, Alpha, "sess_remint", control.SessionBootstrap{ + Hash: second, PlacementGeneration: 2, ExpiresAt: now.Add(time.Minute)}); err != nil { + t.Fatalf("second put: %v", err) + } + if err := s.Bootstraps.ConsumeSessionBootstrap(ctx, Alpha, "sess_remint", first, 1, now); !errors.Is(err, control.ErrBootstrapUnknown) { + t.Fatalf("the retired token: err = %v, want ErrBootstrapUnknown", err) + } + if err := s.Bootstraps.ConsumeSessionBootstrap(ctx, Alpha, "sess_remint", second, 2, now); err != nil { + t.Fatalf("the current token: err = %v, want success", err) + } + + // A re-mint after a spend is unspent again: that is the resume path, and + // a store that kept the consumed mark would refuse every session that + // was ever cold-parked twice. + if err := s.Bootstraps.PutSessionBootstrap(ctx, Alpha, "sess_remint", control.SessionBootstrap{ + Hash: first, PlacementGeneration: 3, ExpiresAt: now.Add(time.Minute)}); err != nil { + t.Fatalf("third put: %v", err) + } + if err := s.Bootstraps.ConsumeSessionBootstrap(ctx, Alpha, "sess_remint", first, 3, now); err != nil { + t.Fatalf("a re-minted token: err = %v, want success", err) + } +} + +// caseSessionBootstrapEmpty (B3) pins that neither method accepts an unscoped +// call, the same rule every other port here keeps. +func caseSessionBootstrapEmpty(t *testing.T, s Stores) { + ctx := context.Background() + now := baseTime() + const hash = "3333333333333333333333333333333333333333333333333333333333333333" + for _, tc := range []struct { + name string + call func() error + }{ + {"Put with no workspace", func() error { + return s.Bootstraps.PutSessionBootstrap(ctx, "", "sess_example", control.SessionBootstrap{Hash: hash, ExpiresAt: now}) + }}, + {"Put with no session", func() error { + return s.Bootstraps.PutSessionBootstrap(ctx, Alpha, "", control.SessionBootstrap{Hash: hash, ExpiresAt: now}) + }}, + {"Put with no hash", func() error { + return s.Bootstraps.PutSessionBootstrap(ctx, Alpha, "sess_example", control.SessionBootstrap{ExpiresAt: now}) + }}, + {"Consume with no workspace", func() error { + return s.Bootstraps.ConsumeSessionBootstrap(ctx, "", "sess_example", hash, 1, now) + }}, + {"Consume with no session", func() error { + return s.Bootstraps.ConsumeSessionBootstrap(ctx, Alpha, "", hash, 1, now) + }}, + {"Consume with no hash", func() error { + return s.Bootstraps.ConsumeSessionBootstrap(ctx, Alpha, "sess_example", "", 1, now) + }}, + } { + if err := tc.call(); !errors.Is(err, control.ErrInvalid) { + t.Errorf("%s: err = %v, want ErrInvalid", tc.name, err) + } + } +} diff --git a/controlapp/repotest/repotest.go b/controlapp/repotest/repotest.go index f670dbc..dfd825f 100644 --- a/controlapp/repotest/repotest.go +++ b/controlapp/repotest/repotest.go @@ -28,7 +28,12 @@ type Stores struct { Sessions control.SessionRepository Environments control.EnvironmentRepository Fleet control.FleetRepository - Provision func(ctx context.Context, ws control.WorkspaceID) error + // Bootstraps is the session bootstrap tokens, over the SAME backing + // store: a token minted against a session created through Sessions must + // be consumable here, and a durable store that keys the two differently + // fails the suite rather than a microVM session's boot. + Bootstraps control.SessionBootstrapStore + Provision func(ctx context.Context, ws control.WorkspaceID) error } // Run drives the contract. open is called once per case and must return an @@ -88,6 +93,10 @@ func cases() []suiteCase { {"F5 sessions on a runner", caseSessionsOnRunner}, {"F6 oldest queued", caseOldestQueued}, {"F7 an empty pool is invalid on every fleet method", caseFleetEmptyPool}, + + {"B1 a bootstrap token is single-use and fenced", caseSessionBootstrap}, + {"B2 a fresh mint retires its predecessor", caseSessionBootstrapRemint}, + {"B3 an empty workspace, session or hash is invalid", caseSessionBootstrapEmpty}, } } diff --git a/controlapp/scheduler.go b/controlapp/scheduler.go index bbf1941..4fdf116 100644 --- a/controlapp/scheduler.go +++ b/controlapp/scheduler.go @@ -113,7 +113,11 @@ func (s *FleetService) drainPool(ctx context.Context, pool control.PoolID) { } return } - go s.dispatchCreate(ctx, pool, row, runnerID, env) + // The runner's announced capabilities travel with the placement + // rather than being re-read at dispatch: they are the claims this + // pass placed ON, and a second read could disagree with the decision + // already made. createSpec keys its withholding on them. + go s.dispatchCreate(ctx, pool, row, runnerID, capabilitiesOf(views, runnerID), env) } } @@ -244,6 +248,20 @@ func overridesEnvironmentImage(row control.Session, env control.Environment) boo return row.Spec.Image != env.Image && row.Spec.Image != env.Snapshot.Ref } +// capabilitiesOf returns the capabilities the placement pass saw on id, or +// nil when the view has gone (a runner that disconnected between the pick and +// the dispatch). Nil is the safe answer: a runner whose claims this pass +// cannot state is a runner nothing is withheld from, which dispatches today's +// Spec — the same thing an older runner gets. +func capabilitiesOf(views []runnerView, id control.RunnerID) []string { + for _, v := range views { + if v.id == id { + return v.caps + } + } + return nil +} + func hasAllCapabilities(caps, reqs []string) bool { for _, want := range reqs { if !slices.Contains(caps, want) { @@ -313,8 +331,13 @@ func cloneSession(s control.Session) control.Session { // dispatchCreate builds the create spec, pins setup provenance, dispatches, // and settles the uncertain-delivery outcome without ever duplicating a // delivered create. -func (s *FleetService) dispatchCreate(ctx context.Context, pool control.PoolID, row control.Session, runnerID control.RunnerID, env *control.Environment) { - spec, fail := s.createSpec(ctx, row, env) +func (s *FleetService) dispatchCreate(ctx context.Context, pool control.PoolID, row control.Session, runnerID control.RunnerID, runnerCaps []string, env *control.Environment) { + // Read BEFORE the spec is built, not after, because a microVM create's + // bootstrap token is FENCED by this number: the hash recorded against the + // session has to name the same placement the create carries, or the first + // exchange is refused as superseded by the very placement that minted it. + gen := s.placedGeneration(ctx, row) + spec, fail := s.createSpec(ctx, row, env, runnerCaps, gen) if fail != "" { s.failCreate(ctx, row, fail) return @@ -326,7 +349,7 @@ func (s *FleetService) dispatchCreate(ctx context.Context, pool control.PoolID, Type: "create", Session: string(row.ID), Spec: spec, - PlacementGeneration: s.placedGeneration(ctx, row), + PlacementGeneration: gen, }) switch { case err != nil: @@ -369,7 +392,13 @@ func (s *FleetService) placedGeneration(ctx context.Context, row control.Session // createSpec builds the runner create spec from the session and its current // environment, resolving sensitive launch material only here and never // storing it. -func (s *FleetService) createSpec(ctx context.Context, row control.Session, env *control.Environment) (*runner.Spec, string) { +// runnerCaps are the capabilities the placement's runner announced, and gen +// the placement generation the create carries. Together they are the whole of +// what this function needs to decide the one question the microVM bootstrap +// design added to it: whether this session's environment secrets travel in +// Spec.Env, as they always have, or stay behind a single-use token the guest +// exchanges for them after it boots (see §3 of the design note). +func (s *FleetService) createSpec(ctx context.Context, row control.Session, env *control.Environment, runnerCaps []string, gen uint64) (*runner.Spec, string) { spec := runner.Spec{ Name: row.Name, Image: row.Spec.Image, @@ -402,7 +431,17 @@ func (s *FleetService) createSpec(ctx context.Context, row control.Session, env spec.Repos = slices.Clone(material.Repos) spec.GitAuthorName = material.GitAuthorName spec.GitAuthorEmail = material.GitAuthorEmail - spec.Env = cloneMap(material.Environment) + // The one branch in this function that is about WHERE the session will + // run rather than what it is. A runner announcing microvm.v1 boots each + // session in its own VM on a shared regional host, where a driver cannot + // hand a value to a daemon through an argv and every channel it does have + // ends in a file; so the values stay here and the create carries their + // NAMES and a token instead (ADR-0003 §2.7 item 1). Every other runner — + // which is every runner today — is dispatched exactly what it always was. + withheld := slices.Contains(runnerCaps, runner.CapabilityMicrovmV1) + if !withheld { + spec.Env = cloneMap(material.Environment) + } // The agent home is the creator's, in this workspace: every session they // start there mounts the same volume, and that sameness is the whole of // "log in once". A session with no creator gets none of this — a home @@ -427,6 +466,38 @@ func (s *FleetService) createSpec(ctx context.Context, row control.Session, env } spec.Env = agentEnv } + // The names, and then the token, in that order: the names are derived + // from the material this function already holds, and they are computed + // AFTER the agent-home block so that the one rule that block states — + // agent paths and the manifest are launch invariants a workspace's own + // configuration may not replace — holds identically on both paths. A + // secret_ref that happens to be spelled CLAUDE_CONFIG_DIR is dropped for + // a Docker session and must not be smuggled back in as a name a microVM + // guest then applies over its own agent home. + if withheld { + spec.SecretNames = withholdableNames(material.Environment, spec.Env) + token, err := s.bootstraps.Mint(ctx, row.WorkspaceID, row.ID, gen) + if err != nil { + // Fail closed. The alternative to refusing here is a session + // dispatched with neither its secrets nor a way to ask for them, + // which boots, reports healthy, and fails at whatever the first + // credential-shaped thing it does is. + return nil, "could not mint this session's bootstrap token" + } + spec.BootstrapToken = token + } + // Either the values or the token, never both — stated as a check rather + // than as a comment, because it is the whole security claim of §3 and it + // is one careless merge away from being false. A spec that reaches here + // carrying both is a bug in this function, and a failed create is a much + // better answer to it than a secret on a shared host's disk. + if spec.BootstrapToken != "" { + for _, name := range spec.SecretNames { + if _, both := spec.Env[name]; both { + return nil, "could not resolve launch material" + } + } + } // The session row stores only the egress its caller or environment // declared; the hosts the resolved material needs are the resolver's // knowledge and are added here, at dispatch, so the row and the view a @@ -502,6 +573,33 @@ func boundOr(declared, fallback int) int { return fallback } +// withholdableNames are the names a withheld create promises its guest: every +// key of the resolved secret environment that the spec's own configuration +// does not already reserve, sorted. +// +// Sorted because the list is on the wire and in a guest's failure message, +// and a set rendered in map order would make two identical creates look +// different. Nil when there is nothing to promise, so a session with no +// secret_refs puts no `secret_names` on the wire at all and its guest reads +// the honest "none declared" rather than "declared and never arrived". +func withholdableNames(material, reserved map[string]string) []string { + if len(material) == 0 { + return nil + } + out := make([]string, 0, len(material)) + for name := range material { + if _, taken := reserved[name]; taken { + continue + } + out = append(out, name) + } + if len(out) == 0 { + return nil + } + slices.Sort(out) + return out +} + func cloneMap(m map[string]string) map[string]string { if m == nil { return nil diff --git a/controlapp/scheduler_test.go b/controlapp/scheduler_test.go index a0d03a4..04584ac 100644 --- a/controlapp/scheduler_test.go +++ b/controlapp/scheduler_test.go @@ -418,7 +418,7 @@ func TestDispatchCreateFailureAndUncertainDelivery(t *testing.T) { fx.transport.dispatchErr = errors.New("connection closed") fx.transport.connectedOverrides["pool_example/vm1"] = false - fx.service.dispatchCreate(context.Background(), "pool_example", row, "vm1", nil) + fx.service.dispatchCreate(context.Background(), "pool_example", row, "vm1", nil, nil) got := fleetGetSessionState(t, fx, "ws_example", "sess_conn") if got.State != control.StateQueued || got.RunnerID != "" { @@ -438,7 +438,7 @@ func TestDispatchCreateFailureAndUncertainDelivery(t *testing.T) { fx.transport.dispatchErr = errors.New("no result before timeout") // Connected stays true: the command was delivered. - fx.service.dispatchCreate(context.Background(), "pool_example", row, "vm1", nil) + fx.service.dispatchCreate(context.Background(), "pool_example", row, "vm1", nil, nil) got := fleetGetSessionState(t, fx, "ws_example", "sess_timeout") if got.State != control.StateCreating || got.RunnerID != "vm1" { @@ -706,7 +706,7 @@ func TestCreateSpecBoundsOnlyTheHooksThatRun(t *testing.T) { if tc.rowImage != "" { r.Spec.Image = tc.rowImage // a row created against a current snapshot boots it } - spec, fail := fx.service.createSpec(fleetCtx, r, &env) + spec, fail := fx.service.createSpec(fleetCtx, r, &env, nil, 0) if fail != "" { t.Fatalf("createSpec failed: %s", fail) } @@ -741,7 +741,7 @@ func TestCreateSpecSendsSetupUnlessTheRowBootsTheSnapshot(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { row := control.Session{ID: "sess_example", WorkspaceID: "ws_example", EnvironmentID: "env_example", Spec: control.PortableSpec{Image: tc.rowImage}} - spec, fail := fx.service.createSpec(fleetCtx, row, &env) + spec, fail := fx.service.createSpec(fleetCtx, row, &env, nil, 0) if fail != "" { t.Fatalf("createSpec failed: %s", fail) } @@ -774,7 +774,7 @@ func TestDispatchCreateCarriesThePlacementGeneration(t *testing.T) { // BEFORE the placement, one generation behind the store. stale := row stale.PlacementGeneration = 2 - fx.service.dispatchCreate(context.Background(), "pool_example", stale, "vm1", nil) + fx.service.dispatchCreate(context.Background(), "pool_example", stale, "vm1", nil, nil) dispatched := fx.transport.dispatchedCommands() if len(dispatched) != 1 { diff --git a/controlapp/uow_test.go b/controlapp/uow_test.go index beb702d..c46db7c 100644 --- a/controlapp/uow_test.go +++ b/controlapp/uow_test.go @@ -490,6 +490,7 @@ func newUOWFleetFixture(t *testing.T) *uowFleetFixture { LaunchMaterial: &fleetFakeResolver{}, UnitOfWork: fx.uow, Checkpoints: locatorStub{}, + Bootstraps: newBootstrapStub(), }) if err != nil { t.Fatalf("NewFleetService: %v", err) diff --git a/internal/controld/controld.go b/internal/controld/controld.go index 3ca56a3..894431b 100644 --- a/internal/controld/controld.go +++ b/internal/controld/controld.go @@ -303,6 +303,11 @@ func (s *Server) compose() error { DefaultInitTimeoutSec: defaultInitTimeoutSec, UnitOfWork: uow, Checkpoints: ckpts, + // The same store, through the port that mints and spends a microVM + // session's bootstrap token. srpc.go reads it through the same + // accessor to answer the exchange, so the mint and the redemption + // are one table and not two. + Bootstraps: s.st.Bootstraps(), }) if err != nil { return fmt.Errorf("controld: composing the fleet service: %w", err) diff --git a/internal/controld/memstore.go b/internal/controld/memstore.go index 8b0550a..4e8615a 100644 --- a/internal/controld/memstore.go +++ b/internal/controld/memstore.go @@ -61,6 +61,11 @@ type memStore struct { // credential set per (user, provider), keyed the same way and holding no // workspace, exactly as the schema does. agentCredentials map[credKey]*AgentCredential + + // bootstraps is the one-shot token a microVM session exchanges for its + // environment's secrets, keyed like every other session row and holding + // only the HASH — the plaintext left the mint and was never stored. + bootstraps map[sessionKey]*bootstrapRow } var _ MemStore = (*memStore)(nil) @@ -116,6 +121,7 @@ func NewMemStore() MemStore { credentials: map[credKey]*Credential{}, agentCredentials: map[credKey]*AgentCredential{}, + bootstraps: map[sessionKey]*bootstrapRow{}, } } @@ -127,12 +133,75 @@ func (m *memStore) Environments() control.EnvironmentRepository { return memEnvi func (m *memStore) Fleet() control.FleetRepository { return memFleet{m} } +// Bootstraps is the fourth: the session bootstrap tokens, a view over this +// store's own rows exactly as the other three are. +func (m *memStore) Bootstraps() control.SessionBootstrapStore { return memBootstraps{m} } + var ( _ control.SessionRepository = memSessions{} _ control.EnvironmentRepository = memEnvironments{} _ control.FleetRepository = memFleet{} + _ control.SessionBootstrapStore = memBootstraps{} ) +// bootstrapRow is one session's minted token as this store keeps it: the +// hash, its fence, its expiry, and whether it has been spent. `spent` is the +// store's own column and no caller's — the whole of single-use is that the +// check and the mark are one step under this store's mutex. +type bootstrapRow struct { + hash string + gen uint64 + expiresAt time.Time + spent bool +} + +// memBootstraps is the in-memory SessionBootstrapStore. +type memBootstraps struct{ m *memStore } + +func (b memBootstraps) PutSessionBootstrap(_ context.Context, ws control.WorkspaceID, id control.SessionID, rec control.SessionBootstrap) error { + if ws == "" || id == "" || rec.Hash == "" { + return control.ErrInvalid + } + b.m.mu.Lock() + defer b.m.mu.Unlock() + // A fresh mint REPLACES its predecessor rather than joining it, which is + // what makes a cold resume's token the only one that works and retires + // the one the previous boot was handed. + b.m.bootstraps[sessionKey{ws: ws, id: id}] = &bootstrapRow{ + hash: rec.Hash, gen: rec.PlacementGeneration, expiresAt: rec.ExpiresAt, + } + return nil +} + +// ConsumeSessionBootstrap checks and spends under one lock. +// +// The order of the four refusals is deliberate: the hash first, so that a +// caller holding no valid token learns nothing about the state of the one +// that exists; then the fence, the expiry, and the spend, from the most +// structural fact to the most transient. +func (b memBootstraps) ConsumeSessionBootstrap(_ context.Context, ws control.WorkspaceID, id control.SessionID, hash string, gen uint64, now time.Time) error { + if ws == "" || id == "" || hash == "" { + return control.ErrInvalid + } + b.m.mu.Lock() + defer b.m.mu.Unlock() + row, ok := b.m.bootstraps[sessionKey{ws: ws, id: id}] + if !ok || row.hash != hash { + return control.ErrBootstrapUnknown + } + if row.gen != gen { + return control.ErrBootstrapFenced + } + if !now.Before(row.expiresAt) { + return control.ErrBootstrapExpired + } + if row.spent { + return control.ErrBootstrapSpent + } + row.spent = true + return nil +} + // --------------------------------------------------------------------------- // clones // --------------------------------------------------------------------------- diff --git a/internal/controld/memstore_test.go b/internal/controld/memstore_test.go index 9912daf..401bcdf 100644 --- a/internal/controld/memstore_test.go +++ b/internal/controld/memstore_test.go @@ -22,6 +22,7 @@ func TestMemStoreRepositories(t *testing.T) { Sessions: st.Sessions(), Environments: st.Environments(), Fleet: st.Fleet(), + Bootstraps: st.Bootstraps(), Provision: st.EnsureWorkspace, } }) diff --git a/internal/controld/pgstore/bootstrap.go b/internal/controld/pgstore/bootstrap.go new file mode 100644 index 0000000..9b91bd6 --- /dev/null +++ b/internal/controld/pgstore/bootstrap.go @@ -0,0 +1,112 @@ +// internal/controld/pgstore/bootstrap.go +package pgstore + +import ( + "context" + "errors" + "time" + + "github.com/jackc/pgx/v5" + + "github.com/tokencanopy/rainier/control" +) + +// pgBootstraps is the durable half of the session bootstrap token: one row +// per session, holding a hash and never a token. +type pgBootstraps struct{ s *Store } + +var _ control.SessionBootstrapStore = pgBootstraps{} + +// Bootstraps is the fourth control repository port, a view over this store's +// own rows exactly as the other three are. +func (s *Store) Bootstraps() control.SessionBootstrapStore { return pgBootstraps{s} } + +// PutSessionBootstrap records rec as id's only acceptable token. +// +// It is an upsert onto the session's own primary key, which is what makes a +// fresh mint retire its predecessor: consumed_at resets to NULL along with +// the hash, so the new token is unspent and the old one is simply not there +// to be presented any more. +func (r pgBootstraps) PutSessionBootstrap(ctx context.Context, ws control.WorkspaceID, id control.SessionID, rec control.SessionBootstrap) error { + if ws == "" || id == "" || rec.Hash == "" { + return control.ErrInvalid + } + _, err := r.s.q(ctx).Exec(ctx, ` + INSERT INTO session_bootstraps + (session_id, workspace_id, token_hash, placement_generation, expires_at, consumed_at) + VALUES ($1, $2, $3, $4, $5, NULL) + ON CONFLICT (session_id) DO UPDATE SET + workspace_id = EXCLUDED.workspace_id, + token_hash = EXCLUDED.token_hash, + placement_generation = EXCLUDED.placement_generation, + expires_at = EXCLUDED.expires_at, + consumed_at = NULL, + created_at = now()`, + string(id), string(ws), rec.Hash, int64(rec.PlacementGeneration), rec.ExpiresAt) + if err != nil { + // The error text is dropped rather than wrapped: a constraint + // violation here quotes the row, and this row's business is a + // capability. The caller fails the create either way. + return unavailable("put session bootstrap", err) + } + return nil +} + +// ConsumeSessionBootstrap spends the token in ONE predicated statement. +// +// The UPDATE is the whole of single-use: every condition a success requires +// is in its WHERE clause, so two exchanges racing on one token produce +// exactly one row affected. A read-then-write would produce two, which is +// the bug this method exists not to have. +// +// The second query runs only on the miss, and only to say WHY. It reads a row +// that no longer has a race to lose — nothing can turn an unspent token into +// a spendable one — so a diagnosis taken after the fact is as true as one +// taken during. It reports nothing it did not already know about a token the +// caller presented. +func (r pgBootstraps) ConsumeSessionBootstrap(ctx context.Context, ws control.WorkspaceID, id control.SessionID, hash string, gen uint64, now time.Time) error { + if ws == "" || id == "" || hash == "" { + return control.ErrInvalid + } + ct, err := r.s.q(ctx).Exec(ctx, ` + UPDATE session_bootstraps SET consumed_at = $1 + WHERE workspace_id = $2 AND session_id = $3 AND token_hash = $4 + AND placement_generation = $5 AND expires_at > $1 AND consumed_at IS NULL`, + now, string(ws), string(id), hash, int64(gen)) + if err != nil { + return unavailable("consume session bootstrap", err) + } + if ct.RowsAffected() == 1 { + return nil + } + + var ( + rowGen int64 + expiresAt time.Time + consumed *time.Time + ) + qerr := r.s.q(ctx).QueryRow(ctx, ` + SELECT placement_generation, expires_at, consumed_at FROM session_bootstraps + WHERE workspace_id = $1 AND session_id = $2 AND token_hash = $3`, + string(ws), string(id), hash).Scan(&rowGen, &expiresAt, &consumed) + switch { + case errors.Is(qerr, pgx.ErrNoRows): + // No row, or a row under a different hash: one answer, because a + // caller holding no valid token must not learn the state of the one + // that exists. + return control.ErrBootstrapUnknown + case qerr != nil: + return unavailable("read session bootstrap", qerr) + case uint64(rowGen) != gen: + return control.ErrBootstrapFenced + case !now.Before(expiresAt): + return control.ErrBootstrapExpired + case consumed != nil: + return control.ErrBootstrapSpent + } + // The row satisfies every condition the UPDATE tested and the UPDATE + // still matched nothing: a concurrent exchange spent it between the two + // statements, which is exactly the race single-use exists to settle, and + // this caller is the one that lost. + return control.ErrBootstrapSpent +} diff --git a/internal/controld/pgstore/migrations/0015_session_bootstrap.sql b/internal/controld/pgstore/migrations/0015_session_bootstrap.sql new file mode 100644 index 0000000..e7550f4 --- /dev/null +++ b/internal/controld/pgstore/migrations/0015_session_bootstrap.sql @@ -0,0 +1,30 @@ +-- 0015_session_bootstrap.sql — the single-use bootstrap token a microVM +-- session exchanges for its environment's decrypted secrets +-- (docs/design/2026-09-20-microvm-bootstrap-token-and-vsock.md §3). +-- +-- One row per session and no history: a fresh mint REPLACES its predecessor, +-- because a cold resume's token must be the only one that works and the one +-- the previous boot was handed must stop being an answer. That is why the +-- primary key is the session rather than the token. +-- +-- token_hash is the hex-encoded SHA-256 of the token; the plaintext left the +-- mint and is stored nowhere, so a reader of this table learns which sessions +-- have a live token and nothing that could redeem one. +-- +-- placement_generation is the fence: a session re-placed onto another runner +-- has moved past the sandbox the token was minted for. expires_at is +-- absolute so every replica agrees without agreeing on a TTL. consumed_at is +-- the spend, and single-use is enforced by the predicated UPDATE that sets it +-- — never by a read followed by a write. +-- +-- ON DELETE CASCADE because a token outliving the session it names is a row +-- nothing will ever consume and nothing will ever clean up. +CREATE TABLE IF NOT EXISTS session_bootstraps ( + session_id text PRIMARY KEY REFERENCES sessions(id) ON DELETE CASCADE, + workspace_id text NOT NULL, + token_hash text NOT NULL, + placement_generation bigint NOT NULL, + expires_at timestamptz NOT NULL, + consumed_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now() +); diff --git a/internal/controld/pgstore/pgstore_test.go b/internal/controld/pgstore/pgstore_test.go index 9c193c8..bfe8d06 100644 --- a/internal/controld/pgstore/pgstore_test.go +++ b/internal/controld/pgstore/pgstore_test.go @@ -529,6 +529,7 @@ func TestPGStoreRepositories(t *testing.T) { Sessions: st.Sessions(), Environments: st.Environments(), Fleet: st.Fleet(), + Bootstraps: st.Bootstraps(), Provision: st.EnsureWorkspace, } }) diff --git a/internal/controld/srpc.go b/internal/controld/srpc.go index 7b25793..8e0f2aa 100644 --- a/internal/controld/srpc.go +++ b/internal/controld/srpc.go @@ -68,6 +68,10 @@ func (s *Server) handleSessionRequest(ctx context.Context, runnerName string, ro return s.answerFetchAgentCredentials(ctx, runnerName, row, env) case runner.MethodPutAgentCredentials: return s.answerPutAgentCredentials(ctx, runnerName, row, env) + case runner.MethodFetchSessionSecrets: + return s.answerFetchSessionSecrets(ctx, runnerName, row, env) + case runner.MethodMintSessionBootstrap: + return s.answerMintSessionBootstrap(ctx, runnerName, row, env) default: log.Printf("controld: runner %s: session %s asked for unknown method %q", runnerName, row.ID, clip(env.Method)) diff --git a/internal/controld/srpc_bootstrap.go b/internal/controld/srpc_bootstrap.go new file mode 100644 index 0000000..b6b2b36 --- /dev/null +++ b/internal/controld/srpc_bootstrap.go @@ -0,0 +1,230 @@ +// internal/controld/srpc_bootstrap.go +package controld + +import ( + "context" + "encoding/json" + "errors" + "log" + + "github.com/tokencanopy/rainier/control" + "github.com/tokencanopy/rainier/controlapp" + "github.com/tokencanopy/rainier/protocol/runner" +) + +// The two methods the microVM bootstrap token rides, answered here beside the +// three that were already on this channel and under the same discipline: the +// placement guard above has established which session asked, every answer is +// derived from the ROW that guard read, and the value appears in exactly one +// place — the payload. +// +// Neither method is gated on the runner having announced microvm.v1, and that +// is deliberate rather than an omission. A runner that did not announce it was +// dispatched the secret values in Spec.Env already, so nothing here is a +// privilege it does not have; adding a capability check would be a second, +// weaker fence in front of the one that actually holds (the placement guard, +// the token hash, the generation, and single use), and two fences that can +// disagree are worse than one that cannot. +// +// See docs/design/2026-09-20-microvm-bootstrap-token-and-vsock.md §3 and §5. + +// sessionBootstrapRequest is the body of both requests: the fetch sends the +// protocol and the token, the mint sends the protocol alone. Decoding both +// through one type is the same economy the agent credential pair makes, and +// the field a mint does not send is simply absent. +type sessionBootstrapRequest struct { + Protocol uint64 `json:"protocol"` + Token string `json:"token"` +} + +// sessionSecretsAnswer and sessionBootstrapAnswer are the two success bodies, +// named types for the same reason mintAnswer is one: sessiond and runnerd read +// these exact keys, so a renamed field here would silently stop a microVM +// session from ever getting its environment rather than failing a build. +type sessionSecretsAnswer struct { + Env map[string]string `json:"env"` +} + +type sessionBootstrapAnswer struct { + Token string `json:"token"` + ExpiresInSec int `json:"expires_in_sec"` +} + +// sessionBootstrapRequestMaxBytes bounds a request body before it is decoded. +// Both bodies are a version and at most one 43-character token; a kilobyte is +// slack rather than a working limit, and bounding before the decode is what +// keeps a misbehaving peer from choosing how much memory this process spends. +const sessionBootstrapRequestMaxBytes = 4 << 10 + +// answerFetchSessionSecrets answers one sandbox's boot-time exchange: +// {"protocol", "token"} → {"env": {name: value}}. +// +// This is the only path in this installation that hands an environment's +// decrypted secret_refs to anything, and every refusal below is a closed one. +// The secrets are re-resolved HERE, from the environment as it stands now, +// rather than remembered from the create: the control plane deliberately never +// held them between the two, which is the same rule that keeps them out of a +// session row. +// +// The value appears in exactly one place: the payload. Not in the log line, +// which names the session, the runner, and a COUNT; not in an error, which is +// why a decode failure is never relayed; and not in a refusal, which names a +// condition. §15.1 of the tenancy specification, pinned by a test that greps +// this package's own log output for its fixture secret. +func (s *Server) answerFetchSessionSecrets(ctx context.Context, runnerName string, row control.Session, env runner.RPCEnvelope) runner.RPCEnvelope { + req, bad := decodeSessionBootstrapRequest(env) + if bad != nil { + log.Printf("controld: runner %s: session %s sent an unusable %s request", + runnerName, row.ID, clip(env.Method)) + return *bad + } + if req.Token == "" { + log.Printf("controld: runner %s: session %s asked for its secrets with no token", runnerName, row.ID) + return rpcRefusal(env.ID, "this request carried no bootstrap token") + } + + // The spend comes BEFORE the secrets are resolved, and single use means + // single use even when what follows fails: a token that was presented has + // been presented, and re-spendability on a failed resolve would turn one + // refused exchange into an unbounded number of attempts. + // + // The generation is the ROW's, read by the guard that authorized this + // request, never anything in the request. + err := s.st.Bootstraps().ConsumeSessionBootstrap(ctx, installWorkspace, row.ID, + controlapp.HashSessionBootstrapToken(req.Token), row.PlacementGeneration, s.clock.Now()) + if err != nil { + sentence, known := controlapp.SessionBootstrapRefusal(err) + if !known { + log.Printf("controld: session %s: spending a bootstrap token on runner %s: %v", + row.ID, runnerName, err) + return rpcRefusal(env.ID, "the bootstrap token could not be checked") + } + log.Printf("controld: session %s: refused a secret fetch on runner %s: %s", + row.ID, runnerName, sentence) + return rpcRefusal(env.ID, sentence) + } + + vars, err := s.sessionSecrets(ctx, row) + if err != nil { + // The resolver's own sentence may name a secret REFERENCE, which is a + // name and not a value, and which a person cannot fix without. It is + // logged and relayed for exactly the reason secretEnvironment says: + // a dangling reference is unfixable without its name. + log.Printf("controld: session %s: resolving the environment's secrets on runner %s: %v", + row.ID, runnerName, err) + return rpcRefusal(env.ID, err.Error()) + } + if vars == nil { + // The key is always present, even when it is empty: the guest reads + // "env", and a missing key would make "this environment declares no + // secrets" and "a malformed answer" look alike on the far side. + vars = map[string]string{} + } + body, err := json.Marshal(sessionSecretsAnswer{Env: vars}) + if err != nil { + // Logged WITHOUT the error: json's own message quotes the value it + // failed on, and that value is the secret. + log.Printf("controld: session %s: encoding the session secrets failed", row.ID) + return rpcRefusal(env.ID, "the session secrets could not be encoded") + } + log.Printf("controld: session %s: delivered %d environment secret(s) to runner %s", + row.ID, len(vars), runnerName) + return runner.RPCEnvelope{ID: env.ID, Method: "resp", OK: true, Payload: body} +} + +// answerMintSessionBootstrap answers a RUNNER's request for a fresh token on +// a cold resume: {"protocol"} → {"token", "expires_in_sec"}. +// +// It is the one method on this channel a sandbox does not originate, and it +// needs no new authorization for it: the request names no session, so the +// session it mints for is FromRunner.Session — the id the placement guard +// already checked — and a runner holding session A cannot mint for B because +// there is nowhere in the message to say B. +// +// The new token retires whatever this session had, which is what makes a +// resume's token the only one that works. The token appears in exactly one +// place: the payload. +func (s *Server) answerMintSessionBootstrap(ctx context.Context, runnerName string, row control.Session, env runner.RPCEnvelope) runner.RPCEnvelope { + if _, bad := decodeSessionBootstrapRequest(env); bad != nil { + log.Printf("controld: runner %s: session %s sent an unusable %s request", + runnerName, row.ID, clip(env.Method)) + return *bad + } + minter := controlapp.SessionBootstrapMinter{Store: s.st.Bootstraps(), Clock: s.clock} + token, err := minter.Mint(ctx, installWorkspace, row.ID, row.PlacementGeneration) + if err != nil { + log.Printf("controld: session %s: minting a bootstrap token for runner %s: %v", + row.ID, runnerName, err) + return rpcRefusal(env.ID, "a bootstrap token could not be minted for this session") + } + body, err := json.Marshal(sessionBootstrapAnswer{ + Token: token, + ExpiresInSec: int(controlapp.SessionBootstrapTTL.Seconds()), + }) + if err != nil { + // Unreachable (a string and an int always marshal) and logged WITHOUT + // the error, which is the one error here whose text could quote the + // value it failed on. + log.Printf("controld: session %s: encoding a bootstrap token failed", row.ID) + return rpcRefusal(env.ID, "the bootstrap token could not be encoded") + } + log.Printf("controld: session %s: minted a bootstrap token for runner %s at placement generation %d", + row.ID, runnerName, row.PlacementGeneration) + return runner.RPCEnvelope{ID: env.ID, Method: "resp", OK: true, Payload: body} +} + +// sessionSecrets resolves the environment secrets row's create would have +// carried, through the same resolver the scheduler dispatches with. +// +// It is the resolver and not a second decryption path on purpose: launchMaterial +// is the only place in the self-hosted adapter set that holds the secrets key, +// and a second one would be a second thing to get wrong. A session with no +// environment, or one whose environment declares no secret_refs, resolves to +// nothing and gets an empty map — a truthful answer, not a refusal. +func (s *Server) sessionSecrets(ctx context.Context, row control.Session) (map[string]string, error) { + env, err := s.sessionEnvironment(ctx, row) + if err != nil { + return nil, err + } + return launchMaterial{st: s.st, key: s.cfg.SecretsKey}.secretEnvironment(ctx, env) +} + +// sessionEnvironment reads the environment row came from, or nil for a +// scratch session. An environment that has been deleted since the create is +// nil too: the session is still running, it simply has no declared secrets to +// be given, and refusing its boot over a row somebody removed would be the +// control plane taking a live session down. +func (s *Server) sessionEnvironment(ctx context.Context, row control.Session) (*control.Environment, error) { + if row.EnvironmentID == "" { + return nil, nil + } + env, err := s.st.Environments().GetEnvironment(ctx, installWorkspace, row.EnvironmentID) + switch { + case errors.Is(err, control.ErrNotFound): + return nil, nil + case err != nil: + return nil, errors.New("this session's environment could not be read") + } + return &env, nil +} + +// decodeSessionBootstrapRequest bounds and decodes one request body, +// returning the refusal to send when it cannot. The decode error is never +// relayed and never logged: a JSON error quotes the bytes it choked on, and +// on this path those bytes are a capability. +func decodeSessionBootstrapRequest(env runner.RPCEnvelope) (sessionBootstrapRequest, *runner.RPCEnvelope) { + var req sessionBootstrapRequest + if len(env.Payload) > sessionBootstrapRequestMaxBytes { + refusal := rpcRefusal(env.ID, "the bootstrap request is too large") + return req, &refusal + } + if err := json.Unmarshal(env.Payload, &req); err != nil { + refusal := rpcRefusal(env.ID, "the bootstrap request could not be decoded") + return req, &refusal + } + if req.Protocol != runner.SessionBootstrapProtocolVersion { + refusal := rpcRefusal(env.ID, "this session must be replaced before it can exchange a bootstrap token") + return req, &refusal + } + return req, nil +} diff --git a/internal/controld/srpc_bootstrap_test.go b/internal/controld/srpc_bootstrap_test.go new file mode 100644 index 0000000..13f3a56 --- /dev/null +++ b/internal/controld/srpc_bootstrap_test.go @@ -0,0 +1,404 @@ +// internal/controld/srpc_bootstrap_test.go +package controld + +import ( + "bytes" + "context" + "encoding/json" + "log" + "strings" + "testing" + "time" + + "github.com/tokencanopy/rainier/control" + "github.com/tokencanopy/rainier/controlapp" +) + +// The two bootstrap methods, driven end to end through the runner plane: a +// scripted fake runner sends the session_req exactly as runnerd forwards one +// out of a sandbox, and the assertions are on the session_rpc that comes back +// down. +// +// The fixture secret is spelled "value_must_not_appear_anywhere" on purpose. +// It is the string the log test greps for, and having every case use it means +// any case that starts leaking it fails that one too. + +const bootstrapFixtureSecret = "value_must_not_appear_anywhere" + +// seedBootstrapSession seeds a running session on runner, with an environment +// declaring one secret ref whose sealed value is in the vault, and returns the +// session id. It is the state a microVM create leaves behind, minus the token. +func seedBootstrapSession(t *testing.T, st MemStore, id, runnerName string) string { + t.Helper() + ctx := context.Background() + ciphertext, nonce, err := Seal(testSecretsKey, []byte(bootstrapFixtureSecret)) + if err != nil { + t.Fatalf("Seal: %v", err) + } + if err := st.PutSecret(ctx, "DEPLOY_KEY", ciphertext, nonce); err != nil { + t.Fatalf("PutSecret: %v", err) + } + if _, err := st.Environments().CreateEnvironment(ctx, installWorkspace, control.Environment{ + ID: "env_example", WorkspaceID: installWorkspace, Name: "example", + SecretRefs: []string{"DEPLOY_KEY"}, + }); err != nil { + t.Fatalf("CreateEnvironment: %v", err) + } + seedSession(t, st, control.Session{ + ID: control.SessionID(id), State: control.StateRunning, + RunnerID: control.RunnerID(runnerName), EnvironmentID: "env_example", + }) + return id +} + +// mintFor records a token for id at the placement generation the row +// currently holds, the way createSpec does, and returns the plaintext. +func mintFor(t *testing.T, s *Server, st MemStore, id string) string { + t.Helper() + row, err := st.Sessions().GetSession(context.Background(), installWorkspace, control.SessionID(id)) + if err != nil { + t.Fatalf("read %s: %v", id, err) + } + minter := controlapp.SessionBootstrapMinter{Store: st.Bootstraps(), Clock: s.clock} + tok, err := minter.Mint(context.Background(), installWorkspace, control.SessionID(id), row.PlacementGeneration) + if err != nil { + t.Fatalf("mint for %s: %v", id, err) + } + return tok +} + +// bootstrapRequest is the body a sandbox sends: the protocol and the token. +func bootstrapRequest(token string) string { + b, _ := json.Marshal(map[string]any{"protocol": 1, "token": token}) + return string(b) +} + +// refusalText reads the {"error": ...} sentence off an ok:false answer, +// failing the test when the answer was a success. +func refusalText(t *testing.T, payload json.RawMessage) string { + t.Helper() + var body struct { + Error string `json:"error"` + } + if err := json.Unmarshal(payload, &body); err != nil { + t.Fatalf("decoding a refusal payload %s: %v", payload, err) + } + return body.Error +} + +// TestFetchSessionSecretsTable is the exchange in its five outcomes: one +// success and four refusals with distinct reasons, over the live plane. +// +// Each row drives the same method against a differently prepared session, so +// what distinguishes them is only ever the state of the token — which is the +// claim §3's "three independent refusals, all in the control plane" makes. +func TestFetchSessionSecretsTable(t *testing.T) { + for _, tc := range []struct { + name string + // prepare returns the token the sandbox will present. + prepare func(t *testing.T, s *Server, st MemStore, id string) string + wantOK bool + // wantReason is a distinguishing fragment of the refusal sentence. + wantReason string + }{ + { + name: "a fresh token is exchanged", + prepare: func(t *testing.T, s *Server, st MemStore, id string) string { + return mintFor(t, s, st, id) + }, + wantOK: true, + }, + { + name: "a replayed token is refused", + prepare: func(t *testing.T, s *Server, st MemStore, id string) string { + tok := mintFor(t, s, st, id) + row, _ := st.Sessions().GetSession(context.Background(), installWorkspace, control.SessionID(id)) + if err := st.Bootstraps().ConsumeSessionBootstrap(context.Background(), installWorkspace, + control.SessionID(id), controlapp.HashSessionBootstrapToken(tok), + row.PlacementGeneration, s.clock.Now()); err != nil { + t.Fatalf("first spend: %v", err) + } + return tok + }, + wantReason: "already been exchanged", + }, + { + name: "an expired token is refused", + prepare: func(t *testing.T, s *Server, st MemStore, id string) string { + row, _ := st.Sessions().GetSession(context.Background(), installWorkspace, control.SessionID(id)) + // Minted as if two hours ago: the mint's own clock is this + // replica's, so an expiry in the past is written directly. + tok := "expired_token_example" + if err := st.Bootstraps().PutSessionBootstrap(context.Background(), installWorkspace, + control.SessionID(id), control.SessionBootstrap{ + Hash: controlapp.HashSessionBootstrapToken(tok), + PlacementGeneration: row.PlacementGeneration, + ExpiresAt: s.clock.Now().Add(-2 * time.Hour), + }); err != nil { + t.Fatalf("put an expired token: %v", err) + } + return tok + }, + wantReason: "expired", + }, + { + name: "a token from a superseded placement is refused", + prepare: func(t *testing.T, s *Server, st MemStore, id string) string { + tok := mintFor(t, s, st, id) + // The session is placed again, which is exactly what a + // re-placement onto another runner does to the row — and the + // token minted for the sandbox that no longer exists must + // stop being an answer. It is placed back onto the SAME + // runner so the placement guard still admits the request and + // the fence is what refuses it. + holder := control.RunnerID("vm1") + if err := st.Sessions().Transition(context.Background(), installWorkspace, control.SessionID(id), + control.NonTerminal, control.StateRunning, + control.TransitionOpts{RunnerID: &holder}); err != nil { + t.Fatalf("re-place the session: %v", err) + } + return tok + }, + wantReason: "placed again", + }, + { + name: "another session's token is refused", + prepare: func(t *testing.T, s *Server, st MemStore, id string) string { + seedSession(t, st, control.Session{ID: "sess_neighbour", State: control.StateRunning, + RunnerID: "vm1", EnvironmentID: "env_example"}) + return mintFor(t, s, st, "sess_neighbour") + }, + wantReason: "no bootstrap token matching", + }, + } { + t.Run(tc.name, func(t *testing.T) { + s, st, ts := newTestControld(t) + f := joinRunner(t, s, ts, runnerScript{Name: "vm1"}) + id := seedBootstrapSession(t, st, "sess_bootstrap", "vm1") + token := tc.prepare(t, s, st, id) + + f.sandboxRequest(t, id, 11, "fetch_session_secrets", bootstrapRequest(token)) + + cmd := nextSessionRPC(t, f) + if cmd.Session != id || cmd.RPC.ID != 11 || cmd.RPC.Method != "resp" { + t.Fatalf("answer = %+v, want a resp for id 11 on %s", cmd.RPC, id) + } + if !tc.wantOK { + if cmd.RPC.OK { + t.Fatalf("a %s was accepted: %s", tc.name, cmd.RPC.Payload) + } + reason := refusalText(t, cmd.RPC.Payload) + if !strings.Contains(reason, tc.wantReason) { + t.Fatalf("refusal = %q, want it to name %q", reason, tc.wantReason) + } + if strings.Contains(reason, bootstrapFixtureSecret) || strings.Contains(reason, token) { + t.Fatalf("a refusal carried the secret or the token: %q", reason) + } + return + } + if !cmd.RPC.OK { + t.Fatalf("a fresh token was refused: %q", refusalText(t, cmd.RPC.Payload)) + } + var answer struct { + Env map[string]string `json:"env"` + } + if err := json.Unmarshal(cmd.RPC.Payload, &answer); err != nil { + t.Fatalf("decoding the answer: %v", err) + } + if answer.Env["DEPLOY_KEY"] != bootstrapFixtureSecret { + t.Fatalf("the answer delivered %d name(s) and not the environment's secret", len(answer.Env)) + } + }) + } +} + +// TestFetchSessionSecretsIsSingleUseOverTheWire is the replay row again, but +// driven the way a replay actually happens: the same token presented twice +// over the plane, with nothing else touching the store in between. +func TestFetchSessionSecretsIsSingleUseOverTheWire(t *testing.T) { + s, st, ts := newTestControld(t) + f := joinRunner(t, s, ts, runnerScript{Name: "vm1"}) + id := seedBootstrapSession(t, st, "sess_single_use", "vm1") + token := mintFor(t, s, st, id) + + f.sandboxRequest(t, id, 1, "fetch_session_secrets", bootstrapRequest(token)) + if first := nextSessionRPC(t, f); !first.RPC.OK { + t.Fatalf("the first exchange was refused: %q", refusalText(t, first.RPC.Payload)) + } + + f.sandboxRequest(t, id, 2, "fetch_session_secrets", bootstrapRequest(token)) + second := nextSessionRPC(t, f) + if second.RPC.OK { + t.Fatalf("the second exchange succeeded: %s", second.RPC.Payload) + } + if reason := refusalText(t, second.RPC.Payload); !strings.Contains(reason, "already been exchanged") { + t.Fatalf("the replay's refusal = %q", reason) + } +} + +// TestFetchSessionSecretsRefusesAnEmptyOrMisversionedRequest pins the two +// pre-checks that run before the store is touched at all. +func TestFetchSessionSecretsRefusesAnEmptyOrMisversionedRequest(t *testing.T) { + s, st, ts := newTestControld(t) + f := joinRunner(t, s, ts, runnerScript{Name: "vm1"}) + id := seedBootstrapSession(t, st, "sess_prechecks", "vm1") + mintFor(t, s, st, id) + + for i, tc := range []struct { + body string + wantReason string + }{ + {`{"protocol":1}`, "carried no bootstrap token"}, + {`{"protocol":2,"token":"x"}`, "must be replaced"}, + {`{"protocol":"one"}`, "could not be decoded"}, + } { + f.sandboxRequest(t, id, uint64(20+i), "fetch_session_secrets", tc.body) + cmd := nextSessionRPC(t, f) + if cmd.RPC.OK { + t.Fatalf("%q was accepted", tc.body) + } + if reason := refusalText(t, cmd.RPC.Payload); !strings.Contains(reason, tc.wantReason) { + t.Fatalf("%q refused with %q, want it to name %q", tc.body, reason, tc.wantReason) + } + } +} + +// TestMintSessionBootstrapAnswersARunner is the cold-resume half: runnerd +// originates the request on a session the guard has placed on it, and gets a +// token that works exactly once. +func TestMintSessionBootstrapAnswersARunner(t *testing.T) { + s, st, ts := newTestControld(t) + f := joinRunner(t, s, ts, runnerScript{Name: "vm1"}) + id := seedBootstrapSession(t, st, "sess_mint", "vm1") + + f.sandboxRequest(t, id, 31, "mint_session_bootstrap", `{"protocol":1}`) + + cmd := nextSessionRPC(t, f) + if cmd.RPC.ID != 31 || !cmd.RPC.OK { + t.Fatalf("mint answer = %+v", cmd.RPC) + } + var answer struct { + Token string `json:"token"` + ExpiresInSec int `json:"expires_in_sec"` + } + if err := json.Unmarshal(cmd.RPC.Payload, &answer); err != nil { + t.Fatalf("decoding the mint answer: %v", err) + } + if answer.Token == "" { + t.Fatal("the mint answered with no token") + } + if answer.ExpiresInSec != int(controlapp.SessionBootstrapTTL.Seconds()) { + t.Fatalf("expires_in_sec = %d, want %d", answer.ExpiresInSec, int(controlapp.SessionBootstrapTTL.Seconds())) + } + + // And the token it minted is the one that works. + f.sandboxRequest(t, id, 32, "fetch_session_secrets", bootstrapRequest(answer.Token)) + used := nextSessionRPC(t, f) + if !used.RPC.OK { + t.Fatalf("the freshly minted token was refused: %q", refusalText(t, used.RPC.Payload)) + } +} + +// TestMintSessionBootstrapRetiresThePreviousToken is what makes a cold resume +// safe: the token the last boot was handed stops working the moment a new one +// is minted, whether or not it was ever spent. +func TestMintSessionBootstrapRetiresThePreviousToken(t *testing.T) { + s, st, ts := newTestControld(t) + f := joinRunner(t, s, ts, runnerScript{Name: "vm1"}) + id := seedBootstrapSession(t, st, "sess_remint", "vm1") + stale := mintFor(t, s, st, id) + + f.sandboxRequest(t, id, 41, "mint_session_bootstrap", `{"protocol":1}`) + if cmd := nextSessionRPC(t, f); !cmd.RPC.OK { + t.Fatalf("the re-mint was refused: %q", refusalText(t, cmd.RPC.Payload)) + } + + f.sandboxRequest(t, id, 42, "fetch_session_secrets", bootstrapRequest(stale)) + cmd := nextSessionRPC(t, f) + if cmd.RPC.OK { + t.Fatalf("the retired token still worked: %s", cmd.RPC.Payload) + } + if reason := refusalText(t, cmd.RPC.Payload); !strings.Contains(reason, "no bootstrap token matching") { + t.Fatalf("the retired token's refusal = %q", reason) + } +} + +// TestBootstrapMethodsAreRefusedForAnotherRunnersSession pins that the guard +// above these arms is the one that decides who may ask. It is the same guard +// the three existing methods keep, and it is what makes "no session id in the +// mint request" a design and not an omission: the id is the one the store +// placed on the asking runner. +func TestBootstrapMethodsAreRefusedForAnotherRunnersSession(t *testing.T) { + s, st, ts := newTestControld(t) + f := joinRunner(t, s, ts, runnerScript{Name: "vm1"}) + id := seedBootstrapSession(t, st, "sess_elsewhere", "vm2") + // A token really does exist for it, so the refusal below is the guard's + // and not an accident of there being nothing to find. + token := mintFor(t, s, st, id) + + for i, method := range []string{"fetch_session_secrets", "mint_session_bootstrap"} { + f.sandboxRequest(t, id, uint64(50+i), method, bootstrapRequest(token)) + cmd := nextSessionRPC(t, f) + if cmd.RPC.OK { + t.Fatalf("%s was answered for a session placed on another runner: %s", method, cmd.RPC.Payload) + } + if reason := refusalText(t, cmd.RPC.Payload); !strings.Contains(reason, "not placed on the runner that asked") { + t.Fatalf("%s refused with %q, want the placement guard's sentence", method, reason) + } + } +} + +// TestTheBootstrapExchangeLeavesNothingInTheLog is §15.1, checked rather than +// claimed: this test captures the package's own log output across a whole +// successful exchange — a mint, a fetch, and a refusal — and greps it for the +// fixture secret and for every token that crossed the wire. +// +// A log line is the easiest place for a value to end up and the hardest place +// to get it back out of, which is why this is a test and not a review note. +func TestTheBootstrapExchangeLeavesNothingInTheLog(t *testing.T) { + var captured bytes.Buffer + prevOut, prevFlags := log.Writer(), log.Flags() + log.SetOutput(&captured) + log.SetFlags(0) + t.Cleanup(func() { log.SetOutput(prevOut); log.SetFlags(prevFlags) }) + + s, st, ts := newTestControld(t) + f := joinRunner(t, s, ts, runnerScript{Name: "vm1"}) + id := seedBootstrapSession(t, st, "sess_log", "vm1") + + // A mint, whose answer carries a token. + f.sandboxRequest(t, id, 61, "mint_session_bootstrap", `{"protocol":1}`) + mintCmd := nextSessionRPC(t, f) + var minted struct { + Token string `json:"token"` + } + if err := json.Unmarshal(mintCmd.RPC.Payload, &minted); err != nil { + t.Fatalf("decoding the mint answer: %v", err) + } + + // A successful fetch, which delivers the fixture secret... + f.sandboxRequest(t, id, 62, "fetch_session_secrets", bootstrapRequest(minted.Token)) + if cmd := nextSessionRPC(t, f); !cmd.RPC.OK { + t.Fatalf("the fetch was refused: %q", refusalText(t, cmd.RPC.Payload)) + } + // ...and a replay, which is refused. + f.sandboxRequest(t, id, 63, "fetch_session_secrets", bootstrapRequest(minted.Token)) + if cmd := nextSessionRPC(t, f); cmd.RPC.OK { + t.Fatal("the replay succeeded") + } + + out := captured.String() + for _, forbidden := range []string{bootstrapFixtureSecret, minted.Token} { + if forbidden == "" { + t.Fatal("the test's own fixture is empty; it would grep for nothing") + } + if strings.Contains(out, forbidden) { + t.Fatalf("the log carries a value it must never carry:\n%s", out) + } + } + // And the log DID say something about this session, so the grep above is + // over real output rather than over an empty buffer. + if !strings.Contains(out, string(id)) { + t.Fatalf("no log line mentions the session at all; the grep proves nothing:\n%s", out) + } +} diff --git a/internal/controld/store.go b/internal/controld/store.go index 5d9f87c..e858b4c 100644 --- a/internal/controld/store.go +++ b/internal/controld/store.go @@ -206,6 +206,12 @@ type Repositories interface { Sessions() control.SessionRepository Environments() control.EnvironmentRepository Fleet() control.FleetRepository + // Bootstraps is the session bootstrap tokens: the hash, the placement it + // was minted under, its expiry, and the atomic spend. It is a port of its + // own rather than four methods on Sessions because minting and spending a + // capability has nothing to do with a session's lifecycle — see + // control.SessionBootstrapStore. + Bootstraps() control.SessionBootstrapStore } // MemStore is the shape the in-memory store has: Store itself, plus the one From 1e0e240c9d6abbb903c730c45e2b39099d332ec9 Mon Sep 17 00:00:00 2001 From: Josh Zhang Date: Sun, 20 Sep 2026 01:34:34 +0000 Subject: [PATCH 3/7] feat(runnerd,driver): vsock is the channel, and the first frame is the session MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 3 of the rollout: the runner announces microvm.v1, the microVM driver opens a virtio-vsock control channel before the VM starts, and a guest's first frame on it is everything it needs to be a session. The first runner to announce the capability is the first session whose secrets are withheld — and by now the plane can answer for them. internal/relay gains NetConn, a Conn over a byte stream with newline-delimited JSON framing. A vsock connection is an AF_VSOCK stream in the guest and an AF_UNIX stream on the host, where the WebSocket this package grew up on delivered whole messages; everything above Conn — the hub, the session side, the control channel, the RPC dispatcher — is untouched, which is the whole reason the seam is there. Writes are serialized, because a Hub writes from every attached client's goroutine and from SendControl, and two concurrent writes to a stream corrupt it permanently for the reader. internal/driver: PUT /vsock with guest_cid 3 and a per-BOOT uds_path inside the session's own state directory, a listener on _1024 opened BEFORE InstanceStart (a listener created after it is a race whose loser is a session that boots and is never configured), and runner.BootConfig as the first FrameControl on whatever connects. session.json and buildGuestEnv's copy of spec.Env are gone entirely: the configuration is composed in memory and written nowhere, and the socket is the only file this path creates (0600). A cold resume gets a new VM, a new socket, and a new token, because the old one is single-use and fenced and because Firecracker's documentation warns one uds_path cannot be multiplexed across VMs. And the driver refuses a create carrying env VALUES with no token, naming the control-plane version. That is the compatibility table's "old controld + new runner" row, and it is a per-driver question with two right answers, which is why the twelfth RunContract subtest asserts both: Docker must keep accepting them (it puts them in an argv and nothing reaches disk, and a control-plane rollback must not take every self-hosted session down), and a driver announcing microvm.v1 must refuse. Subtest 5 now carries a token so it stays a shared subtest rather than becoming a refusal on one driver and a snapshot on the other, and the microVM snapshot manifest now records the boot configuration's keys too, so its strip assertion is about what the session was actually configured with. internal/runnerd: the /register handler's body becomes serveSessionConn, which the WebSocket door and the vsock door now share — so the two cannot drift into meaning different things, and the microVM door is the stronger of the two (tenancy §18 item 53: the session id is the socket path's, not a query parameter the runner believes). A create on a withholding driver carries no dial URL, because there is nothing to dial. A cold suspend on one tells the sandbox it is COLD before the VM goes; the Docker path sends nothing on a cold stop, exactly as it never has, and a test asserts the silence. Two things needed care. The runner now ORIGINATES a session RPC — the mint on a cold resume — so it keeps a pending table of its own and takes its ids from a disjoint space (the high bit), because both directions ride one connection for one session and a response to the runner's id 1 would otherwise be delivered into a sandbox waiting on its own. And the microVM-specific behaviour is keyed on the announced CAPABILITY rather than on the driver's concrete type, through two optional interfaces (CapabilityDriver, HostedDriver), so the claim the control plane acts on and the behaviour the runner performs cannot disagree. Co-Authored-By: Claude Fable 5.1 --- internal/driver/contract.go | 101 +++++++ internal/driver/driver.go | 41 +++ internal/driver/microvm.go | 344 +++++++++++++++------- internal/driver/microvm_test.go | 189 +++++++++--- internal/driver/microvm_vsock.go | 402 ++++++++++++++++++++++++++ internal/driver/microvm_vsock_test.go | 335 +++++++++++++++++++++ internal/relay/netconn.go | 131 +++++++++ internal/relay/netconn_test.go | 138 +++++++++ internal/runnerd/agent.go | 58 +++- internal/runnerd/microvm.go | 224 ++++++++++++++ internal/runnerd/microvm_test.go | 390 +++++++++++++++++++++++++ internal/runnerd/runnerd.go | 73 ++++- 12 files changed, 2261 insertions(+), 165 deletions(-) create mode 100644 internal/driver/microvm_vsock.go create mode 100644 internal/driver/microvm_vsock_test.go create mode 100644 internal/relay/netconn.go create mode 100644 internal/relay/netconn_test.go create mode 100644 internal/runnerd/microvm.go create mode 100644 internal/runnerd/microvm_test.go diff --git a/internal/driver/contract.go b/internal/driver/contract.go index 9940fca..b174587 100644 --- a/internal/driver/contract.go +++ b/internal/driver/contract.go @@ -113,6 +113,20 @@ func workspaceExists(t *testing.T, d Driver, sessionID string) bool { } } +// driverCapabilities names the portable capabilities a driver's runner +// announces on its behalf, so a shared subtest can ask "is this a driver the +// control plane withholds secrets from?" without a per-driver branch. +// +// It asks through the same optional interface the runner asks through +// (CapabilityDriver), so the subtest below and the announce a runner actually +// makes cannot come to different conclusions about the same driver. +func driverCapabilities(d Driver) []string { + if cd, ok := d.(CapabilityDriver); ok { + return cd.Capabilities() + } + return nil +} + func RunContract(t *testing.T, newDriver func(t *testing.T) (Driver, func())) { t.Run("create-inspect-destroy", func(t *testing.T) { d, cleanup := newDriver(t) @@ -254,6 +268,15 @@ func RunContract(t *testing.T, newDriver func(t *testing.T) (Driver, func())) { Name: "t9", Image: "", SessionID: "s9", DialURL: "ws://x", Setup: "true", Env: map[string]string{"CONTRACT_SECRET": "must-not-survive"}, + // The token is here so this subtest can still be written for a + // driver that refuses secret VALUES without one — which is the + // twelfth subtest below, and is the microVM driver. The Docker + // driver ignores the field entirely (nothing in runArgs reads + // it), so its create is the one it has always been. Without a + // token this create would be a refusal on one driver and a + // snapshot on the other, and the shared subtest would have + // stopped being shared. + BootstrapToken: "contract-token-example", }) if err != nil { t.Fatal(err) @@ -273,6 +296,84 @@ func RunContract(t *testing.T, newDriver func(t *testing.T) (Driver, func())) { assertStrippedFromImage(t, d, ref, "must-not-survive", strip) }) + t.Run("a create carrying secret values and no token is refused", func(t *testing.T) { + // The twelfth subtest, and the one that makes the fifth honest for a + // driver that never accepts secret values at all. + // + // The compatibility table's "old controld + new runner" row: a + // control plane that predates the session bootstrap exchange + // dispatches an environment's decrypted secrets in Spec.Env with no + // token. Whether that is acceptable is a per-driver question with + // two different right answers, which is why it is asserted here + // rather than at either driver: + // + // - Docker puts them in a `docker run` argv and nothing reaches the + // host filesystem. It has always accepted them and must keep + // accepting them, or a rollback of the control plane would take + // every self-hosted session down. + // - a microVM host has no argv, and every channel it does have ends + // in a file that outlives the session on a machine shared with + // other tenants. It must refuse, loudly, naming what the control + // plane has to be — a refusal costs one session, and a quiet + // write is the exposure ADR-0003 §2.7 was written against. + // + // The driver announcing microvm.v1 is the one required to refuse, + // because that capability is exactly the claim the control plane + // withholds on: a driver that announces it and then accepts the + // values has told the plane it will do something it does not do. + d, cleanup := newDriver(t) + defer cleanup() + ctx := context.Background() + + spec := Spec{ + Name: "t10", Image: "", SessionID: "s10", DialURL: "ws://x", + Env: map[string]string{"CONTRACT_SECRET": "must-not-be-accepted"}, + } + h, err := d.Create(ctx, spec) + if err == nil { + defer d.Destroy(ctx, h.ID) + } + if !slices.Contains(driverCapabilities(d), "microvm.v1") { + // Not a withholding driver: accepting is the contract, and a + // refusal here would be the rollback failure described above. + if err != nil { + t.Fatalf("a create carrying env values was refused by a driver that does not withhold: %v", err) + } + return + } + if err == nil { + t.Fatal("a driver announcing microvm.v1 accepted a create carrying secret values with no bootstrap token") + } + // The refusal has to say what the operator must change, which is the + // control plane's version and not anything about this host. + for _, want := range []string{"bootstrap token", "microvm.v1"} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("the refusal = %q, want it to name %q", err, want) + } + } + if strings.Contains(err.Error(), "must-not-be-accepted") { + t.Fatalf("the refusal quotes the value it refused: %q", err) + } + // And nothing was half-created: a refused create leaves no session. + listed, lerr := d.List(ctx) + if lerr != nil { + t.Fatal(lerr) + } + for _, l := range listed { + if l.SessionID == "s10" { + t.Fatalf("a refused create left a session behind: %+v", l) + } + } + // The same create WITH a token is accepted, which is what makes the + // refusal about the token rather than about the env block. + spec.BootstrapToken = "contract-token-example" + withToken, err := d.Create(ctx, spec) + if err != nil { + t.Fatalf("a withheld create carrying a token was refused: %v", err) + } + d.Destroy(ctx, withToken.ID) + }) + t.Run("capacity", func(t *testing.T) { d, cleanup := newDriver(t) defer cleanup() diff --git a/internal/driver/driver.go b/internal/driver/driver.go index 1c79566..7751adc 100644 --- a/internal/driver/driver.go +++ b/internal/driver/driver.go @@ -74,6 +74,25 @@ type Spec struct { // no creator and for every create a control plane older than the field // sent. The session's agents then simply ask for a login. Home *HomeMount + // BootstrapToken and SecretNames are the microVM bootstrap exchange + // (runner.Spec's two fields of the same names, carried across this + // boundary unchanged). For a session placed on a runner that announced + // microvm.v1 the control plane leaves the environment's decrypted + // secret_refs OUT of Env and puts these here instead: the names the + // guest should expect, and the single-use capability it exchanges for + // their values once it has booted. + // + // Both are empty on every Docker create and on every create from a + // control plane older than them, which is exactly why the Docker driver + // ignores them and Env keeps its meaning. The microVM driver is the only + // one that reads them, and the only one that refuses a create carrying + // values in Env with no token — see (*Microvm).Create. + // + // Neither is ever written to host disk, logged, or put in a snapshot + // manifest. The token's whole lifetime on a host is the hop from this + // struct into the guest's boot configuration. + BootstrapToken string + SecretNames []string } // HomeMount names the agent home volume and where it lands in the container. @@ -180,6 +199,28 @@ type Listed struct { Handle Handle } +// CapabilityDriver is a driver that names portable capability tokens the +// runner announces on its behalf. +// +// It is an optional interface rather than a method on Driver because most +// drivers have nothing to say: a capability is a claim the control plane +// SCHEDULES on, and inventing one per driver would make the fleet's +// vocabulary an implementation detail of whichever sandbox a runner happens +// to use. The microVM driver is the one that has something to claim +// (microvm.v1), and what it claims is the fence the control plane withholds +// an environment's secret values on — so the announcement and the driver +// that must honour it are the same fact, read from one place. +type CapabilityDriver interface { + Capabilities() []string +} + +// HostedDriver is a driver that needs the runner above it: see MicrovmHost +// for the two things, and why they cannot be constructor arguments (the +// runner is composed OVER the driver, so the driver exists first). +type HostedDriver interface { + SetHost(MicrovmHost) +} + type Driver interface { Create(ctx context.Context, spec Spec) (Handle, error) Suspend(ctx context.Context, id string, warm bool) error // warm=pause, cold=stop diff --git a/internal/driver/microvm.go b/internal/driver/microvm.go index 562bc73..385abdc 100644 --- a/internal/driver/microvm.go +++ b/internal/driver/microvm.go @@ -3,16 +3,22 @@ // The microVM driver: one Firecracker microVM per session, in place of the // docker driver's one container per session. // -// This is the second step of that driver, and it is deliberately not a -// working end-to-end path yet. What it establishes is the seam and the -// invariants: a production `--driver=microvm` either has hardware +// This is the third step of that driver, and it is deliberately not a +// working end-to-end path yet. What the second step established is the seam +// and the invariants: a production `--driver=microvm` either has hardware // virtualization, a kernel, a rootfs, a formatter and the Firecracker binary, // or it refuses to start; nothing decrypted is written to host disk; a // cold-parked session reads as suspended rather than gone; and the simulated // engine behind the tests is reachable only by explicitly injecting it. -// Guest configuration delivery (vsock), the jailer, per-VM network slots, -// nftables, metering, and real image and snapshot work by digest each land in -// their own change — see the TODO markers below and ADR-0003. +// +// What this step adds is the host-to-guest channel those invariants were +// waiting for: virtio-vsock, carrying a session's whole configuration and +// its bootstrap token, in place of a staged file and in place of MMDS. See +// microvm_vsock.go, and docs/design/2026-09-20-microvm-bootstrap-token-and-vsock.md. +// +// The jailer, per-VM network slots, nftables, metering, and real image and +// snapshot work by digest each land in their own change — see the TODO +// markers below and ADR-0003. package driver import ( @@ -37,6 +43,8 @@ import ( "sync/atomic" "syscall" "time" + + "github.com/tokencanopy/rainier/protocol/runner" ) const ( @@ -121,19 +129,27 @@ const ( // enforcement, not a convention — every structure this driver persists either // embeds VMMConfig or is derived from it. type VMMConfig struct { - ID string `json:"id"` - SessionID string `json:"session_id"` - VCPU int `json:"vcpu"` - MemoryMiB int `json:"memory_mib"` - KernelPath string `json:"kernel_path"` - RootfsPath string `json:"rootfs_path"` - WorkspaceDiskPath string `json:"workspace_disk_path"` - HomeDiskPath string `json:"home_disk_path"` - Cmd []string `json:"cmd"` - DialURL string `json:"dial_url"` - ProxyURL string `json:"proxy_url"` - TapDevice string `json:"tap_device"` - Env map[string]string `json:"-"` + ID string `json:"id"` + SessionID string `json:"session_id"` + VCPU int `json:"vcpu"` + MemoryMiB int `json:"memory_mib"` + KernelPath string `json:"kernel_path"` + RootfsPath string `json:"rootfs_path"` + WorkspaceDiskPath string `json:"workspace_disk_path"` + HomeDiskPath string `json:"home_disk_path"` + Cmd []string `json:"cmd"` + DialURL string `json:"dial_url"` + ProxyURL string `json:"proxy_url"` + TapDevice string `json:"tap_device"` + // VsockUDSPath is the host path Firecracker is told to serve this VM's + // virtio-vsock device on: the guest's connections to host port N are + // forwarded to "_N", and 1024 is the only port this design + // uses. It is a path and not a value — recorded like every other path + // here, and per BOOT rather than per instance, because the documentation + // warns that one uds_path cannot be multiplexed across VMs and a cold + // resume is a new VM. + VsockUDSPath string `json:"vsock_uds_path"` + Env map[string]string `json:"-"` // EgressAllow is the session's allowlist. It is carried and recorded, and // nothing enforces it. @@ -186,12 +202,26 @@ type instanceRecord struct { PID int `json:"pid"` Cfg VMMConfig `json:"cfg"` - // envLive marks a record whose Cfg.Env is the live environment THIS - // process built in Create. It is unexported and therefore never - // serialized, which is the point: a record recovered from disk after a - // runnerd restart has no environment behind it, and a cold resume must - // say so rather than boot a guest with a silently empty one. - envLive bool + // boot is the configuration the guest is handed over vsock, and channel + // is the host end of that conn. Both are unexported and therefore never + // serialized, which is the point: the configuration carries the session's + // bootstrap token and ADR-0003 §2.7 item 1 keeps every part of it off a + // shared host's disk. + // + // bootLive says the pair is this process's own. A record recovered from + // disk after a runnerd restart has no configuration behind it, and a cold + // resume must say so rather than boot a guest that will never be told + // what it is. (The bootstrap token would survive such a restart — the + // runner can mint a fresh one — but the rest of the configuration would + // not, and persisting it is precisely what this design does not do.) + boot runner.BootConfig + channel *guestChannel + bootLive bool + + // boots counts launches of this instance, so each one gets a vsock socket + // path of its own. A cold resume is a new VM and Firecracker's own + // documentation warns that one uds_path cannot be multiplexed across two. + boots int // epoch counts mutations of this record, and exists because Inspect and // List ask the hypervisor with the driver mutex RELEASED (its answer is @@ -224,32 +254,24 @@ type snapshotManifest struct { CreatedAt time.Time `json:"created_at"` } -// guestSessionConfig is the structured configuration staged on the host for a -// session, ready for the channel that will carry it. -// -// It carries no environment. What a session needs in order to identify itself -// and reach the relay is not secret; what an environment resolved out of its -// secret refs is, and ADR-0003 §2.7 item 1 has those arriving in the guest -// from cell-gateway against a short-lived bootstrap token, never from a file -// on the host. +// A session's configuration is no longer staged on the host at all. // -// TODO(PR 2): nothing reads this yet — it is staged on the host and there is -// no path into the guest. The channel is virtio-vsock (ADR-0003 §2.7 item 2), -// which also carries the bootstrap token. +// It used to be written to "/instances//session.json", against +// the day something would carry it into the guest. That day is this change, +// and what carries it is virtio-vsock: the configuration is composed in +// memory (bootConfigFor), handed to the guest as the first control frame on +// the conn it opens, and never written anywhere. A file is not needed, and a +// file on a shared host that describes one tenant's session — and, once the +// bootstrap token exists, carries a capability — is exactly what ADR-0003 +// §2.7 asks for there not to be. // -// It is deliberately NOT delivered through the session's own workspace. A -// file inside a volume the agent can write is a file the agent can rewrite, -// and sessiond reading a session id or a dial URL back out of one would let a -// session re-register as another or point its relay somewhere else. That is -// why sessiond has no loader for this file: the guest must receive it from -// the host, over a channel the guest cannot write. -type guestSessionConfig struct { - SessionID string `json:"session_id"` - DialURL string `json:"dial_url"` - ProxyURL string `json:"proxy_url"` - Cmd []string `json:"cmd"` - EgressAllow []string `json:"egress_allow"` -} +// It was also never delivered through the session's own workspace, for a +// reason that still holds and is worth keeping written down: a file inside a +// volume the agent can write is a file the agent can rewrite, and a sessiond +// reading its session id or its proxy back out of one would let a session +// re-register as another or point its egress somewhere else. Over vsock the +// configuration arrives on a socket inside one VM's own directory, which the +// guest cannot write and cannot forge. // Microvm implements driver.Driver for hardware-isolated microVMs. type Microvm struct { @@ -264,6 +286,11 @@ type Microvm struct { instances map[string]*instanceRecord pulls []string strips [][]string + + // host is the runner above this driver: where a guest's control + // connection goes, and who asks the control plane for a fresh bootstrap + // token on a cold resume. nil is a real state — see MicrovmHost. + host MicrovmHost } // NewMicrovm creates a new microVM driver, or fails. @@ -400,13 +427,17 @@ func (m *Microvm) instanceMetaPath(id string) string { return filepath.Join(m.instanceDir(id), "instance.json") } -// persistable copies a record for writing to disk. The copy drops Cfg.Env — -// which `json:"-"` would drop anyway — so that a record handed to a goroutine -// outside the driver mutex carries no reference to the live map either. +// persistable copies a record for writing to disk. The copy drops Cfg.Env and +// the guest's boot configuration — which being unexported, or `json:"-"`, +// would drop anyway — so that a record handed to a goroutine outside the +// driver mutex carries no reference to the live map, the live configuration, +// or the bootstrap token in it either. func persistable(rec *instanceRecord) instanceRecord { cp := *rec cp.Cfg.Env = nil - cp.envLive = false + cp.boot = runner.BootConfig{} + cp.channel = nil + cp.bootLive = false return cp } @@ -710,9 +741,19 @@ func (m *Microvm) resolveRootfs(rootfsRef string) (string, error) { return m.locateRootfs(rootfsRef) } -// buildGuestEnv translates the full Spec into the environment map passed into -// the microVM guest. The result is held in memory for the lifetime of this +// buildGuestEnv is the driver's own record of the CONFIGURATION variables a +// session was created with. It is held in memory for the lifetime of this // process and never written to disk; see VMMConfig.Env. +// +// It no longer copies spec.Env, and that deletion is the point of this +// change. Spec.Env is where an environment's decrypted secret_refs live on +// the Docker path, and on this one they are not dispatched at all — the +// create carries their names and a token instead, and the values reach the +// guest from the control plane over the exchange the boot configuration +// starts. What a guest actually runs on arrives over vsock +// (bootConfigFor), not from this map, which is why nothing here is a +// delivery channel any more: it is what a snapshot's manifest names as +// having been configured, and nothing else reads it. func buildGuestEnv(spec Spec) map[string]string { env := make(map[string]string) @@ -751,26 +792,9 @@ func buildGuestEnv(spec Spec) map[string]string { if spec.GitAuthorEmail != "" { env["RAINIER_GIT_AUTHOR_EMAIL"] = spec.GitAuthorEmail } - for k, v := range spec.Env { - env[k] = v - } return env } -// stageGuestSessionConfig writes the non-secret half of a session's -// configuration for sessiond. -func (m *Microvm) stageGuestSessionConfig(id string, cfg guestSessionConfig) error { - dir := m.instanceDir(id) - if err := os.MkdirAll(dir, microvmDirMode); err != nil { - return err - } - data, err := json.MarshalIndent(cfg, "", " ") - if err != nil { - return err - } - return os.WriteFile(filepath.Join(dir, "session.json"), data, microvmFileMode) -} - // reserveSlot takes the capacity decision and an id under the mutex, so that // the rest of Create — disks, mkfs, the rootfs lookup, the TAP, the launch — // runs with the mutex RELEASED. Holding it across engine.Launch froze @@ -805,6 +829,12 @@ func (m *Microvm) Create(ctx context.Context, spec Spec) (Handle, error) { if err := checkHome(spec.Home); err != nil { return Handle{}, err } + // Before anything with a side effect, and before a slot is even + // reserved: a create this host must not perform is refused rather than + // half-performed. See refuseUnwithheldEnv. + if err := refuseUnwithheldEnv(spec); err != nil { + return Handle{}, err + } id, err := m.reserveSlot() if err != nil { @@ -876,17 +906,23 @@ func (m *Microvm) launch(ctx context.Context, id string, spec Spec) (*instanceRe cmd = []string{"/bin/bash"} } - if err := m.stageGuestSessionConfig(id, guestSessionConfig{ - SessionID: spec.SessionID, - DialURL: spec.DialURL, - ProxyURL: spec.ProxyURL, - Cmd: slices.Clone(cmd), - EgressAllow: slices.Clone(spec.EgressAllow), - }); err != nil { - return nil, fmt.Errorf("stage guest session config: %w", err) - } undo = append(undo, func() { m.deleteInstanceRecord(id) }) + // The guest's control channel, opened BEFORE the VM starts: the guest + // dials host port 1024 as soon as its sessiond is up, and a listener + // created after InstanceStart is a race whose loser is a session that + // boots and is never configured. + bootCfg := bootConfigFor(spec) + udsPath, listenPath, err := m.vsockPaths(id, 1) + if err != nil { + return nil, err + } + channel, err := m.openGuestChannel(spec.SessionID, listenPath, bootCfg) + if err != nil { + return nil, err + } + undo = append(undo, channel.close) + tapDevice := "tap-" + id if err := m.tap.Allocate(tapDevice, m.opts.Network); err != nil { return nil, fmt.Errorf("allocate tap device %s: %w", tapDevice, err) @@ -907,6 +943,7 @@ func (m *Microvm) launch(ctx context.Context, id string, spec Spec) (*instanceRe DialURL: spec.DialURL, ProxyURL: spec.ProxyURL, TapDevice: tapDevice, + VsockUDSPath: udsPath, Env: buildGuestEnv(spec), } @@ -922,7 +959,10 @@ func (m *Microvm) launch(ctx context.Context, id string, spec Spec) (*instanceRe Volume: workspaceVolume(spec.SessionID), PID: m.engine.PID(id), Cfg: cfg, - envLive: true, + boot: bootCfg, + channel: channel, + bootLive: true, + boots: 1, } if err := m.saveRecord(persistable(rec)); err != nil { return nil, fmt.Errorf("save instance metadata %s: %w", id, err) @@ -967,6 +1007,15 @@ func (m *Microvm) Suspend(ctx context.Context, id string, warm bool) error { inst.Cold = !warm if !warm { inst.PID = 0 + // The VM is gone, so its vsock socket is a path nothing serves. It + // is closed and removed here rather than left for the resume to + // overwrite: a stale socket on a shared host is one more file + // describing a session that is not running, and the resume gets a + // path of its own anyway. + if inst.channel != nil { + inst.channel.close() + inst.channel = nil + } } inst.bump() rec := persistable(inst) @@ -986,7 +1035,8 @@ func (m *Microvm) Resume(ctx context.Context, id string) (bool, error) { return false, fmt.Errorf("no such id %s", id) } running := inst.State == StateRunning - cold, envLive, cfg := inst.Cold, inst.envLive, inst.Cfg + cold, bootLive, cfg := inst.Cold, inst.bootLive, inst.Cfg + bootCfg, sessionID, boots := inst.boot, inst.SessionID, inst.boots m.mu.Unlock() // Resuming a session that is already running restarts nothing, and says @@ -1000,27 +1050,52 @@ func (m *Microvm) Resume(ctx context.Context, id string) (bool, error) { } restarted := false + var channel *guestChannel if cold { // A cold resume is a fresh boot, and a fresh boot needs the session's - // environment. This driver holds that in memory only (ADR-0003 §2.7 - // item 1), so a record recovered from disk after a runnerd restart - // has none — including the case where the original session carried no - // Spec.Env at all, which the driver deliberately cannot tell apart, - // having refused to write the evidence down. + // whole configuration. This driver holds that in memory only + // (ADR-0003 §2.7 item 1), so a record recovered from disk after a + // runnerd restart has none — including the case where the original + // session carried nothing secret at all, which the driver + // deliberately cannot tell apart, having refused to write the + // evidence down. // - // The honest answer is to refuse. Relaunching would boot a guest with - // a silently empty environment: no relay dial, no proxy, no - // credentials, and an agent that reports itself healthy. + // The honest answer is to refuse. Relaunching would boot a guest that + // is never told what it is: no session id, no proxy, no boot chain, + // no secrets, and an agent that reports itself healthy. // - // TODO(PR 2): the fix is not to persist the environment. It is the - // bootstrap token over vsock — sessiond asks cell-gateway for fresh - // short-lived credentials at boot and after every resume (ADR-0003 - // §2.7 item 1, §4.3) — after which a cold resume needs nothing from - // the host but the token. - if !envLive { - return false, fmt.Errorf("cold resume of %s: this session's guest environment was held in memory only and did not survive a runnerd restart; a clean relaunch needs the bootstrap token over vsock (ADR-0003 §2.7 item 1), which is not implemented yet", id) + // The bootstrap token is not what is missing here — the runner can + // mint a fresh one, and does, three lines below. What is missing is + // everything else, and persisting THAT is what this design rules out. + // Rebuilding it from the control plane on a resume is the follow-up + // (a create-shaped resume, ADR-0003 §2.3's portable checkpoint). + if !bootLive { + return false, fmt.Errorf("cold resume of %s: this session's guest configuration was held in memory only (ADR-0003 §2.7 item 1) and did not survive a runnerd restart; a clean relaunch needs the control plane to re-resolve it, which is the portable-checkpoint work and not this change", id) + } + // A new VM gets a new token and a new socket. The token because the + // old one is single-use and fenced by a placement generation the + // plane may have moved past; the socket because Firecracker's own + // documentation warns that one uds_path cannot be multiplexed across + // two VMs. + host := m.currentHost() + if host == nil { + return false, fmt.Errorf("cold resume of %s: this driver has no runner above it to mint a bootstrap token", id) + } + token, err := host.MintSessionBootstrap(ctx, sessionID) + if err != nil { + return false, fmt.Errorf("cold resume of %s: minting a bootstrap token: %w", id, err) + } + bootCfg.BootstrapToken = token + udsPath, listenPath, err := m.vsockPaths(id, boots+1) + if err != nil { + return false, err } + if channel, err = m.openGuestChannel(sessionID, listenPath, bootCfg); err != nil { + return false, err + } + cfg.VsockUDSPath = udsPath if err := m.engine.Launch(ctx, cfg); err != nil { + channel.close() return false, fmt.Errorf("relaunch cold microvm %s: %w", id, err) } restarted = true @@ -1032,12 +1107,22 @@ func (m *Microvm) Resume(ctx context.Context, id string) (bool, error) { inst, ok = m.instances[id] if !ok { m.mu.Unlock() + if channel != nil { + channel.close() + } return restarted, fmt.Errorf("no such id %s", id) } inst.State = StateRunning inst.Cold = false if restarted { inst.PID = m.engine.PID(id) + if inst.channel != nil { + inst.channel.close() + } + inst.channel = channel + inst.boot = bootCfg + inst.Cfg.VsockUDSPath = cfg.VsockUDSPath + inst.boots++ } inst.bump() rec := persistable(inst) @@ -1068,8 +1153,23 @@ func (m *Microvm) Snapshot(ctx context.Context, id, ref string, stripEnv []strin m.snapSeq.Add(1) ref = fmt.Sprintf("rainier-mvm:%s-%d", id, m.snapSeq.Load()) } - survivingKeys := make([]string, 0, len(inst.Cfg.Env)) + // What this session was CONFIGURED with, from both places a key can come + // from: the driver's own injection (buildGuestEnv) and the configuration + // block the guest was handed over vsock. Both, because the strip list is + // a promise about the committed image and a key that survived through + // the channel the driver does not happen to be looking at is a key that + // survived. + // + // Keys, never values: see snapshotManifest. + surviving := map[string]struct{}{} for k := range inst.Cfg.Env { + surviving[k] = struct{}{} + } + for k := range inst.boot.Env { + surviving[k] = struct{}{} + } + survivingKeys := make([]string, 0, len(surviving)) + for k := range surviving { if !slices.Contains(stripEnv, k) { survivingKeys = append(survivingKeys, k) } @@ -1155,7 +1255,14 @@ func (m *Microvm) DestroyContainer(ctx context.Context, id string) error { return nil } tapDevice := inst.Cfg.TapDevice + channel := inst.channel + inst.channel = nil m.mu.Unlock() + // Closed before the VM is signalled, so nothing can dial a control + // channel for a session that is being torn down. + if channel != nil { + channel.close() + } if err := m.engine.Stop(ctx, id); err != nil { st, stateErr := m.engine.State(ctx, id) @@ -1553,17 +1660,36 @@ func (f *FirecrackerEngine) Launch(ctx context.Context, cfg VMMConfig) error { } } - // There is deliberately no MMDS configuration here. MMDS answers at - // 169.254.169.254 — the exact address ADR-0003 §4.3 requires the host to - // drop on every TAP — so the config channel and the metadata-denial rule - // could never both hold. The two PUTs also discarded their errors, and - // /mmds/config without network_interfaces is rejected outright, so what - // the guest actually received was nothing. + // 5b. virtio-vsock: the single host-to-guest control channel (ADR-0003 + // §2.7 item 2). It carries the boot configuration, the bootstrap token, + // the sessiond-to-runnerd stream that used to ride a WebSocket through + // the TAP device, the lifecycle handshake, and the credential fetch — all + // over one conn, because internal/relay already multiplexes exactly those + // things over one conn. + // + // The guest's connections to host port N arrive on "_N", and + // the driver is already listening on _1024 by the time this runs. The + // host never sends CONNECT, so there is no host-initiated direction to + // configure. // - // TODO(PR 2): virtio-vsock is the single host-to-guest control channel - // (ADR-0003 §2.7 item 2), carrying the boot configuration and the - // bootstrap token. Until it lands, a guest booted by this engine receives - // no session configuration at all. + // There is deliberately still no MMDS configuration. MMDS answers + // unauthenticated HTTP at 169.254.169.254 — the exact address ADR-0003 + // §4.3 requires the host to drop on every TAP — so a configuration + // channel that needed it would be at war with the rule that exists to + // block it. vsock is invisible to guest routing and to the TAP firewall + // alike, which is why the metadata denial can stay unconditional. + if cfg.VsockUDSPath != "" { + if err := fcClient.putJSON(ctx, "/vsock", map[string]any{ + "guest_cid": guestCID, + "uds_path": cfg.VsockUDSPath, + }); err != nil { + // Not a discarded error, unlike the two MMDS PUTs this replaces: + // a guest with no control channel is a guest that can never be + // configured, and a silently config-less VM reporting itself + // healthy is the failure that made this whole design necessary. + return fmt.Errorf("configure the guest control channel (vsock): %w", err) + } + } // 6. Start the microVM instance if err := fcClient.putJSON(ctx, "/actions", map[string]any{ diff --git a/internal/driver/microvm_test.go b/internal/driver/microvm_test.go index 0b8a149..afe3c58 100644 --- a/internal/driver/microvm_test.go +++ b/internal/driver/microvm_test.go @@ -3,7 +3,6 @@ package driver import ( "context" - "encoding/json" "errors" "fmt" "io" @@ -21,6 +20,8 @@ import ( "syscall" "testing" "time" + + "github.com/tokencanopy/rainier/internal/relay" ) // testMicrovm builds a driver over a fresh state directory with the whole @@ -33,7 +34,7 @@ import ( func testMicrovm(t *testing.T, opts MicrovmOpts) (*Microvm, *SimulatedEngine) { t.Helper() if opts.StateDir == "" { - opts.StateDir = t.TempDir() + opts.StateDir = shortTempDir(t) } if opts.BaseRootfs == "" { opts.BaseRootfs = writeFakeRootfs(t, opts.StateDir) @@ -56,6 +57,23 @@ func testMicrovm(t *testing.T, opts MicrovmOpts) (*Microvm, *SimulatedEngine) { return m, sim } +// shortTempDir is t.TempDir with a name short enough to hang a unix socket +// off. t.TempDir composes the TEST's name into the path, and the guest +// control socket lands at "/instances//v1.sock_1024" — which for a +// test called TestMicrovmSomethingDescriptive is over the 108-byte sun_path +// limit before the driver has done anything wrong. A production state +// directory is nowhere near it; this is a fixture concern and not a +// behaviour one. +func shortTempDir(t *testing.T) string { + t.Helper() + dir, err := os.MkdirTemp("", "mvm") + if err != nil { + t.Fatalf("temp dir: %v", err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + return dir +} + // writeFakeRootfs puts a file where a base rootfs image would be. It is not a // filesystem and nothing boots it; it is there so the driver's "this host has // the image" check has something true to find. @@ -71,10 +89,57 @@ func writeFakeRootfs(t *testing.T, dir string) string { func TestMicrovmSatisfiesContract(t *testing.T) { RunContract(t, func(t *testing.T) (Driver, func()) { d, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4}) + // A microVM driver in production always has a runner above it + // (runnerd.New installs itself), and a cold resume asks it for a + // fresh bootstrap token. The contract drives cold resumes, so the + // fixture supplies the smallest thing that can answer one. + d.SetHost(&stubMicrovmHost{}) return d, func() {} }) } +// stubMicrovmHost is a runner that mints a token and drops every guest +// connection. It is what the contract needs and nothing more: the contract is +// about the driver's own behaviour, and a guest that connects to a fixture +// has nowhere to go. +type stubMicrovmHost struct { + mu sync.Mutex + mints int + conns []string + // mintErr, when set, is what a cold resume's mint reports — the state a + // runner with no control connection is in. + mintErr error +} + +func (h *stubMicrovmHost) GuestConnected(sessionID string, conn relay.Conn) { + h.mu.Lock() + h.conns = append(h.conns, sessionID) + h.mu.Unlock() + _ = conn.Close() +} + +func (h *stubMicrovmHost) MintSessionBootstrap(_ context.Context, sessionID string) (string, error) { + h.mu.Lock() + defer h.mu.Unlock() + if h.mintErr != nil { + return "", h.mintErr + } + h.mints++ + return fmt.Sprintf("token_example_%s_%d", sessionID, h.mints), nil +} + +func (h *stubMicrovmHost) mintCount() int { + h.mu.Lock() + defer h.mu.Unlock() + return h.mints +} + +func (h *stubMicrovmHost) connected() []string { + h.mu.Lock() + defer h.mu.Unlock() + return slices.Clone(h.conns) +} + // TestMicrovmRefusesToStartWithoutAHost is the fail-closed half: there is no // combination of missing pieces that yields a working-looking driver. func TestMicrovmRefusesToStartWithoutAHost(t *testing.T) { @@ -137,7 +202,7 @@ func TestMicrovmRefusesToStartWithoutAHost(t *testing.T) { // drop it, and runnerd.Recover would forget a session whose files are sitting // right there on disk. func TestMicrovmColdSuspendedSessionIsNotGone(t *testing.T) { - stateDir := t.TempDir() + stateDir := shortTempDir(t) m, _ := testMicrovm(t, MicrovmOpts{ TotalSlots: 4, StateDir: stateDir, @@ -206,7 +271,7 @@ func listedState(t *testing.T, m *Microvm, sessionID string) State { } func TestMicrovmRestartRecovery(t *testing.T) { - stateDir := t.TempDir() + stateDir := shortTempDir(t) m1, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4, StateDir: stateDir}) ctx := context.Background() @@ -254,6 +319,7 @@ func TestMicrovmRestartRecovery(t *testing.T) { func TestMicrovmWorkspaceFilesSurviveColdPark(t *testing.T) { m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4}) + m.SetHost(&stubMicrovmHost{}) ctx := context.Background() h, err := m.Create(ctx, Spec{SessionID: "sess-persist"}) @@ -321,7 +387,12 @@ func TestMicrovmGuestEnvTranslationAndBootstrap(t *testing.T) { GitAuthorName: "Test Author", GitAuthorEmail: "author@example.invalid", Cmd: []string{"claude", "--model", "haiku"}, + // Env plus a token is what a create from a control plane that + // withholds looks like: the values that remain are configuration + // (the agent home, the manifest), and this driver refuses a create + // carrying values with no token at all. Env: map[string]string{"APP_ENV": "production"}, + BootstrapToken: "token_example", } h, err := m.Create(ctx, spec) @@ -340,7 +411,6 @@ func TestMicrovmGuestEnvTranslationAndBootstrap(t *testing.T) { "RAINIER_SESSION": "sess-trans", "RAINIER_GIT_AUTHOR_NAME": "Test Author", "RAINIER_GIT_AUTHOR_EMAIL": "author@example.invalid", - "APP_ENV": "production", "RAINIER_SETUP_TIMEOUT": "600", "RAINIER_INIT_TIMEOUT": "180", } @@ -355,25 +425,22 @@ func TestMicrovmGuestEnvTranslationAndBootstrap(t *testing.T) { if cfg.TapDevice == "" { t.Errorf("TapDevice was not allocated on VMMConfig") } - - // The staged session config is the non-secret half, and only that half. - jsonPath := filepath.Join(m.instanceDir(h.ID), "session.json") - data, err := os.ReadFile(jsonPath) - if err != nil { - t.Fatalf("read session.json: %v", err) - } - var gsc guestSessionConfig - if err := json.Unmarshal(data, &gsc); err != nil { - t.Fatalf("unmarshal session.json: %v", err) + // Spec.Env is no longer copied in. This map is the driver's record of + // what a session was CONFIGURED with, read by nothing but a snapshot + // manifest; what the guest actually runs on arrives over vsock, so a + // create's own env block has no business being duplicated here. + if _, copied := cfg.Env["APP_ENV"]; copied { + t.Errorf("the create's env block was copied into the driver's own map: %+v", cfg.Env) } - if gsc.SessionID != "sess-trans" { - t.Errorf("gsc.SessionID = %q, want sess-trans", gsc.SessionID) + if cfg.VsockUDSPath == "" { + t.Error("no vsock uds_path was configured; the guest has no control channel") } - if !reflect.DeepEqual(gsc.Cmd, []string{"claude", "--model", "haiku"}) { - t.Errorf("gsc.Cmd = %v, want claude --model haiku", gsc.Cmd) - } - if strings.Contains(string(data), "APP_ENV") || strings.Contains(string(data), "production") { - t.Errorf("session.json carries the session environment:\n%s", data) + + // And nothing is staged on the host for the guest to read. session.json + // is gone: the configuration goes over the vsock conn and is written + // nowhere. + if _, err := os.Stat(filepath.Join(m.instanceDir(h.ID), "session.json")); !os.IsNotExist(err) { + t.Errorf("the driver staged a session config on the host: %v", err) } } @@ -383,7 +450,7 @@ func TestMicrovmGuestEnvTranslationAndBootstrap(t *testing.T) { // write. func TestMicrovmWritesNoDecryptedEnvironment(t *testing.T) { const secret = "must-not-reach-host-disk" - stateDir := t.TempDir() + stateDir := shortTempDir(t) m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4, StateDir: stateDir}) ctx := context.Background() @@ -391,7 +458,14 @@ func TestMicrovmWritesNoDecryptedEnvironment(t *testing.T) { SessionID: "sess-secret", DialURL: "ws://runner.example.com:8080", Setup: "echo setting up", - Env: map[string]string{"DEPLOY_TOKEN": secret}, + // A withheld create's own env block is CONFIGURATION, and this one + // is a fixture standing in for it. The value below must reach the + // guest over vsock and no file, which is what the walk asserts; the + // token is what makes this an accepted create at all (a create with + // values and no token is refused, and asserted by the twelfth + // contract subtest). + Env: map[string]string{"DEPLOY_TOKEN": secret}, + BootstrapToken: "token_must_not_reach_host_disk", }) if err != nil { t.Fatal(err) @@ -441,18 +515,43 @@ func TestMicrovmWritesNoDecryptedEnvironment(t *testing.T) { t.Fatalf("walk state dir: %v", err) } - // The instance record and the staged guest config carry not even the - // KEYS: the first is the file a runnerd restart reads back, the second is - // bound for the guest, and neither has any business describing an - // environment the driver was told to keep in memory. - for _, name := range []string{"instance.json", "session.json"} { - data, err := os.ReadFile(filepath.Join(m.instanceDir(h.ID), name)) - if err != nil { - t.Fatalf("read %s: %v", name, err) + // The bootstrap token is under the same rule as a value and is checked + // separately, because it is a capability and not just a string: it + // travels from the create into the guest's boot configuration over the + // vsock conn, and a copy of it in any host file would be a credential + // somebody with a shell on a shared host could spend. + err = filepath.WalkDir(stateDir, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + info, ierr := d.Info() + if ierr != nil || info.Size() > 1<<20 { + return ierr } - if strings.Contains(string(data), "DEPLOY_TOKEN") { - t.Errorf("%s names a key from the session's decrypted environment:\n%s", name, data) + data, rerr := os.ReadFile(path) + if rerr != nil { + return rerr } + if strings.Contains(string(data), "token_must_not_reach_host_disk") { + t.Errorf("%s carries the session's bootstrap token", path) + } + return nil + }) + if err != nil { + t.Fatalf("walk state dir for the token: %v", err) + } + + // The instance record carries not even the KEYS: it is the file a runnerd + // restart reads back, and it has no business describing an environment + // the driver was told to keep in memory. There is no staged guest config + // beside it any more — the configuration goes over vsock and is written + // nowhere at all. + data, err := os.ReadFile(filepath.Join(m.instanceDir(h.ID), "instance.json")) + if err != nil { + t.Fatalf("read instance.json: %v", err) + } + if strings.Contains(string(data), "DEPLOY_TOKEN") { + t.Errorf("instance.json names a key from the session's environment:\n%s", data) } } @@ -460,11 +559,11 @@ func TestMicrovmWritesNoDecryptedEnvironment(t *testing.T) { // decision: having refused to write the environment down, the driver says so // rather than boot a guest with an empty one. func TestMicrovmColdResumeAfterRestartRefuses(t *testing.T) { - stateDir := t.TempDir() + stateDir := shortTempDir(t) m1, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4, StateDir: stateDir}) ctx := context.Background() - h, err := m1.Create(ctx, Spec{SessionID: "sess-resume", Env: map[string]string{"TOKEN": "v"}}) + h, err := m1.Create(ctx, Spec{SessionID: "sess-resume", Env: map[string]string{"TOKEN": "v"}, BootstrapToken: "token_example"}) if err != nil { t.Fatal(err) } @@ -473,10 +572,11 @@ func TestMicrovmColdResumeAfterRestartRefuses(t *testing.T) { } m2, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4, StateDir: stateDir}) + m2.SetHost(&stubMicrovmHost{}) if _, err := m2.Resume(ctx, h.ID); err == nil { - t.Fatal("cold resume across a restart succeeded; it would have booted a guest with no environment at all") - } else if !strings.Contains(err.Error(), "vsock") { - t.Errorf("error = %q, want it to name the follow-up work (vsock bootstrap token)", err) + t.Fatal("cold resume across a restart succeeded; it would have booted a guest that is never told what it is") + } else if !strings.Contains(err.Error(), "held in memory only") { + t.Errorf("error = %q, want it to name why the configuration is gone", err) } // And the session is still there to be resumed once that lands. if st := listedState(t, m2, "sess-resume"); st != StateSuspended { @@ -553,6 +653,7 @@ func TestMicrovmSnapshotRefAssociationAndStrip(t *testing.T) { "SECRET_KEY": "super_secret_value", "PUBLIC_KEY": "public_value", }, + BootstrapToken: "token_example", }) if err != nil { t.Fatal(err) @@ -796,7 +897,7 @@ func TestMicrovmStateReconciliation(t *testing.T) { // TestMicrovmPrepullNeverFabricatesAnImage is finding 6's second half: a ref // this host cannot boot is an error, not an empty file and a recorded pull. func TestMicrovmPrepullNeverFabricatesAnImage(t *testing.T) { - stateDir := t.TempDir() + stateDir := shortTempDir(t) m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 2, StateDir: stateDir}) ctx := context.Background() @@ -844,7 +945,7 @@ func TestMicrovmCreateRefusesAnAbsentImage(t *testing.T) { func TestMicrovmRecordsSnapshotStrips(t *testing.T) { m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 2}) ctx := context.Background() - h, err := m.Create(ctx, Spec{SessionID: "mvm-sess-s", Env: map[string]string{"TOKEN": "v"}}) + h, err := m.Create(ctx, Spec{SessionID: "mvm-sess-s", Env: map[string]string{"TOKEN": "v"}, BootstrapToken: "token_example"}) if err != nil { t.Fatal(err) } @@ -927,7 +1028,7 @@ func mustSanitizeRef(t *testing.T, ref string) string { // not produce a silly path, it produces a real one outside the state // directory that RemoveWorkspace would then delete. func TestMicrovmHostileNamesNeverBecomePaths(t *testing.T) { - stateDir := t.TempDir() + stateDir := shortTempDir(t) m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4, StateDir: stateDir}) ctx := context.Background() @@ -981,7 +1082,7 @@ func TestMicrovmHostileNamesNeverBecomePaths(t *testing.T) { // ".." exactly as they are, so a ref of ".." used to name the parent of the // refs directory and a commit wrote its manifest one level up. func TestMicrovmHostileSnapshotRefsAreRefused(t *testing.T) { - stateDir := t.TempDir() + stateDir := shortTempDir(t) m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4, StateDir: stateDir}) ctx := context.Background() @@ -1077,7 +1178,7 @@ func (c *countingResumeEngine) Resume(ctx context.Context, id string) error { // which would leak a slot and send the next Resume down the warm path into a // VM that is no longer there. func TestMicrovmReconcileDoesNotClobberAConcurrentSuspend(t *testing.T) { - stateDir := t.TempDir() + stateDir := shortTempDir(t) gate := make(chan struct{}) m, _ := testMicrovm(t, MicrovmOpts{ TotalSlots: 4, diff --git a/internal/driver/microvm_vsock.go b/internal/driver/microvm_vsock.go new file mode 100644 index 0000000..f7e90bd --- /dev/null +++ b/internal/driver/microvm_vsock.go @@ -0,0 +1,402 @@ +// internal/driver/microvm_vsock.go +// +// virtio-vsock: the single host-to-guest control channel a microVM session +// has, and the whole of how one is configured (ADR-0003 §2.7 item 2, and +// docs/design/2026-09-20-microvm-bootstrap-token-and-vsock.md §4). +// +// It replaces two things the driver spike had and this driver deliberately +// does not: MMDS, which answers unauthenticated HTTP at the one address the +// host firewall exists to drop, and a host-side session.json, which put a +// tenant's configuration on a shared host's disk. Nothing this file writes to +// disk is a value: the only file it creates is a unix socket. +// +// This is the one place the driver layer depends on protocol/runner, and it +// is deliberate rather than a leak of the control-plane vocabulary this +// package otherwise keeps out (see driver.Spec and driver.RepoSpec, which +// have their own twins for exactly that reason). runner.BootConfig is not a +// runner-plane message: it is the host-to-GUEST schema, and the design note +// pins it beside runner.Spec because every field of it is a field of the +// create it came from. Somebody has to depend on it, and a second spelling +// of it here would be a session configured with something the control plane +// never dispatched. +package driver + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log" + "net" + "os" + "path/filepath" + "slices" + "strconv" + "sync" + + "github.com/tokencanopy/rainier/internal/relay" + "github.com/tokencanopy/rainier/protocol/runner" +) + +const ( + // guestCID is the guest's context id. 3 is the value Firecracker's own + // documentation uses; the host is always HOST_CID, which is 2. + guestCID = 3 + // guestControlPort is the ONE port this design uses, and there is not a + // second one. internal/relay already multiplexes configuration, the + // terminal stream, the session RPC and the lifecycle handshake over a + // single conn — that is what the package is for — so three ports would be + // three sockets carrying what one already carries, with three teardowns + // to get wrong. + // + // A guest connection to host port N is forwarded by Firecracker to an + // AF_UNIX socket at "_N", which is why the host listens on a + // path and never calls connect: the host never sends CONNECT. + guestControlPort = 1024 + // unixPathMax is the bound on a sockaddr_un path: Linux's sun_path is 108 + // bytes including its NUL terminator. It is checked here rather than left + // to bind(2) because the path is composed from an operator's + // --microvm-state-dir, and "invalid argument" on a socket is not an error + // anybody can act on, where naming the limit and the path is. + unixPathMax = 107 +) + +// MicrovmHost is what the microVM driver needs from the runner above it. +// +// There are exactly two things, and both are things only the runner can do. +// A guest's connection has to become a relay Hub keyed by the session — which +// means the registry, the boot epoch, the control routing and the event +// callbacks, all of which live in runnerd and none of which belong in a +// driver. And a cold resume needs a fresh bootstrap token, which means a +// session RPC upward to the control plane on the runner's own connection. +// +// nil is a real state and not an unconfigured one: the local dev surface and +// the driver contract suite have no runner above them, and a driver with no +// host simply has no guest channel and refuses a cold resume that would need +// one, rather than inventing either. +type MicrovmHost interface { + // GuestConnected hands over one guest's control connection, already + // carrying its boot configuration as the first frame it will read. The + // implementation builds the session's relay hub over it exactly as the + // WebSocket register path does. + // + // It must not block: the driver calls it from its accept loop. + GuestConnected(sessionID string, conn relay.Conn) + // MintSessionBootstrap asks the control plane for a fresh single-use + // token for sessionID, which is what a cold resume boots with. The + // session id is the runner's own — the control plane answers from the row + // its placement guard read, never from anything in the request. + MintSessionBootstrap(ctx context.Context, sessionID string) (string, error) +} + +var ( + _ CapabilityDriver = (*Microvm)(nil) + _ HostedDriver = (*Microvm)(nil) +) + +// Capabilities is what the runner announces on this driver's behalf, and it +// is the fence rather than a hint: the control plane withholds an +// environment's decrypted secret values from a placement whose runner +// announced microvm.v1, and this driver refuses a create that carries them +// anyway. The claim and the refusal are two halves of one promise, so they +// are read from one place. +func (m *Microvm) Capabilities() []string { return []string{runner.CapabilityMicrovmV1} } + +// SetHost installs the runner above this driver. It is a setter rather than a +// field on MicrovmOpts because the runner is composed OVER the driver +// (runnerd.New takes one), so the two cannot both be constructed first. +// +// Called once, before the driver serves anything. +func (m *Microvm) SetHost(h MicrovmHost) { + m.mu.Lock() + defer m.mu.Unlock() + m.host = h +} + +func (m *Microvm) currentHost() MicrovmHost { + m.mu.Lock() + defer m.mu.Unlock() + return m.host +} + +// vsockPaths returns the Firecracker uds_path for one boot of an instance and +// the socket the guest's port-1024 connections are forwarded to. +// +// The path is per BOOT, not per instance: Firecracker's own documentation +// warns that one uds_path cannot be multiplexed across VMs, and a cold resume +// is a new VM. boot is the instance's launch counter. +func (m *Microvm) vsockPaths(id string, boot int) (udsPath, listenPath string, err error) { + if err := checkPathSegment("instance id", id); err != nil { + return "", "", err + } + udsPath = filepath.Join(m.instanceDir(id), "v"+strconv.Itoa(boot)+".sock") + listenPath = udsPath + "_" + strconv.Itoa(guestControlPort) + if len(listenPath) > unixPathMax { + return "", "", fmt.Errorf( + "microvm: the vsock socket path %q is %d bytes, over the %d-byte kernel limit; "+ + "a shorter --microvm-state-dir is the fix, because a truncated path is a guest that boots and is never configured", + listenPath, len(listenPath), unixPathMax) + } + return udsPath, listenPath, nil +} + +// GuestSocketPath returns the host path a session's guest control channel is +// currently listening on, and whether there is one — a cold-parked session +// has none, because its VM is gone and so is its socket. +// +// It is exported for the same reason SnapshotEnvKeys and Volumes are: it is +// how a test above this package can act like a guest, which is the only way +// to assert what a guest actually receives without a VM. Nothing in +// production reads it. +func (m *Microvm) GuestSocketPath(sessionID string) (string, bool) { + m.mu.Lock() + defer m.mu.Unlock() + for _, inst := range m.instances { + if inst.SessionID == sessionID && inst.channel != nil { + return inst.channel.path, true + } + } + return "", false +} + +// guestChannel is one instance's host end of the vsock control channel: the +// listener Firecracker forwards the guest's port-1024 connections to, and the +// boot configuration whoever connects is handed as their first frame. +// +// The configuration is held HERE, in memory, and nowhere else. It carries the +// bootstrap token, and §2.7 item 1's whole point is that no part of a +// session's configuration reaches a shared host's disk — which is why this +// struct has no persisted twin and why a runnerd restart loses it (see +// (*Microvm).Resume). +type guestChannel struct { + listener net.Listener + path string + + mu sync.Mutex + boot runner.BootConfig + // closed makes the accept loop's exit quiet: a listener closed by + // teardown reports an error like any other, and logging that as a + // failure would put a line in an operator's log for every session that + // ends normally. + closed bool +} + +// setBoot replaces the configuration the next guest connection is handed. A +// cold resume mints a new token, and the guest that comes up must get THAT +// one — the retired token no longer works. +func (g *guestChannel) setBoot(cfg runner.BootConfig) { + g.mu.Lock() + defer g.mu.Unlock() + g.boot = cfg +} + +func (g *guestChannel) currentBoot() runner.BootConfig { + g.mu.Lock() + defer g.mu.Unlock() + return g.boot +} + +func (g *guestChannel) close() { + g.mu.Lock() + g.closed = true + g.mu.Unlock() + if g.listener != nil { + _ = g.listener.Close() + } + // The socket file is the listener's own and goes with it. An absent one + // is not an error: a teardown that runs twice is ordinary. + _ = os.Remove(g.path) +} + +func (g *guestChannel) isClosed() bool { + g.mu.Lock() + defer g.mu.Unlock() + return g.closed +} + +// openGuestChannel creates the host end of the channel before the VM starts. +// +// Before, deliberately: the guest dials port 1024 as soon as its sessiond is +// up, and Firecracker refuses a guest connection whose "_1024" does +// not exist. A listener created after InstanceStart is a race whose loser is +// a session that boots and is never configured. +func (m *Microvm) openGuestChannel(sessionID, listenPath string, cfg runner.BootConfig) (*guestChannel, error) { + if err := os.MkdirAll(filepath.Dir(listenPath), microvmDirMode); err != nil { + return nil, fmt.Errorf("create the guest channel directory: %w", err) + } + // A socket left behind by a VM that is gone would make Listen fail with + // "address already in use" for a path nothing is listening on. + _ = os.Remove(listenPath) + ln, err := net.Listen("unix", listenPath) + if err != nil { + return nil, fmt.Errorf("listen on the guest control socket %s: %w", listenPath, err) + } + // 0600: the socket is one tenant's control channel on a shared host, and + // anything that can connect to it can be that session's sandbox. Listen + // creates it with the process umask, which is not a policy this driver + // gets to inherit. + if err := os.Chmod(listenPath, microvmFileMode); err != nil { + _ = ln.Close() + _ = os.Remove(listenPath) + return nil, fmt.Errorf("restrict the guest control socket %s: %w", listenPath, err) + } + g := &guestChannel{listener: ln, path: listenPath, boot: cfg} + go m.acceptGuests(sessionID, g) + return g, nil +} + +// acceptGuests serves the channel for the life of the instance. +// +// It loops rather than accepting once because a sandbox redials: sessiond +// survives losing its connection and comes back (cmd/sessiond's dial loop), +// and the guest that reconnects needs its configuration again — it is a new +// process as often as not, on the far side of a cold resume. +func (m *Microvm) acceptGuests(sessionID string, g *guestChannel) { + for { + c, err := g.listener.Accept() + if err != nil { + if !g.isClosed() { + log.Printf("microvm: session %s: the guest control socket stopped accepting: %v", sessionID, err) + } + return + } + m.serveGuest(sessionID, g, c) + } +} + +// serveGuest hands one guest connection its configuration and then hands the +// connection to the runner. +// +// The boot configuration is the FIRST frame on the conn, written here before +// anything else can write to it, which is what makes "nothing has to predate +// the stream" true: the guest reads its whole configuration off the same +// connection it will then serve its terminal and its RPC over. +// +// A host that has not been installed closes the connection rather than +// holding it. There is nothing to attach it to, and a guest that is dropped +// redials, where a guest held by a channel nobody serves waits forever. +func (m *Microvm) serveGuest(sessionID string, g *guestChannel, c net.Conn) { + conn := relay.NetConn(c) + if err := writeBootConfig(conn, g.currentBoot()); err != nil { + log.Printf("microvm: session %s: sending the boot configuration: %v", sessionID, err) + _ = conn.Close() + return + } + host := m.currentHost() + if host == nil { + log.Printf("microvm: session %s: a guest connected but this driver has no runner above it to serve it", sessionID) + _ = conn.Close() + return + } + host.GuestConnected(sessionID, conn) +} + +// writeBootConfig sends one boot configuration as an ordinary FrameControl — +// the same frame shape every event, request and response on this channel +// uses, so the guest needs no special reader for it. +// +// The context is Background and not the caller's: this is the one write whose +// failure means the guest is unconfigurable, and inheriting a create's +// context would make a create that has just returned cancel the configuration +// of the guest it started. +func writeBootConfig(conn relay.Conn, cfg runner.BootConfig) error { + body, err := json.Marshal(cfg) + if err != nil { + // Unreachable — every field is a string, an int, or a slice of them — + // and logged without the error, because a json message quotes what it + // choked on and this value carries the bootstrap token. + return errors.New("the boot configuration could not be encoded") + } + payload, err := json.Marshal(relay.ControlEvent{Kind: relay.KindBootConfig, Payload: body}) + if err != nil { + return errors.New("the boot configuration frame could not be encoded") + } + frame, err := relay.Encode(relay.Frame{Type: relay.FrameControl, Payload: payload}) + if err != nil { + return errors.New("the boot configuration frame could not be encoded") + } + return conn.Write(context.Background(), frame) +} + +// bootConfigFor composes what a guest is told about itself. +// +// Every field is one the create resolved, carried across unchanged. There are +// two things deliberately NOT in it: any environment secret value (they are +// exactly what the token buys, and they never reach this host), and a dial +// URL (there is nothing to dial — the connection this rides on IS the +// channel, which is why createWithID leaves Spec.DialURL empty on this +// driver). +// +// The proxy URL carries the session's identity as URL userinfo, composed here +// with the same helper the Docker driver uses, so egressd reads one thing +// whichever driver the session is on. +func bootConfigFor(spec Spec) runner.BootConfig { + cfg := runner.BootConfig{ + Protocol: runner.SessionBootstrapProtocolVersion, + SessionID: spec.SessionID, + Cmd: slices.Clone(spec.Cmd), + EgressAllow: slices.Clone(spec.EgressAllow), + Setup: spec.Setup, + SetupTimeoutSec: spec.SetupTimeoutSec, + Init: spec.Init, + InitTimeoutSec: spec.InitTimeoutSec, + GitAuthorName: spec.GitAuthorName, + GitAuthorEmail: spec.GitAuthorEmail, + SecretNames: slices.Clone(spec.SecretNames), + BootstrapToken: spec.BootstrapToken, + } + if len(cfg.Cmd) == 0 { + cfg.Cmd = []string{"/bin/bash"} + } + if spec.ProxyURL != "" { + cfg.ProxyURL = withSessionUserinfo(spec.ProxyURL, spec.SessionID) + // No dial host to exempt, because there is no dial: NO_PROXY is the + // base list and nothing else. + cfg.NoProxy = noProxyFor("") + } + for _, r := range spec.Repos { + cfg.Repos = append(cfg.Repos, runner.RepoSpec{ + Owner: r.Owner, Name: r.Name, BaseBranch: r.BaseBranch, + SessionBranch: r.SessionBranch, Dir: r.Dir, + }) + } + // Env is the NON-SECRET configuration block, which for a microVM create + // is the agent-home path variables and the agent manifest — paths and + // names. Create has already refused any create whose Env carries values + // with no token, and a create WITH a token is one the control plane + // withheld the values from, so what is left here is configuration by + // construction. + if len(spec.Env) > 0 { + cfg.Env = make(map[string]string, len(spec.Env)) + for k, v := range spec.Env { + cfg.Env[k] = v + } + } + return cfg +} + +// refuseUnwithheldEnv is the microVM driver's half of the compatibility +// table's "old controld + new runner" row. +// +// A control plane that predates the bootstrap token dispatches an +// environment's decrypted secrets in Spec.Env and no token. On the Docker +// path that is fine and always has been — the values reach a `docker run` +// argv and no file. On this one it is not: this driver has no argv to hand +// them to, and every channel it does have ends in something that outlives the +// session on a host shared with other tenants. +// +// So it refuses, loudly, naming what the control plane has to be. A refusal +// costs one session; the alternative — writing them somewhere and booting +// anyway — is the exposure §2.7 was written against, and it would be silent. +func refuseUnwithheldEnv(spec Spec) error { + if len(spec.Env) == 0 || spec.BootstrapToken != "" { + return nil + } + return fmt.Errorf( + "microvm: refusing to create session %s: its create carries %d environment value(s) and no bootstrap token, "+ + "which means a control plane older than the session bootstrap exchange "+ + "(docs/design/2026-09-20-microvm-bootstrap-token-and-vsock.md). A microVM host has no argv to hand a "+ + "value to and will not write one to disk, so this session cannot be booted here until the control plane "+ + "announces it withholds them for a runner announcing %q", + spec.SessionID, len(spec.Env), runner.CapabilityMicrovmV1) +} diff --git a/internal/driver/microvm_vsock_test.go b/internal/driver/microvm_vsock_test.go new file mode 100644 index 0000000..40de556 --- /dev/null +++ b/internal/driver/microvm_vsock_test.go @@ -0,0 +1,335 @@ +// internal/driver/microvm_vsock_test.go +package driver + +import ( + "context" + "encoding/json" + "errors" + "net" + "os" + "strings" + "testing" + "time" + + "github.com/tokencanopy/rainier/internal/relay" + "github.com/tokencanopy/rainier/protocol/runner" +) + +// dialGuest connects to the host end of a session's vsock control channel the +// way Firecracker's forwarding does: an AF_UNIX connection to +// "_1024". No VM and no KVM are involved — the socket is the part +// of this that runs anywhere, and the part a host harness has to prove is +// that Firecracker forwards to it. +func dialGuest(t *testing.T, m *Microvm, instanceID string, boot int) relay.Conn { + t.Helper() + _, listenPath, err := m.vsockPaths(instanceID, boot) + if err != nil { + t.Fatalf("vsock paths: %v", err) + } + // The driver opens the listener before it launches, so it is there by the + // time Create returns; a short retry covers nothing but scheduling. + deadline := time.Now().Add(2 * time.Second) + for { + c, err := net.Dial("unix", listenPath) + if err == nil { + t.Cleanup(func() { _ = c.Close() }) + return relay.NetConn(c) + } + if time.Now().After(deadline) { + t.Fatalf("dial the guest control socket %s: %v", listenPath, err) + } + time.Sleep(5 * time.Millisecond) + } +} + +// readBootConfig reads the first frame off a guest connection and requires it +// to be the boot configuration. +func readBootConfig(t *testing.T, conn relay.Conn) runner.BootConfig { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + raw, err := conn.Read(ctx) + if err != nil { + t.Fatalf("read the first frame: %v", err) + } + frame, err := relay.Decode(raw) + if err != nil { + t.Fatalf("decode the first frame: %v", err) + } + if frame.Type != relay.FrameControl || frame.AttachID != 0 { + t.Fatalf("the first frame is %+v, want a control frame on attachment 0", frame) + } + var ev relay.ControlEvent + if err := json.Unmarshal(frame.Payload, &ev); err != nil { + t.Fatalf("decode the control event: %v", err) + } + if ev.Kind != relay.KindBootConfig { + t.Fatalf("the first control frame is %q, want %q", ev.Kind, relay.KindBootConfig) + } + var cfg runner.BootConfig + if err := json.Unmarshal(ev.Payload, &cfg); err != nil { + t.Fatalf("decode the boot configuration: %v", err) + } + return cfg +} + +// TestMicrovmSendsTheBootConfigFirst is the whole configuration path in one +// test: a guest connects to the socket Firecracker forwards port 1024 to, and +// the first thing it reads is everything it needs to be a session. +// +// The assertions are on what the GUEST sees, not on what the driver stored, +// because the guest is the only consumer and a field it does not receive is a +// session that boots wrong however neatly the host recorded it. +func TestMicrovmSendsTheBootConfigFirst(t *testing.T) { + m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4}) + host := &stubMicrovmHost{} + m.SetHost(host) + ctx := context.Background() + + h, err := m.Create(ctx, Spec{ + SessionID: "sess-boot", + ProxyURL: "http://proxy.example.invalid:3128", + EgressAllow: []string{"registry.npmjs.org"}, + Setup: "npm ci\n", + SetupTimeoutSec: 600, + Init: "make dev\n", + InitTimeoutSec: 180, + Repos: []RepoSpec{{Owner: "acme", Name: "app", BaseBranch: "main", SessionBranch: "rainier/work", Dir: "app"}}, + GitAuthorName: "example", + GitAuthorEmail: "42+example@users.noreply.github.com", + Cmd: []string{"claude"}, + Env: map[string]string{"CLAUDE_CONFIG_DIR": "/rainier/agents/claude"}, + SecretNames: []string{"DEPLOY_KEY"}, + BootstrapToken: "token_example", + }) + if err != nil { + t.Fatal(err) + } + defer m.Destroy(ctx, h.ID) + + cfg := readBootConfig(t, dialGuest(t, m, h.ID, 1)) + + if cfg.Protocol != runner.SessionBootstrapProtocolVersion { + t.Errorf("protocol = %d, want %d", cfg.Protocol, runner.SessionBootstrapProtocolVersion) + } + // The session id is the HOST's statement, taken from the socket path the + // connection arrived on, and not something the guest asserted. + if cfg.SessionID != "sess-boot" { + t.Errorf("session id = %q, want sess-boot", cfg.SessionID) + } + if len(cfg.Cmd) != 1 || cfg.Cmd[0] != "claude" { + t.Errorf("cmd = %v", cfg.Cmd) + } + if cfg.Setup != "npm ci\n" || cfg.SetupTimeoutSec != 600 || cfg.Init != "make dev\n" || cfg.InitTimeoutSec != 180 { + t.Errorf("the boot chain did not survive: %+v", cfg) + } + if len(cfg.Repos) != 1 || cfg.Repos[0].Dir != "app" || cfg.Repos[0].SessionBranch != "rainier/work" { + t.Errorf("repos = %+v", cfg.Repos) + } + if cfg.GitAuthorName != "example" || cfg.GitAuthorEmail != "42+example@users.noreply.github.com" { + t.Errorf("git identity = %q <%q>", cfg.GitAuthorName, cfg.GitAuthorEmail) + } + if cfg.Env["CLAUDE_CONFIG_DIR"] != "/rainier/agents/claude" { + t.Errorf("the agent-home configuration did not reach the guest: %v", cfg.Env) + } + if len(cfg.SecretNames) != 1 || cfg.SecretNames[0] != "DEPLOY_KEY" || cfg.BootstrapToken != "token_example" { + t.Errorf("the bootstrap pair did not reach the guest: names=%v", cfg.SecretNames) + } + // The proxy carries the session's identity as URL userinfo, exactly as + // the Docker driver composes it, so egressd reads one thing on both + // paths. And NO_PROXY has no dial host to exempt, because there is no + // dial: the conn this rode in on IS the channel. + if !strings.Contains(cfg.ProxyURL, "sess-boot:") { + t.Errorf("proxy url = %q, want the session's identity as userinfo", cfg.ProxyURL) + } + if cfg.NoProxy == "" || strings.Contains(cfg.NoProxy, "runner") { + t.Errorf("no_proxy = %q, want the base list with no dial host in it", cfg.NoProxy) + } + + // And the connection was handed to the runner above the driver, which is + // what turns it into the session's relay hub. + waitFor(t, func() bool { + for _, s := range host.connected() { + if s == "sess-boot" { + return true + } + } + return false + }, "the guest connection was never handed to the runner") +} + +// TestMicrovmGuestChannelIsPrivateToItsSession pins the mode on the one file +// this path creates. The socket is a session's control channel on a host +// shared with other tenants, and anything that can connect to it can be that +// session's sandbox. +func TestMicrovmGuestChannelIsPrivateToItsSession(t *testing.T) { + m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4}) + m.SetHost(&stubMicrovmHost{}) + ctx := context.Background() + + h, err := m.Create(ctx, Spec{SessionID: "sess-mode"}) + if err != nil { + t.Fatal(err) + } + defer m.Destroy(ctx, h.ID) + + _, listenPath, err := m.vsockPaths(h.ID, 1) + if err != nil { + t.Fatal(err) + } + info, err := os.Stat(listenPath) + if err != nil { + t.Fatalf("stat the guest control socket: %v", err) + } + if info.Mode().Perm() != microvmFileMode { + t.Errorf("the guest control socket has mode %04o, want %04o", info.Mode().Perm(), microvmFileMode) + } +} + +// TestMicrovmColdResumeMintsAFreshTokenAndSocket is §4.4's resume, in the two +// things that make it safe: the token the previous boot was handed is +// single-use and fenced, so a new VM gets a new one; and Firecracker's own +// documentation warns that one uds_path cannot be multiplexed across VMs, so +// a new VM gets a new socket. +func TestMicrovmColdResumeMintsAFreshTokenAndSocket(t *testing.T) { + m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4}) + host := &stubMicrovmHost{} + m.SetHost(host) + ctx := context.Background() + + h, err := m.Create(ctx, Spec{SessionID: "sess-resume", BootstrapToken: "token_from_the_create"}) + if err != nil { + t.Fatal(err) + } + defer m.Destroy(ctx, h.ID) + if got := readBootConfig(t, dialGuest(t, m, h.ID, 1)).BootstrapToken; got != "token_from_the_create" { + t.Fatalf("the first boot got token %q, want the create's", got) + } + + if err := m.Suspend(ctx, h.ID, false); err != nil { + t.Fatal(err) + } + // The cold-parked VM's socket goes with it: a path nothing serves is one + // more file on a shared host describing a session that is not running. + _, firstPath, _ := m.vsockPaths(h.ID, 1) + if _, err := os.Stat(firstPath); !os.IsNotExist(err) { + t.Errorf("the cold-parked session left its control socket behind: %v", err) + } + + restarted, err := m.Resume(ctx, h.ID) + if err != nil { + t.Fatalf("cold resume: %v", err) + } + if !restarted { + t.Error("a cold resume reported no restart") + } + if host.mintCount() != 1 { + t.Fatalf("the cold resume made %d mint(s), want exactly 1", host.mintCount()) + } + + // A new socket, and the new token on it. + cfg := readBootConfig(t, dialGuest(t, m, h.ID, 2)) + if cfg.BootstrapToken == "token_from_the_create" { + t.Fatal("the resumed guest was handed the previous boot's token, which is spent and fenced") + } + if cfg.BootstrapToken == "" { + t.Fatal("the resumed guest was handed no token at all") + } + if cfg.SessionID != "sess-resume" { + t.Errorf("the resumed guest's session id = %q", cfg.SessionID) + } +} + +// TestMicrovmColdResumeFailsWhenTheTokenCannotBeMinted is the fail-closed +// half: a resume that cannot get a token reports that, rather than booting a +// guest that will ask for its secrets and be refused. +func TestMicrovmColdResumeFailsWhenTheTokenCannotBeMinted(t *testing.T) { + m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4}) + host := &stubMicrovmHost{mintErr: errors.New("this runner has no controld connection")} + m.SetHost(host) + ctx := context.Background() + + h, err := m.Create(ctx, Spec{SessionID: "sess-nomint", BootstrapToken: "token_example"}) + if err != nil { + t.Fatal(err) + } + defer m.Destroy(ctx, h.ID) + if err := m.Suspend(ctx, h.ID, false); err != nil { + t.Fatal(err) + } + + if _, err := m.Resume(ctx, h.ID); err == nil { + t.Fatal("a cold resume with no token succeeded") + } else if !strings.Contains(err.Error(), "no controld connection") { + t.Errorf("error = %q, want the mint's own reason", err) + } + // And the session is still parked, so a later resume can try again. + if st := listedState(t, m, "sess-nomint"); st != StateSuspended { + t.Errorf("a refused resume changed the session's state to %q", st) + } +} + +// TestMicrovmRefusesSecretValuesWithNoToken is the driver's own half of the +// compatibility table's "old controld + new runner" row. The shared contract +// asserts the same refusal for any driver announcing microvm.v1; this one +// asserts the message says what an operator has to change. +func TestMicrovmRefusesSecretValuesWithNoToken(t *testing.T) { + m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4}) + m.SetHost(&stubMicrovmHost{}) + ctx := context.Background() + + _, err := m.Create(ctx, Spec{ + SessionID: "sess-unwithheld", + Env: map[string]string{"DEPLOY_KEY": "value_must_not_be_accepted"}, + }) + if err == nil { + t.Fatal("a create carrying secret values with no token was accepted") + } + for _, want := range []string{"bootstrap token", "microvm.v1", "sess-unwithheld"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the refusal = %q, want it to name %q", err, want) + } + } + if strings.Contains(err.Error(), "value_must_not_be_accepted") { + t.Errorf("the refusal quotes the value it refused: %q", err) + } + // Nothing was reserved, created, or left behind: the check runs before + // the slot is taken. + if used, _, _ := m.Capacity(ctx); used != 0 { + t.Errorf("a refused create left %d slot(s) occupied", used) + } + // And an empty env with no token is a perfectly ordinary create: the + // refusal is about VALUES, not about the field existing. + if _, err := m.Create(ctx, Spec{SessionID: "sess-plain"}); err != nil { + t.Fatalf("a create with no env and no token was refused: %v", err) + } +} + +// TestMicrovmVsockPathRefusesAnOverlongStateDir pins the one configuration +// mistake that would otherwise surface as "invalid argument" from bind(2). +func TestMicrovmVsockPathRefusesAnOverlongStateDir(t *testing.T) { + long := "/" + strings.Repeat("d", 120) + m := &Microvm{opts: MicrovmOpts{StateDir: long}} + if _, _, err := m.vsockPaths("mvm-1", 1); err == nil { + t.Fatal("an overlong state directory produced a socket path") + } else if !strings.Contains(err.Error(), "--microvm-state-dir") { + t.Errorf("error = %q, want it to name the flag an operator can change", err) + } +} + +// waitFor polls cond until it holds or the test gives up. It exists for the +// one assertion here that crosses a goroutine — the driver hands a guest +// connection to the runner from its accept loop. +func waitFor(t *testing.T, cond func() bool, msg string) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for { + if cond() { + return + } + if time.Now().After(deadline) { + t.Fatal(msg) + } + time.Sleep(5 * time.Millisecond) + } +} diff --git a/internal/relay/netconn.go b/internal/relay/netconn.go new file mode 100644 index 0000000..8cf5bef --- /dev/null +++ b/internal/relay/netconn.go @@ -0,0 +1,131 @@ +// internal/relay/netconn.go +package relay + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "net" + "sync" + "time" +) + +// NetConn adapts a byte STREAM to Conn, which is a message interface. +// +// It exists for virtio-vsock, the single host-to-guest control channel a +// microVM session has (ADR-0003 §2.7 item 2): a vsock connection is an +// ordinary AF_VSOCK stream on the guest side and an AF_UNIX stream on the +// host side, with no framing of its own, where the WebSocket this package +// grew up on delivered whole messages. Everything above Conn — the hub, the +// session side, the control channel, the RPC dispatcher — is unchanged by +// this, which is the whole reason the seam is at Conn. +// +// The framing is one JSON value per LINE. It is chosen rather than a length +// prefix because every message on this conn is already a JSON object produced +// by encoding/json, which never emits a bare newline inside one: a framing +// that needs no escaping and no second encoder cannot disagree with the +// encoder above it. maxFrameBytes bounds a line, so a peer cannot make this +// end allocate without limit by never sending one. +// +// Writes are serialized. That is not belt-and-braces: a Hub writes from the +// goroutine of every attached client AND from SendControl, and two concurrent +// Write calls on a stream interleave the bytes of two frames and corrupt it +// permanently for the reader. A *websocket.Conn serializes internally and a +// net.Conn does not, so the discipline has to be here. +func NetConn(c net.Conn) Conn { + return &netConn{ + c: c, + // The same 16 MiB bound both WebSocket ends set with SetReadLimit, so + // a frame that crosses this transport and a frame that crosses that + // one are the same size of thing. + r: bufio.NewReaderSize(c, netConnReadBuffer), + } +} + +const ( + // netConnReadBuffer is the reader's initial buffer. A control frame is + // a few hundred bytes and a terminal frame a few KiB; the buffer grows + // for the rare large one rather than being sized for it. + netConnReadBuffer = 64 << 10 + // maxFrameBytes is the largest single message this transport will + // assemble, matching the WebSocket ends' SetReadLimit. + maxFrameBytes = 16 << 20 +) + +type netConn struct { + c net.Conn + r *bufio.Reader + + // wmu serializes writers — see NetConn's doc comment. + wmu sync.Mutex +} + +// ErrFrameTooLarge is returned when a peer sends a line longer than this +// transport will assemble. It ends the connection rather than skipping the +// frame: a reader that resynchronized mid-stream would hand the decoder the +// tail of one message as though it were a whole one. +var ErrFrameTooLarge = errors.New("relay: frame over the transport's size limit") + +func (n *netConn) Read(ctx context.Context) ([]byte, error) { + // A cancelled ctx unblocks the read by poisoning the conn's deadline, + // which is the only thing that can interrupt a blocking stream read. It + // is one-shot by design: a cancelled context means this conn is over, and + // AfterFunc costs nothing on the path where it never fires. + if ctx.Done() != nil { + stop := context.AfterFunc(ctx, func() { _ = n.c.SetReadDeadline(time.Now()) }) + defer stop() + } + var line []byte + for { + chunk, err := n.r.ReadSlice('\n') + if len(line)+len(chunk) > maxFrameBytes { + return nil, ErrFrameTooLarge + } + line = append(line, chunk...) + if err == nil { + break + } + if errors.Is(err, bufio.ErrBufferFull) { + continue + } + if len(line) > 0 && errors.Is(err, net.ErrClosed) { + // A partial line at a closed conn is not a message. + return nil, err + } + return nil, err + } + // A JSON value never contains a bare newline or carriage return, so + // trimming them is free and makes the framing tolerant of a peer that + // writes CRLF. + return bytes.TrimRight(line[:len(line)-1], "\r"), nil +} + +func (n *netConn) Write(ctx context.Context, b []byte) error { + if len(b) > maxFrameBytes { + return fmt.Errorf("%w: %d bytes", ErrFrameTooLarge, len(b)) + } + if bytes.ContainsRune(b, '\n') { + // Unreachable for anything encoding/json produced, and a hard error + // rather than an escape: a message with a newline in it would frame + // as two, and the second would be garbage the peer could not decode + // and could not resynchronize from. + return errors.New("relay: a frame containing a newline cannot be framed by line") + } + if ctx.Done() != nil { + stop := context.AfterFunc(ctx, func() { _ = n.c.SetWriteDeadline(time.Now()) }) + defer stop() + } + n.wmu.Lock() + defer n.wmu.Unlock() + // One Write call, not two: a frame and its terminator must not be + // separable by a concurrent writer or by a short write in between. + out := make([]byte, 0, len(b)+1) + out = append(out, b...) + out = append(out, '\n') + _, err := n.c.Write(out) + return err +} + +func (n *netConn) Close() error { return n.c.Close() } diff --git a/internal/relay/netconn_test.go b/internal/relay/netconn_test.go new file mode 100644 index 0000000..22d86e4 --- /dev/null +++ b/internal/relay/netconn_test.go @@ -0,0 +1,138 @@ +package relay + +import ( + "bytes" + "context" + "errors" + "net" + "strings" + "sync" + "testing" +) + +// TestNetConnRoundTripsFrames pins that a stream transport delivers the same +// MESSAGES a WebSocket one does: what goes in as one Write comes out as one +// Read, whatever the stream does with the boundaries in between. +func TestNetConnRoundTripsFrames(t *testing.T) { + a, b := net.Pipe() + defer a.Close() + defer b.Close() + left, right := NetConn(a), NetConn(b) + ctx := context.Background() + + frames := [][]byte{ + []byte(`{"t":4,"a":0,"p":"eyJraW5kIjoiYm9vdF9jb25maWcifQ=="}`), + []byte(`{"t":3,"a":7,"p":"` + strings.Repeat("A", 200<<10) + `"}`), + []byte(`{"t":1,"a":7}`), + } + go func() { + for _, f := range frames { + if err := left.Write(ctx, f); err != nil { + return + } + } + }() + for i, want := range frames { + got, err := right.Read(ctx) + if err != nil { + t.Fatalf("frame %d: %v", i, err) + } + if !bytes.Equal(got, want) { + t.Fatalf("frame %d: got %d bytes, want %d", i, len(got), len(want)) + } + } +} + +// TestNetConnSerializesConcurrentWrites is the property a raw stream needs +// and a WebSocket supplies for free. A Hub writes from the goroutine of every +// attached client and from SendControl; two interleaved writes would corrupt +// the stream permanently for the reader, so every frame must arrive whole. +func TestNetConnSerializesConcurrentWrites(t *testing.T) { + a, b := net.Pipe() + defer a.Close() + defer b.Close() + left, right := NetConn(a), NetConn(b) + ctx := context.Background() + + const writers, each = 8, 16 + var wg sync.WaitGroup + for w := 0; w < writers; w++ { + wg.Add(1) + go func(w int) { + defer wg.Done() + payload := []byte(strings.Repeat(string(rune('a'+w)), 32<<10)) + for i := 0; i < each; i++ { + if err := left.Write(ctx, payload); err != nil { + return + } + } + }(w) + } + go func() { wg.Wait(); a.Close() }() + + for n := 0; n < writers*each; n++ { + got, err := right.Read(ctx) + if err != nil { + t.Fatalf("frame %d: %v", n, err) + } + if len(got) != 32<<10 { + t.Fatalf("frame %d is %d bytes: two writes interleaved", n, len(got)) + } + if bytes.Count(got, got[:1]) != len(got) { + t.Fatalf("frame %d mixes two writers' bytes", n) + } + } +} + +// TestNetConnRefusesAnUnframeableWrite pins the one shape this framing cannot +// carry. It is unreachable for anything encoding/json produces, and it is an +// error rather than an escape because a silently split frame would leave the +// peer decoding garbage it could never resynchronize from. +func TestNetConnRefusesAnUnframeableWrite(t *testing.T) { + a, b := net.Pipe() + defer a.Close() + defer b.Close() + if err := NetConn(a).Write(context.Background(), []byte("one\ntwo")); err == nil { + t.Fatal("a frame containing a newline was accepted") + } + _ = b +} + +// TestNetConnReadEndsOnACancelledContext pins that a blocking stream read is +// interruptible, which is what a Hub whose context ends relies on. +func TestNetConnReadEndsOnACancelledContext(t *testing.T) { + a, b := net.Pipe() + defer a.Close() + defer b.Close() + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + _, err := NetConn(a).Read(ctx) + done <- err + }() + cancel() + if err := <-done; err == nil { + t.Fatal("a read on a cancelled context returned no error") + } +} + +// TestNetConnRefusesAnOversizedFrame pins the bound: a peer cannot make this +// end assemble an unbounded line. +func TestNetConnRefusesAnOversizedFrame(t *testing.T) { + a, b := net.Pipe() + defer a.Close() + defer b.Close() + go func() { + // Written straight to the stream, bypassing the writer's own bound, + // because it is the READER's bound this pins. + chunk := bytes.Repeat([]byte("x"), 1<<20) + for i := 0; i < 20; i++ { + if _, err := a.Write(chunk); err != nil { + return + } + } + }() + if _, err := NetConn(b).Read(context.Background()); !errors.Is(err, ErrFrameTooLarge) { + t.Fatalf("an oversized frame gave %v, want ErrFrameTooLarge", err) + } +} diff --git a/internal/runnerd/agent.go b/internal/runnerd/agent.go index 773a112..e5a140e 100644 --- a/internal/runnerd/agent.go +++ b/internal/runnerd/agent.go @@ -332,7 +332,7 @@ func (s *Server) agentSession(ctx context.Context, cfg AgentConfig) (established active, idleExited := s.reg.counts() ann := runner.FromRunner{Type: "announce", Proto: runner.ProtocolVersion, Runner: cfg.RunnerName, Sessions: s.Announce(), Used: used, Total: total, Active: active, IdleExited: idleExited, - Capabilities: buildCapabilities(cfg.Capabilities)} + Capabilities: buildCapabilities(cfg.Capabilities, s.driverCapabilities()...)} if err := wsjson.Write(connCtx, c, ann); err != nil { return false, err // nothing can have been accepted before the announce } @@ -449,6 +449,14 @@ func (s *Server) execute(ctx context.Context, m runner.ToRunner, send func(runne Init: m.Spec.Init, InitTimeoutSec: m.Spec.InitTimeoutSec, GitAuthorName: m.Spec.GitAuthorName, GitAuthorEmail: m.Spec.GitAuthorEmail, Home: driverHome(m.Spec.Home), + // The microVM bootstrap pair, carried through like + // everything else. This runner does not read the token — it + // goes into the guest's boot configuration and is dropped — + // and it does not check the names against Env. The DRIVER + // decides what an unwithheld create means, because the answer + // is different for each one: Docker has always accepted the + // values and still does. + BootstrapToken: m.Spec.BootstrapToken, SecretNames: m.Spec.SecretNames, } allow = m.Spec.EgressAllow } @@ -598,6 +606,18 @@ func (s *Server) forwardSessionRPC(m runner.ToRunner, send func(runner.FromRunne log.Printf("agent: session_rpc for %s carried no id or method; ignoring", m.Session) return } + // A response to a request this RUNNER originated stops here: the runner + // is a pure forwarder for everything a sandbox asked, and the one thing + // it asks for itself (a bootstrap token on a cold resume) is answered to + // a caller inside this process, not to the guest. The id spaces are + // disjoint so the two can share one connection — see + // runnerOriginatedIDBase. + if env.Method == "resp" && isRunnerOriginated(env.ID) { + if !s.runnerRPC.deliver(env) { + log.Printf("agent: an answer for this runner's own request %d has no caller waiting (timed out?); dropping", env.ID) + } + return + } err := s.sendSessionRPC(m.Session, env) if err == nil { return @@ -738,19 +758,45 @@ func (s *Server) dialAttachBack(ctx context.Context, m runner.ToRunner, cfg Agen // far worse failure than the 501 this append exists to avoid for what is a // cheap pre-check and not the fence. The sandbox's own `exec_started` is the // fence, so the only cost of dropping the claim is one wasted round trip. -func buildCapabilities(declared []string) []string { +// `microvm.v1` is the second such fact, and the one place this comment needs +// a caveat: it is not merely a pre-check. The control plane WITHHOLDS an +// environment's decrypted secret values from a runner that announces it, so +// announcing it is what makes the withholding happen and failing to announce +// it is what gets the values dispatched to a driver that will refuse them. +// It comes from the driver this runner was started with rather than from a +// flag, for the same reason exec.v1 does: it is already decided by +// --driver=microvm, and a second flag beside it is a second thing to get +// wrong. See (*Server).driverCapabilities. +func buildCapabilities(declared []string, fromBuild ...string) []string { + out := declared + for _, c := range append([]string{runner.CapabilityExecV1}, fromBuild...) { + out = appendCapability(out, c) + } + return out +} + +// appendCapability adds one build fact to an operator's list, unless it is +// already there or there is no room for it. +// +// The no-room case is why this is not one line. runnerplane refuses a WHOLE +// registration whose claim carries more than MaxCapabilities, so an operator +// already passing the maximum would announce one too many after this rolls +// and never reconnect — a working runner out of the fleet permanently, which +// is far worse than the capability being absent. The operator's list is +// otherwise left exactly as given, order included: it is a claim the control +// plane decides whether to schedule on, not something to tidy. +func appendCapability(declared []string, capability string) []string { for _, c := range declared { - if c == runner.CapabilityExecV1 { + if c == capability { return declared } } if len(declared) >= runnerplane.MaxCapabilities { log.Printf("agent: %d capabilities declared, which is runnerplane's maximum; "+ - "announcing without %s (exec is still fenced by the sandbox's own handshake)", - len(declared), runner.CapabilityExecV1) + "announcing without %s", len(declared), capability) return declared } - return append(append([]string(nil), declared...), runner.CapabilityExecV1) + return append(append([]string(nil), declared...), capability) } // attachDialTimeout bounds one attach-back handshake. It sits below diff --git a/internal/runnerd/microvm.go b/internal/runnerd/microvm.go new file mode 100644 index 0000000..e3667ab --- /dev/null +++ b/internal/runnerd/microvm.go @@ -0,0 +1,224 @@ +// internal/runnerd/microvm.go +// +// The runner's half of the microVM driver's two needs: where a guest's vsock +// connection goes, and where a cold resume's bootstrap token comes from. +// Both are implementations of driver.MicrovmHost, and both are here rather +// than in the driver because both are things only the runner can do — one +// needs the session registry, the other needs the control connection. +package runnerd + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log" + "slices" + "sync" + "sync/atomic" + "time" + + "github.com/tokencanopy/rainier/internal/driver" + "github.com/tokencanopy/rainier/internal/relay" + "github.com/tokencanopy/rainier/protocol/runner" +) + +var _ driver.MicrovmHost = (*Server)(nil) + +// GuestConnected serves one microVM guest's control connection, exactly as +// the /register handler serves a container's WebSocket. +// +// On its own goroutine because the driver calls this from its accept loop and +// serveSessionConn blocks for the life of the conn — a loop that waited would +// never accept the guest's next dial. +// +// The session id is the DRIVER's, taken from the socket path Firecracker +// forwarded the connection to, and not anything the guest said. That is the +// whole difference between this door and the WebSocket one, where `register` +// believes a query parameter with no authentication on the hop at all. +func (s *Server) GuestConnected(sessionID string, conn relay.Conn) { + if _, ok := s.reg.get(sessionID); !ok { + // A guest for a session this runner no longer holds: a destroy that + // raced the boot. Closing it is what tells the guest to stop; leaving + // it open would leak the conn and its goroutine with no registry + // entry that could ever reap them. + log.Printf("microvm guest connected for unknown session %s; closing", sessionID) + _ = conn.Close() + return + } + go s.serveSessionConn(context.Background(), sessionID, conn) +} + +// runnerOriginatedIDBase separates the request ids this RUNNER assigns from +// the ones a sandbox assigns. +// +// Both travel up the same connection as a "session_req" for the same session, +// and the control plane echoes whichever id it was given, so the two spaces +// must not collide: a response to the runner's mint carrying id 1 would +// otherwise be delivered into a sandbox that is waiting on its own id 1. The +// high bit is the cheapest disjoint space there is, and a sandbox counting +// from 1 will not reach it. +const runnerOriginatedIDBase uint64 = 1 << 63 + +// isRunnerOriginated reports whether an id came from this runner's own +// counter. forwardSessionRPC asks before it routes a response into a sandbox. +func isRunnerOriginated(id uint64) bool { return id&runnerOriginatedIDBase != 0 } + +// runnerRPCTable is the pending table for requests this runner originated. +// +// It is deliberately tiny and deliberately separate from the sandbox's: the +// runner asks the control plane exactly one thing (a bootstrap token, on a +// cold resume), and a table shared with the forwarding path would make the +// forwarder a participant in the conversation it is supposed to be a pipe +// for. +type runnerRPCTable struct { + seq atomic.Uint64 + mu sync.Mutex + // waiting is id → the one-slot channel its caller is selecting on. Every + // caller removes its own entry, so nothing sweeps this map. + waiting map[uint64]chan runner.RPCEnvelope +} + +func newRunnerRPCTable() *runnerRPCTable { + return &runnerRPCTable{waiting: map[uint64]chan runner.RPCEnvelope{}} +} + +func (t *runnerRPCTable) begin() (uint64, chan runner.RPCEnvelope) { + id := runnerOriginatedIDBase | t.seq.Add(1) + ch := make(chan runner.RPCEnvelope, 1) + t.mu.Lock() + t.waiting[id] = ch + t.mu.Unlock() + return id, ch +} + +func (t *runnerRPCTable) end(id uint64) { + t.mu.Lock() + delete(t.waiting, id) + t.mu.Unlock() +} + +// deliver hands a response to the call waiting on its id, reporting whether +// anyone was. The channel is buffered by one and every caller removes its own +// entry, so this never blocks the reader that called it. +func (t *runnerRPCTable) deliver(env runner.RPCEnvelope) bool { + t.mu.Lock() + ch, ok := t.waiting[env.ID] + t.mu.Unlock() + if !ok { + return false + } + select { + case ch <- env: + default: + } + return true +} + +// mintBootstrapTimeout bounds one upward mint. It sits well under a cold +// resume's own patience and well over a round trip to a control plane that is +// answering: a resume that waits longer than this is waiting on a plane that +// is not going to answer, and reporting that is better than holding the +// session's resume open. +const mintBootstrapTimeout = 10 * time.Second + +// MintSessionBootstrap asks the control plane for a fresh single-use token +// for sessionID, on the runner's own control connection. +// +// It is the one request this runner originates on the session RPC. The +// message carries no session id of its own — FromRunner.Session is the id, +// and the control plane answers from the row its placement guard read — which +// is what stops a runner holding session A from minting for session B. +// +// A runner with no control connection fails immediately rather than waiting: +// there is no queue behind this channel by design, and a cold resume that +// cannot get a token has to report that rather than boot a guest that will +// ask for its secrets and be refused. +func (s *Server) MintSessionBootstrap(ctx context.Context, sessionID string) (string, error) { + body, err := json.Marshal(struct { + Protocol int `json:"protocol"` + }{runner.SessionBootstrapProtocolVersion}) + if err != nil { + return "", fmt.Errorf("encoding the bootstrap mint request: %w", err) + } + + id, ch := s.runnerRPC.begin() + defer s.runnerRPC.end(id) + + if !s.fireSessionRPC(sessionID, runner.RPCEnvelope{ + ID: id, Method: runner.MethodMintSessionBootstrap, Payload: body}) { + return "", errors.New("this runner has no controld connection to mint a bootstrap token on") + } + + timer := time.NewTimer(mintBootstrapTimeout) + defer timer.Stop() + var answer runner.RPCEnvelope + select { + case answer = <-ch: + case <-ctx.Done(): + return "", fmt.Errorf("minting a bootstrap token for %s: %w", sessionID, ctx.Err()) + case <-timer.C: + return "", fmt.Errorf("minting a bootstrap token for %s: no answer within %s", sessionID, mintBootstrapTimeout) + } + if !answer.OK { + // The control plane's own sentence, relayed: it names the condition + // a person can act on, and it never carries a token or a value. + return "", errors.New(rpcErrorText(answer.Payload)) + } + var minted struct { + Token string `json:"token"` + } + if err := json.Unmarshal(answer.Payload, &minted); err != nil { + // Logged without the error and without the payload: a json message + // quotes what it choked on, and what it choked on is the token. + return "", errors.New("the control plane's bootstrap token could not be decoded") + } + if minted.Token == "" { + return "", errors.New("the control plane answered a bootstrap mint with no token") + } + return minted.Token, nil +} + +// rpcErrorText reads the {"error": ...} sentence a failed response carries, +// falling back to a flat one so a refusal with no body is still a reason. +func rpcErrorText(payload json.RawMessage) string { + var body struct { + Error string `json:"error"` + } + if len(payload) > 0 && json.Unmarshal(payload, &body) == nil && body.Error != "" { + return body.Error + } + return "the control plane refused a bootstrap mint without a reason" +} + +// driverCapabilities are the capability tokens that are facts about which +// DRIVER this runner was started with, appended to the operator's own claims +// exactly as exec.v1 is and for the same reason: whether a session on this +// runner is a microVM is decided by a flag the operator already passed +// (--driver=microvm), not by a second flag they have to remember beside it. +// +// It is the fence, not merely a pre-check, which is the one way it differs +// from exec.v1: the control plane WITHHOLDS an environment's secret values +// from a runner that announces microvm.v1, so a runner that announced it +// falsely would be dispatched creates it cannot fulfil, and one that failed +// to announce it would be dispatched the values it must not accept — which +// its driver then refuses (driver.refuseUnwithheldEnv). +func (s *Server) driverCapabilities() []string { + if cd, ok := s.drv.(driver.CapabilityDriver); ok { + return cd.Capabilities() + } + return nil +} + +// withholdsSecrets reports whether this runner's driver is one the control +// plane withholds an environment's secret values from. +// +// It is read off the announced capability rather than off the driver's +// concrete type, and that is the point: the two places this runner behaves +// differently for a microVM session — a create carries no dial URL, and a +// cold suspend tells the sandbox it is cold — are consequences of the same +// claim the control plane acts on. Keying them on a second fact would let +// the announcement and the behaviour disagree. +func (s *Server) withholdsSecrets() bool { + return slices.Contains(s.driverCapabilities(), runner.CapabilityMicrovmV1) +} diff --git a/internal/runnerd/microvm_test.go b/internal/runnerd/microvm_test.go new file mode 100644 index 0000000..ca3c516 --- /dev/null +++ b/internal/runnerd/microvm_test.go @@ -0,0 +1,390 @@ +// internal/runnerd/microvm_test.go +package runnerd + +import ( + "context" + "encoding/json" + "errors" + "net" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/tokencanopy/rainier/internal/driver" + "github.com/tokencanopy/rainier/internal/relay" + "github.com/tokencanopy/rainier/protocol/runner" +) + +// withholdingDriver is a driver that announces microvm.v1 and takes a host, +// over the ordinary fake. +// +// It is NOT a *driver.Microvm, and it deliberately cannot be: the microVM +// driver's simulated engine lives in a _test.go file of its own package +// precisely so no code outside it can construct a simulated hypervisor, and +// that rule is worth more than the convenience of reusing it here. What this +// file tests is the RUNNER's half — what it announces, where a guest's +// connection goes, what a create carries, what a cold suspend says — and +// every one of those is keyed on the announced capability rather than on a +// concrete type, so a driver that announces it is exactly the right fixture. +// The driver's own half is tested in internal/driver. +type withholdingDriver struct { + *driver.Fake + mu sync.Mutex + host driver.MicrovmHost +} + +func (d *withholdingDriver) Capabilities() []string { return []string{runner.CapabilityMicrovmV1} } + +func (d *withholdingDriver) SetHost(h driver.MicrovmHost) { + d.mu.Lock() + defer d.mu.Unlock() + d.host = h +} + +func (d *withholdingDriver) installedHost() driver.MicrovmHost { + d.mu.Lock() + defer d.mu.Unlock() + return d.host +} + +// testMicrovmServer builds a runner whose driver withholds, with the dial +// base and proxy a real one is configured with. +func testMicrovmServer(t *testing.T) (*Server, *withholdingDriver) { + t.Helper() + d := &withholdingDriver{Fake: driver.NewFake(4)} + return New(d, "ws://runner.example.invalid:8080", "", "http://proxy.example.invalid:3128"), d +} + +// TestMicrovmRunnerAnnouncesTheCapability is the fence the control plane keys +// its withholding on. It is a fact about which driver this runner was started +// with, not a flag an operator has to remember beside --driver=microvm, and a +// runner that failed to announce it would be dispatched the secret values its +// own driver then refuses. +func TestMicrovmRunnerAnnouncesTheCapability(t *testing.T) { + s, _ := testMicrovmServer(t) + caps := buildCapabilities(nil, s.driverCapabilities()...) + if !slices.Contains(caps, runner.CapabilityMicrovmV1) { + t.Fatalf("a microVM runner announces %v, want %s among them", caps, runner.CapabilityMicrovmV1) + } + if !slices.Contains(caps, runner.CapabilityExecV1) { + t.Fatalf("announcing microvm.v1 dropped exec.v1: %v", caps) + } + + // And a Docker runner announces neither it nor anything new: the whole + // compatibility floor is that a runner without this capability is + // dispatched exactly what it always was. + docker := New(driver.NewFake(4), "", "", "") + if got := buildCapabilities(nil, docker.driverCapabilities()...); !slices.Equal(got, []string{runner.CapabilityExecV1}) { + t.Fatalf("a non-microVM runner announces %v, want only exec.v1", got) + } + + // The operator's own claims are kept, in order, and not reordered by the + // build facts appended after them. + declared := []string{"gpu", "docker.rootless"} + got := buildCapabilities(declared, s.driverCapabilities()...) + if len(got) != 4 || got[0] != "gpu" || got[1] != "docker.rootless" { + t.Fatalf("buildCapabilities(%v) = %v; it reordered the operator's list", declared, got) + } +} + +// TestMicrovmCreateLeavesNoDialURL is §4's step 2. There is no URL to dial: +// the guest's control channel is the vsock conn IT opens, so a dial URL would +// be a promise of an endpoint the guest has no route to — and an empty one is +// what makes NO_PROXY inside the guest the base list with no dial host in it. +func TestMicrovmCreateLeavesNoDialURL(t *testing.T) { + s, d := testMicrovmServer(t) + if err := s.CreateWithID(context.Background(), "sess-nodial", driver.Spec{}, nil); err != nil { + t.Fatal(err) + } + got := d.LastSpec() + if got.DialURL != "" { + t.Fatalf("a withholding driver's create carries dial URL %q; there is nothing to dial", got.DialURL) + } + // Everything else a create carries is unchanged: the id and the proxy + // are still this runner's own concerns and still set here. + if got.SessionID != "sess-nodial" || got.ProxyURL != "http://proxy.example.invalid:3128" { + t.Fatalf("the create lost something else: %+v", got) + } + + // The Docker path keeps its dial URL, byte for byte: that is the floor + // this whole change promises not to move. + dockerRunner := New(driver.NewFake(4), "ws://runner.example.invalid:8080", "", "") + if err := dockerRunner.CreateWithID(context.Background(), "sess-docker", driver.Spec{}, nil); err != nil { + t.Fatal(err) + } + fake := dockerRunner.drv.(*driver.Fake) + if got := fake.LastSpec().DialURL; got != "ws://runner.example.invalid:8080/register" { + t.Fatalf("the docker path's dial URL = %q", got) + } +} + +// TestGuestConnectedBecomesTheSessionsHub is the vsock half of `register`: +// the connection Firecracker forwarded becomes the session's relay hub, +// keyed by the session the SOCKET PATH named, and the session reports +// running. +func TestGuestConnectedBecomesTheSessionsHub(t *testing.T) { + s, _ := testMicrovmServer(t) + events := make(chan string, 8) + s.SetOnEvent(func(id, state, _ string) { events <- id + ":" + state }) + + s.reg.put("sess-guest", &sessionEntry{id: "sess-guest", state: "running"}) + + guest, host := net.Pipe() + defer guest.Close() + s.GuestConnected("sess-guest", relay.NetConn(host)) + + select { + case got := <-events: + if got != "sess-guest:running" { + t.Fatalf("event = %q, want sess-guest:running", got) + } + case <-time.After(3 * time.Second): + t.Fatal("a guest connection produced no running event") + } + if _, ok := s.reg.hub("sess-guest"); !ok { + t.Fatal("a guest connection did not become the session's hub") + } + + // A guest for a session this runner does not hold is closed rather than + // served: a destroy that raced the boot must not leave a conn and a + // goroutine with no entry that could ever reap them. + other, otherHost := net.Pipe() + defer other.Close() + s.GuestConnected("sess-unknown", relay.NetConn(otherHost)) + if _, err := other.Read(make([]byte, 1)); err == nil { + t.Fatal("a guest for an unknown session was left connected") + } +} + +// TestMintSessionBootstrapRidesTheControlConnection is the cold-resume mint, +// end to end over a fake control plane: the runner originates the request, +// the plane answers it, and the answer reaches the caller inside this process +// rather than being forwarded into the sandbox. +func TestMintSessionBootstrapRidesTheControlConnection(t *testing.T) { + s, _ := testMicrovmServer(t) + s.reg.put("sess-mint", &sessionEntry{id: "sess-mint", state: "running"}) + + fc := newFakeControld(t, testToken) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go s.RunAgent(ctx, AgentConfig{ControldURL: fc.wsURL(), Token: testToken, RunnerName: "vm1"}) + conn := fc.nextConn(t) + conn.readAnnounce(t) + + minted := make(chan string, 1) + mintErr := make(chan error, 1) + go func() { + tok, err := s.MintSessionBootstrap(context.Background(), "sess-mint") + if err != nil { + mintErr <- err + return + } + minted <- tok + }() + + req := conn.readMsg(t) + if req.Type != "session_req" || req.Session != "sess-mint" || req.RPC == nil { + t.Fatalf("the runner sent %+v, want a session_req for sess-mint", req) + } + if req.RPC.Method != runner.MethodMintSessionBootstrap { + t.Fatalf("method = %q, want %q", req.RPC.Method, runner.MethodMintSessionBootstrap) + } + // The id is in the runner's own space, so it cannot collide with a + // sandbox's — both travel up this one connection for this one session. + if !isRunnerOriginated(req.RPC.ID) { + t.Fatalf("the runner's own request carries id %d, which is in the sandbox's space", req.RPC.ID) + } + var body struct { + Protocol int `json:"protocol"` + } + if err := json.Unmarshal(req.RPC.Payload, &body); err != nil || body.Protocol != runner.SessionBootstrapProtocolVersion { + t.Fatalf("request body = %s (%v)", req.RPC.Payload, err) + } + + answer, _ := json.Marshal(map[string]any{"token": "token_example", "expires_in_sec": 120}) + conn.send(t, runner.ToRunner{Type: "session_rpc", Session: "sess-mint", + RPC: &runner.RPCEnvelope{ID: req.RPC.ID, Method: "resp", OK: true, Payload: answer}}) + + select { + case tok := <-minted: + if tok != "token_example" { + t.Fatalf("minted token = %q", tok) + } + case err := <-mintErr: + t.Fatalf("mint: %v", err) + case <-time.After(5 * time.Second): + t.Fatal("the mint never returned") + } + + // And the answer did NOT reach the sandbox: the runner is a forwarder + // for everything the sandbox asked and a participant only in this one. + if _, ok := s.reg.hub("sess-mint"); ok { + t.Fatal("this fixture has no sandbox; the assertion below would prove nothing") + } +} + +// TestMintSessionBootstrapFailsClosed pins the two ways a mint can go wrong, +// both of which a cold resume has to hear about rather than boot through. +func TestMintSessionBootstrapFailsClosed(t *testing.T) { + t.Run("no control connection", func(t *testing.T) { + s, _ := testMicrovmServer(t) + if _, err := s.MintSessionBootstrap(context.Background(), "sess-x"); err == nil { + t.Fatal("a mint with no controld connection succeeded") + } else if !strings.Contains(err.Error(), "no controld connection") { + t.Fatalf("error = %q", err) + } + }) + + t.Run("a refusal is relayed verbatim", func(t *testing.T) { + s, _ := testMicrovmServer(t) + s.reg.put("sess-refused", &sessionEntry{id: "sess-refused", state: "running"}) + fc := newFakeControld(t, testToken) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go s.RunAgent(ctx, AgentConfig{ControldURL: fc.wsURL(), Token: testToken, RunnerName: "vm1"}) + conn := fc.nextConn(t) + conn.readAnnounce(t) + + out := make(chan error, 1) + go func() { + _, err := s.MintSessionBootstrap(context.Background(), "sess-refused") + out <- err + }() + req := conn.readMsg(t) + refusal, _ := json.Marshal(map[string]string{"error": "this session is not placed on the runner that asked"}) + conn.send(t, runner.ToRunner{Type: "session_rpc", Session: "sess-refused", + RPC: &runner.RPCEnvelope{ID: req.RPC.ID, Method: "resp", Payload: refusal}}) + + select { + case err := <-out: + if err == nil || !strings.Contains(err.Error(), "not placed on the runner that asked") { + t.Fatalf("error = %v, want the control plane's own sentence", err) + } + case <-time.After(5 * time.Second): + t.Fatal("the mint never returned") + } + }) + + t.Run("a cancelled context ends the wait", func(t *testing.T) { + s, _ := testMicrovmServer(t) + s.reg.put("sess-cancel", &sessionEntry{id: "sess-cancel", state: "running"}) + fc := newFakeControld(t, testToken) + agentCtx, stopAgent := context.WithCancel(context.Background()) + defer stopAgent() + go s.RunAgent(agentCtx, AgentConfig{ControldURL: fc.wsURL(), Token: testToken, RunnerName: "vm1"}) + fc.nextConn(t).readAnnounce(t) + + ctx, cancel := context.WithCancel(context.Background()) + out := make(chan error, 1) + go func() { + _, err := s.MintSessionBootstrap(ctx, "sess-cancel") + out <- err + }() + cancel() + select { + case err := <-out: + if !errors.Is(err, context.Canceled) { + t.Fatalf("error = %v, want a cancellation", err) + } + case <-time.After(5 * time.Second): + t.Fatal("a cancelled mint never returned") + } + }) +} + +// TestColdSuspendTellsAMicrovmSandboxItIsCold is §4.4's handshake. A cold +// microVM suspend terminates the VM with no memory image anywhere, so the +// sandbox is told first and told that it is cold — flush, unmount, forget the +// delivered secrets — where a warm one is a freeze and says nothing new. +func TestColdSuspendTellsAMicrovmSandboxItIsCold(t *testing.T) { + s, d := testMicrovmServer(t) + if d.installedHost() == nil { + t.Fatal("New did not install the runner as the driver's host") + } + s.suspendAckWait = 200 * time.Millisecond + s.suspendReadyWait = 200 * time.Millisecond + ctx := context.Background() + if err := s.CreateWithID(ctx, "sess-cold", driver.Spec{}, nil); err != nil { + t.Fatal(err) + } + + // A sandbox on the far side of the session's conn, reading control + // frames. The vsock transport is the same relay.Conn the WebSocket one + // is, so a pipe stands in for it exactly. + guest, host := net.Pipe() + defer guest.Close() + s.GuestConnected("sess-cold", relay.NetConn(host)) + waitForHub(t, s, "sess-cold") + sandbox := relay.NetConn(guest) + + done := make(chan error, 1) + go func() { done <- s.Op(ctx, "sess-cold", "suspend", false) }() + + ev := readControlEvent(t, sandbox) + if ev.Kind != relay.KindSuspending { + t.Fatalf("the sandbox was sent %q, want %q", ev.Kind, relay.KindSuspending) + } + if !ev.Cold { + t.Fatal("a cold microVM suspend was announced as a freeze; the guest would not flush or unmount") + } + if ev.ID == 0 { + t.Fatal("the suspend notice carries no nonce") + } + if err := <-done; err != nil { + t.Fatalf("cold suspend: %v", err) + } +} + +// TestColdSuspendOnDockerIsUnchanged is the other half, and the one that +// matters most: the Docker path sends no notice on a cold stop, exactly as it +// never has. `docker stop` delivers the SIGTERM sessiond's own handler +// answers, and a frame added here would be a behaviour change on the one path +// this design promises to leave alone. +func TestColdSuspendOnDockerIsUnchanged(t *testing.T) { + fd := driver.NewFake(4) + s := New(fd, "", "", "") + ctx := context.Background() + if err := s.CreateWithID(ctx, "sess-docker", driver.Spec{}, nil); err != nil { + t.Fatal(err) + } + guest, host := net.Pipe() + defer guest.Close() + go s.serveSessionConn(context.Background(), "sess-docker", relay.NetConn(host)) + waitForHub(t, s, "sess-docker") + + if err := s.Op(ctx, "sess-docker", "suspend", false); err != nil { + t.Fatalf("cold suspend: %v", err) + } + // Nothing was written to the sandbox at all. A read with a deadline is + // how "no frame" is asserted; the conn is live throughout. + _ = guest.SetReadDeadline(time.Now().Add(200 * time.Millisecond)) + if n, err := guest.Read(make([]byte, 64)); err == nil { + t.Fatalf("the docker path sent the sandbox %d bytes on a cold stop", n) + } +} + +// readControlEvent reads one frame off a sandbox-side conn and decodes the +// control event in it. +func readControlEvent(t *testing.T, conn relay.Conn) relay.ControlEvent { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + raw, err := conn.Read(ctx) + if err != nil { + t.Fatalf("read a frame: %v", err) + } + f, err := relay.Decode(raw) + if err != nil { + t.Fatalf("decode a frame: %v", err) + } + if f.Type != relay.FrameControl { + t.Fatalf("frame type = %d, want a control frame", f.Type) + } + var ev relay.ControlEvent + if err := json.Unmarshal(f.Payload, &ev); err != nil { + t.Fatalf("decode a control event: %v", err) + } + return ev +} diff --git a/internal/runnerd/runnerd.go b/internal/runnerd/runnerd.go index e07439d..044f650 100644 --- a/internal/runnerd/runnerd.go +++ b/internal/runnerd/runnerd.go @@ -121,6 +121,10 @@ type Server struct { // value New leaves — means "derive it from the timeout" (see // idleSweepInterval); a test sets it directly to keep its loop short. idleSweep time.Duration + // runnerRPC is the pending table for the session-RPC requests this + // RUNNER originates, which today is exactly one: the bootstrap token a + // microVM cold resume needs. See runnerRPCTable and microvm.go. + runnerRPC *runnerRPCTable } // suspendWaiter is one in-flight suspend's two answers, matched by nonce. @@ -211,10 +215,19 @@ func (e *egressError) Error() string { return "egress setup: " + e.err.Error() } func (e *egressError) Unwrap() error { return e.err } func New(drv driver.Driver, dialBase, egressAdmin, proxyURL string) *Server { - return &Server{drv: drv, reg: newRegistry(), dialBase: dialBase, egressAdmin: egressAdmin, + s := &Server{drv: drv, reg: newRegistry(), dialBase: dialBase, egressAdmin: egressAdmin, proxyURL: proxyURL, hubWait: defaultHubWait, now: time.Now, suspendAckWait: defaultSuspendAckWait, suspendReadyWait: defaultSuspendReadyWait, - suspends: map[string]*suspendWaiter{}} + suspends: map[string]*suspendWaiter{}, runnerRPC: newRunnerRPCTable()} + // The microVM driver is composed BELOW this server and needs two things + // from above it — where a guest's control conn goes, and where a cold + // resume's bootstrap token comes from — so the runner installs itself + // here rather than being passed into the driver's constructor, which + // runs first. See driver.MicrovmHost. + if hd, ok := drv.(driver.HostedDriver); ok { + hd.SetHost(s) + } + return s } // Recover rebuilds the in-memory registry from the driver's labeled @@ -384,6 +397,16 @@ func (s *Server) createWithID(ctx context.Context, id string, spec driver.Spec, spec.SessionID = id spec.DialURL = s.dialBase + "/register" spec.ProxyURL = s.proxyURL + if s.withholdsSecrets() { + // There is no URL to dial. A microVM guest's control channel is the + // vsock conn IT opens to the host, which the driver is already + // listening for by the time the VM starts, so a dial URL here would + // be a promise of an endpoint the guest has no route to and no + // business reaching. Empty is also what makes NO_PROXY inside the + // guest the base list and nothing else: there is no dial host to + // exempt (design note §4, step 2). + spec.DialURL = "" + } h, err := s.drv.Create(ctx, spec) if err != nil { s.reg.remove(id) @@ -650,9 +673,22 @@ func (s *Server) Op(ctx context.Context, id, op string, warm bool) error { // design's lifetime rule. Sent only for the warm case, so the cold // path keeps exactly the shape it has today. if warm { - s.quiesceExecs(ctx, id) + s.quiesceExecs(ctx, id, false) } if !warm { + // A cold microVM suspend TERMINATES the VM, and no memory image + // is written anywhere (ADR-0003 §2.2), so the sandbox is told + // first — and told that it is COLD: flush what it has, unmount + // the agent home, and forget every secret the bootstrap exchange + // delivered, before the VM goes. + // + // The Docker path is deliberately untouched here. `docker stop` + // delivers the SIGTERM sessiond's own handler already answers, + // and a notice sent there would be a behaviour change on the one + // path this design promises to leave exactly as it is. + if s.withholdsSecrets() { + s.quiesceExecs(ctx, id, true) + } // Cold suspend (docker stop) kills the container's sessiond, // which closes its /register conn — the exact same socket-level // event as a crash. Mark "suspending" BEFORE calling Suspend so @@ -930,6 +966,26 @@ func (s *Server) register(w http.ResponseWriter, r *http.Request) { return } c.SetReadLimit(16 << 20) + s.serveSessionConn(r.Context(), id, relay.WSConn(c)) +} + +// serveSessionConn is what a registered session's conn MEANS to this runner, +// with the transport that carried it already resolved: build the hub, publish +// it, report the session running, and then hold the goroutine until the conn +// dies so the entry can be settled. +// +// It is shared by the two doors a sandbox can arrive at. The WebSocket one +// above is the Docker path and every session that exists today, where the +// container dials in over the network and asserts its session id in a query +// parameter. The vsock one is a microVM guest, whose connection Firecracker +// forwards to a socket inside that VM's own directory — so the session id is +// the PATH's and not the guest's claim, which is strictly stronger than the +// hop it replaces (tenancy §18 item 53). +// +// Everything below this line was in the /register handler and is unchanged; +// the extraction is what keeps the two doors from drifting into meaning two +// different things. +func (s *Server) serveSessionConn(ctx context.Context, id string, conn relay.Conn) { // Read BEFORE the hub, because NewHubWithControl starts the read loop. // Every control frame from this conn carries it, and the registry drops // the ones that name a boot the session has moved past. Note READ, not @@ -944,7 +1000,7 @@ func (s *Server) register(w http.ResponseWriter, r *http.Request) { // process on the other end may not be the one that numbered the last // report. See sessionEntry.execReg. reg := s.reg.registration() - hub := relay.NewHubWithControl(r.Context(), relay.WSConn(c), func(payload []byte) { + hub := relay.NewHubWithControl(ctx, conn, func(payload []byte) { // On its own goroutine, deliberately: this runs on the hub's read // loop, the single goroutine demultiplexing every attachment // multiplexed over this session's conn, and routeControl ends in the @@ -1226,7 +1282,12 @@ const ( // sessiond that predates the notice drops it, a conn that has died takes the // whole question with it, and a cancelled dispatch is one whose suspend is // about to fail on its own. None of those is worse than not sending it at all. -func (s *Server) quiesceExecs(ctx context.Context, id string) { +// cold says which kind of suspend is coming. False is a freeze and is the +// notice this function has always sent, byte for byte: `cold` is omitempty, +// so a warm notice is unchanged for every sandbox already running. True is +// the end of this VM, and asks the sandbox for the two things a freeze does +// not need — a flush and an unmount — before it answers. +func (s *Server) quiesceExecs(ctx context.Context, id string, cold bool) { hub, ok := s.reg.hub(id) if !ok { return @@ -1235,7 +1296,7 @@ func (s *Server) quiesceExecs(ctx context.Context, id string) { w := s.armSuspendWaiter(id, nonce) defer s.disarmSuspendWaiter(id, w) - b, err := json.Marshal(relay.ControlEvent{Kind: relay.KindSuspending, ID: nonce}) + b, err := json.Marshal(relay.ControlEvent{Kind: relay.KindSuspending, ID: nonce, Cold: cold}) if err != nil { log.Printf("session %s: encoding the suspend notice: %v", id, err) return From cb1b7deea057d8a4ce88e69d17c03e990444cdc3 Mon Sep 17 00:00:00 2001 From: Josh Zhang Date: Sun, 20 Sep 2026 01:45:52 +0000 Subject: [PATCH 4/7] feat(sessiond): a guest that is told what it is, over the channel it opened MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 4, and the last: the transport seam behind dialLoop, the microVM boot over vsock, and the cold-suspend handshake from inside the guest. A session keeps the sessiond it booted with for life, which is why this is last for the microVM path and irrelevant for the Docker one. The seam is a func(context.Context) (relay.Conn, error), because that is all either transport needs. The Docker path is byte for byte what dialLoop did inline — the same URL, the same nil options, the same read limit, pinned by a test that asserts the request URI a runner actually receives. The microVM path opens AF_VSOCK to (2, 1024); everything above the seam — ServeSessionWithExec, the RPC dispatcher, the agent sync — is untouched, because all of it already took a relay.Conn. A microVM guest boots before it reads any environment at all, because it has none: it dials, reads its whole configuration off the first control frame, exchanges its bootstrap token for the secrets the create withheld, and applies both to this process's environment in the shape bootEnvFromOS already reads. Translating into that shape rather than threading the struct through is deliberate: prepareBoot, the agent sync, the exec runner's inherited environment and the agent itself all read exactly these variables on the other path, and a second way of saying the same thing is a second way for the two to diverge. The connection it bootstrapped on is the connection dialLoop serves — one channel, by design. The exchange happens when the configuration brings a token this session has not spent, and that one rule is both halves of "at boot and again after every resume". A bootstrap token is single-use: a sessiond that re-exchanged on every connection would be refused on its first redial inside a live VM and stay refused forever, and one that never re-exchanged would come back from a cold resume with nothing. A resume is exactly the connection that brings a new token, because runnerd mints one before it boots the new VM — so no second signal is needed, and a test drives a boot, a redial and a resume in one case to pin all three. A refusal, or declared secrets with no token, fails the BOOT CHAIN rather than this process: it becomes the chain's first stage, which only fails, so the existing machinery reports stage_failed with a readable tail and the agent is never exec'd. The tail names the count of undelivered names and the control plane's own condition — never a name, never a value, never the token — and a test greps it for all three. And `suspending` with cold: true is not a freeze. The sandbox acknowledges at once, flushes the agent sync (the last thing an agent wrote is usually the thing worth keeping), ends every exec, lazily unmounts /rainier/agents so the home's block device is consistent when it goes away, forgets every delivered secret by name, and only then answers ready. A warm notice is unchanged and keeps them, because a frozen session comes back with the same process and the same agent — pinned in both directions. Two things only a real host can test, and they are named in the PR body: that Firecracker forwards _1024 to the socket the driver is listening on, and that the guest kernel has CONFIG_VIRTIO_VSOCKETS and /dev/vsock. Everything above that is exercised here over a net.Pipe standing in for the conn, which is what the seam is for. Co-Authored-By: Claude Fable 5.1 --- cmd/sessiond/bootconfig.go | 419 +++++++++++++++++++++++ cmd/sessiond/bootconfig_test.go | 555 +++++++++++++++++++++++++++++++ cmd/sessiond/coldsuspend_test.go | 97 ++++++ cmd/sessiond/gitchain.go | 59 +++- cmd/sessiond/main.go | 167 +++++++++- cmd/sessiond/transport.go | 93 ++++++ cmd/sessiond/transport_test.go | 124 +++++++ cmd/sessiond/unmount_linux.go | 45 +++ cmd/sessiond/unmount_other.go | 8 + cmd/sessiond/vsock_linux.go | 56 ++++ cmd/sessiond/vsock_other.go | 16 + 11 files changed, 1619 insertions(+), 20 deletions(-) create mode 100644 cmd/sessiond/bootconfig.go create mode 100644 cmd/sessiond/bootconfig_test.go create mode 100644 cmd/sessiond/coldsuspend_test.go create mode 100644 cmd/sessiond/transport.go create mode 100644 cmd/sessiond/transport_test.go create mode 100644 cmd/sessiond/unmount_linux.go create mode 100644 cmd/sessiond/unmount_other.go create mode 100644 cmd/sessiond/vsock_linux.go create mode 100644 cmd/sessiond/vsock_other.go diff --git a/cmd/sessiond/bootconfig.go b/cmd/sessiond/bootconfig.go new file mode 100644 index 0000000..dc0ebce --- /dev/null +++ b/cmd/sessiond/bootconfig.go @@ -0,0 +1,419 @@ +package main + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "log" + "os" + "slices" + "strconv" + "sync" + "time" + + "github.com/tokencanopy/rainier/internal/relay" + "github.com/tokencanopy/rainier/protocol/runner" +) + +// The microVM boot: what a guest reads off its vsock connection before it is +// a session at all, and the one exchange that turns a capability into the +// environment secrets its create deliberately did not carry. +// +// See docs/design/2026-09-20-microvm-bootstrap-token-and-vsock.md §4. On the +// Docker path none of this runs: the container's environment block IS the +// configuration, sessiond reads it with bootEnvFromOS as it always has, and +// no secret ever has to be asked for. + +const ( + // bootConfigWait bounds how long a guest waits for its configuration. + // The host writes it as the first frame the instant the guest connects, + // so this is a bound on a host that is not going to answer rather than + // on anything that legitimately takes time. + bootConfigWait = 30 * time.Second + // secretsExchangeWait bounds the round trip to the control plane. It + // sits under the bootstrap token's own 120-second lifetime, so a refusal + // is an answer rather than an expiry. + secretsExchangeWait = 30 * time.Second + // secretsRequestID is the id this end assigns its boot-time exchange. + // The request is made before the RPC dispatcher is serving — there is no + // session to serve for yet — so it is numbered by hand, and the + // dispatcher's counter is seeded past it (see bootstrapper.exchange). + secretsRequestID = 1 +) + +// bootstrapper is the guest side of the boot: it reads a configuration off +// each connection and, when that configuration brings a token it has not +// already spent, exchanges it for the session's secrets. +// +// It is stateful for one reason, and it is the reason a redial is not a +// resume: a bootstrap token is SINGLE-USE. A sessiond that re-exchanged on +// every connection would be refused on the first redial inside a live VM and +// would keep being refused forever. What distinguishes a resume from a +// redial is that a resume brings a NEW token — runnerd mints one before it +// boots the new VM — so "exchange when the token is one I have not spent" is +// exactly "at boot and again after every resume", with no second signal +// needed. +type bootstrapper struct { + mu sync.Mutex + // spent is the token whose exchange succeeded, if any. + spent string + // delivered are the names this session has been given values for, kept + // so a cold suspend can forget them by name. NAMES, never values: what + // was delivered lives in the process environment and nowhere else. + delivered []string +} + +// bootFailure is a boot that cannot proceed, rendered as the sentence its +// failing stage prints. +// +// It names a COUNT and never a name's value — and never the names either, +// beyond how many there were, because a refusal that listed them would put +// an environment's shape in a session's error text where a workspace's +// members can read it. +type bootFailure struct{ reason string } + +func (f *bootFailure) Error() string { return f.reason } + +// undeliveredSecrets is the sentence a boot fails with when the values a +// create promised did not arrive. why is the control plane's own refusal, or +// this end's description of what was missing; neither carries a value. +func undeliveredSecrets(n int, why string) *bootFailure { + return &bootFailure{reason: fmt.Sprintf( + "rainier: this session's environment declares %d secret(s) that could not be delivered: %s. "+ + "The agent has not been started, because an environment that promised a credential and does not have it "+ + "is not the environment this session was created from.", n, why)} +} + +// readBootConfig waits for the first control frame on a fresh connection and +// requires it to be the boot configuration. +// +// Anything else on the way is dropped rather than refused: this conn is the +// session's whole channel, and a host that sent something before the +// configuration is a host this guest should keep listening to. +func readBootConfig(ctx context.Context, conn relay.Conn) (runner.BootConfig, error) { + ctx, cancel := context.WithTimeout(ctx, bootConfigWait) + defer cancel() + for { + raw, err := conn.Read(ctx) + if err != nil { + return runner.BootConfig{}, fmt.Errorf("waiting for the boot configuration: %w", err) + } + f, err := relay.Decode(raw) + if err != nil || f.Type != relay.FrameControl { + continue + } + var ev relay.ControlEvent + if err := json.Unmarshal(f.Payload, &ev); err != nil || ev.Kind != relay.KindBootConfig { + continue + } + var cfg runner.BootConfig + if err := json.Unmarshal(ev.Payload, &cfg); err != nil { + return runner.BootConfig{}, fmt.Errorf("the boot configuration could not be decoded: %w", err) + } + if cfg.Protocol != runner.SessionBootstrapProtocolVersion { + return runner.BootConfig{}, fmt.Errorf( + "this host speaks boot-configuration protocol %d and this session image speaks %d; the session image must be replaced", + cfg.Protocol, runner.SessionBootstrapProtocolVersion) + } + return cfg, nil + } +} + +// exchange turns cfg's token into the session's secrets, or says why it +// could not. +// +// It returns (nil, nil) for the two cases that are not failures: a +// configuration declaring no secrets at all, and a token this session has +// already spent — which is every redial inside one live VM. +// +// The request is written straight onto the conn rather than through the RPC +// dispatcher because the dispatcher is not serving yet: the relay it rides on +// is started later, by dialLoop, once there is a session to serve. +func (b *bootstrapper) exchange(ctx context.Context, conn relay.Conn, cfg runner.BootConfig) (map[string]string, error) { + if len(cfg.SecretNames) == 0 { + // A clean boot. "No secrets declared" and "declared and never + // arrived" are different facts, and this is the first. + return nil, nil + } + b.mu.Lock() + alreadySpent := cfg.BootstrapToken != "" && cfg.BootstrapToken == b.spent + b.mu.Unlock() + if alreadySpent { + return nil, nil + } + if cfg.BootstrapToken == "" { + return nil, undeliveredSecrets(len(cfg.SecretNames), + "this session's create carried no bootstrap token, which means a control plane that does not withhold them") + } + + body, err := json.Marshal(struct { + Protocol int `json:"protocol"` + Token string `json:"token"` + }{runner.SessionBootstrapProtocolVersion, cfg.BootstrapToken}) + if err != nil { + return nil, undeliveredSecrets(len(cfg.SecretNames), "the request could not be encoded") + } + payload, err := json.Marshal(relay.ControlEvent{ + Kind: "req:" + runner.MethodFetchSessionSecrets, ID: secretsRequestID, Payload: body}) + if err != nil { + return nil, undeliveredSecrets(len(cfg.SecretNames), "the request could not be encoded") + } + frame, err := relay.Encode(relay.Frame{Type: relay.FrameControl, Payload: payload}) + if err != nil { + return nil, undeliveredSecrets(len(cfg.SecretNames), "the request could not be encoded") + } + + ctx, cancel := context.WithTimeout(ctx, secretsExchangeWait) + defer cancel() + if err := conn.Write(ctx, frame); err != nil { + return nil, undeliveredSecrets(len(cfg.SecretNames), "the request could not be sent to the runner") + } + + for { + raw, err := conn.Read(ctx) + if err != nil { + return nil, undeliveredSecrets(len(cfg.SecretNames), "the runner did not answer") + } + f, derr := relay.Decode(raw) + if derr != nil || f.Type != relay.FrameControl { + continue + } + var ev relay.ControlEvent + if json.Unmarshal(f.Payload, &ev) != nil || ev.Kind != "resp" || ev.ID != secretsRequestID { + continue + } + if !ev.OK { + // The control plane's own sentence, relayed: it names which of + // the four conditions holds — spent, expired, fenced, unknown — + // and never a value. + return nil, undeliveredSecrets(len(cfg.SecretNames), rpcErrorText(ev.Payload)) + } + var answer struct { + Env map[string]string `json:"env"` + } + if json.Unmarshal(ev.Payload, &answer) != nil { + // Logged without the error, and reported without it: a json + // message quotes what it choked on, and that is the secret. + return nil, undeliveredSecrets(len(cfg.SecretNames), "the answer could not be decoded") + } + b.mu.Lock() + b.spent = cfg.BootstrapToken + b.mu.Unlock() + return answer.Env, nil + } +} + +// remember records which names this session has been given values for, so a +// cold suspend can unset exactly those and nothing else. +func (b *bootstrapper) remember(names []string) { + b.mu.Lock() + defer b.mu.Unlock() + for _, n := range names { + if !slices.Contains(b.delivered, n) { + b.delivered = append(b.delivered, n) + } + } +} + +// forget unsets every delivered secret from this process's environment and +// reports how many there were. +// +// It is what a cold suspend asks for. The VM is about to be terminated with +// no memory image written anywhere, so this is belt and braces rather than +// the guarantee — but it is the half this process can make, and it is free: +// a sessiond that is frozen instead of terminated (a host that sent `cold` +// and then changed its mind) has no secret left in its environment for +// whatever reads it next. +// +// It cannot reach a child that already inherited them. That is stated rather +// than papered over: the agent has the values, by design, and the thing that +// takes them away is the VM ending. +func (b *bootstrapper) forget() int { + b.mu.Lock() + names := b.delivered + b.delivered = nil + b.spent = "" + b.mu.Unlock() + for _, n := range names { + _ = os.Unsetenv(n) + } + return len(names) +} + +// applyBootConfig puts a boot configuration into this process's environment, +// in the shape the boot chain already reads (bootEnvFromOS). +// +// Translating into the environment block rather than threading the struct +// through is deliberate: everything downstream — prepareBoot, the agent +// sync, the exec runner's inherited environment, the child the agent becomes +// — already reads exactly these variables on the Docker path, and a second +// way of saying the same thing is a second way for the two paths to diverge. +// The scripts arrive as plain strings on the wire and are base64-encoded +// here for the same reason they are on the other path: that is what the +// reader expects. +// +// RAINIER_DIAL is deliberately NOT set. There is nothing to dial. +func applyBootConfig(cfg runner.BootConfig, secrets map[string]string) error { + set := func(k, v string) error { + if v == "" { + return nil + } + return os.Setenv(k, v) + } + b64 := func(s string) string { + if s == "" { + return "" + } + return base64.StdEncoding.EncodeToString([]byte(s)) + } + + // The secrets go FIRST, so that the configuration below wins over them. + // That is the same precedence the control plane already applies on the + // Docker path, where the agent-home keys are launch invariants a + // workspace's own configuration may not replace; a delivered value + // spelled like one of them must not redirect credential custody here + // either. (The plane also drops such a name from SecretNames, so this is + // the second of two fences rather than the only one.) + for k, v := range secrets { + if err := os.Setenv(k, v); err != nil { + return fmt.Errorf("applying this session's environment: %w", err) + } + } + + if err := set("RAINIER_SESSION", cfg.SessionID); err != nil { + return err + } + if cfg.ProxyURL != "" { + for _, k := range []string{"HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy"} { + if err := set(k, cfg.ProxyURL); err != nil { + return err + } + } + for _, k := range []string{"NO_PROXY", "no_proxy"} { + if err := set(k, cfg.NoProxy); err != nil { + return err + } + } + } + if cfg.Setup != "" { + if err := set("RAINIER_SETUP_B64", b64(cfg.Setup)); err != nil { + return err + } + if err := set("RAINIER_SETUP_TIMEOUT", strconv.Itoa(cfg.SetupTimeoutSec)); err != nil { + return err + } + } + if cfg.Init != "" { + if err := set("RAINIER_INIT_B64", b64(cfg.Init)); err != nil { + return err + } + if err := set("RAINIER_INIT_TIMEOUT", strconv.Itoa(cfg.InitTimeoutSec)); err != nil { + return err + } + } + if len(cfg.Repos) > 0 { + blob, err := json.Marshal(cfg.Repos) + if err != nil { + return fmt.Errorf("encoding this session's repository list: %w", err) + } + if err := set("RAINIER_REPOS_B64", base64.StdEncoding.EncodeToString(blob)); err != nil { + return err + } + } + if err := set("RAINIER_GIT_AUTHOR_NAME", cfg.GitAuthorName); err != nil { + return err + } + if err := set("RAINIER_GIT_AUTHOR_EMAIL", cfg.GitAuthorEmail); err != nil { + return err + } + for k, v := range cfg.Env { + if err := os.Setenv(k, v); err != nil { + return fmt.Errorf("applying this session's configuration: %w", err) + } + } + return nil +} + +// bootOverVsock is the whole microVM boot preamble, run once before this +// process has a session at all: dial, read the configuration, exchange the +// token, apply both. +// +// It returns the connection it bootstrapped on, which dialLoop then serves +// its relay over — the SAME conn, because the boot configuration and the +// terminal stream and the session RPC all ride one channel by design, and +// because a second dial would be a second guest for a socket that serves one. +// +// A failed exchange is returned as a *bootFailure and NOT as a dial error: +// the connection is good, the session is real, and what has to happen is +// that the boot chain fails loudly enough for a person to see why. The +// caller turns it into a failing stage. +func bootOverVsock(ctx context.Context, dial dialSession, b *bootstrapper) (relay.Conn, runner.BootConfig, *bootFailure, error) { + conn, err := dial(ctx) + if err != nil { + return nil, runner.BootConfig{}, nil, err + } + cfg, err := readBootConfig(ctx, conn) + if err != nil { + _ = conn.Close() + return nil, runner.BootConfig{}, nil, err + } + secrets, xerr := b.exchange(ctx, conn, cfg) + var failure *bootFailure + if xerr != nil { + if bf, ok := xerr.(*bootFailure); ok { + failure = bf + } else { + _ = conn.Close() + return nil, cfg, nil, xerr + } + } + if err := applyBootConfig(cfg, secrets); err != nil { + _ = conn.Close() + return nil, cfg, nil, err + } + b.remember(namesOf(secrets)) + log.Printf("sessiond booted as %s over vsock with %d declared secret(s), %d delivered", + cfg.SessionID, len(cfg.SecretNames), len(secrets)) + return conn, cfg, failure, nil +} + +// reBootstrap is the preamble on every LATER connection: read the +// configuration again, and exchange again when it brings a token this +// session has not spent — which is what a cold resume brings and a redial +// does not. +// +// A failure here is an error rather than a failed boot chain: the session is +// already running, its agent already has whatever it was given, and the +// answer to a refused re-exchange is to try the next connection rather than +// to tear down a live session. +func reBootstrap(ctx context.Context, conn relay.Conn, b *bootstrapper) error { + cfg, err := readBootConfig(ctx, conn) + if err != nil { + return err + } + secrets, xerr := b.exchange(ctx, conn, cfg) + if xerr != nil { + return xerr + } + if len(secrets) == 0 { + return nil + } + if err := applyBootConfig(cfg, secrets); err != nil { + return err + } + b.remember(namesOf(secrets)) + log.Printf("sessiond re-applied %d environment secret(s) after a resume", len(secrets)) + return nil +} + +// namesOf returns a map's keys. Names, which are not values — the same +// distinction the launch-material resolver and the create both make. +func namesOf(m map[string]string) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + slices.Sort(out) + return out +} diff --git a/cmd/sessiond/bootconfig_test.go b/cmd/sessiond/bootconfig_test.go new file mode 100644 index 0000000..75e7f94 --- /dev/null +++ b/cmd/sessiond/bootconfig_test.go @@ -0,0 +1,555 @@ +package main + +import ( + "context" + "encoding/base64" + "encoding/json" + "net" + "os" + "strings" + "testing" + "time" + + "github.com/tokencanopy/rainier/internal/relay" + "github.com/tokencanopy/rainier/protocol/runner" +) + +// The microVM boot, over a net.Pipe standing in for the vsock conn. +// +// There is no VM and no KVM here, and that is the point of the seam: the +// transport is a func returning a relay.Conn, so everything above it — the +// boot configuration, the exchange, what lands in this process's environment +// — is exercised end to end against a fake host. What only a real host can +// answer is that Firecracker forwards _1024 to the socket and that +// the guest kernel has /dev/vsock; see the PR body. + +// fakeHost is the runner's end of a guest's control channel: it writes the +// boot configuration as the first frame and answers (or refuses) the one +// exchange the guest makes. +type fakeHost struct { + conn relay.Conn + t *testing.T +} + +func (h *fakeHost) sendBootConfig(cfg runner.BootConfig) { + h.t.Helper() + body, err := json.Marshal(cfg) + if err != nil { + h.t.Fatal(err) + } + h.sendControl(relay.ControlEvent{Kind: relay.KindBootConfig, Payload: body}) +} + +func (h *fakeHost) sendControl(ev relay.ControlEvent) { + h.t.Helper() + payload, err := json.Marshal(ev) + if err != nil { + h.t.Fatal(err) + } + frame, err := relay.Encode(relay.Frame{Type: relay.FrameControl, Payload: payload}) + if err != nil { + h.t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := h.conn.Write(ctx, frame); err != nil { + h.t.Fatalf("write to the guest: %v", err) + } +} + +// nextRequest reads the next control REQUEST the guest sent, failing the +// test if none arrives. +func (h *fakeHost) nextRequest() relay.ControlEvent { + h.t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + for { + raw, err := h.conn.Read(ctx) + if err != nil { + h.t.Fatalf("read from the guest: %v", err) + } + f, err := relay.Decode(raw) + if err != nil || f.Type != relay.FrameControl { + continue + } + var ev relay.ControlEvent + if json.Unmarshal(f.Payload, &ev) != nil { + continue + } + if strings.HasPrefix(ev.Kind, "req:") { + return ev + } + } +} + +// nothingMore asserts the guest sent no further request within a short +// window. It is how "exactly once" is checked. +func (h *fakeHost) nothingMore(within time.Duration) bool { + h.t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), within) + defer cancel() + for { + raw, err := h.conn.Read(ctx) + if err != nil { + return true // the window closed with nothing on it + } + f, derr := relay.Decode(raw) + if derr != nil || f.Type != relay.FrameControl { + continue + } + var ev relay.ControlEvent + if json.Unmarshal(f.Payload, &ev) != nil { + continue + } + if strings.HasPrefix(ev.Kind, "req:") { + return false + } + } +} + +func (h *fakeHost) answer(id uint64, env map[string]string) { + h.t.Helper() + body, err := json.Marshal(struct { + Env map[string]string `json:"env"` + }{env}) + if err != nil { + h.t.Fatal(err) + } + h.sendControl(relay.ControlEvent{Kind: "resp", ID: id, OK: true, Payload: body}) +} + +func (h *fakeHost) refuse(id uint64, reason string) { + h.t.Helper() + body, err := json.Marshal(struct { + Error string `json:"error"` + }{reason}) + if err != nil { + h.t.Fatal(err) + } + h.sendControl(relay.ControlEvent{Kind: "resp", ID: id, Payload: body}) +} + +// fakeTransport returns a dialer that hands out one pipe per dial and the +// host ends of those pipes, in order. +func fakeTransport(t *testing.T) (dialSession, <-chan *fakeHost) { + hosts := make(chan *fakeHost, 4) + return func(context.Context) (relay.Conn, error) { + guest, host := net.Pipe() + t.Cleanup(func() { _ = guest.Close(); _ = host.Close() }) + hosts <- &fakeHost{conn: relay.NetConn(host), t: t} + return relay.NetConn(guest), nil + }, hosts +} + +// cleanEnv unsets every variable the boot might set, before and after, so a +// case reads what THIS boot applied and not what the last one left. +func cleanEnv(t *testing.T) { + t.Helper() + names := []string{ + "RAINIER_SESSION", "RAINIER_DIAL", + "HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy", "NO_PROXY", "no_proxy", + "RAINIER_SETUP_B64", "RAINIER_SETUP_TIMEOUT", "RAINIER_INIT_B64", "RAINIER_INIT_TIMEOUT", + "RAINIER_REPOS_B64", "RAINIER_GIT_AUTHOR_NAME", "RAINIER_GIT_AUTHOR_EMAIL", + "CLAUDE_CONFIG_DIR", "RAINIER_AGENTS_B64", "DEPLOY_KEY", "NPM_TOKEN", + } + unset := func() { + for _, n := range names { + _ = os.Unsetenv(n) + } + } + unset() + t.Cleanup(unset) +} + +// TestBootOverVsockAppliesTheConfigurationAndTheSecrets is the whole boot in +// one case: the guest reads a configuration off its first frame, exchanges +// its token exactly once, and ends up with an environment block in the shape +// the boot chain already reads — which is what makes everything downstream +// (prepareBoot, the agent sync, the exec runner, the agent itself) identical +// to the Docker path. +func TestBootOverVsockAppliesTheConfigurationAndTheSecrets(t *testing.T) { + cleanEnv(t) + dial, hosts := fakeTransport(t) + boots := &bootstrapper{} + + cfg := runner.BootConfig{ + Protocol: runner.SessionBootstrapProtocolVersion, + SessionID: "sess_example", + Cmd: []string{"claude"}, + ProxyURL: "http://sess_example:rainier@proxy.invalid:3128", + NoProxy: "127.0.0.1,localhost", + Setup: "npm ci\n", + SetupTimeoutSec: 600, + Init: "make dev\n", + InitTimeoutSec: 180, + Repos: []runner.RepoSpec{{Owner: "acme", Name: "app", BaseBranch: "main", + SessionBranch: "rainier/work", Dir: "app"}}, + GitAuthorName: "example", + GitAuthorEmail: "42+example@users.noreply.github.com", + Env: map[string]string{"CLAUDE_CONFIG_DIR": "/rainier/agents/claude"}, + SecretNames: []string{"DEPLOY_KEY", "NPM_TOKEN"}, + BootstrapToken: "token_example", + } + + done := make(chan struct{}) + var ( + conn relay.Conn + failure *bootFailure + bootErr error + ) + go func() { + defer close(done) + conn, _, failure, bootErr = bootOverVsock(context.Background(), dial, boots) + }() + + host := <-hosts + host.sendBootConfig(cfg) + req := host.nextRequest() + if req.Kind != "req:"+runner.MethodFetchSessionSecrets { + t.Fatalf("the guest asked %q, want the secret fetch", req.Kind) + } + var body struct { + Protocol int `json:"protocol"` + Token string `json:"token"` + } + if err := json.Unmarshal(req.Payload, &body); err != nil { + t.Fatal(err) + } + if body.Protocol != runner.SessionBootstrapProtocolVersion || body.Token != "token_example" { + t.Fatalf("the request carried %+v", body) + } + host.answer(req.ID, map[string]string{"DEPLOY_KEY": "value_example", "NPM_TOKEN": "other_value_example"}) + + <-done + if bootErr != nil { + t.Fatalf("boot: %v", bootErr) + } + if failure != nil { + t.Fatalf("the boot reported a failure: %v", failure) + } + if conn == nil { + t.Fatal("the boot returned no connection to serve") + } + + // The configuration, in the shape bootEnvFromOS reads. + env := bootEnvFromOS() + if env.SetupB64 != base64.StdEncoding.EncodeToString([]byte("npm ci\n")) || env.SetupTimeout != "600" { + t.Errorf("setup = %q/%q", env.SetupB64, env.SetupTimeout) + } + if env.InitB64 != base64.StdEncoding.EncodeToString([]byte("make dev\n")) || env.InitTimeout != "180" { + t.Errorf("init = %q/%q", env.InitB64, env.InitTimeout) + } + if env.GitAuthorName != "example" || env.GitAuthorEmail != "42+example@users.noreply.github.com" { + t.Errorf("git identity = %q <%q>", env.GitAuthorName, env.GitAuthorEmail) + } + repos, err := decodeRepos(env.ReposB64) + if err != nil || len(repos) != 1 || repos[0].Dir != "app" { + t.Errorf("repos = %+v (%v)", repos, err) + } + if os.Getenv("RAINIER_SESSION") != "sess_example" { + t.Errorf("RAINIER_SESSION = %q", os.Getenv("RAINIER_SESSION")) + } + if os.Getenv("CLAUDE_CONFIG_DIR") != "/rainier/agents/claude" { + t.Errorf("the agent home configuration was not applied") + } + for _, k := range []string{"HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy"} { + if os.Getenv(k) != cfg.ProxyURL { + t.Errorf("%s = %q, want the proxy", k, os.Getenv(k)) + } + } + if os.Getenv("NO_PROXY") != "127.0.0.1,localhost" { + t.Errorf("NO_PROXY = %q", os.Getenv("NO_PROXY")) + } + // There is nothing to dial, and the guest must not think there is. + if v := os.Getenv("RAINIER_DIAL"); v != "" { + t.Errorf("RAINIER_DIAL = %q on a session with no URL to dial", v) + } + // And the secrets, which arrived over the exchange and from nowhere else. + if os.Getenv("DEPLOY_KEY") != "value_example" || os.Getenv("NPM_TOKEN") != "other_value_example" { + t.Error("the delivered secrets did not reach this process's environment") + } + + // Exactly once: the token is single-use, so a second exchange on the same + // token would be refused and would fail a boot that had already worked. + if !host.nothingMore(200 * time.Millisecond) { + t.Fatal("the guest made a second request on one boot") + } +} + +// TestBootOverVsockWithNoSecretsDeclaredIsACleanBoot pins the difference +// between "none declared" and "declared and never arrived": the first asks +// for nothing at all. +func TestBootOverVsockWithNoSecretsDeclaredIsACleanBoot(t *testing.T) { + cleanEnv(t) + dial, hosts := fakeTransport(t) + boots := &bootstrapper{} + + done := make(chan struct{}) + var failure *bootFailure + var bootErr error + go func() { + defer close(done) + _, _, failure, bootErr = bootOverVsock(context.Background(), dial, boots) + }() + + host := <-hosts + host.sendBootConfig(runner.BootConfig{ + Protocol: runner.SessionBootstrapProtocolVersion, SessionID: "sess_example", + BootstrapToken: "token_example", + }) + <-done + if bootErr != nil || failure != nil { + t.Fatalf("a session declaring no secrets did not boot cleanly: %v / %v", bootErr, failure) + } + if !host.nothingMore(200 * time.Millisecond) { + t.Fatal("a session declaring no secrets asked for some anyway") + } +} + +// TestARefusedExchangeFailsTheBootChain is the compatibility table's last +// row, and the one that decides what a user sees. A refusal must not boot an +// agent into an environment that promised a credential it does not have; it +// must fail the boot chain, as the existing stage_failed, naming the COUNT +// of undelivered names and none of their values. +func TestARefusedExchangeFailsTheBootChain(t *testing.T) { + cleanEnv(t) + dial, hosts := fakeTransport(t) + boots := &bootstrapper{} + + done := make(chan struct{}) + var failure *bootFailure + var bootErr error + go func() { + defer close(done) + _, _, failure, bootErr = bootOverVsock(context.Background(), dial, boots) + }() + + host := <-hosts + host.sendBootConfig(runner.BootConfig{ + Protocol: runner.SessionBootstrapProtocolVersion, SessionID: "sess_example", + SecretNames: []string{"DEPLOY_KEY", "NPM_TOKEN"}, BootstrapToken: "token_example", + }) + req := host.nextRequest() + host.refuse(req.ID, "this session's bootstrap token has already been exchanged") + + <-done + if bootErr != nil { + t.Fatalf("a refusal was reported as a boot error rather than a failed chain: %v", bootErr) + } + if failure == nil { + t.Fatal("a refused exchange did not fail the boot") + } + // The count, the plane's own condition, and no value or name. + if !strings.Contains(failure.Error(), "2 secret(s)") { + t.Errorf("the failure does not name the count: %q", failure) + } + if !strings.Contains(failure.Error(), "already been exchanged") { + t.Errorf("the failure drops the control plane's reason: %q", failure) + } + for _, leaked := range []string{"DEPLOY_KEY", "NPM_TOKEN", "token_example"} { + if strings.Contains(failure.Error(), leaked) { + t.Errorf("the failure carries %q: %q", leaked, failure) + } + } + + // And it becomes a STAGE: the first one, which only fails, so the agent + // is never exec'd and the watcher reports stage_failed with this tail. + dir := t.TempDir() + stages, _, err := prepareBoot(dir, "/workspace", bootEnv{SecretsFailure: failure.Error(), SetupB64: "ZWNobyBoaQo="}) + if err != nil { + t.Fatal(err) + } + if len(stages) != 2 || stages[0].Name != stageSecrets { + t.Fatalf("stages = %v, want the secrets stage first", stageNames(stages)) + } + script, err := os.ReadFile(stages[0].ScriptPath) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(script), "exit 1") { + t.Fatalf("the secrets stage does not fail:\n%s", script) + } +} + +// TestAMissingTokenWithDeclaredSecretsFailsTheBootChain is the same row from +// the other direction: an older control plane that declared nothing and +// withheld nothing sends no token, and a guest whose configuration names +// secrets it was given no way to fetch must not start either. +func TestAMissingTokenWithDeclaredSecretsFailsTheBootChain(t *testing.T) { + cleanEnv(t) + dial, hosts := fakeTransport(t) + boots := &bootstrapper{} + + done := make(chan struct{}) + var failure *bootFailure + go func() { + defer close(done) + _, _, failure, _ = bootOverVsock(context.Background(), dial, boots) + }() + + host := <-hosts + host.sendBootConfig(runner.BootConfig{ + Protocol: runner.SessionBootstrapProtocolVersion, SessionID: "sess_example", + SecretNames: []string{"DEPLOY_KEY"}, + }) + <-done + if failure == nil { + t.Fatal("a configuration declaring secrets with no token booted anyway") + } + if !strings.Contains(failure.Error(), "1 secret(s)") || !strings.Contains(failure.Error(), "no bootstrap token") { + t.Errorf("the failure = %q", failure) + } + if !host.nothingMore(200 * time.Millisecond) { + t.Fatal("the guest asked for its secrets with no token to ask with") + } +} + +// TestAResumeReExchangesAndARedialDoesNot is the rule that makes single use +// workable. +// +// A bootstrap token is spent by its first exchange. A sessiond that +// re-exchanged on every connection would be refused on the first redial +// inside a live VM and stay refused forever; one that never re-exchanged +// would come back from a cold resume with no secrets. What tells the two +// apart is the token itself: a resume brings a new one, a redial brings the +// one already spent. +func TestAResumeReExchangesAndARedialDoesNot(t *testing.T) { + cleanEnv(t) + dial, hosts := fakeTransport(t) + boots := &bootstrapper{} + + // Boot. + done := make(chan struct{}) + go func() { + defer close(done) + _, _, _, _ = bootOverVsock(context.Background(), dial, boots) + }() + host := <-hosts + cfg := runner.BootConfig{ + Protocol: runner.SessionBootstrapProtocolVersion, SessionID: "sess_example", + SecretNames: []string{"DEPLOY_KEY"}, BootstrapToken: "token_from_the_create", + } + host.sendBootConfig(cfg) + first := host.nextRequest() + host.answer(first.ID, map[string]string{"DEPLOY_KEY": "value_example"}) + <-done + + // A REDIAL inside the same VM: the same configuration, the same token. + // Nothing is asked for, because the token has been spent and the values + // are already in this process. + redialConn, err := dial(context.Background()) + if err != nil { + t.Fatal(err) + } + redialHost := <-hosts + redialDone := make(chan error, 1) + go func() { redialDone <- reBootstrap(context.Background(), redialConn, boots) }() + redialHost.sendBootConfig(cfg) + if err := <-redialDone; err != nil { + t.Fatalf("a redial failed: %v", err) + } + if !redialHost.nothingMore(200 * time.Millisecond) { + t.Fatal("a redial re-exchanged a spent token, which would be refused forever") + } + + // A RESUME: a new VM, a new socket, a new token. The secrets are fetched + // again, because the process on this side may be a completely new one + // and the values it holds may be from a boot that no longer exists. + _ = os.Unsetenv("DEPLOY_KEY") + resumeConn, err := dial(context.Background()) + if err != nil { + t.Fatal(err) + } + resumeHost := <-hosts + resumeDone := make(chan error, 1) + go func() { resumeDone <- reBootstrap(context.Background(), resumeConn, boots) }() + resumed := cfg + resumed.BootstrapToken = "token_from_the_resume" + resumeHost.sendBootConfig(resumed) + req := resumeHost.nextRequest() + var body struct { + Token string `json:"token"` + } + if err := json.Unmarshal(req.Payload, &body); err != nil { + t.Fatal(err) + } + if body.Token != "token_from_the_resume" { + t.Fatalf("the resume exchanged %q, want the freshly minted token", body.Token) + } + resumeHost.answer(req.ID, map[string]string{"DEPLOY_KEY": "value_after_the_resume"}) + if err := <-resumeDone; err != nil { + t.Fatalf("a resume failed: %v", err) + } + if os.Getenv("DEPLOY_KEY") != "value_after_the_resume" { + t.Errorf("the resumed session's secret = %q", os.Getenv("DEPLOY_KEY")) + } +} + +// TestForgetUnsetsEveryDeliveredSecret is the cold suspend's third act. The +// VM is about to end with no memory image anywhere, so this is belt and +// braces — but it is the half this process can make, and a sessiond that +// was frozen instead of terminated has nothing left in its environment. +func TestForgetUnsetsEveryDeliveredSecret(t *testing.T) { + cleanEnv(t) + boots := &bootstrapper{} + cfg := runner.BootConfig{Protocol: runner.SessionBootstrapProtocolVersion, SessionID: "sess_example"} + if err := applyBootConfig(cfg, map[string]string{"DEPLOY_KEY": "value_example", "NPM_TOKEN": "other"}); err != nil { + t.Fatal(err) + } + boots.remember([]string{"DEPLOY_KEY", "NPM_TOKEN"}) + if os.Getenv("DEPLOY_KEY") == "" { + t.Fatal("the fixture did not apply anything, so forgetting it proves nothing") + } + if n := boots.forget(); n != 2 { + t.Fatalf("forgot %d secret(s), want 2", n) + } + for _, k := range []string{"DEPLOY_KEY", "NPM_TOKEN"} { + if v, set := os.LookupEnv(k); set { + t.Errorf("%s survived the cold suspend as %q", k, v) + } + } + // And the session id, which is configuration and not a secret, is still + // there: forgetting is by name and not a blanket wipe. + if os.Getenv("RAINIER_SESSION") != "sess_example" { + t.Error("forgetting the secrets also dropped the session's own configuration") + } +} + +// TestAConfigurationWinsOverADeliveredSecret pins the precedence the control +// plane already applies on the Docker path: the agent-home keys are launch +// invariants, and a delivered value spelled like one of them must not +// redirect credential custody. The plane also drops such a name from +// SecretNames, so this is the second of two fences. +func TestAConfigurationWinsOverADeliveredSecret(t *testing.T) { + cleanEnv(t) + cfg := runner.BootConfig{ + Protocol: runner.SessionBootstrapProtocolVersion, SessionID: "sess_example", + Env: map[string]string{"CLAUDE_CONFIG_DIR": "/rainier/agents/claude"}, + } + if err := applyBootConfig(cfg, map[string]string{"CLAUDE_CONFIG_DIR": "/workspace/attacker"}); err != nil { + t.Fatal(err) + } + if got := os.Getenv("CLAUDE_CONFIG_DIR"); got != "/rainier/agents/claude" { + t.Fatalf("CLAUDE_CONFIG_DIR = %q; a delivered value replaced the agent home", got) + } +} + +// TestAMisversionedBootConfigIsRefused pins that a guest booted from an +// image older than the host's configuration protocol says so rather than +// running on a configuration it has half understood. +func TestAMisversionedBootConfigIsRefused(t *testing.T) { + cleanEnv(t) + dial, hosts := fakeTransport(t) + conn, err := dial(context.Background()) + if err != nil { + t.Fatal(err) + } + host := <-hosts + go host.sendBootConfig(runner.BootConfig{Protocol: 99, SessionID: "sess_example"}) + + if _, err := readBootConfig(context.Background(), conn); err == nil { + t.Fatal("a configuration from an unknown protocol was accepted") + } else if !strings.Contains(err.Error(), "must be replaced") { + t.Errorf("error = %q, want it to say which side has to change", err) + } +} diff --git a/cmd/sessiond/coldsuspend_test.go b/cmd/sessiond/coldsuspend_test.go new file mode 100644 index 0000000..3a0a95b --- /dev/null +++ b/cmd/sessiond/coldsuspend_test.go @@ -0,0 +1,97 @@ +package main + +import ( + "encoding/json" + "os" + "testing" + + "github.com/tokencanopy/rainier/internal/relay" + "github.com/tokencanopy/rainier/protocol/runner" +) + +// The cold half of the suspend handshake, wired the way main wires it, so +// what is exercised is the frame's whole journey — OnControl's kind +// dispatch, the `cold` flag, the handler, and the two answers — rather than +// quiesceCold called directly. +func coldWired(execs execKiller, boots *bootstrapper) (*rpcDispatcher, *recordingSender) { + d := newRPCDispatcher() + sender := &recordingSender{} + d.online(sender) + d.RegisterEventHandler(relay.KindSuspending, func(ev relay.ControlEvent) { + if ev.Cold { + quiesceCold(execs, d, nil, boots, ev.ID) + return + } + quiesceExecs(execs, d, ev.ID) + }) + return d, sender +} + +func suspendingFrame(t *testing.T, cold bool, nonce uint64) []byte { + t.Helper() + b, err := json.Marshal(relay.ControlEvent{Kind: relay.KindSuspending, ID: nonce, Cold: cold}) + if err != nil { + t.Fatal(err) + } + return b +} + +// TestAColdSuspendForgetsTheDeliveredSecrets is §4.4 from inside the guest: +// a cold notice is not a freeze, so the sandbox ends its execs AND forgets +// every secret the bootstrap exchange delivered before it answers ready. +func TestAColdSuspendForgetsTheDeliveredSecrets(t *testing.T) { + cleanEnv(t) + boots := &bootstrapper{} + if err := applyBootConfig( + runner.BootConfig{Protocol: runner.SessionBootstrapProtocolVersion, SessionID: "sess_example"}, + map[string]string{"DEPLOY_KEY": "value_example"}); err != nil { + t.Fatal(err) + } + boots.remember([]string{"DEPLOY_KEY"}) + + execs := &recordingExecs{} + d, sender := coldWired(execs, boots) + d.OnControl(suspendingFrame(t, true, 91)) + + if _, set := os.LookupEnv("DEPLOY_KEY"); set { + t.Error("a cold suspend left a delivered secret in this process's environment") + } + if calls, budget := execs.state(); calls != 1 || budget != execQuiesceBudget { + t.Fatalf("the cold suspend killed the execs %d time(s) with budget %s", calls, budget) + } + got := sender.events() + if len(got) != 2 || got[0].Kind != relay.KindSuspendAck || got[1].Kind != relay.KindSuspendReady { + t.Fatalf("the sandbox answered %+v, want an ack and then a ready", got) + } + for _, ev := range got { + if ev.ID != 91 { + t.Fatalf("an answer carries nonce %d, want the notice's 91", ev.ID) + } + } +} + +// TestAWarmSuspendKeepsTheDeliveredSecrets is the other side of the same +// flag, and the reason it exists: a freeze is not the end of this VM, so the +// session comes back with the same process, the same agent, and the same +// environment. Forgetting there would break a warm resume for nothing. +func TestAWarmSuspendKeepsTheDeliveredSecrets(t *testing.T) { + cleanEnv(t) + boots := &bootstrapper{} + if err := applyBootConfig( + runner.BootConfig{Protocol: runner.SessionBootstrapProtocolVersion, SessionID: "sess_example"}, + map[string]string{"DEPLOY_KEY": "value_example"}); err != nil { + t.Fatal(err) + } + boots.remember([]string{"DEPLOY_KEY"}) + + execs := &recordingExecs{} + d, sender := coldWired(execs, boots) + d.OnControl(suspendingFrame(t, false, 92)) + + if os.Getenv("DEPLOY_KEY") != "value_example" { + t.Error("a warm suspend forgot a delivered secret the resumed session still needs") + } + if got := sender.events(); len(got) != 2 { + t.Fatalf("the sandbox answered %+v, want an ack and then a ready", got) + } +} diff --git a/cmd/sessiond/gitchain.go b/cmd/sessiond/gitchain.go index fabbab1..c4931ad 100644 --- a/cmd/sessiond/gitchain.go +++ b/cmd/sessiond/gitchain.go @@ -44,19 +44,28 @@ import ( // The stage names. They are wire-visible: they travel in ControlEvent.Stage // and controld composes a session's error text out of them. const ( - stageSetup = "setup" - stageClone = "clone" - stageAgents = "agents" - stageInit = "init" + // stageSecrets is FIRST in the chain and exists only to fail. A microVM + // session whose environment declared secrets it could not be given is + // not the environment it was created from, so the agent must not start — + // and the way a boot chain refuses to start the agent is a stage that + // exits non-zero, which the watcher then reports as stage_failed with a + // tail a person can read. See bootconfig.go. + stageSecrets = "secrets" + stageSetup = "setup" + stageClone = "clone" + stageAgents = "agents" + stageInit = "init" ) // The chain's files, all in the session's own .rainier directory beside the // setup script Plan 4 put there. const ( - clonesScriptName = "clones.sh" - cloneRCName = "clone.rc" - agentsScriptName = "agents.sh" - agentsRCName = "agents.rc" + secretsScriptName = "secrets.sh" + secretsRCName = "secrets.rc" + clonesScriptName = "clones.sh" + cloneRCName = "clone.rc" + agentsScriptName = "agents.sh" + agentsRCName = "agents.rc" // agentsDoneName is the marker sessiond writes when the agent homes are as // ready as they are going to get, and the file the agents stage waits for. // It lives beside the rc files, on the same persistent volume, which is why @@ -147,6 +156,20 @@ type bootEnv struct { GitAuthorName string GitAuthorEmail string + + // SecretsFailure is the one field that does not come from the + // environment block: it is set by the microVM boot when the values this + // session's create promised could not be delivered (bootconfig.go). + // + // It becomes the first stage of the chain, and that stage only fails. + // Putting it here rather than handling it in main is what makes it a + // STAGE — reported as stage_failed with a tail, with the agent never + // exec'd — through exactly the machinery a failed setup or a failed + // clone already goes through, rather than through a second path that + // would have to be kept in step with it. + // + // It names a count and never a value; see undeliveredSecrets. + SecretsFailure string } // bootEnvFromOS reads the variables the driver injects (internal/driver @@ -175,7 +198,8 @@ func bootEnvFromOS() bootEnv { // design's intent: a home nobody waited for is a login that lands after the // agent already read its configuration. func (e bootEnv) any() bool { - return e.SetupB64 != "" || e.ReposB64 != "" || e.InitB64 != "" || e.AgentsB64 != "" + return e.SetupB64 != "" || e.ReposB64 != "" || e.InitB64 != "" || e.AgentsB64 != "" || + e.SecretsFailure != "" } // git reports whether git will run in this session — the clone stage, or an @@ -337,6 +361,23 @@ func prepareBoot(dir, root string, env bootEnv) ([]bootStage, []envVar, error) { } var stages []bootStage + // First, and only when the microVM boot could not get what this + // session's create promised it. Everything after it is skipped by the + // chain's own `exit $rc`, which is the point: an environment missing its + // credentials must not run a setup script, clone a repository, or start + // an agent that will fail at whatever it reaches for first. + if env.SecretsFailure != "" { + if err := writeStageScript(dir, secretsScriptName, secretsRCName, + []byte(failingScript(env.SecretsFailure))); err != nil { + return nil, nil, err + } + stages = append(stages, bootStage{ + Name: stageSecrets, + ScriptPath: dir + "/" + secretsScriptName, + RCPath: dir + "/" + secretsRCName, + Timeout: stageTimeout(""), + }) + } if env.SetupB64 != "" { if err := prepareSetup(dir, env.SetupB64); err != nil { return nil, nil, err diff --git a/cmd/sessiond/main.go b/cmd/sessiond/main.go index 4929c72..e7237ac 100644 --- a/cmd/sessiond/main.go +++ b/cmd/sessiond/main.go @@ -18,8 +18,6 @@ import ( "syscall" "time" - "github.com/coder/websocket" - "github.com/tokencanopy/rainier/internal/eventlog" "github.com/tokencanopy/rainier/internal/reap" "github.com/tokencanopy/rainier/internal/relay" @@ -60,6 +58,8 @@ func main() { rows := flag.Int("rows", 32, "initial rows") dial := flag.String("dial", "", "runnerd URL to dial and register with (relay mode)") sessionID := flag.String("session", "", "session id to register as (relay mode)") + transport := flag.String("transport", envOr("RAINIER_TRANSPORT", transportWebSocket), + "how to reach the runner: websocket (the default, and every Docker session) or vsock (a microVM guest, which receives its whole configuration over the channel rather than in its environment)") flag.Parse() argv := flag.Args() if len(argv) == 0 { @@ -82,6 +82,49 @@ func main() { } } + // The microVM boot, before anything reads the environment block — + // because on this path there IS no environment block until this runs. + // The guest opens its vsock channel, reads its whole configuration off + // the first frame, exchanges its bootstrap token for the secrets its + // create deliberately withheld, and puts both into this process's + // environment in the shape everything below already reads. + // + // The connection it bootstrapped on is kept and served: the + // configuration, the terminal stream and the session RPC all ride one + // channel by design, and a second dial would be a second guest for a + // socket that serves one. + var ( + dialer dialSession + preamble func(context.Context, relay.Conn) error + firstConn relay.Conn + boots = &bootstrapper{} + // secretsFailure is a boot that must not start an agent: the + // environment declared credentials this session could not be given. + // It becomes the chain's first stage, which only fails. + secretsFailure string + ) + if *transport == transportVsock { + dialer = vsockTransport() + preamble = func(ctx context.Context, c relay.Conn) error { return reBootstrap(ctx, c, boots) } + conn, cfg, failure, err := bootOverVsock(context.Background(), dialer, boots) + if err != nil { + // Deliberately fatal, and the same judgement prepareBoot's own + // failure gets: a guest that could not read its configuration + // does not know what session it is, what to run, or where its + // egress goes. Dying is what makes the runner notice. + log.Fatalf("microvm boot: %v", err) + } + firstConn = conn + if *sessionID == "" { + *sessionID = cfg.SessionID + } + if failure != nil { + secretsFailure = failure.Error() + } + } else if *transport != transportWebSocket { + log.Fatalf("unknown --transport %q (valid: %s, %s)", *transport, transportWebSocket, transportVsock) + } + // The boot chain (design §4.3). An environment's setup script (Plan 4), the // repositories controld resolved, and the environment's per-boot init hook // all arrive as base64 in the environment block, injected by the driver. @@ -97,8 +140,13 @@ func main() { // exists on a dialed conn, the credential the clone stage needs can only be // minted over it, and the local dev listener has no runnerd to reach. bootEnvironment := bootEnvFromOS() + bootEnvironment.SecretsFailure = secretsFailure + // A vsock session is in relay mode by construction: it HAS a connection, + // it just did not get it from a URL. Everything gated on relay mode + // below reads this rather than the flag. + relayMode := *dial != "" || firstConn != nil var stages []bootStage - if bootEnvironment.any() && *dial == "" { + if bootEnvironment.any() && !relayMode { // Can't happen from the driver (it injects RAINIER_DIAL alongside), but // a human running sessiond by hand with the vars set would otherwise // get an environment silently missing its setup and its repositories. @@ -134,7 +182,7 @@ func main() { // carried no manifest — a session with no creator, or an older controld — // and then nothing in agents.go runs at all. var agents *agentSync - if *dial != "" { + if relayMode { rpc = newRPCDispatcher() startAgentSocket(context.Background(), agentSocketPath, rpc, events) // The workspace-inspection methods controld drives INTO this sandbox: @@ -234,6 +282,14 @@ func main() { // the path users actually take. if rpc != nil { rpc.RegisterEventHandler(relay.KindSuspending, func(ev relay.ControlEvent) { + if ev.Cold { + // Not a freeze: this VM is ending, with no memory image + // written anywhere. Everything that has to survive has to be + // on a disk by the time this answers, and everything that + // must not survive has to be gone. + quiesceCold(execs, rpc, agents, boots, ev.ID) + return + } quiesceExecs(execs, rpc, ev.ID) }) } @@ -278,11 +334,16 @@ func main() { os.Exit(0) }() - if *dial != "" { + if relayMode { if len(stages) > 0 { startStageWatcher(stageCtx, s.Stop, stages, *logPath, events) } - dialLoop(context.Background(), *dial, *sessionID, s, events, execCounts, rpc, execs) + if dialer == nil { + dialer = websocketTransport(*dial, *sessionID) + } + dialLoop(context.Background(), sessionTransport{ + dial: dialer, preamble: preamble, first: firstConn, name: *sessionID, + }, s, events, execCounts, rpc, execs) return } @@ -317,15 +378,53 @@ func main() { // conn's sender for as long as it lives. The asymmetry with events is // deliberate — an event queues across a reconnect, a request does not (see // rpcConn). -func dialLoop(ctx context.Context, dial, sessionID string, s *session.Session, events <-chan []byte, +// sessionTransport is how one dialLoop reaches its runner: the dialer, the +// per-connection preamble (nil on the WebSocket path), the connection the +// microVM boot already established, and the session's name for the log. +// +// It is a struct rather than four parameters because the four are one +// decision — which transport this session has — and a caller that could +// supply a vsock preamble with a WebSocket dialer would be describing a +// session that does not exist. +type sessionTransport struct { + dial dialSession + preamble func(context.Context, relay.Conn) error + first relay.Conn + name string +} + +// connect returns the next connection to serve, running the preamble on it. +// The connection the boot established is used once, first, and then dropped: +// the boot already read its configuration and made its exchange, and running +// the preamble over it again would read a second configuration that is not +// coming. +func (t *sessionTransport) connect(ctx context.Context) (relay.Conn, error) { + if t.first != nil { + c := t.first + t.first = nil + return c, nil + } + c, err := t.dial(ctx) + if err != nil { + return nil, err + } + if t.preamble != nil { + if err := t.preamble(ctx, c); err != nil { + _ = c.Close() + return nil, err + } + } + return c, nil +} + +func dialLoop(ctx context.Context, tr sessionTransport, s *session.Session, events <-chan []byte, execCounts *execCountMailbox, rpc *rpcDispatcher, execs *sandboxexec.Runner) { backoff := time.Second var pending [][]byte // control payloads no connection has accepted yet for { - c, _, err := websocket.Dial(ctx, dial+"?session="+sessionID, nil) + conn, err := tr.connect(ctx) if err == nil { - c.SetReadLimit(16 << 20) - log.Printf("sessiond registered with runnerd as %s", sessionID) + log.Printf("sessiond registered with runnerd as %s", tr.name) backoff = time.Second // WithControl, not plain ServeSession: the returned sender shares // the relay's single writer, so a control event and a terminal @@ -345,7 +444,7 @@ func dialLoop(ctx context.Context, dial, sessionID string, s *session.Session, e // cannot arrive, and a handler that outlives the conn answers over // the (now dead) conn its request came in on rather than over a // later one — see rpc.go. - sender, errc := relay.ServeSessionWithExec(ctx, relay.WSConn(c), s, rpc.OnControl, execs) + sender, errc := relay.ServeSessionWithExec(ctx, conn, s, rpc.OnControl, execs) rpc.online(sender) var relayErr error pending, relayErr = serveConn(sender, errc, events, execCounts.c(), execs, pending) @@ -463,6 +562,52 @@ func quiesceExecs(execs execKiller, notifier eventNotifier, nonce uint64) { } } +// agentHomeMount is where the agent homes are mounted inside a session. It +// is controlapp.HomeMountPath, spelled here because this process must not +// import the control plane's application package — the two ends of the same +// mount, like every other wire word this file shares with it. +const agentHomeMount = "/rainier/agents" + +// quiesceCold is the cold half of the suspend handshake: this VM is ending, +// with no memory image written anywhere (ADR-0003 §2.2), so a resume is a +// fresh boot from disks and nothing else survives. +// +// Three things, in the order they have to happen: +// +// 1. FLUSH. The last thing an agent wrote is usually the thing worth +// keeping — a login completed seconds ago — and the sync's own tick must +// not be what decides whether it makes it into custody. +// 2. END the execs, which is what the warm path does too and for the same +// reason: a detached command's lifetime is its session's. +// 3. UNMOUNT the agent home and FORGET the delivered secrets. The unmount +// is what makes the home's filesystem consistent before the block device +// goes away under it; forgetting is belt and braces beside a VM that is +// about to cease to exist, and it costs nothing. +// +// Every step is best effort and the answer goes out regardless, for the same +// reason the warm path's does: the host terminates the VM when it hears +// nothing, so staying silent would only make the stop slower and put the +// reason nowhere. +func quiesceCold(execs execKiller, notifier eventNotifier, agents *agentSync, boots *bootstrapper, nonce uint64) { + if err := notifier.Notify(relay.ControlEvent{Kind: relay.KindSuspendAck, ID: nonce}); err != nil { + log.Printf("acknowledging the cold suspend notice: %v", err) + } + if agents != nil { + agents.flush() + } + if n := execs.KillAllAndWait(execQuiesceBudget); n > 0 { + log.Printf("%d exec(s) had not ended %s after the cold suspend notice; answering anyway", + n, execQuiesceBudget) + } + unmountAgentHome(agentHomeMount) + if n := boots.forget(); n > 0 { + log.Printf("forgot %d delivered environment secret(s) before this VM ends", n) + } + if err := notifier.Notify(relay.ControlEvent{Kind: relay.KindSuspendReady, ID: nonce}); err != nil { + log.Printf("reporting the cold suspend ready: %v", err) + } +} + // execReporter is the one method serveConn needs from the exec runner, named // as an interface so the delivery rules can be tested without a sandbox to // have processes in. diff --git a/cmd/sessiond/transport.go b/cmd/sessiond/transport.go new file mode 100644 index 0000000..1f72e59 --- /dev/null +++ b/cmd/sessiond/transport.go @@ -0,0 +1,93 @@ +package main + +import ( + "context" + "fmt" + "os" + + "github.com/coder/websocket" + + "github.com/tokencanopy/rainier/internal/relay" +) + +// The transport seam: how this sessiond reaches the runner holding it. +// +// There are two, and everything above them is identical. A Docker session +// dials a WebSocket over the container network and asserts its session id in +// a query parameter; a microVM session opens an AF_VSOCK connection to the +// host, which Firecracker forwards to a socket inside that VM's own +// directory — so the id is the host's statement rather than the guest's +// claim, and the channel works before the guest's network does. +// +// relay.ServeSessionWithExec, the RPC dispatcher and the agent sync are +// untouched by the difference, because every one of them already takes a +// relay.Conn. + +const ( + // transportWebSocket is the default and the compatibility floor: every + // session that exists today, and every Docker session there will ever be. + transportWebSocket = "websocket" + // transportVsock is the microVM path. It is a flag (or RAINIER_TRANSPORT) + // rather than something inferred, and it comes from the session IMAGE + // rather than from a create: a microVM guest receives no environment at + // all until its boot configuration arrives, so there is nothing per-session + // for it to be inferred from. Which transport a guest has is a property of + // the rootfs it booted, which is exactly where the flag is set. + transportVsock = "vsock" +) + +// dialSession opens one connection to the runner. It is the whole seam: a +// function of a context, because that is all either transport needs. +type dialSession func(context.Context) (relay.Conn, error) + +// websocketTransport is the Docker path, byte for byte what dialLoop did +// before the seam existed: the same URL, the same nil options, the same read +// limit. A session keeps the sessiond it booted with for life, so this hop +// is a contract with images months old and is not a place to tidy anything. +func websocketTransport(dial, sessionID string) dialSession { + return func(ctx context.Context) (relay.Conn, error) { + c, _, err := websocket.Dial(ctx, dial+"?session="+sessionID, nil) + if err != nil { + return nil, err + } + c.SetReadLimit(16 << 20) + return relay.WSConn(c), nil + } +} + +// vsockTransport is the microVM path: AF_VSOCK to (HOST_CID, 1024). +// +// One port, because internal/relay already multiplexes the configuration, +// the terminal stream, the session RPC and the lifecycle handshake over one +// conn. The host never dials in; the guest opens this and everything rides +// it. +func vsockTransport() dialSession { + return func(ctx context.Context) (relay.Conn, error) { + c, err := dialVsock(ctx, vsockHostCID, vsockControlPort) + if err != nil { + return nil, fmt.Errorf("dial the host control channel over vsock: %w", err) + } + return relay.NetConn(c), nil + } +} + +// envOr reads a variable, falling back to def when it is unset or empty. +// It is how --transport takes its default from the session IMAGE: a microVM +// guest has no per-session environment to be configured from, but it does +// have the rootfs it booted, and that is the right place for a fact about +// which transport this guest has. +func envOr(name, def string) string { + if v := os.Getenv(name); v != "" { + return v + } + return def +} + +const ( + // vsockHostCID is HOST_CID: the host's context id is always 2. + vsockHostCID = 2 + // vsockControlPort is the host port the runner listens on. It matches + // the driver's guestControlPort, and the two are the same wire fact + // spelled in two binaries that ship in different artifacts. + vsockControlPort = 1024 +) diff --git a/cmd/sessiond/transport_test.go b/cmd/sessiond/transport_test.go new file mode 100644 index 0000000..db45a14 --- /dev/null +++ b/cmd/sessiond/transport_test.go @@ -0,0 +1,124 @@ +package main + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/coder/websocket" + + "github.com/tokencanopy/rainier/internal/relay" +) + +// TestTheWebSocketTransportDialsTheURLItAlwaysHas is the compatibility floor +// of this whole change, pinned at the one hop that moved. +// +// dialLoop used to call websocket.Dial inline; it now calls a dialSession. +// A session keeps the sessiond it booted with for life, so the URL that hop +// produces is a contract with a runnerd that may be months newer — and the +// seam must have changed where the call lives and nothing about what it +// sends. +func TestTheWebSocketTransportDialsTheURLItAlwaysHas(t *testing.T) { + got := make(chan string, 1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got <- r.URL.RequestURI() + c, err := websocket.Accept(w, r, nil) + if err != nil { + return + } + defer c.CloseNow() + <-r.Context().Done() + })) + defer srv.Close() + + base := strings.Replace(srv.URL, "http", "ws", 1) + "/register" + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + conn, err := websocketTransport(base, "sess_example")(ctx) + if err != nil { + t.Fatalf("dial: %v", err) + } + defer conn.Close() + + select { + case uri := <-got: + if uri != "/register?session=sess_example" { + t.Fatalf("the transport dialed %q, want /register?session=sess_example", uri) + } + case <-time.After(5 * time.Second): + t.Fatal("the transport never reached the server") + } +} + +// TestTheBootConnectionIsServedOnceAndThenTheDialerIs pins the one piece of +// state sessionTransport holds: the connection the microVM boot already +// established is served first and then dropped, because its configuration +// has been read and its exchange made. Running the preamble over it again +// would wait for a second configuration that is not coming. +func TestTheBootConnectionIsServedOnceAndThenTheDialerIs(t *testing.T) { + dial, _ := fakeTransport(t) + first, _ := fakeTransport(t) + boot, err := first(context.Background()) + if err != nil { + t.Fatal(err) + } + + preambles := 0 + tr := sessionTransport{ + dial: dial, + first: boot, + preamble: func(context.Context, relay.Conn) error { preambles++; return nil }, + } + + got, err := tr.connect(context.Background()) + if err != nil { + t.Fatal(err) + } + if got != boot { + t.Fatal("the first connect did not serve the connection the boot established") + } + if preambles != 0 { + t.Fatalf("the boot connection was put through the preamble %d time(s)", preambles) + } + + next, err := tr.connect(context.Background()) + if err != nil { + t.Fatal(err) + } + if next == boot { + t.Fatal("the boot connection was served twice") + } + if preambles != 1 { + t.Fatalf("a redial ran the preamble %d time(s), want exactly once", preambles) + } +} + +// TestAFailedPreambleClosesTheConnection: a connection whose configuration +// could not be read is not a connection to serve, and leaving it open would +// leak one per redial for the life of the session. +func TestAFailedPreambleClosesTheConnection(t *testing.T) { + dial, hosts := fakeTransport(t) + tr := sessionTransport{ + dial: dial, + preamble: func(context.Context, relay.Conn) error { return errPreambleFailed }, + } + if _, err := tr.connect(context.Background()); err != errPreambleFailed { + t.Fatalf("connect = %v, want the preamble's own error", err) + } + host := <-hosts + // The guest end is closed, so the host's read ends rather than blocking. + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + if _, err := host.conn.Read(ctx); err == nil { + t.Fatal("a connection whose preamble failed was left open") + } +} + +var errPreambleFailed = errStub("the boot configuration never arrived") + +type errStub string + +func (e errStub) Error() string { return string(e) } diff --git a/cmd/sessiond/unmount_linux.go b/cmd/sessiond/unmount_linux.go new file mode 100644 index 0000000..9ffe5a3 --- /dev/null +++ b/cmd/sessiond/unmount_linux.go @@ -0,0 +1,45 @@ +//go:build linux + +package main + +import ( + "errors" + "log" + "os" + + "golang.org/x/sys/unix" +) + +// unmountAgentHome unmounts the agent home before this VM ends. +// +// It is the one filesystem act a cold suspend needs from inside the guest: +// the home is a block device of its own (ADR-0003 §4.1 keeps it out of the +// workspace disk and its checkpoint), and a device detached with dirty pages +// above it is a home that comes back needing a fsck for a credential set +// somebody logged in for once. +// +// MNT_DETACH is deliberate. A lazy unmount detaches the tree immediately and +// lets the kernel finish when the last reference goes, which is what a +// process still holding a file under it turns an ordinary unmount into +// EBUSY over — and the alternative to detaching is refusing to answer the +// host, which ends the VM anyway with nothing flushed at all. +// +// Every failure is a log line and not an error. A session with no home +// mounted (no creator, or an older control plane) is the ordinary case and +// reports ENOENT or EINVAL; a failure here must not be what stops this +// process answering the host. +func unmountAgentHome(path string) { + if _, err := os.Stat(path); err != nil { + return + } + if err := unix.Unmount(path, unix.MNT_DETACH); err != nil { + if errors.Is(err, unix.EINVAL) || errors.Is(err, unix.ENOENT) { + // Not a mount point: nothing was mounted here, which is what a + // session with no agent home looks like from inside. + return + } + log.Printf("unmounting the agent home at %s before this VM ends: %v", path, err) + return + } + log.Printf("unmounted the agent home at %s", path) +} diff --git a/cmd/sessiond/unmount_other.go b/cmd/sessiond/unmount_other.go new file mode 100644 index 0000000..d9a4143 --- /dev/null +++ b/cmd/sessiond/unmount_other.go @@ -0,0 +1,8 @@ +//go:build !linux + +package main + +// unmountAgentHome on anything but Linux. A cold suspend only ever happens +// to a microVM guest, which is Linux by construction; this exists so the +// package builds on a developer's machine. +func unmountAgentHome(string) {} diff --git a/cmd/sessiond/vsock_linux.go b/cmd/sessiond/vsock_linux.go new file mode 100644 index 0000000..643fd0b --- /dev/null +++ b/cmd/sessiond/vsock_linux.go @@ -0,0 +1,56 @@ +//go:build linux + +package main + +import ( + "context" + "fmt" + "net" + "os" + + "golang.org/x/sys/unix" +) + +// dialVsock opens an AF_VSOCK stream to (cid, port). +// +// There is no net.Dial("vsock", …): the address family is not in the +// standard library, so the socket is made by hand and handed to net.FileConn, +// which gives back an ordinary net.Conn with the runtime's poller behind it +// — deadlines, cancellation and all, which relay.NetConn relies on. +// +// The guest needs CONFIG_VIRTIO_VSOCKETS and /dev/vsock for this to work at +// all, and whether the session image's kernel has them is one of the things +// only a real host can answer (design note §7). +func dialVsock(ctx context.Context, cid, port uint32) (net.Conn, error) { + fd, err := unix.Socket(unix.AF_VSOCK, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0) + if err != nil { + return nil, fmt.Errorf("vsock socket: %w", err) + } + // From here on the fd is owned by this function until os.NewFile takes + // it: every failure below closes it, or a guest that retries its dial + // leaks one per attempt for the life of the session. + if err := unix.Connect(fd, &unix.SockaddrVM{CID: cid, Port: port}); err != nil { + _ = unix.Close(fd) + return nil, fmt.Errorf("vsock connect to (%d,%d): %w", cid, port, err) + } + f := os.NewFile(uintptr(fd), fmt.Sprintf("vsock:%d:%d", cid, port)) + if f == nil { + _ = unix.Close(fd) + return nil, fmt.Errorf("vsock: fd %d is not a file", fd) + } + // FileConn DUPLICATES the descriptor, so the original is closed here + // whether it succeeded or not. + conn, err := net.FileConn(f) + _ = f.Close() + if err != nil { + return nil, fmt.Errorf("vsock: adopting the connection: %w", err) + } + // A context that is already done must not leave a live connection + // behind; there is nothing to cancel mid-connect, because unix.Connect + // on a blocking socket does not take one. + if err := ctx.Err(); err != nil { + _ = conn.Close() + return nil, err + } + return conn, nil +} diff --git a/cmd/sessiond/vsock_other.go b/cmd/sessiond/vsock_other.go new file mode 100644 index 0000000..65622eb --- /dev/null +++ b/cmd/sessiond/vsock_other.go @@ -0,0 +1,16 @@ +//go:build !linux + +package main + +import ( + "context" + "errors" + "net" +) + +// dialVsock on anything but Linux. A microVM guest is Linux by construction +// — Firecracker boots a Linux kernel — so this exists only so the package +// builds on a developer's machine, and it refuses rather than pretending. +func dialVsock(context.Context, uint32, uint32) (net.Conn, error) { + return nil, errors.New("vsock is a Linux facility; this sessiond was built for another platform") +} From 94e34debdf2a27aacafdd65f8d521cffb8cdcbf9 Mon Sep 17 00:00:00 2001 From: Josh Zhang Date: Sun, 20 Sep 2026 02:22:48 +0000 Subject: [PATCH 5/7] =?UTF-8?q?fix:=20the=20review's=20findings=20?= =?UTF-8?q?=E2=80=94=20a=20sandbox=20may=20not=20mint,=20and=20a=20refusal?= =?UTF-8?q?=20is=20not=20a=20teardown?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An independent adversarial review of this branch found nine substantive issues. Every one is fixed here, with the test that would have caught it. **A sandbox could mint itself unlimited bootstrap tokens.** routeControl forwards any `req:` a sandbox puts on its control conn, and mint_session_bootstrap was in that set: a compromised agent could ask for a fresh token whenever it liked, spend it, and repeat — an unbounded, self-renewing capability to re-read its environment's current secrets, which is the property this design exists to remove. Single use and the 120-second TTL were enforced against nobody. controld cannot make this check — a session_req proves only that SOME runner sent it — so the runner refuses it, and refuses a request carrying an id from its own reserved space with it, since an id space the untrusted end can write into is not a space: a guest choosing 1<<63|n could have controld's echo delivered to a cold resume waiting on that number. The refusal is not conditional on the driver, because a Docker sandbox is on the same channel. **A refused exchange took the session off the air.** The control plane spends a token when it receives it, so a refused or timed-out exchange has spent it. sessiond keyed its "already done" on SUCCESS, so every later connection re-presented a dead token, the preamble failed, dialLoop treated it as a dial failure and backed off — forever. A session that was merely missing its secrets, and had already said so as a failed boot stage, became unreachable at the first conn blip. It now keys on ATTEMPTED, and a refused re-exchange on a live session is logged and the connection served, which is what reBootstrap's own comment always claimed. **relay.NetConn poisoned its deadline instead of failing.** A cancelled context set a read/write deadline and never cleared it, so a bounded write (connWriter.writeWithin's exec budget) left the conn readable and unwritable — a session silently dead upward while both ends believed it live. It now CLOSES the conn, which is exactly what *websocket.Conn does and what writeWithin's doc comment relies on. It follows that no caller may bound an operation on a conn it wants to keep, so the boot preamble closes and re-dials rather than handing on a conn it may have broken. **The boot exchange used id 1**, which is also the RPC dispatcher's first — so a late answer would be matched to the dispatcher's next call and decoded as an empty agent credential set. The comment claiming the counter was seeded past it was simply false. It is now numbered out of the dispatcher's space entirely, and clear of the runner's. **A failed cold resume made a session unresumable.** Firecracker binds `` at PUT /vsock and the driver binds `_1024`; only the second was ever removed, and the boot counter does not advance past a failed attempt — so the next resume recomputed the same path and bind failed EADDRINUSE on a socket nothing served, forever. Both paths are removed, at open and at close. **An unreadable placement made a token that could never be redeemed.** placedGeneration returns 0 when the read fails, documented as safe because "zero fences nothing" — true of an event, and no longer true of the number a token is minted against. A token at generation 0 against a row at 3 is refused as superseded by the very placement that minted it, and a guest cannot re-mint. It now reports whether it read anything, and a withholding create refuses rather than dispatching a session that cannot get its secrets. Every other create is dispatched exactly as before, which is asserted too. **Two tests were not testing what they said.** A runnerd case asserted that a hub it never created did not exist, with a failure message referring to an assertion that was not there; it now builds a sandbox and asserts nothing reaches it. And the microVM arm of assertStrippedFromImage could pass over an empty key list, so subtest 5 now carries a key it does NOT strip and the arm requires it to survive — if the driver stopped recording what a session was configured with, the strip checks would have gone quiet rather than failing. Also: a one-byte off-by-one made a frame of exactly maxFrameBytes writable and unreadable; the first dial now retries like every other one in the process rather than being the one whose transient failure is fatal; and four comments that claimed more than the code did (a seeded counter, a growing bufio buffer, an independent invariant check, a cleanup that removed one path of two) now say what is true. The store's refusal ORDER is written into the port contract, so two implementations cannot describe the same failure two ways. Co-Authored-By: Claude Fable 5.1 --- cmd/sessiond/bootconfig.go | 132 ++++++++++++++++++++++---- cmd/sessiond/bootconfig_test.go | 90 ++++++++++++++++-- cmd/sessiond/main.go | 10 +- control/bootstrap.go | 10 ++ controlapp/bootstrap_test.go | 52 ++++++++++ controlapp/scheduler.go | 37 ++++++-- internal/driver/contract.go | 41 ++++++-- internal/driver/microvm.go | 4 +- internal/driver/microvm_sim_test.go | 13 +++ internal/driver/microvm_vsock.go | 79 ++++++--------- internal/driver/microvm_vsock_test.go | 59 ++++++++++++ internal/relay/netconn.go | 53 +++++++---- internal/runnerd/microvm.go | 39 ++++++++ internal/runnerd/microvm_test.go | 117 ++++++++++++++++++++++- internal/runnerd/runnerd.go | 15 +++ 15 files changed, 637 insertions(+), 114 deletions(-) diff --git a/cmd/sessiond/bootconfig.go b/cmd/sessiond/bootconfig.go index dc0ebce..1b88fec 100644 --- a/cmd/sessiond/bootconfig.go +++ b/cmd/sessiond/bootconfig.go @@ -35,11 +35,30 @@ const ( // sits under the bootstrap token's own 120-second lifetime, so a refusal // is an answer rather than an expiry. secretsExchangeWait = 30 * time.Second + // bootDialAttempts and bootDialBackoff bound the retry on the very first + // dial. Everything else in this process redials with backoff for the + // life of the session, and the one dial that could not was the one whose + // failure is fatal — a guest whose kernel brought /dev/vsock up a moment + // after sessiond started, or a host whose accept loop was one scheduling + // quantum behind, would have taken the session down for a condition that + // resolves itself. The bound exists because a guest that genuinely has + // no channel must fail rather than sit in a loop nobody can see. + bootDialAttempts = 10 + bootDialBackoff = 500 * time.Millisecond // secretsRequestID is the id this end assigns its boot-time exchange. + // // The request is made before the RPC dispatcher is serving — there is no - // session to serve for yet — so it is numbered by hand, and the - // dispatcher's counter is seeded past it (see bootstrapper.exchange). - secretsRequestID = 1 + // session to serve for yet — so it is numbered by hand, and it is + // numbered OUT OF the dispatcher's space rather than at the bottom of + // it. The dispatcher counts from 1, so a boot exchange numbered 1 whose + // answer arrived late would be matched to the first call the dispatcher + // made afterwards (an agent credential fetch, typically) and answered + // with a body that method cannot read. + // + // It stays clear of runnerd's own reserved space too: that is the high + // BIT (1<<63), and a request carrying it is refused at the runner as a + // sandbox using an id that is not its to use. + secretsRequestID = 1 << 62 ) // bootstrapper is the guest side of the boot: it reads a configuration off @@ -56,8 +75,19 @@ const ( // needed. type bootstrapper struct { mu sync.Mutex - // spent is the token whose exchange succeeded, if any. - spent string + // attempted is the last token this session sent a request for, whatever + // the answer was. + // + // ATTEMPTED and not "spent", which is the difference between a session + // that recovers and one that does not. A token is consumed by the + // control plane BEFORE the secrets are resolved, so a refused or + // timed-out exchange has spent it just as surely as a successful one; + // retrying it can only ever be refused. A sessiond that keyed on success + // would re-exchange a dead token on every redial, fail the preamble + // every time, and take a session that was merely missing its secrets — + // and had already said so, loudly, as a failed boot stage — off the air + // entirely. + attempted string // delivered are the names this session has been given values for, kept // so a cold suspend can forget them by name. NAMES, never values: what // was delivered lives in the process environment and nowhere else. @@ -137,9 +167,15 @@ func (b *bootstrapper) exchange(ctx context.Context, conn relay.Conn, cfg runner return nil, nil } b.mu.Lock() - alreadySpent := cfg.BootstrapToken != "" && cfg.BootstrapToken == b.spent + already := cfg.BootstrapToken != "" && cfg.BootstrapToken == b.attempted + if !already { + // Recorded BEFORE the request goes out, not after it succeeds: the + // control plane spends the token when it receives it, so a request + // that was sent has spent it whatever comes back. + b.attempted = cfg.BootstrapToken + } b.mu.Unlock() - if alreadySpent { + if already { return nil, nil } if cfg.BootstrapToken == "" { @@ -197,9 +233,6 @@ func (b *bootstrapper) exchange(ctx context.Context, conn relay.Conn, cfg runner // message quotes what it choked on, and that is the secret. return nil, undeliveredSecrets(len(cfg.SecretNames), "the answer could not be decoded") } - b.mu.Lock() - b.spent = cfg.BootstrapToken - b.mu.Unlock() return answer.Env, nil } } @@ -233,7 +266,10 @@ func (b *bootstrapper) forget() int { b.mu.Lock() names := b.delivered b.delivered = nil - b.spent = "" + // `attempted` is deliberately NOT cleared. The token this session was + // handed is spent whether or not this process still holds its values, + // and a resume brings a new one — so forgetting which token was already + // presented would only make the next connection re-present a dead one. b.mu.Unlock() for _, n := range names { _ = os.Unsetenv(n) @@ -349,7 +385,7 @@ func applyBootConfig(cfg runner.BootConfig, secrets map[string]string) error { // that the boot chain fails loudly enough for a person to see why. The // caller turns it into a failing stage. func bootOverVsock(ctx context.Context, dial dialSession, b *bootstrapper) (relay.Conn, runner.BootConfig, *bootFailure, error) { - conn, err := dial(ctx) + conn, err := dialBoot(ctx, dial) if err != nil { return nil, runner.BootConfig{}, nil, err } @@ -361,15 +397,30 @@ func bootOverVsock(ctx context.Context, dial dialSession, b *bootstrapper) (rela secrets, xerr := b.exchange(ctx, conn, cfg) var failure *bootFailure if xerr != nil { - if bf, ok := xerr.(*bootFailure); ok { - failure = bf - } else { + bf, ok := xerr.(*bootFailure) + if !ok { _ = conn.Close() return nil, cfg, nil, xerr } + failure = bf + // The conn is NOT handed back on a failed exchange. It may be + // perfectly good (a refusal) or it may not (a timeout, which ends + // the read by poisoning the conn's deadline), and the boot has no + // way to tell — so it is closed and dialLoop dials a fresh one. The + // cost is one extra connection on a session that is about to fail + // its boot chain anyway; the alternative is serving a conn that may + // already be dead and discovering it one frame later. + // + // The re-dial is safe precisely because the token is recorded as + // ATTEMPTED: the preamble on the new connection reads the + // configuration again and asks for nothing. + _ = conn.Close() + conn = nil } if err := applyBootConfig(cfg, secrets); err != nil { - _ = conn.Close() + if conn != nil { + _ = conn.Close() + } return nil, cfg, nil, err } b.remember(namesOf(secrets)) @@ -383,10 +434,19 @@ func bootOverVsock(ctx context.Context, dial dialSession, b *bootstrapper) (rela // session has not spent — which is what a cold resume brings and a redial // does not. // -// A failure here is an error rather than a failed boot chain: the session is -// already running, its agent already has whatever it was given, and the -// answer to a refused re-exchange is to try the next connection rather than -// to tear down a live session. +// The two failures it can meet are answered differently, and the difference +// is the difference between a session that recovers and one that vanishes. +// +// A configuration that never arrived means this conn is not usable, so the +// error is returned and dialLoop backs off and dials another. +// +// A refused EXCHANGE does not: the session is already running, its agent +// already has whatever it was given, and it has already reported a failed +// boot chain if it was given nothing. Tearing the conn down there would take +// a session that was merely missing its secrets off the air completely — no +// terminal, no attach, no events — for as long as the refusal persisted, +// which for a spent token is forever. So it is logged and the connection is +// served. func reBootstrap(ctx context.Context, conn relay.Conn, b *bootstrapper) error { cfg, err := readBootConfig(ctx, conn) if err != nil { @@ -394,7 +454,9 @@ func reBootstrap(ctx context.Context, conn relay.Conn, b *bootstrapper) error { } secrets, xerr := b.exchange(ctx, conn, cfg) if xerr != nil { - return xerr + log.Printf("this session's secrets were not re-delivered on a new connection (%v); "+ + "serving it anyway — the session is running and has already reported what it was given", xerr) + return nil } if len(secrets) == 0 { return nil @@ -407,6 +469,34 @@ func reBootstrap(ctx context.Context, conn relay.Conn, b *bootstrapper) error { return nil } +// dialBoot is the first dial, with the retry every later one already has. +// +// A failure here is fatal to the session, which is why it is the one dial +// that must not give up on the first refusal: a guest whose /dev/vsock came +// up a moment after this process did, or a host whose accept loop was a +// scheduling quantum behind, is a condition that resolves itself in +// milliseconds. A guest that genuinely has no channel still fails, and still +// fails quickly. +func dialBoot(ctx context.Context, dial dialSession) (relay.Conn, error) { + var err error + for attempt := 1; ; attempt++ { + var conn relay.Conn + conn, err = dial(ctx) + if err == nil { + return conn, nil + } + if attempt >= bootDialAttempts || ctx.Err() != nil { + return nil, fmt.Errorf("after %d attempt(s): %w", attempt, err) + } + log.Printf("the host control channel is not answering yet (%v); retrying", err) + select { + case <-time.After(bootDialBackoff): + case <-ctx.Done(): + return nil, ctx.Err() + } + } +} + // namesOf returns a map's keys. Names, which are not values — the same // distinction the launch-material resolver and the create both make. func namesOf(m map[string]string) []string { diff --git a/cmd/sessiond/bootconfig_test.go b/cmd/sessiond/bootconfig_test.go index 75e7f94..22b9d98 100644 --- a/cmd/sessiond/bootconfig_test.go +++ b/cmd/sessiond/bootconfig_test.go @@ -317,18 +317,22 @@ func TestARefusedExchangeFailsTheBootChain(t *testing.T) { boots := &bootstrapper{} done := make(chan struct{}) - var failure *bootFailure - var bootErr error + var ( + conn relay.Conn + failure *bootFailure + bootErr error + ) go func() { defer close(done) - _, _, failure, bootErr = bootOverVsock(context.Background(), dial, boots) + conn, _, failure, bootErr = bootOverVsock(context.Background(), dial, boots) }() - host := <-hosts - host.sendBootConfig(runner.BootConfig{ + cfg := runner.BootConfig{ Protocol: runner.SessionBootstrapProtocolVersion, SessionID: "sess_example", SecretNames: []string{"DEPLOY_KEY", "NPM_TOKEN"}, BootstrapToken: "token_example", - }) + } + host := <-hosts + host.sendBootConfig(cfg) req := host.nextRequest() host.refuse(req.ID, "this session's bootstrap token has already been exchanged") @@ -339,6 +343,9 @@ func TestARefusedExchangeFailsTheBootChain(t *testing.T) { if failure == nil { t.Fatal("a refused exchange did not fail the boot") } + if conn != nil { + t.Fatal("a failed boot handed back a connection it may already have broken") + } // The count, the plane's own condition, and no value or name. if !strings.Contains(failure.Error(), "2 secret(s)") { t.Errorf("the failure does not name the count: %q", failure) @@ -369,6 +376,77 @@ func TestARefusedExchangeFailsTheBootChain(t *testing.T) { if !strings.Contains(string(script), "exit 1") { t.Fatalf("the secrets stage does not fail:\n%s", script) } + + // And the session still comes up. This is the half that decides whether + // a user can see WHY: the boot chain has failed and said so, and the + // session now has to register so that the failure, the terminal and an + // attach are reachable at all. + // + // The token is spent — the control plane consumes it before it resolves + // anything — so the connection that carries the session must not + // re-present it. A sessiond that retried would be refused on every + // redial, `connect` would drop every connection, and a session that was + // merely missing its secrets would be off the air completely. + redial, err := dial(context.Background()) + if err != nil { + t.Fatal(err) + } + redialHost := <-hosts + redialDone := make(chan error, 1) + go func() { redialDone <- reBootstrap(context.Background(), redial, boots) }() + redialHost.sendBootConfig(cfg) + if err := <-redialDone; err != nil { + t.Fatalf("the connection after a failed boot was refused: %v", err) + } + if !redialHost.nothingMore(200 * time.Millisecond) { + t.Fatal("the redial re-presented a token the control plane had already spent") + } +} + +// TestTheBootExchangeIsNumberedOutOfTheDispatchersSpace pins the id the boot +// exchange uses, which is the one request this process makes before its RPC +// dispatcher exists. +// +// It cannot be 1. The dispatcher counts from 1, so a boot exchange that +// timed out and was answered late would have its `{"env": …}` matched to the +// dispatcher's first call — an agent credential fetch, typically — which +// would decode it as an empty credential set and report a shape it cannot +// read for a request that was never answered. +// +// And it cannot carry the high bit, which runnerd reserves for its own +// requests and refuses a sandbox for using. +func TestTheBootExchangeIsNumberedOutOfTheDispatchersSpace(t *testing.T) { + cleanEnv(t) + dial, hosts := fakeTransport(t) + boots := &bootstrapper{} + + done := make(chan struct{}) + go func() { + defer close(done) + _, _, _, _ = bootOverVsock(context.Background(), dial, boots) + }() + host := <-hosts + host.sendBootConfig(runner.BootConfig{ + Protocol: runner.SessionBootstrapProtocolVersion, SessionID: "sess_example", + SecretNames: []string{"DEPLOY_KEY"}, BootstrapToken: "token_example", + }) + req := host.nextRequest() + host.answer(req.ID, map[string]string{"DEPLOY_KEY": "value_example"}) + <-done + + if req.ID == 0 { + t.Fatal("the boot exchange carried no id") + } + // The dispatcher's own first id, which this must not collide with. + d := newRPCDispatcher() + d.online(&recordingSender{}) + firstDispatcherID := d.seq.Add(1) + if req.ID == firstDispatcherID { + t.Fatalf("the boot exchange uses id %d, which is the dispatcher's first", req.ID) + } + if req.ID&(1<<63) != 0 { + t.Fatalf("the boot exchange uses id %d, which is in the runner's reserved space", req.ID) + } } // TestAMissingTokenWithDeclaredSecretsFailsTheBootChain is the same row from diff --git a/cmd/sessiond/main.go b/cmd/sessiond/main.go index e7237ac..c741477 100644 --- a/cmd/sessiond/main.go +++ b/cmd/sessiond/main.go @@ -97,6 +97,12 @@ func main() { dialer dialSession preamble func(context.Context, relay.Conn) error firstConn relay.Conn + // overVsock is what puts this process in relay mode on the microVM + // path, and it is NOT "firstConn != nil": a boot whose exchange was + // refused closes its connection and dials again, and such a session + // is still very much in relay mode — it has a boot chain to fail + // loudly through. + overVsock bool boots = &bootstrapper{} // secretsFailure is a boot that must not start an agent: the // environment declared credentials this session could not be given. @@ -114,7 +120,7 @@ func main() { // egress goes. Dying is what makes the runner notice. log.Fatalf("microvm boot: %v", err) } - firstConn = conn + firstConn, overVsock = conn, true if *sessionID == "" { *sessionID = cfg.SessionID } @@ -144,7 +150,7 @@ func main() { // A vsock session is in relay mode by construction: it HAS a connection, // it just did not get it from a URL. Everything gated on relay mode // below reads this rather than the flag. - relayMode := *dial != "" || firstConn != nil + relayMode := *dial != "" || overVsock var stages []bootStage if bootEnvironment.any() && !relayMode { // Can't happen from the driver (it injects RAINIER_DIAL alongside), but diff --git a/control/bootstrap.go b/control/bootstrap.go index 613edb7..5adc3d7 100644 --- a/control/bootstrap.go +++ b/control/bootstrap.go @@ -87,5 +87,15 @@ type SessionBootstrapStore interface { // It returns nothing but an error on purpose. The VALUES the token buys // are not the store's to know — they are resolved above it, from the // environment, by the one component that holds the secrets key. + // + // When more than one refusal applies, they are reported in this order: + // Unknown, Fenced, Expired, Spent. The order is part of the contract + // rather than each store's own choice, because it is what an operator + // reads: two implementations that disagreed about whether a replayed, + // expired token is "expired" or "already exchanged" would make the same + // fleet describe the same failure two ways. It runs from the most + // structural fact to the most transient, and it puts the hash first so + // that a caller holding no valid token learns nothing at all about the + // state of the one that exists. ConsumeSessionBootstrap(ctx context.Context, ws WorkspaceID, id SessionID, hash string, gen uint64, now time.Time) error } diff --git a/controlapp/bootstrap_test.go b/controlapp/bootstrap_test.go index aa7a5eb..2fd0c03 100644 --- a/controlapp/bootstrap_test.go +++ b/controlapp/bootstrap_test.go @@ -169,6 +169,58 @@ func TestCreateSpecFailsClosedWhenTheTokenCannotBeRecorded(t *testing.T) { } } +// TestAnUnreadablePlacementFailsAWithholdingCreateOnly is finding 6 of the +// branch's own review, and the reason placedGeneration now reports whether +// it could read anything. +// +// Zero has always been "not carried", and on an EVENT it fences nothing. The +// same number is now also the generation a bootstrap token is minted +// against, and there it is not inert: a token recorded at 0 against a row at +// 3 is refused on its one and only exchange, as superseded by the very +// placement that minted it — a store blip becoming a dead session with a +// misleading reason and no way back, because a guest cannot re-mint. +// +// So a withholding create refuses, and every other create is dispatched +// exactly as it always was. Both halves are asserted, because failing the +// second would be a regression for the whole fleet. +func TestAnUnreadablePlacementFailsAWithholdingCreateOnly(t *testing.T) { + // The row is deliberately never seeded, so the read-back GetSession + // makes cannot answer — the same shape as a store that is down. + row := control.Session{ + ID: "sess_unplaced", WorkspaceID: "ws_example", State: control.StateCreating, + PoolID: "pool_example", RunnerID: "vm1", + Spec: control.PortableSpec{Image: "img:latest"}, + } + + t.Run("a withholding create refuses", func(t *testing.T) { + fx := newFleetFixture(t) + fx.st.seedRunner(fleetSeededRunner("vm1", 2, 0, true)) + fx.service.dispatchCreate(fleetCtx, "pool_example", row, "vm1", + []string{runner.CapabilityMicrovmV1}, nil) + + if got := fx.transport.dispatchedCommands(); len(got) != 0 { + t.Fatalf("dispatched %d command(s) with a placement it could not read: %+v", len(got), got) + } + if _, minted := fx.bootstraps.minted("sess_unplaced"); minted { + t.Fatal("a token was minted against a placement generation nobody could read") + } + }) + + t.Run("every other create is dispatched", func(t *testing.T) { + fx := newFleetFixture(t) + fx.st.seedRunner(fleetSeededRunner("vm1", 2, 0, true)) + fx.service.dispatchCreate(fleetCtx, "pool_example", row, "vm1", nil, nil) + + got := fx.transport.dispatchedCommands() + if len(got) != 1 { + t.Fatalf("dispatched %d command(s), want 1 — an unreadable placement fences nothing here", len(got)) + } + if got[0].PlacementGeneration != 0 { + t.Fatalf("placement generation = %d, want 0 (not carried)", got[0].PlacementGeneration) + } + }) +} + // TestWithholdableNamesRespectsTheAgentHomeReservation pins that a // secret_ref spelled like an agent-home variable is dropped on BOTH paths. // diff --git a/controlapp/scheduler.go b/controlapp/scheduler.go index 4fdf116..b2b9145 100644 --- a/controlapp/scheduler.go +++ b/controlapp/scheduler.go @@ -336,7 +336,15 @@ func (s *FleetService) dispatchCreate(ctx context.Context, pool control.PoolID, // bootstrap token is FENCED by this number: the hash recorded against the // session has to name the same placement the create carries, or the first // exchange is refused as superseded by the very placement that minted it. - gen := s.placedGeneration(ctx, row) + gen, known := s.placedGeneration(ctx, row) + if !known && slices.Contains(runnerCaps, runner.CapabilityMicrovmV1) { + // Only for a withholding placement, and only because the token's + // fence is this number: everywhere else an unknown generation is + // inert, and failing a create over a transient read would be a + // regression for every session in the fleet. + s.failCreate(ctx, row, "could not read this session's placement to mint its bootstrap token") + return + } spec, fail := s.createSpec(ctx, row, env, runnerCaps, gen) if fail != "" { s.failCreate(ctx, row, fail) @@ -381,17 +389,27 @@ func (s *FleetService) dispatchCreate(ctx context.Context, pool control.PoolID, // A read that cannot be made carries nothing rather than the value it knows // to be stale: zero is "not carried" on the wire and fences nothing, while a // wrong number would fence every event this sandbox ever sends. -func (s *FleetService) placedGeneration(ctx context.Context, row control.Session) uint64 { +// +// The bool is the same fact said out loud, and it exists because that +// reasoning stopped being universally true. Zero fences nothing on an EVENT, +// which is what this was written for — but the same number is now also the +// generation a microVM session's bootstrap token is minted against, and +// there a wrong value is not inert: a token recorded at 0 against a row at 3 +// is refused on its one and only exchange, as superseded by the very +// placement that minted it. So the caller is told, and refuses the create +// rather than dispatching a session that cannot get its secrets. +func (s *FleetService) placedGeneration(ctx context.Context, row control.Session) (uint64, bool) { placed, err := s.sessions.GetSession(ctx, row.WorkspaceID, row.ID) if err != nil { - return 0 + return 0, false } - return placed.PlacementGeneration + return placed.PlacementGeneration, true } // createSpec builds the runner create spec from the session and its current // environment, resolving sensitive launch material only here and never // storing it. +// // runnerCaps are the capabilities the placement's runner announced, and gen // the placement generation the create carries. Together they are the whole of // what this function needs to decide the one question the microVM bootstrap @@ -488,9 +506,14 @@ func (s *FleetService) createSpec(ctx context.Context, row control.Session, env } // Either the values or the token, never both — stated as a check rather // than as a comment, because it is the whole security claim of §3 and it - // is one careless merge away from being false. A spec that reaches here - // carrying both is a bug in this function, and a failed create is a much - // better answer to it than a secret on a shared host's disk. + // is one careless merge away from being false. + // + // It is a BACKSTOP and not an independent proof: withholdableNames + // already excludes every name spec.Env holds, so as this function stands + // it cannot fire. What it is for is the edit that adds a third writer of + // spec.Env below the withholding branch, or reorders the two — at which + // point a failed create is a much better answer than a secret on a + // shared host's disk. if spec.BootstrapToken != "" { for _, name := range spec.SecretNames { if _, both := spec.Env[name]; both { diff --git a/internal/driver/contract.go b/internal/driver/contract.go index b174587..e32a058 100644 --- a/internal/driver/contract.go +++ b/internal/driver/contract.go @@ -36,7 +36,10 @@ func cleanupSnapshotRef(d Driver, ref string) { // rather than carrying anything at all. Docker's semantics are // strip-to-empty — the key survives the commit set to "" — which is why an // empty value passes here and not only an absent one. -func assertStrippedFromImage(t *testing.T, d Driver, ref, value string, stripped []string) { +// survivor is a key the create set that the caller did NOT strip, or "" when +// there is none to name. It is what makes the microVM arm an assertion rather +// than a statement about an empty list — see below. +func assertStrippedFromImage(t *testing.T, d Driver, ref, value string, stripped []string, survivor string) { t.Helper() switch dd := d.(type) { case *Docker: @@ -68,11 +71,23 @@ func assertStrippedFromImage(t *testing.T, d Driver, ref, value string, stripped if !ok { t.Fatalf("no snapshot manifest for ref %q: nothing to assert the strip against", ref) } + // The manifest has to actually DESCRIBE the create, or the strip + // assertion below is a statement about an empty list. survivor is a + // key the create set and the caller did not strip: if the driver + // stopped recording what a session was configured with, this fires + // first and the strip checks stop being vacuous silently. + if survivor != "" && !slices.Contains(keys, survivor) { + t.Fatalf("the committed manifest does not describe the create at all "+ + "(no %s among %v), so the strip below would assert nothing", survivor, keys) + } for _, k := range stripped { if slices.Contains(keys, k) { t.Fatalf("stripped key %s survived into the committed manifest: %v", k, keys) } } + // Keys and no values, checked against the raw bytes rather than the + // decoded keys: the manifest type has no field a value could live in + // today, and this is what would notice if one were added. if raw := dd.snapshotManifestBytes(ref); strings.Contains(string(raw), value) { t.Fatalf("the committed manifest carries a stripped value:\n%s", raw) } @@ -267,15 +282,27 @@ func RunContract(t *testing.T, newDriver func(t *testing.T) (Driver, func())) { h, err := d.Create(ctx, Spec{ Name: "t9", Image: "", SessionID: "s9", DialURL: "ws://x", Setup: "true", - Env: map[string]string{"CONTRACT_SECRET": "must-not-survive"}, + // CONTRACT_KEPT stands for the configuration a create legitimately + // carries — an agent-home path, a manifest — and is what proves + // the committed configuration describes this create at all. + // CONTRACT_SECRET is the one being stripped. + Env: map[string]string{ + "CONTRACT_SECRET": "must-not-survive", + "CONTRACT_KEPT": "configuration-not-a-credential", + }, // The token is here so this subtest can still be written for a // driver that refuses secret VALUES without one — which is the // twelfth subtest below, and is the microVM driver. The Docker // driver ignores the field entirely (nothing in runArgs reads - // it), so its create is the one it has always been. Without a - // token this create would be a refusal on one driver and a - // snapshot on the other, and the shared subtest would have - // stopped being shared. + // it), so its create is the one it has always been. + // + // The PAIRING is synthetic and worth saying so: the control plane + // never sends a secret value together with a token — that is the + // whole of §3, and createSpec checks it. What it does send with a + // token is the configuration CONTRACT_KEPT stands for. This + // subtest keeps the secret-shaped value because stripping is what + // it is about, and the rule the plane actually keeps is pinned by + // the twelfth subtest below and by controlapp's own matrix. BootstrapToken: "contract-token-example", }) if err != nil { @@ -293,7 +320,7 @@ func RunContract(t *testing.T, newDriver func(t *testing.T) (Driver, func())) { if snap.Ref != ref { t.Fatalf("snapshot ref = %q, want %q verbatim", snap.Ref, ref) } - assertStrippedFromImage(t, d, ref, "must-not-survive", strip) + assertStrippedFromImage(t, d, ref, "must-not-survive", strip, "CONTRACT_KEPT") }) t.Run("a create carrying secret values and no token is refused", func(t *testing.T) { diff --git a/internal/driver/microvm.go b/internal/driver/microvm.go index 385abdc..c14cdca 100644 --- a/internal/driver/microvm.go +++ b/internal/driver/microvm.go @@ -917,7 +917,7 @@ func (m *Microvm) launch(ctx context.Context, id string, spec Spec) (*instanceRe if err != nil { return nil, err } - channel, err := m.openGuestChannel(spec.SessionID, listenPath, bootCfg) + channel, err := m.openGuestChannel(spec.SessionID, udsPath, listenPath, bootCfg) if err != nil { return nil, err } @@ -1090,7 +1090,7 @@ func (m *Microvm) Resume(ctx context.Context, id string) (bool, error) { if err != nil { return false, err } - if channel, err = m.openGuestChannel(sessionID, listenPath, bootCfg); err != nil { + if channel, err = m.openGuestChannel(sessionID, udsPath, listenPath, bootCfg); err != nil { return false, err } cfg.VsockUDSPath = udsPath diff --git a/internal/driver/microvm_sim_test.go b/internal/driver/microvm_sim_test.go index dfdd57c..a86c558 100644 --- a/internal/driver/microvm_sim_test.go +++ b/internal/driver/microvm_sim_test.go @@ -78,6 +78,7 @@ type SimulatedEngine struct { states map[string]VMMState configs map[string]VMMConfig failOnStop map[string]error + failLaunch error } func NewSimulatedEngine() *SimulatedEngine { @@ -158,9 +159,21 @@ func (s *SimulatedEngine) FailOnStop(id string, err error) { s.failOnStop[id] = err } +// FailLaunch makes every Launch fail with err, standing in for a VMM that +// will not start — the one case that leaves a half-configured boot behind. +// A nil err clears it. +func (s *SimulatedEngine) FailLaunch(err error) { + s.mu.Lock() + defer s.mu.Unlock() + s.failLaunch = err +} + func (s *SimulatedEngine) Launch(_ context.Context, cfg VMMConfig) error { s.mu.Lock() defer s.mu.Unlock() + if s.failLaunch != nil { + return s.failLaunch + } s.states[cfg.ID] = VMMStateRunning s.configs[cfg.ID] = cfg s.saveState(cfg.ID, VMMStateRunning) diff --git a/internal/driver/microvm_vsock.go b/internal/driver/microvm_vsock.go index f7e90bd..0927545 100644 --- a/internal/driver/microvm_vsock.go +++ b/internal/driver/microvm_vsock.go @@ -140,25 +140,6 @@ func (m *Microvm) vsockPaths(id string, boot int) (udsPath, listenPath string, e return udsPath, listenPath, nil } -// GuestSocketPath returns the host path a session's guest control channel is -// currently listening on, and whether there is one — a cold-parked session -// has none, because its VM is gone and so is its socket. -// -// It is exported for the same reason SnapshotEnvKeys and Volumes are: it is -// how a test above this package can act like a guest, which is the only way -// to assert what a guest actually receives without a VM. Nothing in -// production reads it. -func (m *Microvm) GuestSocketPath(sessionID string) (string, bool) { - m.mu.Lock() - defer m.mu.Unlock() - for _, inst := range m.instances { - if inst.SessionID == sessionID && inst.channel != nil { - return inst.channel.path, true - } - } - return "", false -} - // guestChannel is one instance's host end of the vsock control channel: the // listener Firecracker forwards the guest's port-1024 connections to, and the // boot configuration whoever connects is handed as their first frame. @@ -170,10 +151,20 @@ func (m *Microvm) GuestSocketPath(sessionID string) (string, bool) { // (*Microvm).Resume). type guestChannel struct { listener net.Listener - path string - - mu sync.Mutex + // listenPath is "_1024", the socket this end serves; udsPath is + // the one FIRECRACKER binds when it is told about the device. Both are + // held because both have to be removed: the second is not this process's + // to create, but a VM that died leaves it behind, and a later boot that + // found it would fail its PUT /vsock with "address already in use" for a + // path nothing is serving. + listenPath string + udsPath string + // boot is the configuration whoever connects is handed. It is written + // once, at construction, and never again: a cold resume mints a new + // token and gets a new CHANNEL, because the socket path is per-boot too. boot runner.BootConfig + + mu sync.Mutex // closed makes the accept loop's exit quiet: a listener closed by // teardown reports an error like any other, and logging that as a // failure would put a line in an operator's log for every session that @@ -181,21 +172,6 @@ type guestChannel struct { closed bool } -// setBoot replaces the configuration the next guest connection is handed. A -// cold resume mints a new token, and the guest that comes up must get THAT -// one — the retired token no longer works. -func (g *guestChannel) setBoot(cfg runner.BootConfig) { - g.mu.Lock() - defer g.mu.Unlock() - g.boot = cfg -} - -func (g *guestChannel) currentBoot() runner.BootConfig { - g.mu.Lock() - defer g.mu.Unlock() - return g.boot -} - func (g *guestChannel) close() { g.mu.Lock() g.closed = true @@ -203,9 +179,10 @@ func (g *guestChannel) close() { if g.listener != nil { _ = g.listener.Close() } - // The socket file is the listener's own and goes with it. An absent one - // is not an error: a teardown that runs twice is ordinary. - _ = os.Remove(g.path) + // Both paths, and an absent one is not an error: a teardown that runs + // twice is ordinary, and a VM that never started leaves only one of them. + _ = os.Remove(g.listenPath) + _ = os.Remove(g.udsPath) } func (g *guestChannel) isClosed() bool { @@ -220,13 +197,18 @@ func (g *guestChannel) isClosed() bool { // up, and Firecracker refuses a guest connection whose "_1024" does // not exist. A listener created after InstanceStart is a race whose loser is // a session that boots and is never configured. -func (m *Microvm) openGuestChannel(sessionID, listenPath string, cfg runner.BootConfig) (*guestChannel, error) { +func (m *Microvm) openGuestChannel(sessionID, udsPath, listenPath string, cfg runner.BootConfig) (*guestChannel, error) { if err := os.MkdirAll(filepath.Dir(listenPath), microvmDirMode); err != nil { return nil, fmt.Errorf("create the guest channel directory: %w", err) } - // A socket left behind by a VM that is gone would make Listen fail with - // "address already in use" for a path nothing is listening on. + // Sockets left behind by a VM that is gone would make this Listen, or + // Firecracker's own bind at PUT /vsock, fail with "address already in + // use" for paths nothing is serving. A launch that failed after the + // device was configured leaves exactly that, and the boot counter does + // not advance past a failed attempt — so without this a session could + // never be resumed again on this runner. _ = os.Remove(listenPath) + _ = os.Remove(udsPath) ln, err := net.Listen("unix", listenPath) if err != nil { return nil, fmt.Errorf("listen on the guest control socket %s: %w", listenPath, err) @@ -240,7 +222,7 @@ func (m *Microvm) openGuestChannel(sessionID, listenPath string, cfg runner.Boot _ = os.Remove(listenPath) return nil, fmt.Errorf("restrict the guest control socket %s: %w", listenPath, err) } - g := &guestChannel{listener: ln, path: listenPath, boot: cfg} + g := &guestChannel{listener: ln, listenPath: listenPath, udsPath: udsPath, boot: cfg} go m.acceptGuests(sessionID, g) return g, nil } @@ -277,7 +259,7 @@ func (m *Microvm) acceptGuests(sessionID string, g *guestChannel) { // redials, where a guest held by a channel nobody serves waits forever. func (m *Microvm) serveGuest(sessionID string, g *guestChannel, c net.Conn) { conn := relay.NetConn(c) - if err := writeBootConfig(conn, g.currentBoot()); err != nil { + if err := writeBootConfig(conn, g.boot); err != nil { log.Printf("microvm: session %s: sending the boot configuration: %v", sessionID, err) _ = conn.Close() return @@ -395,8 +377,9 @@ func refuseUnwithheldEnv(spec Spec) error { return fmt.Errorf( "microvm: refusing to create session %s: its create carries %d environment value(s) and no bootstrap token, "+ "which means a control plane older than the session bootstrap exchange "+ - "(docs/design/2026-09-20-microvm-bootstrap-token-and-vsock.md). A microVM host has no argv to hand a "+ - "value to and will not write one to disk, so this session cannot be booted here until the control plane "+ - "announces it withholds them for a runner announcing %q", + "(docs/design/2026-09-20-microvm-bootstrap-token-and-vsock.md). Not all of those values are necessarily "+ + "secret — a session with a creator carries its agent-home configuration in the same block — but this host "+ + "cannot tell them apart, has no argv to hand any of them to, and will not write one to disk. Upgrade the "+ + "control plane to one that withholds them for a runner announcing %q", spec.SessionID, len(spec.Env), runner.CapabilityMicrovmV1) } diff --git a/internal/driver/microvm_vsock_test.go b/internal/driver/microvm_vsock_test.go index 40de556..f490151 100644 --- a/internal/driver/microvm_vsock_test.go +++ b/internal/driver/microvm_vsock_test.go @@ -240,6 +240,65 @@ func TestMicrovmColdResumeMintsAFreshTokenAndSocket(t *testing.T) { } } +// TestAFailedBootLeavesNoSocketBehind is finding 5 of this branch's own +// review, and it is the difference between a session that can be resumed +// again and one that cannot. +// +// Firecracker binds "" itself at PUT /vsock; the driver binds +// "_1024". A launch that fails after the device was configured +// leaves the first behind, and the boot counter does not advance past a +// failed attempt — so the next resume would recompute the same path, and +// Firecracker's bind would fail EADDRINUSE on a socket nothing is serving, +// forever. +func TestAFailedBootLeavesNoSocketBehind(t *testing.T) { + m, sim := testMicrovm(t, MicrovmOpts{TotalSlots: 4}) + host := &stubMicrovmHost{} + m.SetHost(host) + ctx := context.Background() + + h, err := m.Create(ctx, Spec{SessionID: "sess-stale", BootstrapToken: "token_example"}) + if err != nil { + t.Fatal(err) + } + defer m.Destroy(ctx, h.ID) + udsPath, listenPath, err := m.vsockPaths(h.ID, 1) + if err != nil { + t.Fatal(err) + } + // Stand in for what Firecracker leaves at PUT /vsock: the simulated + // engine does not bind anything, so the file is created here and the + // assertion is about whether the DRIVER cleans it up. + if err := os.WriteFile(udsPath, nil, microvmFileMode); err != nil { + t.Fatal(err) + } + if err := m.Suspend(ctx, h.ID, false); err != nil { + t.Fatal(err) + } + for _, p := range []string{udsPath, listenPath} { + if _, err := os.Stat(p); !os.IsNotExist(err) { + t.Errorf("a cold park left %s behind: %v", p, err) + } + } + + // And a resume whose launch fails must not poison the path it was going + // to use either: the next one recomputes the same one. + sim.FailLaunch(errors.New("the VMM refused to start")) + if _, err := m.Resume(ctx, h.ID); err == nil { + t.Fatal("a resume whose launch fails reported success") + } + _, secondListen, err := m.vsockPaths(h.ID, 2) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(secondListen); !os.IsNotExist(err) { + t.Errorf("a failed resume left %s behind", secondListen) + } + sim.FailLaunch(nil) + if _, err := m.Resume(ctx, h.ID); err != nil { + t.Fatalf("the resume after a failed one: %v", err) + } +} + // TestMicrovmColdResumeFailsWhenTheTokenCannotBeMinted is the fail-closed // half: a resume that cannot get a token reports that, rather than booting a // guest that will ask for its secrets and be refused. diff --git a/internal/relay/netconn.go b/internal/relay/netconn.go index 8cf5bef..1168ef2 100644 --- a/internal/relay/netconn.go +++ b/internal/relay/netconn.go @@ -9,7 +9,6 @@ import ( "fmt" "net" "sync" - "time" ) // NetConn adapts a byte STREAM to Conn, which is a message interface. @@ -45,9 +44,12 @@ func NetConn(c net.Conn) Conn { } const ( - // netConnReadBuffer is the reader's initial buffer. A control frame is - // a few hundred bytes and a terminal frame a few KiB; the buffer grows - // for the rare large one rather than being sized for it. + // netConnReadBuffer is the reader's buffer, and it is FIXED — + // bufio.Reader never grows one. A frame larger than it is assembled a + // bufferful at a time by the ErrBufferFull loop in Read, which is what + // keeps a 64 KiB buffer from being a 64 KiB frame limit. A control frame + // is a few hundred bytes and a terminal frame a few KiB, so the common + // case takes one pass. netConnReadBuffer = 64 << 10 // maxFrameBytes is the largest single message this transport will // assemble, matching the WebSocket ends' SetReadLimit. @@ -69,18 +71,33 @@ type netConn struct { var ErrFrameTooLarge = errors.New("relay: frame over the transport's size limit") func (n *netConn) Read(ctx context.Context) ([]byte, error) { - // A cancelled ctx unblocks the read by poisoning the conn's deadline, - // which is the only thing that can interrupt a blocking stream read. It - // is one-shot by design: a cancelled context means this conn is over, and - // AfterFunc costs nothing on the path where it never fires. + // A cancelled context CLOSES the conn, which is the only thing that can + // interrupt a blocking stream read — and it is deliberately the same + // answer *websocket.Conn gives, rather than merely poisoning a deadline. + // + // The difference matters at one caller. connWriter.writeWithin bounds an + // exec frame's write and documents the transport's answer to an expired + // write context as closing the conn, "because closing it is what makes + // sessiond redial". A transport that instead left the conn readable and + // unwritable would leave a session whose output and RPC were silently + // dead upward while both ends still believed the conn was live — which + // is worse than either failing or working. + // + // It follows that no caller may bound a read or a write on a conn it + // wants to keep. cmd/sessiond's boot preamble is the one that tries, and + // it closes and re-dials rather than handing on a conn it may have + // broken. if ctx.Done() != nil { - stop := context.AfterFunc(ctx, func() { _ = n.c.SetReadDeadline(time.Now()) }) + stop := context.AfterFunc(ctx, func() { _ = n.c.Close() }) defer stop() } var line []byte for { chunk, err := n.r.ReadSlice('\n') - if len(line)+len(chunk) > maxFrameBytes { + // The terminator is not part of the message, so a frame of exactly + // maxFrameBytes is readable: the budget is the message's, and the + // one byte of framing is this transport's own. + if len(line)+len(chunk) > maxFrameBytes+1 { return nil, ErrFrameTooLarge } line = append(line, chunk...) @@ -90,10 +107,9 @@ func (n *netConn) Read(ctx context.Context) ([]byte, error) { if errors.Is(err, bufio.ErrBufferFull) { continue } - if len(line) > 0 && errors.Is(err, net.ErrClosed) { - // A partial line at a closed conn is not a message. - return nil, err - } + // Anything else ends the read, whatever was accumulated: a line with + // no terminator is not a message, and handing half of one to the + // decoder would be worse than reporting the conn's own error. return nil, err } // A JSON value never contains a bare newline or carriage return, so @@ -113,12 +129,15 @@ func (n *netConn) Write(ctx context.Context, b []byte) error { // and could not resynchronize from. return errors.New("relay: a frame containing a newline cannot be framed by line") } + n.wmu.Lock() + defer n.wmu.Unlock() + // Same answer as Read's, and for the same reason: a cancelled write + // context closes the conn, which is what *websocket.Conn does and what + // connWriter.writeWithin's own doc comment relies on. if ctx.Done() != nil { - stop := context.AfterFunc(ctx, func() { _ = n.c.SetWriteDeadline(time.Now()) }) + stop := context.AfterFunc(ctx, func() { _ = n.c.Close() }) defer stop() } - n.wmu.Lock() - defer n.wmu.Unlock() // One Write call, not two: a frame and its terminator must not be // separable by a concurrent writer or by a short write in between. out := make([]byte, 0, len(b)+1) diff --git a/internal/runnerd/microvm.go b/internal/runnerd/microvm.go index e3667ab..e7c1b88 100644 --- a/internal/runnerd/microvm.go +++ b/internal/runnerd/microvm.go @@ -32,6 +32,12 @@ var _ driver.MicrovmHost = (*Server)(nil) // serveSessionConn blocks for the life of the conn — a loop that waited would // never accept the guest's next dial. // +// The context is this runner's own and not a request's, which is the one +// difference from the WebSocket door: there is no request. The hub's life is +// then bounded by the conn and by hub.Close(), which is what +// serveSessionConn's own teardown already relies on — `register` blocks on +// hub.Done() exactly because r.Context() does not reflect the socket dying. +// // The session id is the DRIVER's, taken from the socket path Firecracker // forwarded the connection to, and not anything the guest said. That is the // whole difference between this door and the WebSocket one, where `register` @@ -64,6 +70,39 @@ const runnerOriginatedIDBase uint64 = 1 << 63 // counter. forwardSessionRPC asks before it routes a response into a sandbox. func isRunnerOriginated(id uint64) bool { return id&runnerOriginatedIDBase != 0 } +// refuseSandboxOrigin reports why an upward request must not be forwarded, +// or "" when it may be. It is the fence on the one door an untrusted peer +// has into the control plane's method table. +// +// Two things are refused, and only this hop can refuse either of them. +// +// A sandbox may not mint its own bootstrap token. The design's whole point +// is that the token is SINGLE-USE and lives 120 seconds: a guest that could +// ask for a fresh one whenever it liked would hold an unbounded, self- +// renewing capability to re-read its environment's current secrets, which is +// the property being removed rather than added. The method exists for a COLD +// RESUME, which is a thing the runner does and the guest cannot observe, and +// controld cannot tell the two apart — a "session_req" proves only that some +// runner sent it, and which end of the runner originated it is a fact only +// the runner has. (This applies to a Docker sandbox too, which is why it is +// not conditional on the driver.) +// +// And a sandbox may not use an id from the runner's own space. The high bit +// is how a response is routed back to a caller inside this process rather +// than into the sandbox (see forwardSessionRPC), and an id space the +// untrusted end can write into is not a space: a guest choosing +// 1<<63|n could have controld's echo delivered to a cold resume that is +// waiting on that number. +func refuseSandboxOrigin(method string, id uint64) string { + switch { + case method == runner.MethodMintSessionBootstrap: + return "a sandbox may not mint its own bootstrap token; a fresh one is minted by the runner on a cold resume" + case isRunnerOriginated(id): + return "this request id is reserved for the runner's own requests" + } + return "" +} + // runnerRPCTable is the pending table for requests this runner originated. // // It is deliberately tiny and deliberately separate from the sandbox's: the diff --git a/internal/runnerd/microvm_test.go b/internal/runnerd/microvm_test.go index ca3c516..4f18669 100644 --- a/internal/runnerd/microvm_test.go +++ b/internal/runnerd/microvm_test.go @@ -165,6 +165,13 @@ func TestGuestConnectedBecomesTheSessionsHub(t *testing.T) { func TestMintSessionBootstrapRidesTheControlConnection(t *testing.T) { s, _ := testMicrovmServer(t) s.reg.put("sess-mint", &sessionEntry{id: "sess-mint", state: "running"}) + // A real sandbox on the far side, so "the answer did not reach the + // guest" is something this test can actually observe rather than + // something the absence of a hub makes true by accident. + guest, host := net.Pipe() + defer guest.Close() + s.GuestConnected("sess-mint", relay.NetConn(host)) + waitForHub(t, s, "sess-mint") fc := newFakeControld(t, testToken) ctx, cancel := context.WithCancel(context.Background()) @@ -218,10 +225,112 @@ func TestMintSessionBootstrapRidesTheControlConnection(t *testing.T) { t.Fatal("the mint never returned") } - // And the answer did NOT reach the sandbox: the runner is a forwarder - // for everything the sandbox asked and a participant only in this one. - if _, ok := s.reg.hub("sess-mint"); ok { - t.Fatal("this fixture has no sandbox; the assertion below would prove nothing") + // And the answer did NOT reach the sandbox. The runner is a pure + // forwarder for everything the sandbox asked and a participant only in + // this one, so a response on its own id must be consumed here — if it + // were forwarded, the guest would receive a `resp` for a request it + // never made and its dispatcher would log an unknown id. + _ = guest.SetReadDeadline(time.Now().Add(300 * time.Millisecond)) + if n, err := guest.Read(make([]byte, 256)); err == nil { + t.Fatalf("the runner forwarded its own answer into the sandbox (%d bytes)", n) + } +} + +// TestASandboxMayNotMintItsOwnBootstrapToken is the fence on the one door an +// untrusted peer has into the control plane's method table. +// +// The token is single-use and lives 120 seconds, and both of those mean +// nothing if a guest can ask for a fresh one whenever it likes: it would hold +// an unbounded, self-renewing capability to re-read its environment's current +// secrets. controld cannot make this check — a session_req proves only that +// SOME runner sent it — so the runner refuses it here, and the refusal is not +// conditional on the driver, because a Docker sandbox is on the same channel. +func TestASandboxMayNotMintItsOwnBootstrapToken(t *testing.T) { + s, _ := testMicrovmServer(t) + s.reg.put("sess-guard", &sessionEntry{id: "sess-guard", state: "running"}) + guest, host := net.Pipe() + defer guest.Close() + s.GuestConnected("sess-guard", relay.NetConn(host)) + waitForHub(t, s, "sess-guard") + + // Nothing may be forwarded upstream, so the sink records what was. + forwarded := make(chan runner.RPCEnvelope, 4) + s.SetOnSessionRPC(func(_ string, env runner.RPCEnvelope) { forwarded <- env }) + defer s.SetOnSessionRPC(nil) + + sandbox := relay.NetConn(guest) + for _, tc := range []struct { + name string + ev relay.ControlEvent + wantReason string + }{ + { + name: "the mint method itself", + ev: relay.ControlEvent{Kind: "req:" + runner.MethodMintSessionBootstrap, ID: 7, Payload: []byte(`{"protocol":1}`)}, + wantReason: "may not mint its own bootstrap token", + }, + { + name: "an id from the runner's own space", + ev: relay.ControlEvent{Kind: "req:" + runner.MethodFetchSessionSecrets, + ID: runnerOriginatedIDBase | 9, Payload: []byte(`{"protocol":1,"token":"x"}`)}, + wantReason: "reserved for the runner's own requests", + }, + } { + t.Run(tc.name, func(t *testing.T) { + payload, err := json.Marshal(tc.ev) + if err != nil { + t.Fatal(err) + } + frame, err := relay.Encode(relay.Frame{Type: relay.FrameControl, Payload: payload}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := sandbox.Write(ctx, frame); err != nil { + t.Fatal(err) + } + + answer := readControlEvent(t, sandbox) + if answer.Kind != "resp" || answer.ID != tc.ev.ID || answer.OK { + t.Fatalf("the sandbox got %+v, want an ok:false resp for its own id", answer) + } + if reason := rpcErrorText(answer.Payload); !strings.Contains(reason, tc.wantReason) { + t.Fatalf("refusal = %q, want it to name %q", reason, tc.wantReason) + } + select { + case env := <-forwarded: + t.Fatalf("the request was forwarded to the control plane anyway: %+v", env) + default: + } + }) + } + + // And the methods a sandbox IS allowed to originate still go up + // untouched — the guard is two names and an id range, not a new + // allowlist that every future method has to be added to. + allowed := relay.ControlEvent{Kind: "req:" + runner.MethodFetchSessionSecrets, ID: 11, + Payload: []byte(`{"protocol":1,"token":"x"}`)} + payload, err := json.Marshal(allowed) + if err != nil { + t.Fatal(err) + } + frame, err := relay.Encode(relay.Frame{Type: relay.FrameControl, Payload: payload}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := sandbox.Write(ctx, frame); err != nil { + t.Fatal(err) + } + select { + case env := <-forwarded: + if env.ID != 11 || env.Method != runner.MethodFetchSessionSecrets { + t.Fatalf("the forwarded request = %+v", env) + } + case <-time.After(3 * time.Second): + t.Fatal("an ordinary sandbox request was not forwarded") } } diff --git a/internal/runnerd/runnerd.go b/internal/runnerd/runnerd.go index 044f650..e3b0ffb 100644 --- a/internal/runnerd/runnerd.go +++ b/internal/runnerd/runnerd.go @@ -1240,6 +1240,21 @@ func (s *Server) routeControl(id string, boot, reg uint64, payload []byte) { log.Printf("session %s: unknown control kind %q", id, ev.Kind) return } + // The forwarder is generic, and that is its whole value — but it is + // also an untrusted peer's door into the control plane's method + // table, so the two things that are NOT a sandbox's to send are + // refused here rather than upstream. controld cannot make this check + // itself: a "session_req" proves only that some runner sent it, and + // which end of the runner originated it is a fact only the runner + // has. + if refusal := refuseSandboxOrigin(method, ev.ID); refusal != "" { + log.Printf("session %s: refusing %q from a sandbox: %s", id, method, refusal) + if err := s.sendSessionRPC(id, runner.RPCEnvelope{ID: ev.ID, Method: "resp", + Payload: rpcErrorPayload(refusal)}); err != nil { + log.Printf("session %s: refusing %q locally: %v", id, method, err) + } + return + } if s.fireSessionRPC(id, runner.RPCEnvelope{ID: ev.ID, Method: method, Payload: ev.Payload}) { return } From 64ec9c7cf8ddbcae3e1b9b28f0852393b0b9ba94 Mon Sep 17 00:00:00 2001 From: jiashuoz Date: Mon, 21 Sep 2026 00:10:17 +0900 Subject: [PATCH 6/7] fix(pgstore): key the bootstrap-token row by (workspace_id, session_id) sessions has been keyed by (workspace_id, id) since migration 0007, so a foreign key to sessions(id) alone is refused by Postgres and migration 0015 never applied; every store test then reported the database as never ready, which is what CI showed three runs in a row. The token row is now keyed and referenced by the same pair, the upsert conflicts on it, and the migration-head assertion counts 0015. Co-Authored-By: Claude Fable 5.1 --- internal/controld/pgstore/bootstrap.go | 2 +- .../pgstore/migrations/0015_session_bootstrap.sql | 9 +++++++-- internal/controld/pgstore/pgstore_test.go | 11 ++++++----- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/internal/controld/pgstore/bootstrap.go b/internal/controld/pgstore/bootstrap.go index 9b91bd6..8ab6317 100644 --- a/internal/controld/pgstore/bootstrap.go +++ b/internal/controld/pgstore/bootstrap.go @@ -35,7 +35,7 @@ func (r pgBootstraps) PutSessionBootstrap(ctx context.Context, ws control.Worksp INSERT INTO session_bootstraps (session_id, workspace_id, token_hash, placement_generation, expires_at, consumed_at) VALUES ($1, $2, $3, $4, $5, NULL) - ON CONFLICT (session_id) DO UPDATE SET + ON CONFLICT (workspace_id, session_id) DO UPDATE SET workspace_id = EXCLUDED.workspace_id, token_hash = EXCLUDED.token_hash, placement_generation = EXCLUDED.placement_generation, diff --git a/internal/controld/pgstore/migrations/0015_session_bootstrap.sql b/internal/controld/pgstore/migrations/0015_session_bootstrap.sql index e7550f4..da4f00b 100644 --- a/internal/controld/pgstore/migrations/0015_session_bootstrap.sql +++ b/internal/controld/pgstore/migrations/0015_session_bootstrap.sql @@ -19,12 +19,17 @@ -- -- ON DELETE CASCADE because a token outliving the session it names is a row -- nothing will ever consume and nothing will ever clean up. +-- sessions has been keyed by (workspace_id, id) since migration 0007, so the +-- token row is keyed and referenced the same way: a session id alone is not +-- unique across workspaces and Postgres rightly refuses a foreign key to it. CREATE TABLE IF NOT EXISTS session_bootstraps ( - session_id text PRIMARY KEY REFERENCES sessions(id) ON DELETE CASCADE, workspace_id text NOT NULL, + session_id text NOT NULL, token_hash text NOT NULL, placement_generation bigint NOT NULL, expires_at timestamptz NOT NULL, consumed_at timestamptz, - created_at timestamptz NOT NULL DEFAULT now() + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, session_id), + FOREIGN KEY (workspace_id, session_id) REFERENCES sessions(workspace_id, id) ON DELETE CASCADE ); diff --git a/internal/controld/pgstore/pgstore_test.go b/internal/controld/pgstore/pgstore_test.go index bfe8d06..7bb056e 100644 --- a/internal/controld/pgstore/pgstore_test.go +++ b/internal/controld/pgstore/pgstore_test.go @@ -272,13 +272,14 @@ func TestMigrate0003To0004AddsColumnsToLegacyRows(t *testing.T) { if want := embeddedMigrationVersions(t); !slices.Equal(applied, want) { t.Fatalf("schema_migrations = %v, want every embedded migration in order %v", applied, want) } - // This release's head is 14: a database that stopped at 0003 runs the + // This release's head is 15: a database that stopped at 0003 runs the // expand step (0007), the contract step (0008), the events table // (0009), the agent credentials table (0010), the tombstone (0011), the - // durable revoke fence (0012), the controller lease (0013), and the exec - // event's command name (0014) in the same start. - if head := applied[len(applied)-1]; head != 14 { - t.Fatalf("head migration = %d, want 14", head) + // durable revoke fence (0012), the controller lease (0013), the exec + // event's command name (0014), and the session bootstrap token (0015) in + // the same start. + if head := applied[len(applied)-1]; head != 15 { + t.Fatalf("head migration = %d, want 15", head) } // The legacy session survived, and its new columns read as "never exited" From 7fa9044f583be37c4cb651652169c84df60c072d Mon Sep 17 00:00:00 2001 From: jiashuoz Date: Mon, 21 Sep 2026 16:42:19 +0900 Subject: [PATCH 7/7] =?UTF-8?q?fix(microvm):=20review=20round=202=20?= =?UTF-8?q?=E2=80=94=20the=20guest=20channel,=20the=20id=20space,=20and=20?= =?UTF-8?q?three=20budgets?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Seven findings from the second review of the bootstrap-token and vsock change, fixed in one commit because four of them are the same guest channel seen from different sides. 1. The vsock accept loop served inline. serveGuest was called without `go` and immediately wrote a boot config that can exceed 1.3 MiB (two scripts at MaxSetupBytes), so the first guest that connected and did not drain wedged the loop for good: later dials were accepted by the kernel and never served, sessiond's 30s wait expired, and the channel tracked no connections so Destroy could not break the write. Each accepted conn now gets its own goroutine, the boot-config write is bounded (bootConfigWriteTimeout), and guestChannel tracks the conn it serves so close() ends it. 2. Every connection got the configuration, the token and the hub. Exactly one guest connection is now served per boot generation and every later one is closed having received nothing — /dev/vsock is world-accessible inside the guest, so any sandbox process could dial (2, 1024), read the session's whole configuration plus a live token, and take over the hub. A sessiond that crashes inside a live VM is a new boot generation with a fresh mint (design note open question 2), not a re-serve. On the WebSocket side, setHub now REPORTS the hub it displaces, and retireDisplacedHub closes it after a grace — a displaced hub still has frames to deliver (a child_exited across a redial can be the only copy), but it cannot be held open forever by a peer that will not hang up. 3. A data race in TestTheBootstrapExchangeLeavesNothingInTheLog: log.SetOutput over an unsynchronized bytes.Buffer while runner-plane goroutines were still logging, caught by -race about one run in three. The sink is mutex-guarded, and the runner is stopped before the buffer is read. 4. The cold suspend was paid for at a freeze's price. The cold handler wraps a synchronous flush and an unmount around the same 10s exec kill the warm 12s budget was sized for, and the VM ends with no memory image, so anything the budget cuts short is lost rather than deferred: the cold path gets its own 30s ready budget (the ack stays at 2s, because it is answered before any of that work). The guest's lazy MNT_DETACH is now preceded by sync(2), which is what makes "no fsck needed" true rather than claimed. 5. A vacuous test: mutating the spent-token check so a redial re-presents its token still passed, because the re-exchange blocked on an undrained pipe and nothingMore() returned true on any read error. fakeHost now drains its conn from the moment it exists, nothingMore distinguishes silence from a broken connection, and the redial assertions run before the preamble is waited on so a re-exchange fails fast instead of passing slowly. 6. The runner's reserved id space was closed for requests and open for responses, so a sandbox could put a runner-space id on the wire in a `resp` — which travels further than a `req:`, being forwarded to the control plane rather than refused locally. The fence now reads both arms. The runner's pending table also matched on the id alone: it records the session a call was made for and deliver() checks it, so an answer naming another session cannot wake a cold resume. 7. A boots TOCTOU: two concurrent cold Resumes could compute the same next boot number, and openGuestChannel unlinks before it listens, so one would remove the other's live socket out from under a VM already told about it. The boot number is taken under the driver mutex, and a second Resume for an instance with one in flight is refused. Gates: make verify, go vet ./..., git diff --check, three -race runs over the changed packages, and the Postgres store tests against a real database. cmd/rainier-latency's TestMeasureSampleFallsBackToNameCleanupBeforeCreateAck fails intermittently under the full -race suite on origin/main too; it is a 1s subprocess budget in a package this change does not touch. Co-Authored-By: Claude Fable 5.1 --- cmd/sessiond/bootconfig.go | 17 +- cmd/sessiond/bootconfig_test.go | 176 ++++++++---- cmd/sessiond/transport_test.go | 8 +- cmd/sessiond/unmount_linux.go | 24 +- ...09-20-microvm-bootstrap-token-and-vsock.md | 20 +- internal/controld/srpc_bootstrap_test.go | 47 +++- internal/driver/microvm.go | 52 +++- internal/driver/microvm_test.go | 9 + internal/driver/microvm_vsock.go | 143 ++++++++-- internal/driver/microvm_vsock_test.go | 255 +++++++++++++++++- internal/runnerd/agent.go | 10 +- internal/runnerd/microvm.go | 57 ++-- internal/runnerd/microvm_test.go | 110 ++++++++ internal/runnerd/registry.go | 20 +- internal/runnerd/registry_test.go | 93 ++++++- internal/runnerd/runnerd.go | 118 +++++++- 16 files changed, 1027 insertions(+), 132 deletions(-) diff --git a/cmd/sessiond/bootconfig.go b/cmd/sessiond/bootconfig.go index 1b88fec..fe82111 100644 --- a/cmd/sessiond/bootconfig.go +++ b/cmd/sessiond/bootconfig.go @@ -404,12 +404,17 @@ func bootOverVsock(ctx context.Context, dial dialSession, b *bootstrapper) (rela } failure = bf // The conn is NOT handed back on a failed exchange. It may be - // perfectly good (a refusal) or it may not (a timeout, which ends - // the read by poisoning the conn's deadline), and the boot has no - // way to tell — so it is closed and dialLoop dials a fresh one. The - // cost is one extra connection on a session that is about to fail - // its boot chain anyway; the alternative is serving a conn that may - // already be dead and discovering it one frame later. + // perfectly good (a refusal) or it may already be gone: a timeout + // expires the context this exchange bounded its read and write + // with, and relay.NetConn answers an expired context by CLOSING the + // conn — deliberately, and the same answer *websocket.Conn gives + // (netconn.go). It is a close, not a poisoned deadline, so there is + // nothing here that could be cleared and reused. The boot cannot + // tell the two cases apart from the error, so it closes and dialLoop + // dials a fresh one. The cost is one extra connection on a session + // that is about to fail its boot chain anyway; the alternative is + // serving a conn that may already be dead and discovering it one + // frame later. // // The re-dial is safe precisely because the token is recorded as // ATTEMPTED: the preamble on the new connection reads the diff --git a/cmd/sessiond/bootconfig_test.go b/cmd/sessiond/bootconfig_test.go index 22b9d98..f358adf 100644 --- a/cmd/sessiond/bootconfig_test.go +++ b/cmd/sessiond/bootconfig_test.go @@ -26,9 +26,60 @@ import ( // fakeHost is the runner's end of a guest's control channel: it writes the // boot configuration as the first frame and answers (or refuses) the one // exchange the guest makes. +// +// It READS the connection continuously, from the moment it exists, and that +// is load-bearing rather than tidy. net.Pipe is synchronous: a guest that +// writes a request nobody is reading blocks in Write until its own deadline +// expires, and relay.NetConn answers an expired write context by closing the +// conn. A host that only looked for the request afterwards would therefore +// find a broken connection and read it as silence — which is how an +// "exactly once" assertion passes for a guest that asked twice. With a +// reader running the whole time, a request the guest makes is read off the +// wire as it is written, and is still here to be counted when the assertion +// runs. type fakeHost struct { conn relay.Conn t *testing.T + + // reqs carries every control REQUEST the guest sent, in order. + reqs chan relay.ControlEvent + // readErr carries the one error that ended the drain, which is what lets + // an assertion tell "the guest sent nothing" from "the connection died". + readErr chan error +} + +func newFakeHost(t *testing.T, conn relay.Conn) *fakeHost { + h := &fakeHost{ + conn: conn, t: t, + reqs: make(chan relay.ControlEvent, 8), + readErr: make(chan error, 1), + } + go h.drain() + return h +} + +// drain is the reader. It runs until the connection ends, and it never +// touches *testing.T: a failure reported from this goroutine would be +// reported after the test that owns it may have finished. +func (h *fakeHost) drain() { + for { + raw, err := h.conn.Read(context.Background()) + if err != nil { + h.readErr <- err + return + } + f, derr := relay.Decode(raw) + if derr != nil || f.Type != relay.FrameControl { + continue + } + var ev relay.ControlEvent + if json.Unmarshal(f.Payload, &ev) != nil { + continue + } + if strings.HasPrefix(ev.Kind, "req:") { + h.reqs <- ev + } + } } func (h *fakeHost) sendBootConfig(cfg runner.BootConfig) { @@ -57,53 +108,65 @@ func (h *fakeHost) sendControl(ev relay.ControlEvent) { } } -// nextRequest reads the next control REQUEST the guest sent, failing the -// test if none arrives. +// nextRequest returns the next control REQUEST the guest sent, failing the +// test if none arrives or if the connection ends first. func (h *fakeHost) nextRequest() relay.ControlEvent { h.t.Helper() - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - for { - raw, err := h.conn.Read(ctx) - if err != nil { - h.t.Fatalf("read from the guest: %v", err) - } - f, err := relay.Decode(raw) - if err != nil || f.Type != relay.FrameControl { - continue - } - var ev relay.ControlEvent - if json.Unmarshal(f.Payload, &ev) != nil { - continue - } - if strings.HasPrefix(ev.Kind, "req:") { - return ev + select { + case ev := <-h.reqs: + return ev + case err := <-h.readErr: + h.t.Fatalf("the guest's connection ended before it asked for anything: %v", err) + case <-time.After(5 * time.Second): + h.t.Fatal("the guest made no request within 5s") + } + return relay.ControlEvent{} +} + +// nothingMore asserts the guest made no further request within a short +// window. It is how "exactly once" is checked, and it fails the test itself +// so that the reason is the assertion's own sentence. +// +// The three ways the window can end are three different facts and only one +// of them is silence: +// +// - a request arrives: the guest asked for something it must not have +// asked for, and `what` says why that matters; +// - the window expires with nothing on the conn: silence, which is what +// the caller is claiming; +// - the connection ends: NOT silence by itself. It is only evidence +// because the drain has been reading since this host existed, so a +// request the guest did write is already in h.reqs and is checked for +// once more before this returns. Without that reader, a guest whose +// write timed out on an undrained pipe would break its own connection +// and be reported as quiet — which is exactly the assertion this +// replaces. +func (h *fakeHost) nothingMore(within time.Duration, what string) { + h.t.Helper() + select { + case ev := <-h.reqs: + h.t.Fatalf("%s: the guest sent %q", what, ev.Kind) + case err := <-h.readErr: + select { + case ev := <-h.reqs: + h.t.Fatalf("%s: the guest sent %q", what, ev.Kind) + default: } + h.t.Logf("the guest's connection ended within the window (%v) having asked for nothing", err) + case <-time.After(within): } } -// nothingMore asserts the guest sent no further request within a short -// window. It is how "exactly once" is checked. -func (h *fakeHost) nothingMore(within time.Duration) bool { +// waitClosed returns the error that ended the drain — how a test observes +// the guest hanging up. +func (h *fakeHost) waitClosed(within time.Duration) error { h.t.Helper() - ctx, cancel := context.WithTimeout(context.Background(), within) - defer cancel() - for { - raw, err := h.conn.Read(ctx) - if err != nil { - return true // the window closed with nothing on it - } - f, derr := relay.Decode(raw) - if derr != nil || f.Type != relay.FrameControl { - continue - } - var ev relay.ControlEvent - if json.Unmarshal(f.Payload, &ev) != nil { - continue - } - if strings.HasPrefix(ev.Kind, "req:") { - return false - } + select { + case err := <-h.readErr: + return err + case <-time.After(within): + h.t.Fatalf("the guest's connection was still open after %s", within) + return nil } } @@ -136,7 +199,7 @@ func fakeTransport(t *testing.T) (dialSession, <-chan *fakeHost) { return func(context.Context) (relay.Conn, error) { guest, host := net.Pipe() t.Cleanup(func() { _ = guest.Close(); _ = host.Close() }) - hosts <- &fakeHost{conn: relay.NetConn(host), t: t} + hosts <- newFakeHost(t, relay.NetConn(host)) return relay.NetConn(guest), nil }, hosts } @@ -271,9 +334,7 @@ func TestBootOverVsockAppliesTheConfigurationAndTheSecrets(t *testing.T) { // Exactly once: the token is single-use, so a second exchange on the same // token would be refused and would fail a boot that had already worked. - if !host.nothingMore(200 * time.Millisecond) { - t.Fatal("the guest made a second request on one boot") - } + host.nothingMore(200*time.Millisecond, "the guest made a second request on one boot") } // TestBootOverVsockWithNoSecretsDeclaredIsACleanBoot pins the difference @@ -301,9 +362,7 @@ func TestBootOverVsockWithNoSecretsDeclaredIsACleanBoot(t *testing.T) { if bootErr != nil || failure != nil { t.Fatalf("a session declaring no secrets did not boot cleanly: %v / %v", bootErr, failure) } - if !host.nothingMore(200 * time.Millisecond) { - t.Fatal("a session declaring no secrets asked for some anyway") - } + host.nothingMore(200*time.Millisecond, "a session declaring no secrets asked for some anyway") } // TestARefusedExchangeFailsTheBootChain is the compatibility table's last @@ -395,12 +454,15 @@ func TestARefusedExchangeFailsTheBootChain(t *testing.T) { redialDone := make(chan error, 1) go func() { redialDone <- reBootstrap(context.Background(), redial, boots) }() redialHost.sendBootConfig(cfg) + // Asserted BEFORE the preamble is waited on, and that ordering is the + // test: a guest that re-presented the token would be sitting in + // exchange() waiting out its whole 30-second answer budget, so waiting + // on reBootstrap first would turn a bug into a slow pass. + redialHost.nothingMore(200*time.Millisecond, + "the redial re-presented a token the control plane had already spent") if err := <-redialDone; err != nil { t.Fatalf("the connection after a failed boot was refused: %v", err) } - if !redialHost.nothingMore(200 * time.Millisecond) { - t.Fatal("the redial re-presented a token the control plane had already spent") - } } // TestTheBootExchangeIsNumberedOutOfTheDispatchersSpace pins the id the boot @@ -477,9 +539,7 @@ func TestAMissingTokenWithDeclaredSecretsFailsTheBootChain(t *testing.T) { if !strings.Contains(failure.Error(), "1 secret(s)") || !strings.Contains(failure.Error(), "no bootstrap token") { t.Errorf("the failure = %q", failure) } - if !host.nothingMore(200 * time.Millisecond) { - t.Fatal("the guest asked for its secrets with no token to ask with") - } + host.nothingMore(200*time.Millisecond, "the guest asked for its secrets with no token to ask with") } // TestAResumeReExchangesAndARedialDoesNot is the rule that makes single use @@ -511,6 +571,9 @@ func TestAResumeReExchangesAndARedialDoesNot(t *testing.T) { first := host.nextRequest() host.answer(first.ID, map[string]string{"DEPLOY_KEY": "value_example"}) <-done + // Exactly one on the boot connection, so the redial below is being + // compared against a known number and not against "at least one". + host.nothingMore(200*time.Millisecond, "the boot exchanged more than once") // A REDIAL inside the same VM: the same configuration, the same token. // Nothing is asked for, because the token has been spent and the values @@ -523,12 +586,13 @@ func TestAResumeReExchangesAndARedialDoesNot(t *testing.T) { redialDone := make(chan error, 1) go func() { redialDone <- reBootstrap(context.Background(), redialConn, boots) }() redialHost.sendBootConfig(cfg) + // Before the wait, for the same reason as above: a guest that + // re-exchanged would be blocked on an answer, not finished. + redialHost.nothingMore(200*time.Millisecond, + "a redial re-exchanged a spent token, which would be refused forever") if err := <-redialDone; err != nil { t.Fatalf("a redial failed: %v", err) } - if !redialHost.nothingMore(200 * time.Millisecond) { - t.Fatal("a redial re-exchanged a spent token, which would be refused forever") - } // A RESUME: a new VM, a new socket, a new token. The secrets are fetched // again, because the process on this side may be a completely new one diff --git a/cmd/sessiond/transport_test.go b/cmd/sessiond/transport_test.go index db45a14..0dce349 100644 --- a/cmd/sessiond/transport_test.go +++ b/cmd/sessiond/transport_test.go @@ -109,10 +109,10 @@ func TestAFailedPreambleClosesTheConnection(t *testing.T) { t.Fatalf("connect = %v, want the preamble's own error", err) } host := <-hosts - // The guest end is closed, so the host's read ends rather than blocking. - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - if _, err := host.conn.Read(ctx); err == nil { + // The guest end is closed, so the host's drain ends rather than + // blocking. It is read through the host's own reader rather than off the + // conn here: two readers on one net.Pipe end would race. + if err := host.waitClosed(2 * time.Second); err == nil { t.Fatal("a connection whose preamble failed was left open") } } diff --git a/cmd/sessiond/unmount_linux.go b/cmd/sessiond/unmount_linux.go index 9ffe5a3..112a611 100644 --- a/cmd/sessiond/unmount_linux.go +++ b/cmd/sessiond/unmount_linux.go @@ -18,11 +18,23 @@ import ( // above it is a home that comes back needing a fsck for a credential set // somebody logged in for once. // -// MNT_DETACH is deliberate. A lazy unmount detaches the tree immediately and -// lets the kernel finish when the last reference goes, which is what a -// process still holding a file under it turns an ordinary unmount into -// EBUSY over — and the alternative to detaching is refusing to answer the -// host, which ends the VM anyway with nothing flushed at all. +// MNT_DETACH is deliberate, and the sync(2) in front of it is what makes it +// safe. A lazy unmount detaches the tree immediately and lets the kernel +// finish when the last reference goes, which is what turns an ordinary +// unmount into EBUSY when a process is still holding a file under it — and +// the alternative to detaching is refusing to answer the host, which ends the +// VM anyway with nothing flushed at all. But "finish when the last reference +// goes" is precisely a writeback this VM may not live to see: the host +// terminates it on the ready answer, so a detach on its own is NOT a promise +// that the device is clean. sync() before it is: it returns once every dirty +// page on every mounted filesystem has been handed to its device, so what the +// lazy detach has left to do afterwards is teardown rather than data. +// +// (sync() is whole-machine rather than per-mount, which inside a microVM +// whose only writable devices are this session's own is exactly the right +// scope — the workspace disk wants flushing before this VM ends for the same +// reason. syncfs(2) on a descriptor under the mount would be narrower and +// buys nothing here.) // // Every failure is a log line and not an error. A session with no home // mounted (no creator, or an older control plane) is the ordinary case and @@ -32,6 +44,8 @@ func unmountAgentHome(path string) { if _, err := os.Stat(path); err != nil { return } + // Before the detach: see above. It cannot fail and returns nothing. + unix.Sync() if err := unix.Unmount(path, unix.MNT_DETACH); err != nil { if errors.Is(err, unix.EINVAL) || errors.Is(err, unix.ENOENT) { // Not a mount point: nothing was mounted here, which is what a diff --git a/docs/design/2026-09-20-microvm-bootstrap-token-and-vsock.md b/docs/design/2026-09-20-microvm-bootstrap-token-and-vsock.md index 78cac9e..64863ad 100644 --- a/docs/design/2026-09-20-microvm-bootstrap-token-and-vsock.md +++ b/docs/design/2026-09-20-microvm-bootstrap-token-and-vsock.md @@ -257,7 +257,14 @@ socket is inside one VM's jail directory, so the connection's identity is its pa 4. The guest boots. `sessiond` opens `AF_VSOCK` to (2, 1024). 5. Firecracker forwards it to `_1024`; runnerd accepts and builds the relay hub with `relay.NewHubWithControl`, as `register` does today, keyed by the session the - socket path names. + socket path names. **Exactly one** connection is served per boot generation, and every + later one on that socket is closed having received nothing at all: `/dev/vsock` is + world-accessible inside an ordinary guest, so any process in the sandbox can dial + (2, 1024), and step 6's frame is the session's whole configuration plus a live token. + Each accepted connection is handled on a goroutine of its own and that frame's write is + bounded, so a peer that connects and never reads holds nothing but its own connection. + A sessiond that crashes and restarts inside a live VM is therefore NOT re-served: it is + a new boot generation, with a new socket and a fresh mint (open question 2). 6. runnerd's first frame is `FrameControl{Kind: "boot_config"}`: session id, command, proxy URL, egress allowlist, agent manifest, git author, repos, setup and init scripts with their bounds, `SecretNames`, and the token. @@ -271,8 +278,15 @@ socket is inside one VM's jail directory, so the connection's identity is its pa Cold suspend reuses the handshake that exists, with one additive field: `ControlEvent` gains ``Cold bool `json:"cold,omitempty"` `` on `KindSuspending`, meaning "this is not a freeze — flush, unmount `/rainier/agents`, forget every delivered secret". `sessiond` -answers `KindSuspendAck` at once and `KindSuspendReady` when done, under the budgets that -already exist (2s and 12s). runnerd then terminates the microVM — no memory image, per +answers `KindSuspendAck` at once — before any of that work, so the ack budget is the warm +path's **2s**, unchanged — and `KindSuspendReady` when done, under a cold ready budget of +**30s** rather than the warm 12s. The warm 12s covers one thing: the sandbox's own 10s +exec kill (`execQuiesceBudget`), with two to spare. The cold path wraps a synchronous +agent-home flush and an unmount (which syncs the device before it detaches it) around +that same 10s kill, and this VM ends with no memory image — so work the budget cuts short +is lost rather than deferred, where a warm freeze merely postpones it. 30s is the kill +with a factor of three over it. A sandbox that answers spends none of it; one that +predates `Cold` answers as fast as it always did. runnerd then terminates the microVM — no memory image, per ADR §2.2 — detaches the disk, drops the token, and produces the checkpoint behind the §4.4 barrier. A sessiond predating `Cold` reads a plain `suspending`, quiesces its execs, and the host-side unmount still happens; the guest simply did not help. diff --git a/internal/controld/srpc_bootstrap_test.go b/internal/controld/srpc_bootstrap_test.go index 13f3a56..f71e2ec 100644 --- a/internal/controld/srpc_bootstrap_test.go +++ b/internal/controld/srpc_bootstrap_test.go @@ -5,8 +5,10 @@ import ( "bytes" "context" "encoding/json" + "errors" "log" "strings" + "sync" "testing" "time" @@ -355,10 +357,36 @@ func TestBootstrapMethodsAreRefusedForAnotherRunnersSession(t *testing.T) { // // A log line is the easiest place for a value to end up and the hardest place // to get it back out of, which is why this is a test and not a review note. +// syncBuffer is a log sink that can be READ while something is still writing +// to it. +// +// log.Logger serializes its own writers under its own mutex, so concurrent +// log.Printf calls are safe against each other — but the test's read of the +// buffer is not one of those writers, and this package's runner-plane +// goroutines log on their own schedule. A plain bytes.Buffer here is a data +// race that `go test -race` catches in roughly one run in three, which is +// the worst frequency a race can have. +type syncBuffer struct { + mu sync.Mutex + b bytes.Buffer +} + +func (s *syncBuffer) Write(p []byte) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.b.Write(p) +} + +func (s *syncBuffer) String() string { + s.mu.Lock() + defer s.mu.Unlock() + return s.b.String() +} + func TestTheBootstrapExchangeLeavesNothingInTheLog(t *testing.T) { - var captured bytes.Buffer + captured := &syncBuffer{} prevOut, prevFlags := log.Writer(), log.Flags() - log.SetOutput(&captured) + log.SetOutput(captured) log.SetFlags(0) t.Cleanup(func() { log.SetOutput(prevOut); log.SetFlags(prevFlags) }) @@ -387,6 +415,21 @@ func TestTheBootstrapExchangeLeavesNothingInTheLog(t *testing.T) { t.Fatal("the replay succeeded") } + // Everything this test set in motion is stopped before the log is read. + // The runner plane holds a goroutine per connected runner and it logs — + // so a grep taken while one is still running is a grep over a buffer + // that is still being written, and the question "did anything leak + // across this whole exchange" has no answer until the exchange's own + // machinery has finished saying things. (syncBuffer makes reading it + // safe either way; this makes the answer complete.) + f.close() + eventually(t, 3*time.Second, func() error { + if s.runnerConnected("vm1") { + return errors.New("controld has not noticed the runner leaving yet") + } + return nil + }) + out := captured.String() for _, forbidden := range []string{bootstrapFixtureSecret, minted.Token} { if forbidden == "" { diff --git a/internal/driver/microvm.go b/internal/driver/microvm.go index c14cdca..968acbc 100644 --- a/internal/driver/microvm.go +++ b/internal/driver/microvm.go @@ -221,8 +221,18 @@ type instanceRecord struct { // boots counts launches of this instance, so each one gets a vsock socket // path of its own. A cold resume is a new VM and Firecracker's own // documentation warns that one uds_path cannot be multiplexed across two. + // + // It is read AND advanced under the driver mutex by the one resume that + // claimed the instance (see resuming), so no two boots of one instance + // can ever be handed the same path. boots int + // resuming is the claim one Resume holds over this instance while it is + // in flight. A second Resume for the same id is refused rather than run + // beside it: see Resume for what two concurrent cold ones would do to + // each other's socket. + resuming bool + // epoch counts mutations of this record, and exists because Inspect and // List ask the hypervisor with the driver mutex RELEASED (its answer is // socket I/O, and holding the mutex across it froze every other session @@ -1036,8 +1046,7 @@ func (m *Microvm) Resume(ctx context.Context, id string) (bool, error) { } running := inst.State == StateRunning cold, bootLive, cfg := inst.Cold, inst.bootLive, inst.Cfg - bootCfg, sessionID, boots := inst.boot, inst.SessionID, inst.boots - m.mu.Unlock() + bootCfg, sessionID := inst.boot, inst.SessionID // Resuming a session that is already running restarts nothing, and says // so without touching the hypervisor (driver.go's Resume contract). The @@ -1046,9 +1055,43 @@ func (m *Microvm) Resume(ctx context.Context, id string) (bool, error) { // resume turned "nothing to do" into a failed Resume — and runnerd reads // a failed Resume as a session it could not bring back. if running { + m.mu.Unlock() return false, nil } + // One resume at a time per instance, claimed under the same lock that + // reads the state it decided on. + // + // Two concurrent cold resumes would otherwise both see the same state, + // both compute the same next boot number, and both open a channel on the + // same socket path — and openGuestChannel UNLINKS before it listens, so + // the second would remove the first's live socket out from under a VM + // that had already been told about it. That session then boots and is + // never configured, which is the one outcome this whole path exists to + // prevent. The pair of them would also mint two tokens and launch twice. + if inst.resuming { + m.mu.Unlock() + return false, fmt.Errorf("resume of %s: a resume is already in flight for this instance", id) + } + inst.resuming = true + // The boot number is taken HERE, under the same lock, and it advances + // even for an attempt that fails: a failed launch may have left a socket + // behind at that path, and an attempt that reuses a number is an attempt + // that inherits it. The counter is only ever a source of distinct paths. + boots := inst.boots + if cold { + inst.boots++ + boots = inst.boots + } + m.mu.Unlock() + defer func() { + m.mu.Lock() + if e, ok := m.instances[id]; ok { + e.resuming = false + } + m.mu.Unlock() + }() + restarted := false var channel *guestChannel if cold { @@ -1086,7 +1129,7 @@ func (m *Microvm) Resume(ctx context.Context, id string) (bool, error) { return false, fmt.Errorf("cold resume of %s: minting a bootstrap token: %w", id, err) } bootCfg.BootstrapToken = token - udsPath, listenPath, err := m.vsockPaths(id, boots+1) + udsPath, listenPath, err := m.vsockPaths(id, boots) if err != nil { return false, err } @@ -1122,7 +1165,8 @@ func (m *Microvm) Resume(ctx context.Context, id string) (bool, error) { inst.channel = channel inst.boot = bootCfg inst.Cfg.VsockUDSPath = cfg.VsockUDSPath - inst.boots++ + // inst.boots was advanced when this resume claimed the instance, so + // that the path it opened could not collide with a concurrent one. } inst.bump() rec := persistable(inst) diff --git a/internal/driver/microvm_test.go b/internal/driver/microvm_test.go index afe3c58..5dcf08d 100644 --- a/internal/driver/microvm_test.go +++ b/internal/driver/microvm_test.go @@ -128,6 +128,15 @@ func (h *stubMicrovmHost) MintSessionBootstrap(_ context.Context, sessionID stri return fmt.Sprintf("token_example_%s_%d", sessionID, h.mints), nil } +// connCount is how many guest connections the runner was handed. One per +// BOOT is the rule the driver enforces; anything more is a second process in +// the sandbox having been served a session's configuration. +func (h *stubMicrovmHost) connCount() int { + h.mu.Lock() + defer h.mu.Unlock() + return len(h.conns) +} + func (h *stubMicrovmHost) mintCount() int { h.mu.Lock() defer h.mu.Unlock() diff --git a/internal/driver/microvm_vsock.go b/internal/driver/microvm_vsock.go index 0927545..eab8948 100644 --- a/internal/driver/microvm_vsock.go +++ b/internal/driver/microvm_vsock.go @@ -33,6 +33,7 @@ import ( "slices" "strconv" "sync" + "time" "github.com/tokencanopy/rainier/internal/relay" "github.com/tokencanopy/rainier/protocol/runner" @@ -59,6 +60,21 @@ const ( // --microvm-state-dir, and "invalid argument" on a socket is not an error // anybody can act on, where naming the limit and the path is. unixPathMax = 107 + // bootConfigWriteTimeout bounds the ONE write this driver makes onto a + // guest connection, and it exists because the thing written is large and + // the peer is untrusted. + // + // A boot configuration carries two scripts of up to MaxSetupBytes each + // (driver.go), so it can reach well over a megabyte — far past any socket + // buffer. A guest that connects and never reads would park an unbounded + // write forever, which is a session's control channel held open by + // whoever dialled it first. Ten seconds is orders of magnitude more than + // a guest that IS reading needs, and it is finite. + // + // relay.NetConn answers an expired write context by CLOSING the conn, + // which is exactly the right answer here: a guest that will not take its + // own configuration has nothing further to say on this connection. + bootConfigWriteTimeout = 10 * time.Second ) // MicrovmHost is what the microVM driver needs from the runner above it. @@ -170,15 +186,76 @@ type guestChannel struct { // failure would put a line in an operator's log for every session that // ends normally. closed bool + // served records that this boot generation's ONE guest connection has + // been taken. /dev/vsock is world-accessible inside an ordinary guest, + // so every process in the sandbox can dial (2, 1024) — and what the + // first frame carries is the session's whole configuration and a LIVE + // bootstrap token. Serving every connection would hand that to whoever + // asked, as many times as they asked, and let the last one become the + // session's hub. + // + // So the model is the design note's: one guest-initiated connection per + // boot (§4). The first is sessiond; every later one is refused and closed + // having received nothing at all. A sessiond that crashed and came back + // is a NEW boot generation — a new VM, a new socket, a new mint (open + // question 2) — and not something to re-serve this token to. + served bool + // conns are the connections this channel is still responsible for. It + // holds at most the one served guest, and it exists so close() can end + // it: a Destroy that closed only the listener would leave a wedged + // boot-config write (a guest that never drains) holding a goroutine and + // a socket for as long as its deadline, with nothing able to interrupt + // it. + conns map[net.Conn]struct{} +} + +// claim reserves this boot generation's one guest connection for c and takes +// responsibility for closing it. It reports false for a second connection and +// for one that arrived after teardown — in both cases the caller closes c +// without writing a byte to it. +func (g *guestChannel) claim(c net.Conn) bool { + g.mu.Lock() + defer g.mu.Unlock() + if g.closed || g.served { + return false + } + g.served = true + if g.conns == nil { + g.conns = map[net.Conn]struct{}{} + } + g.conns[c] = struct{}{} + return true +} + +// drop closes a claimed connection this channel will not be handing on, and +// stops tracking it. The claim is NOT released: a boot generation gets one +// attempt, and a guest that could not be configured needs a fresh boot and a +// fresh token rather than a second go at a spent one. +func (g *guestChannel) drop(c net.Conn) { + g.mu.Lock() + delete(g.conns, c) + g.mu.Unlock() + _ = c.Close() } func (g *guestChannel) close() { g.mu.Lock() g.closed = true + conns := make([]net.Conn, 0, len(g.conns)) + for c := range g.conns { + conns = append(conns, c) + } + g.conns = nil g.mu.Unlock() if g.listener != nil { _ = g.listener.Close() } + // The accepted connection goes too, and it is what makes Destroy + // unblockable: a boot-config write into a guest that is not draining is + // only interrupted by the conn dying. + for _, c := range conns { + _ = c.Close() + } // Both paths, and an absent one is not an error: a teardown that runs // twice is ordinary, and a VM that never started leaves only one of them. _ = os.Remove(g.listenPath) @@ -204,9 +281,12 @@ func (m *Microvm) openGuestChannel(sessionID, udsPath, listenPath string, cfg ru // Sockets left behind by a VM that is gone would make this Listen, or // Firecracker's own bind at PUT /vsock, fail with "address already in // use" for paths nothing is serving. A launch that failed after the - // device was configured leaves exactly that, and the boot counter does - // not advance past a failed attempt — so without this a session could - // never be resumed again on this runner. + // device was configured leaves exactly that. A resume no longer reuses + // the failed attempt's number (Resume advances the counter under the + // driver mutex, so no two boots of one instance share a path) — but a + // create is always boot 1, and a record recovered after a runnerd + // restart counts from whatever it was, so neither path gets to assume + // the path it was handed is unused. _ = os.Remove(listenPath) _ = os.Remove(udsPath) ln, err := net.Listen("unix", listenPath) @@ -227,12 +307,15 @@ func (m *Microvm) openGuestChannel(sessionID, udsPath, listenPath string, cfg ru return g, nil } -// acceptGuests serves the channel for the life of the instance. +// acceptGuests keeps accepting for the life of the instance. // -// It loops rather than accepting once because a sandbox redials: sessiond -// survives losing its connection and comes back (cmd/sessiond's dial loop), -// and the guest that reconnects needs its configuration again — it is a new -// process as often as not, on the far side of a cold resume. +// It keeps accepting even though exactly one connection is ever SERVED, +// because the alternative is worse in both directions: an accept loop that +// stopped would leave later dials queued in the kernel with nobody to refuse +// them, and a loop that served inline would be wedged for good by the first +// guest that connected and did not read (the boot config can exceed a +// megabyte — see bootConfigWriteTimeout). So each connection gets a goroutine +// of its own, and all but the first are refused in it. func (m *Microvm) acceptGuests(sessionID string, g *guestChannel) { for { c, err := g.listener.Accept() @@ -242,11 +325,11 @@ func (m *Microvm) acceptGuests(sessionID string, g *guestChannel) { } return } - m.serveGuest(sessionID, g, c) + go m.serveGuest(sessionID, g, c) } } -// serveGuest hands one guest connection its configuration and then hands the +// serveGuest hands ONE guest connection its configuration and then hands the // connection to the runner. // // The boot configuration is the FIRST frame on the conn, written here before @@ -254,20 +337,36 @@ func (m *Microvm) acceptGuests(sessionID string, g *guestChannel) { // the stream" true: the guest reads its whole configuration off the same // connection it will then serve its terminal and its RPC over. // +// Every connection after the first is closed having received nothing — not +// the configuration, not the token, not a byte. See guestChannel.served: the +// socket is reachable by every process in the sandbox, and this is the one +// place that decides the first dial is the session's and no other is. +// // A host that has not been installed closes the connection rather than -// holding it. There is nothing to attach it to, and a guest that is dropped -// redials, where a guest held by a channel nobody serves waits forever. +// holding it. There is nothing to attach it to, and the claim stays spent: +// this boot has had its one connection. func (m *Microvm) serveGuest(sessionID string, g *guestChannel, c net.Conn) { + if !g.claim(c) { + // Not an error the operator can act on and not a rarity worth a + // line per occurrence — but it IS somebody in the guest dialling a + // socket that is not theirs, so it is said once per attempt and + // names nothing about the session but its id. + log.Printf("microvm: session %s: refusing a second connection on a control channel that serves one guest per boot", sessionID) + _ = c.Close() + return + } conn := relay.NetConn(c) - if err := writeBootConfig(conn, g.boot); err != nil { + ctx, cancel := context.WithTimeout(context.Background(), bootConfigWriteTimeout) + defer cancel() + if err := writeBootConfig(ctx, conn, g.boot); err != nil { log.Printf("microvm: session %s: sending the boot configuration: %v", sessionID, err) - _ = conn.Close() + g.drop(c) return } host := m.currentHost() if host == nil { log.Printf("microvm: session %s: a guest connected but this driver has no runner above it to serve it", sessionID) - _ = conn.Close() + g.drop(c) return } host.GuestConnected(sessionID, conn) @@ -277,11 +376,13 @@ func (m *Microvm) serveGuest(sessionID string, g *guestChannel, c net.Conn) { // the same frame shape every event, request and response on this channel // uses, so the guest needs no special reader for it. // -// The context is Background and not the caller's: this is the one write whose -// failure means the guest is unconfigurable, and inheriting a create's -// context would make a create that has just returned cancel the configuration -// of the guest it started. -func writeBootConfig(conn relay.Conn, cfg runner.BootConfig) error { +// The context is the caller's own bounded one rather than a create's: this is +// the one write whose failure means the guest is unconfigurable, so +// inheriting a create's context would let a create that has just returned +// cancel the configuration of the guest it started — but leaving it unbounded +// would let a guest that never reads hold the write open forever. See +// bootConfigWriteTimeout. +func writeBootConfig(ctx context.Context, conn relay.Conn, cfg runner.BootConfig) error { body, err := json.Marshal(cfg) if err != nil { // Unreachable — every field is a string, an int, or a slice of them — @@ -297,7 +398,7 @@ func writeBootConfig(conn relay.Conn, cfg runner.BootConfig) error { if err != nil { return errors.New("the boot configuration frame could not be encoded") } - return conn.Write(context.Background(), frame) + return conn.Write(ctx, frame) } // bootConfigFor composes what a guest is told about itself. diff --git a/internal/driver/microvm_vsock_test.go b/internal/driver/microvm_vsock_test.go index f490151..0509975 100644 --- a/internal/driver/microvm_vsock_test.go +++ b/internal/driver/microvm_vsock_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "errors" + "io" "net" "os" "strings" @@ -246,10 +247,11 @@ func TestMicrovmColdResumeMintsAFreshTokenAndSocket(t *testing.T) { // // Firecracker binds "" itself at PUT /vsock; the driver binds // "_1024". A launch that fails after the device was configured -// leaves the first behind, and the boot counter does not advance past a -// failed attempt — so the next resume would recompute the same path, and -// Firecracker's bind would fail EADDRINUSE on a socket nothing is serving, -// forever. +// leaves the first behind, on a path a later boot can meet again — a create +// is always boot 1, and a record recovered after a runnerd restart counts +// from whatever it was — and Firecracker's bind would then fail EADDRINUSE +// on a socket nothing is serving, forever. So a failed attempt takes both +// paths with it. func TestAFailedBootLeavesNoSocketBehind(t *testing.T) { m, sim := testMicrovm(t, MicrovmOpts{TotalSlots: 4}) host := &stubMicrovmHost{} @@ -392,3 +394,248 @@ func waitFor(t *testing.T, cond func() bool, msg string) { time.Sleep(5 * time.Millisecond) } } + +// dialRawGuest is dialGuest without the relay framing: the tests below care +// about BYTES on the socket — whether any arrive at all, and whether reading +// them is what the far end is waiting for — which a relay.Conn's reader +// hides. +func dialRawGuest(t *testing.T, m *Microvm, instanceID string, boot int) net.Conn { + t.Helper() + _, listenPath, err := m.vsockPaths(instanceID, boot) + if err != nil { + t.Fatalf("vsock paths: %v", err) + } + deadline := time.Now().Add(2 * time.Second) + for { + c, err := net.Dial("unix", listenPath) + if err == nil { + t.Cleanup(func() { _ = c.Close() }) + return c + } + if time.Now().After(deadline) { + t.Fatalf("dial the guest control socket %s: %v", listenPath, err) + } + time.Sleep(5 * time.Millisecond) + } +} + +// oversizedSpec is a create whose boot configuration cannot fit in a socket +// buffer: two scripts at the driver's own cap, so the JSON the host writes is +// comfortably past a megabyte. A guest that does not read one of these is a +// guest the write is genuinely parked on, which is the condition the two +// tests below are about. +func oversizedSpec(sessionID string) Spec { + script := strings.Repeat("x", MaxSetupBytes) + return Spec{ + SessionID: sessionID, + Setup: script, + Init: script, + SecretNames: []string{"DEPLOY_KEY"}, + BootstrapToken: "token_example", + } +} + +// TestASilentGuestDoesNotWedgeTheControlChannel is review round 2, finding 1. +// +// The accept loop used to write the boot configuration inline. That +// configuration can exceed a megabyte (two scripts at MaxSetupBytes), so the +// first process in the guest to connect and then NOT read parked the loop for +// good: every later connection sat in the kernel's backlog accepted by nobody, +// the real sessiond's 30-second wait for its configuration expired, and the +// session died — with no way for a Destroy to break the write, because the +// channel tracked no connections at all. +// +// So: a silent guest holds nothing but its own connection. A second +// connection is still ANSWERED (refused, per the one-guest-per-boot rule +// below — but answered, which is the part that proves the loop is alive), and +// Destroy returns. +func TestASilentGuestDoesNotWedgeTheControlChannel(t *testing.T) { + m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4}) + m.SetHost(&stubMicrovmHost{}) + ctx := context.Background() + + h, err := m.Create(ctx, oversizedSpec("sess-silent")) + if err != nil { + t.Fatal(err) + } + defer m.Destroy(ctx, h.ID) + + // Connect, take ONE byte, and then stop reading. That byte is what makes + // this deterministic rather than a race with the dial below: it proves + // this connection is the one the channel claimed and that its + // configuration is already on its way. The remaining megabyte-odd has + // nowhere to go, so the host's write is now parked for good. + silent := dialRawGuest(t, m, h.ID, 1) + defer silent.Close() + if err := silent.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil { + t.Fatal(err) + } + var one [1]byte + if _, err := io.ReadFull(silent, one[:]); err != nil { + t.Fatalf("the first guest was sent nothing at all: %v", err) + } + + // The accept loop must still be serving. A second connection is closed + // with nothing on it, and — this is the assertion — it gets that answer + // promptly rather than after the silent guest's write deadline. + second := dialRawGuest(t, m, h.ID, 1) + if err := second.SetReadDeadline(time.Now().Add(3 * time.Second)); err != nil { + t.Fatal(err) + } + var buf [1]byte + n, err := second.Read(buf[:]) + if n != 0 { + t.Fatalf("a second connection was sent %d byte(s); it must receive nothing at all", n) + } + if !errors.Is(err, io.EOF) { + t.Fatalf("a second connection ended with %v, want EOF — it must be closed, not left waiting", err) + } + + // And the session can still be torn down with that write still parked. + done := make(chan error, 1) + go func() { done <- m.Destroy(ctx, h.ID) }() + select { + case err := <-done: + if err != nil { + t.Fatalf("Destroy: %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("Destroy never returned while a guest held the control channel open") + } +} + +// TestOnlyOneGuestIsServedPerBoot is review round 2, finding 2. +// +// /dev/vsock is world-accessible inside an ordinary guest, so every process +// in the sandbox can dial (2, 1024). The first frame on that connection is +// the session's whole configuration AND a live, single-use bootstrap token — +// so a host that served every connection handed both to whoever asked, as +// often as they asked, and let the last one become the session's hub. +// +// The design's model is one guest-initiated connection per boot (§4). This +// pins it: the first connection is served, every later one receives NOTHING +// and is closed, the hub the first produced is not displaced, and a Destroy +// afterwards still cleans the socket up. +func TestOnlyOneGuestIsServedPerBoot(t *testing.T) { + m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4}) + host := &stubMicrovmHost{} + m.SetHost(host) + ctx := context.Background() + + h, err := m.Create(ctx, Spec{ + SessionID: "sess-once", SecretNames: []string{"DEPLOY_KEY"}, + BootstrapToken: "token_example", + }) + if err != nil { + t.Fatal(err) + } + defer m.Destroy(ctx, h.ID) + + // The first connection is sessiond, and it gets everything. + if got := readBootConfig(t, dialGuest(t, m, h.ID, 1)).BootstrapToken; got != "token_example" { + t.Fatalf("the first guest was handed token %q, want the create's", got) + } + waitFor(t, func() bool { return host.connCount() == 1 }, + "the first guest connection never reached the runner") + + // Every later one is a process inside the sandbox dialling a socket that + // is not its to dial. It reads EOF, having been sent nothing. + for i := 0; i < 2; i++ { + other := dialRawGuest(t, m, h.ID, 1) + if err := other.SetReadDeadline(time.Now().Add(3 * time.Second)); err != nil { + t.Fatal(err) + } + b, err := io.ReadAll(other) + if len(b) != 0 { + t.Fatalf("connection %d received %d byte(s) of a session's configuration", i+2, len(b)) + } + if err != nil { + t.Fatalf("connection %d was left open rather than closed: %v", i+2, err) + } + } + + // And it did not become the session's hub: the runner was handed exactly + // one connection, the first. + if n := host.connCount(); n != 1 { + t.Fatalf("the runner was handed %d guest connection(s) for one boot, want 1", n) + } + + // A Destroy after all that still tears the channel down. + if err := m.Destroy(ctx, h.ID); err != nil { + t.Fatalf("Destroy: %v", err) + } + _, listenPath, err := m.vsockPaths(h.ID, 1) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(listenPath); !os.IsNotExist(err) { + t.Errorf("Destroy left the guest control socket behind: %v", err) + } +} + +// TestTwoConcurrentColdResumesDoNotShareASocket is review round 2, finding 7. +// +// The boot counter used to be read with the driver mutex released, so two +// cold resumes of one instance could compute the same next boot number — and +// openGuestChannel unlinks before it listens, so the second would remove the +// first's live socket out from under a VM that had already been told about +// it. That session boots and is never configured, which is the one outcome +// this whole path exists to prevent. +// +// One resume wins; the other is refused; the winner's socket is there +// afterwards. +func TestTwoConcurrentColdResumesDoNotShareASocket(t *testing.T) { + m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4}) + m.SetHost(&stubMicrovmHost{}) + ctx := context.Background() + + h, err := m.Create(ctx, Spec{SessionID: "sess-race", BootstrapToken: "token_example"}) + if err != nil { + t.Fatal(err) + } + defer m.Destroy(ctx, h.ID) + if err := m.Suspend(ctx, h.ID, false); err != nil { + t.Fatal(err) + } + + type outcome struct { + restarted bool + err error + } + out := make(chan outcome, 2) + start := make(chan struct{}) + for i := 0; i < 2; i++ { + go func() { + <-start + restarted, err := m.Resume(ctx, h.ID) + out <- outcome{restarted, err} + }() + } + close(start) + + restarts, refusals := 0, 0 + for i := 0; i < 2; i++ { + o := <-out + switch { + case o.err == nil && o.restarted: + restarts++ + case o.err != nil && strings.Contains(o.err.Error(), "already in flight"): + refusals++ + default: + t.Fatalf("a concurrent resume returned (%v, %v), want a restart or an in-flight refusal", o.restarted, o.err) + } + } + if restarts != 1 || refusals != 1 { + t.Fatalf("%d restart(s) and %d refusal(s) from two concurrent resumes, want 1 and 1", restarts, refusals) + } + + // The winner's guest channel is live: a guest can still be configured, + // which is what the losing unlink used to take away. + _, _, err = m.vsockPaths(h.ID, 2) + if err != nil { + t.Fatal(err) + } + if got := readBootConfig(t, dialGuest(t, m, h.ID, 2)).SessionID; got != "sess-race" { + t.Fatalf("the resumed guest read session %q off its channel", got) + } +} diff --git a/internal/runnerd/agent.go b/internal/runnerd/agent.go index e5a140e..b28e558 100644 --- a/internal/runnerd/agent.go +++ b/internal/runnerd/agent.go @@ -612,9 +612,15 @@ func (s *Server) forwardSessionRPC(m runner.ToRunner, send func(runner.FromRunne // a caller inside this process, not to the guest. The id spaces are // disjoint so the two can share one connection — see // runnerOriginatedIDBase. + // + // The SESSION is part of the match, not just the id: an answer naming a + // different session than the call was made for is not that call's answer, + // and delivering it would hand a cold resume of one session a token + // minted against another's row. if env.Method == "resp" && isRunnerOriginated(env.ID) { - if !s.runnerRPC.deliver(env) { - log.Printf("agent: an answer for this runner's own request %d has no caller waiting (timed out?); dropping", env.ID) + if !s.runnerRPC.deliver(m.Session, env) { + log.Printf("agent: an answer for this runner's own request %d on session %s has no caller waiting (timed out, or a different session than it was made for); dropping", + env.ID, m.Session) } return } diff --git a/internal/runnerd/microvm.go b/internal/runnerd/microvm.go index e7c1b88..32ae768 100644 --- a/internal/runnerd/microvm.go +++ b/internal/runnerd/microvm.go @@ -70,10 +70,18 @@ const runnerOriginatedIDBase uint64 = 1 << 63 // counter. forwardSessionRPC asks before it routes a response into a sandbox. func isRunnerOriginated(id uint64) bool { return id&runnerOriginatedIDBase != 0 } -// refuseSandboxOrigin reports why an upward request must not be forwarded, +// refuseSandboxOrigin reports why an upward message must not be forwarded, // or "" when it may be. It is the fence on the one door an untrusted peer // has into the control plane's method table. // +// It guards RESPONSES as well as requests, and that is not a formality: an +// id space is closed at both ends or it is not closed. A sandbox that may +// not send `req:` numbered 1<<63|n but may send `resp` numbered 1<<63|n has +// put an id from the runner's space on the wire either way — and the `resp` +// travels further, because the runner forwards it to the control plane +// rather than answering it there. `method` is the envelope's method, which +// is "resp" for a response, so both arms consult the same fence. +// // Two things are refused, and only this hop can refuse either of them. // // A sandbox may not mint its own bootstrap token. The design's whole point @@ -98,7 +106,7 @@ func refuseSandboxOrigin(method string, id uint64) string { case method == runner.MethodMintSessionBootstrap: return "a sandbox may not mint its own bootstrap token; a fresh one is minted by the runner on a cold resume" case isRunnerOriginated(id): - return "this request id is reserved for the runner's own requests" + return "this id is reserved for the runner's own requests" } return "" } @@ -113,20 +121,34 @@ func refuseSandboxOrigin(method string, id uint64) string { type runnerRPCTable struct { seq atomic.Uint64 mu sync.Mutex - // waiting is id → the one-slot channel its caller is selecting on. Every - // caller removes its own entry, so nothing sweeps this map. - waiting map[uint64]chan runner.RPCEnvelope + // waiting is id → the one call waiting on it. Every caller removes its + // own entry, so nothing sweeps this map. + waiting map[uint64]pendingRunnerCall +} + +// pendingRunnerCall is one in-flight request this runner made, and it records +// the SESSION it was made for as well as the channel to answer on. +// +// The session is half of the correlation, not decoration. An id alone says +// "some call is waiting on this number"; a mint for session A answered by a +// message naming session B would otherwise be delivered to A's caller, which +// would boot A's new VM with a token minted against B's row. The id spaces +// are per-process, the sessions are not, so the pair is what identifies a +// call. +type pendingRunnerCall struct { + session string + ch chan runner.RPCEnvelope } func newRunnerRPCTable() *runnerRPCTable { - return &runnerRPCTable{waiting: map[uint64]chan runner.RPCEnvelope{}} + return &runnerRPCTable{waiting: map[uint64]pendingRunnerCall{}} } -func (t *runnerRPCTable) begin() (uint64, chan runner.RPCEnvelope) { +func (t *runnerRPCTable) begin(session string) (uint64, chan runner.RPCEnvelope) { id := runnerOriginatedIDBase | t.seq.Add(1) ch := make(chan runner.RPCEnvelope, 1) t.mu.Lock() - t.waiting[id] = ch + t.waiting[id] = pendingRunnerCall{session: session, ch: ch} t.mu.Unlock() return id, ch } @@ -137,16 +159,21 @@ func (t *runnerRPCTable) end(id uint64) { t.mu.Unlock() } -// deliver hands a response to the call waiting on its id, reporting whether -// anyone was. The channel is buffered by one and every caller removes its own -// entry, so this never blocks the reader that called it. -func (t *runnerRPCTable) deliver(env runner.RPCEnvelope) bool { +// deliver hands a response to the call waiting on its id AND its session, +// reporting whether one was. The channel is buffered by one and every caller +// removes its own entry, so this never blocks the reader that called it. +// +// A mismatched session is not delivered anywhere: the caller learns nothing +// arrived and times out, which is the honest outcome for an answer that does +// not belong to the question. +func (t *runnerRPCTable) deliver(session string, env runner.RPCEnvelope) bool { t.mu.Lock() - ch, ok := t.waiting[env.ID] + p, ok := t.waiting[env.ID] t.mu.Unlock() - if !ok { + if !ok || p.session != session { return false } + ch := p.ch select { case ch <- env: default: @@ -181,7 +208,7 @@ func (s *Server) MintSessionBootstrap(ctx context.Context, sessionID string) (st return "", fmt.Errorf("encoding the bootstrap mint request: %w", err) } - id, ch := s.runnerRPC.begin() + id, ch := s.runnerRPC.begin(sessionID) defer s.runnerRPC.end(id) if !s.fireSessionRPC(sessionID, runner.RPCEnvelope{ diff --git a/internal/runnerd/microvm_test.go b/internal/runnerd/microvm_test.go index 4f18669..be214a8 100644 --- a/internal/runnerd/microvm_test.go +++ b/internal/runnerd/microvm_test.go @@ -414,6 +414,10 @@ func TestColdSuspendTellsAMicrovmSandboxItIsCold(t *testing.T) { } s.suspendAckWait = 200 * time.Millisecond s.suspendReadyWait = 200 * time.Millisecond + // The cold path has a budget of its own, and this test's sandbox never + // answers — so without shortening it too, this test would sit out the + // production thirty seconds. + s.coldSuspendReadyWait = 200 * time.Millisecond ctx := context.Background() if err := s.CreateWithID(ctx, "sess-cold", driver.Spec{}, nil); err != nil { t.Fatal(err) @@ -497,3 +501,109 @@ func readControlEvent(t *testing.T, conn relay.Conn) relay.ControlEvent { } return ev } + +// TestASandboxMayNotAnswerWithARunnerSpaceID is review round 2, finding 6. +// +// The guard used to read only the `req:` arm, so a sandbox could not ASK with +// an id out of the runner's reserved space but could ANSWER with one — and an +// answer travels further than a request does, because the runner forwards it +// to the control plane rather than refusing it locally. An id space the +// untrusted end can write into is not a space, and it is not a space at +// either end of it. +func TestASandboxMayNotAnswerWithARunnerSpaceID(t *testing.T) { + s, _ := testMicrovmServer(t) + s.reg.put("sess-resp", &sessionEntry{id: "sess-resp", state: "running"}) + guest, host := net.Pipe() + defer guest.Close() + s.GuestConnected("sess-resp", relay.NetConn(host)) + waitForHub(t, s, "sess-resp") + + forwarded := make(chan runner.RPCEnvelope, 4) + s.SetOnSessionRPC(func(_ string, env runner.RPCEnvelope) { forwarded <- env }) + defer s.SetOnSessionRPC(nil) + + sandbox := relay.NetConn(guest) + send := func(ev relay.ControlEvent) { + t.Helper() + payload, err := json.Marshal(ev) + if err != nil { + t.Fatal(err) + } + frame, err := relay.Encode(relay.Frame{Type: relay.FrameControl, Payload: payload}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := sandbox.Write(ctx, frame); err != nil { + t.Fatal(err) + } + } + + // A response numbered out of the runner's own space: dropped, not + // forwarded. It is not refused back down either — answering an answer is + // meaningless, and there was never a caller for this one. + send(relay.ControlEvent{Kind: "resp", ID: runnerOriginatedIDBase | 5, OK: true, + Payload: []byte(`{"token":"token_the_sandbox_chose"}`)}) + + // An ordinary response after it, to prove the channel still works and + // that the drop above is about the id and not about `resp` at all. It + // arriving is also what makes the "nothing was forwarded" check below + // read over real traffic rather than over an empty channel. + send(relay.ControlEvent{Kind: "resp", ID: 12, OK: true, Payload: []byte(`{"ok":true}`)}) + + select { + case env := <-forwarded: + if isRunnerOriginated(env.ID) { + t.Fatalf("a sandbox put a runner-space id on the wire: %+v", env) + } + if env.ID != 12 { + t.Fatalf("the forwarded response = %+v, want the sandbox's own id 12", env) + } + case <-time.After(3 * time.Second): + t.Fatal("no response reached the control plane at all") + } + select { + case env := <-forwarded: + t.Fatalf("a second response was forwarded: %+v", env) + default: + } +} + +// TestARunnerAnswerForAnotherSessionIsNotDelivered is the other half of +// review round 2, finding 6: the runner's pending table used to match on the +// id alone and ignore the session the answer named. +// +// The id space is per-process and the session is not, so a mint for one +// session answered by a message naming another would have been delivered to +// the first's caller — which is a cold resume booting a VM with a token +// minted against a different session's row. +func TestARunnerAnswerForAnotherSessionIsNotDelivered(t *testing.T) { + tbl := newRunnerRPCTable() + id, ch := tbl.begin("sess-a") + defer tbl.end(id) + + answer, err := json.Marshal(map[string]any{"token": "token_for_b"}) + if err != nil { + t.Fatal(err) + } + env := runner.RPCEnvelope{ID: id, Method: "resp", OK: true, Payload: answer} + + if tbl.deliver("sess-b", env) { + t.Fatal("an answer naming another session was delivered to this call") + } + select { + case got := <-ch: + t.Fatalf("the caller was woken by another session's answer: %+v", got) + default: + } + + // The same envelope on the right session is the same call's answer, so + // the check above is the session and not a blanket refusal. + if !tbl.deliver("sess-a", env) { + t.Fatal("the call's own answer was not delivered") + } + if got := <-ch; got.ID != id { + t.Fatalf("delivered %+v, want the call's own id", got) + } +} diff --git a/internal/runnerd/registry.go b/internal/runnerd/registry.go index 9375245..b687de3 100644 --- a/internal/runnerd/registry.go +++ b/internal/runnerd/registry.go @@ -341,15 +341,29 @@ func (r *registry) list() []sessionEntry { // still booting/dialing in) — the caller has a live hub with a running // readLoop goroutine that will now never be found through the registry, so // it must close that hub itself instead of leaking it. -func (r *registry) setHub(id string, h *relay.Hub) bool { +// +// It also RETURNS the hub it displaced, when there was one and it is not the +// same hub again, because a displaced hub is nobody's otherwise: nothing in +// the registry points at it any more, so if its conn never dies its readLoop +// never errors and its conn, its fd and its attachments are held by a +// structure nothing can reach. That is not hypothetical — the peer on the +// other end of a displaced conn is a sandbox, and a sandbox can simply +// decline to hang up. It is returned rather than closed here because the +// registry is not where a hub's lifetime is decided and because a displaced +// hub is not immediately useless: see retireDisplacedHub. +func (r *registry) setHub(id string, h *relay.Hub) (displaced *relay.Hub, ok bool) { r.mu.Lock() defer r.mu.Unlock() e, ok := r.items[id] if !ok { - return false + return nil, false } + displaced = e.hub e.hub = h - return true + if displaced == h { + displaced = nil + } + return displaced, true } // hub reads an entry's hub field under the registry lock. attach's diff --git a/internal/runnerd/registry_test.go b/internal/runnerd/registry_test.go index 283c97f..7064e18 100644 --- a/internal/runnerd/registry_test.go +++ b/internal/runnerd/registry_test.go @@ -1,7 +1,14 @@ // internal/runnerd/registry_test.go package runnerd -import "testing" +import ( + "context" + "net" + "testing" + "time" + + "github.com/tokencanopy/rainier/internal/relay" +) func TestRegistryTracksAndWaits(t *testing.T) { r := newRegistry() @@ -18,3 +25,87 @@ func TestRegistryTracksAndWaits(t *testing.T) { t.Fatal("removed entry still present") } } + +// TestSetHubReportsTheHubItDisplacesSoItCanBeRetired is review round 2, +// finding 2. +// +// setHub used to overwrite the field and drop the previous hub on the floor, +// naming nobody. The ordinary re-register is a hub whose conn already died, +// so nothing was visibly wrong — but a SECOND live connection claiming one +// session left the first hub running with nothing able to reach it: its +// readLoop never errored, and its conn, its fd and its attachments were held +// for the life of the process, because the peer that decides when that conn +// dies is the sandbox. +// +// It is reported rather than closed here, because a displaced hub still has +// frames to deliver — see retireDisplacedHub, which is what ends it. +func TestSetHubReportsTheHubItDisplacesSoItCanBeRetired(t *testing.T) { + r := newRegistry() + r.put("s1", &sessionEntry{id: "s1", state: "running"}) + + firstGuest, firstHost := net.Pipe() + defer firstGuest.Close() + defer firstHost.Close() + first := relay.NewHub(context.Background(), relay.NetConn(firstHost)) + if displaced, ok := r.setHub("s1", first); !ok || displaced != nil { + t.Fatalf("the first setHub displaced %p (ok=%v); there was nothing to displace", displaced, ok) + } + + secondGuest, secondHost := net.Pipe() + defer secondGuest.Close() + defer secondHost.Close() + second := relay.NewHub(context.Background(), relay.NetConn(secondHost)) + displaced, ok := r.setHub("s1", second) + if !ok { + t.Fatal("the entry did not take the second hub") + } + if displaced != first { + t.Fatalf("setHub reported %p as displaced, want the first hub %p — an unreported hub is a leaked one", displaced, first) + } + got, ok := r.hub("s1") + if !ok || got != second { + t.Fatalf("the registry holds %p, want the second hub %p", got, second) + } + + // Setting the SAME hub again displaces nothing: a caller told to retire + // it would be closing the session's own live connection. + if displaced, ok := r.setHub("s1", second); !ok || displaced != nil { + t.Fatalf("re-setting a session's own hub reported it as displaced (%p, ok=%v)", displaced, ok) + } + + // And an entry that vanished reports neither. + r.remove("s1") + if displaced, ok := r.setHub("s1", second); ok || displaced != nil { + t.Fatalf("setHub on a removed entry = (%p, %v), want (nil, false)", displaced, ok) + } +} + +// TestRetireDisplacedHubWaitsAndThenCloses pins both halves of the grace. +// +// A hub that is going to drain what it already read gets to: the child_exited +// on a conn a redial replaced can be the only copy of it +// (TestAChildExitAcrossARedialIsRecordedEndToEnd). A hub whose peer simply +// never hangs up does not get to hold a conn, an fd and its attachments +// forever. +func TestRetireDisplacedHubWaitsAndThenCloses(t *testing.T) { + guest, host := net.Pipe() + defer guest.Close() + defer host.Close() + h := relay.NewHub(context.Background(), relay.NetConn(host)) + + // Within the grace it is left alone, live conn and all. + go retireDisplacedHub("s1", h, time.Hour) + select { + case <-h.Done(): + t.Fatal("a displaced hub was closed inside its grace, dropping what it had already read") + case <-time.After(100 * time.Millisecond): + } + + // And when the grace is up it goes, peer or no peer. + go retireDisplacedHub("s1", h, time.Millisecond) + select { + case <-h.Done(): + case <-time.After(3 * time.Second): + t.Fatal("a displaced hub outlived its grace; its conn, its fd and its attachments are leaked") + } +} diff --git a/internal/runnerd/runnerd.go b/internal/runnerd/runnerd.go index e3b0ffb..c2eb662 100644 --- a/internal/runnerd/runnerd.go +++ b/internal/runnerd/runnerd.go @@ -105,6 +105,14 @@ type Server struct { // written immediately after New. suspendAckWait time.Duration suspendReadyWait time.Duration + // coldSuspendReadyWait is the "are they gone" budget for a COLD suspend, + // which asks the sandbox for strictly more work than a freeze does. Same + // reason it is a field: a test must not spend the production one. + coldSuspendReadyWait time.Duration + // displacedHubGrace is how long a hub a re-register replaced keeps + // delivering before it is closed. A field for the same reason, and + // written only immediately after New. + displacedHubGrace time.Duration // now is the clock every idle-stop decision reads: time.Now in // production, a test's own function in tests, so the thirty minutes a // session has to sit idle can be a table row rather than a sleep. A field @@ -218,6 +226,8 @@ func New(drv driver.Driver, dialBase, egressAdmin, proxyURL string) *Server { s := &Server{drv: drv, reg: newRegistry(), dialBase: dialBase, egressAdmin: egressAdmin, proxyURL: proxyURL, hubWait: defaultHubWait, now: time.Now, suspendAckWait: defaultSuspendAckWait, suspendReadyWait: defaultSuspendReadyWait, + coldSuspendReadyWait: defaultColdSuspendReadyWait, + displacedHubGrace: defaultDisplacedHubGrace, suspends: map[string]*suspendWaiter{}, runnerRPC: newRunnerRPCTable()} // The microVM driver is composed BELOW this server and needs two things // from above it — where a guest's control conn goes, and where a cold @@ -1021,7 +1031,8 @@ func (s *Server) serveSessionConn(ctx context.Context, id string, conn relay.Con // its own and this hop stayed as it is. go s.routeControl(id, boot, reg, payload) }) - if !s.reg.setHub(id, hub) { + displaced, ok := s.reg.setHub(id, hub) + if !ok { // The entry vanished between our existence check above and now — a // concurrent DELETE raced this dial-in (session torn down while its // container was still booting). No registry entry will ever exist to @@ -1030,6 +1041,9 @@ func (s *Server) serveSessionConn(ctx context.Context, id string, conn relay.Con hub.Close() return } + if displaced != nil { + go retireDisplacedHub(id, displaced, s.displacedHubGrace) + } log.Printf("session %s registered", id) s.fireEvent(id, "running") // Block on the hub's own liveness signal, not r.Context(). websocket.Accept @@ -1227,6 +1241,16 @@ func (s *Server) routeControl(id string, boot, reg uint64, payload []byte) { log.Printf("session %s: control response with no id; dropping", id) return } + // The id space is fenced for responses too. A response is not a + // request, so there is no method here to refuse — but the number on + // it is the same number, and a sandbox answering with one out of the + // runner's own space has put a runner-space id on the wire for the + // control plane to echo. Dropped rather than refused: answering an + // answer is meaningless, and there was never a caller for this one. + if refusal := refuseSandboxOrigin("resp", ev.ID); refusal != "" { + log.Printf("session %s: dropping a control response from a sandbox: %s", id, refusal) + return + } if !s.fireSessionRPC(id, runner.RPCEnvelope{ID: ev.ID, Method: "resp", OK: ev.OK, Payload: ev.Payload}) { // Nothing to report to and nothing to answer: answering an answer // is meaningless, and whoever asked has already given up (its @@ -1284,9 +1308,35 @@ func (s *Server) routeControl(id string, boot, reg uint64, payload []byte) { // arrived — so only a sandbox that actually speaks the vocabulary can spend // it. It is comfortably longer than the sandbox's own quiesce budget, so the // ordinary answer is the acknowledgement rather than this timeout. +// defaultColdSuspendReadyWait is the third budget, and it exists because a +// cold suspend is not a freeze and cannot be paid for at a freeze's price. +// +// A warm suspend asks the sandbox for one thing: end your execs. Twelve +// seconds is comfortably over the ten the sandbox gives that +// (execQuiesceBudget in cmd/sessiond), because the ten is itself the +// SIGTERM-then-SIGKILL escalation and nothing follows it. +// +// A cold suspend asks for three things, in series, and this VM is about to +// cease to exist with no memory image (ADR-0003 §2.2), so anything not +// finished is lost rather than deferred: +// +// - a synchronous agent-home flush, which is a write to a block device; +// - the same ten-second exec kill; +// - an unmount of /rainier/agents, which syncs that device before it goes. +// +// Ten seconds of that is the kill alone, so twelve leaves two for a flush and +// an unmount — and the first of those two to run slowly is the one whose work +// is thrown away. Thirty is the kill with a factor of three over it, which is +// headroom rather than a second stopwatch: a sandbox that answers takes +// milliseconds, and only one that is genuinely stuck spends any of this. +// +// The ACK budget is shared with the warm path deliberately. The cold handler +// answers the ack before it does any of the work above, so "did you hear me" +// is the same question at the same price on both paths. const ( - defaultSuspendAckWait = 2 * time.Second - defaultSuspendReadyWait = 12 * time.Second + defaultSuspendAckWait = 2 * time.Second + defaultSuspendReadyWait = 12 * time.Second + defaultColdSuspendReadyWait = 30 * time.Second ) // quiesceExecs tells the sandbox it is about to be frozen and waits for it to @@ -1339,17 +1389,73 @@ func (s *Server) quiesceExecs(ctx context.Context, id string, cold bool) { return } - // "Are they gone." Only a sandbox that answered above can spend this. - readyTimer := time.NewTimer(s.suspendReadyWait) + // "Are they gone." Only a sandbox that answered above can spend this, + // and a COLD suspend is given its own budget: it asked for a flush and + // an unmount around the same kill, and work this one does not finish is + // lost rather than deferred, because there is no memory image and no VM + // on the other side of it. + readyWait := s.suspendReadyWait + if cold { + readyWait = s.coldSuspendReadyWait + } + readyTimer := time.NewTimer(readyWait) defer readyTimer.Stop() select { case <-w.ready: case <-hub.Done(): case <-ctx.Done(): case <-readyTimer.C: + what := "ending its commands" + if cold { + what = "flushing, ending its commands and unmounting the agent home" + } log.Printf("session %s: the sandbox heard the suspend but had not finished "+ - "ending its commands within %s; suspending anyway", id, s.suspendReadyWait) + "%s within %s; suspending anyway", id, what, readyWait) + } +} + +// defaultDisplacedHubGrace is how long a hub a re-register replaced has to +// finish delivering what it had already read, before it is closed. +// +// A displaced hub is not immediately useless, which is why this is a grace +// and not a close. sessiond survives losing its conn and redials, and a hub +// read loop that was stalled writing to a wedged viewer drains its buffered +// frames whenever it comes back — and the child_exited on that older conn can +// be the ONLY copy, because sessiond re-sends only what it never delivered. +// Dropping it leaves a finished session holding its slot for the life of the +// runner (TestAChildExitAcrossARedialIsRecordedEndToEnd). +// +// But it cannot be kept forever either, and "until its conn dies" is forever +// when the peer is a sandbox that declines to hang up: nothing in the +// registry points at a displaced hub, so its readLoop, its conn, its fd and +// its attachments are held by a structure nothing can reach (review round 2, +// finding 2). +// +// Ninety seconds is over clientWriteBase (70s, internal/relay), which is the +// longest one stalled write can legitimately hold that read loop up — so a +// hub that is going to drain has drained, and one that is not is closed. The +// ordinary case costs nothing: a redial happens because the old conn broke, +// so the old hub is usually already done before this timer is armed. +const defaultDisplacedHubGrace = 90 * time.Second + +// retireDisplacedHub ends a hub a newer connection replaced, once its grace +// is up. It returns without closing anything if the hub ends on its own +// first, which is what nearly every redial looks like. +// +// The goroutine that was serving the displaced hub is parked on hub.Done(), +// so closing here is what lets it go; its hubDied finds a newer hub in the +// registry and settles nothing, which is already how a stale hub is handled. +func retireDisplacedHub(id string, h *relay.Hub, grace time.Duration) { + timer := time.NewTimer(grace) + defer timer.Stop() + select { + case <-h.Done(): + return + case <-timer.C: } + log.Printf("session %s: closing the connection a re-register replaced; it had %s to deliver what it had already read", + id, grace) + h.Close() } // armSuspendWaiter registers this suspend's waiter, replacing any stale entry: