diff --git a/cmd/sessiond/bootconfig.go b/cmd/sessiond/bootconfig.go new file mode 100644 index 00000000..fe821117 --- /dev/null +++ b/cmd/sessiond/bootconfig.go @@ -0,0 +1,514 @@ +package main + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "log" + "os" + "slices" + "strconv" + "sync" + "time" + + "github.com/tokencanopy/rainier/internal/relay" + "github.com/tokencanopy/rainier/protocol/runner" +) + +// The microVM boot: what a guest reads off its vsock connection before it is +// a session at all, and the one exchange that turns a capability into the +// environment secrets its create deliberately did not carry. +// +// See docs/design/2026-09-20-microvm-bootstrap-token-and-vsock.md §4. On the +// Docker path none of this runs: the container's environment block IS the +// configuration, sessiond reads it with bootEnvFromOS as it always has, and +// no secret ever has to be asked for. + +const ( + // bootConfigWait bounds how long a guest waits for its configuration. + // The host writes it as the first frame the instant the guest connects, + // so this is a bound on a host that is not going to answer rather than + // on anything that legitimately takes time. + bootConfigWait = 30 * time.Second + // secretsExchangeWait bounds the round trip to the control plane. It + // sits under the bootstrap token's own 120-second lifetime, so a refusal + // is an answer rather than an expiry. + secretsExchangeWait = 30 * time.Second + // bootDialAttempts and bootDialBackoff bound the retry on the very first + // dial. Everything else in this process redials with backoff for the + // life of the session, and the one dial that could not was the one whose + // failure is fatal — a guest whose kernel brought /dev/vsock up a moment + // after sessiond started, or a host whose accept loop was one scheduling + // quantum behind, would have taken the session down for a condition that + // resolves itself. The bound exists because a guest that genuinely has + // no channel must fail rather than sit in a loop nobody can see. + bootDialAttempts = 10 + bootDialBackoff = 500 * time.Millisecond + // secretsRequestID is the id this end assigns its boot-time exchange. + // + // The request is made before the RPC dispatcher is serving — there is no + // session to serve for yet — so it is numbered by hand, and it is + // numbered OUT OF the dispatcher's space rather than at the bottom of + // it. The dispatcher counts from 1, so a boot exchange numbered 1 whose + // answer arrived late would be matched to the first call the dispatcher + // made afterwards (an agent credential fetch, typically) and answered + // with a body that method cannot read. + // + // It stays clear of runnerd's own reserved space too: that is the high + // BIT (1<<63), and a request carrying it is refused at the runner as a + // sandbox using an id that is not its to use. + secretsRequestID = 1 << 62 +) + +// bootstrapper is the guest side of the boot: it reads a configuration off +// each connection and, when that configuration brings a token it has not +// already spent, exchanges it for the session's secrets. +// +// It is stateful for one reason, and it is the reason a redial is not a +// resume: a bootstrap token is SINGLE-USE. A sessiond that re-exchanged on +// every connection would be refused on the first redial inside a live VM and +// would keep being refused forever. What distinguishes a resume from a +// redial is that a resume brings a NEW token — runnerd mints one before it +// boots the new VM — so "exchange when the token is one I have not spent" is +// exactly "at boot and again after every resume", with no second signal +// needed. +type bootstrapper struct { + mu sync.Mutex + // attempted is the last token this session sent a request for, whatever + // the answer was. + // + // ATTEMPTED and not "spent", which is the difference between a session + // that recovers and one that does not. A token is consumed by the + // control plane BEFORE the secrets are resolved, so a refused or + // timed-out exchange has spent it just as surely as a successful one; + // retrying it can only ever be refused. A sessiond that keyed on success + // would re-exchange a dead token on every redial, fail the preamble + // every time, and take a session that was merely missing its secrets — + // and had already said so, loudly, as a failed boot stage — off the air + // entirely. + attempted string + // delivered are the names this session has been given values for, kept + // so a cold suspend can forget them by name. NAMES, never values: what + // was delivered lives in the process environment and nowhere else. + delivered []string +} + +// bootFailure is a boot that cannot proceed, rendered as the sentence its +// failing stage prints. +// +// It names a COUNT and never a name's value — and never the names either, +// beyond how many there were, because a refusal that listed them would put +// an environment's shape in a session's error text where a workspace's +// members can read it. +type bootFailure struct{ reason string } + +func (f *bootFailure) Error() string { return f.reason } + +// undeliveredSecrets is the sentence a boot fails with when the values a +// create promised did not arrive. why is the control plane's own refusal, or +// this end's description of what was missing; neither carries a value. +func undeliveredSecrets(n int, why string) *bootFailure { + return &bootFailure{reason: fmt.Sprintf( + "rainier: this session's environment declares %d secret(s) that could not be delivered: %s. "+ + "The agent has not been started, because an environment that promised a credential and does not have it "+ + "is not the environment this session was created from.", n, why)} +} + +// readBootConfig waits for the first control frame on a fresh connection and +// requires it to be the boot configuration. +// +// Anything else on the way is dropped rather than refused: this conn is the +// session's whole channel, and a host that sent something before the +// configuration is a host this guest should keep listening to. +func readBootConfig(ctx context.Context, conn relay.Conn) (runner.BootConfig, error) { + ctx, cancel := context.WithTimeout(ctx, bootConfigWait) + defer cancel() + for { + raw, err := conn.Read(ctx) + if err != nil { + return runner.BootConfig{}, fmt.Errorf("waiting for the boot configuration: %w", err) + } + f, err := relay.Decode(raw) + if err != nil || f.Type != relay.FrameControl { + continue + } + var ev relay.ControlEvent + if err := json.Unmarshal(f.Payload, &ev); err != nil || ev.Kind != relay.KindBootConfig { + continue + } + var cfg runner.BootConfig + if err := json.Unmarshal(ev.Payload, &cfg); err != nil { + return runner.BootConfig{}, fmt.Errorf("the boot configuration could not be decoded: %w", err) + } + if cfg.Protocol != runner.SessionBootstrapProtocolVersion { + return runner.BootConfig{}, fmt.Errorf( + "this host speaks boot-configuration protocol %d and this session image speaks %d; the session image must be replaced", + cfg.Protocol, runner.SessionBootstrapProtocolVersion) + } + return cfg, nil + } +} + +// exchange turns cfg's token into the session's secrets, or says why it +// could not. +// +// It returns (nil, nil) for the two cases that are not failures: a +// configuration declaring no secrets at all, and a token this session has +// already spent — which is every redial inside one live VM. +// +// The request is written straight onto the conn rather than through the RPC +// dispatcher because the dispatcher is not serving yet: the relay it rides on +// is started later, by dialLoop, once there is a session to serve. +func (b *bootstrapper) exchange(ctx context.Context, conn relay.Conn, cfg runner.BootConfig) (map[string]string, error) { + if len(cfg.SecretNames) == 0 { + // A clean boot. "No secrets declared" and "declared and never + // arrived" are different facts, and this is the first. + return nil, nil + } + b.mu.Lock() + already := cfg.BootstrapToken != "" && cfg.BootstrapToken == b.attempted + if !already { + // Recorded BEFORE the request goes out, not after it succeeds: the + // control plane spends the token when it receives it, so a request + // that was sent has spent it whatever comes back. + b.attempted = cfg.BootstrapToken + } + b.mu.Unlock() + if already { + return nil, nil + } + if cfg.BootstrapToken == "" { + return nil, undeliveredSecrets(len(cfg.SecretNames), + "this session's create carried no bootstrap token, which means a control plane that does not withhold them") + } + + body, err := json.Marshal(struct { + Protocol int `json:"protocol"` + Token string `json:"token"` + }{runner.SessionBootstrapProtocolVersion, cfg.BootstrapToken}) + if err != nil { + return nil, undeliveredSecrets(len(cfg.SecretNames), "the request could not be encoded") + } + payload, err := json.Marshal(relay.ControlEvent{ + Kind: "req:" + runner.MethodFetchSessionSecrets, ID: secretsRequestID, Payload: body}) + if err != nil { + return nil, undeliveredSecrets(len(cfg.SecretNames), "the request could not be encoded") + } + frame, err := relay.Encode(relay.Frame{Type: relay.FrameControl, Payload: payload}) + if err != nil { + return nil, undeliveredSecrets(len(cfg.SecretNames), "the request could not be encoded") + } + + ctx, cancel := context.WithTimeout(ctx, secretsExchangeWait) + defer cancel() + if err := conn.Write(ctx, frame); err != nil { + return nil, undeliveredSecrets(len(cfg.SecretNames), "the request could not be sent to the runner") + } + + for { + raw, err := conn.Read(ctx) + if err != nil { + return nil, undeliveredSecrets(len(cfg.SecretNames), "the runner did not answer") + } + f, derr := relay.Decode(raw) + if derr != nil || f.Type != relay.FrameControl { + continue + } + var ev relay.ControlEvent + if json.Unmarshal(f.Payload, &ev) != nil || ev.Kind != "resp" || ev.ID != secretsRequestID { + continue + } + if !ev.OK { + // The control plane's own sentence, relayed: it names which of + // the four conditions holds — spent, expired, fenced, unknown — + // and never a value. + return nil, undeliveredSecrets(len(cfg.SecretNames), rpcErrorText(ev.Payload)) + } + var answer struct { + Env map[string]string `json:"env"` + } + if json.Unmarshal(ev.Payload, &answer) != nil { + // Logged without the error, and reported without it: a json + // message quotes what it choked on, and that is the secret. + return nil, undeliveredSecrets(len(cfg.SecretNames), "the answer could not be decoded") + } + return answer.Env, nil + } +} + +// remember records which names this session has been given values for, so a +// cold suspend can unset exactly those and nothing else. +func (b *bootstrapper) remember(names []string) { + b.mu.Lock() + defer b.mu.Unlock() + for _, n := range names { + if !slices.Contains(b.delivered, n) { + b.delivered = append(b.delivered, n) + } + } +} + +// forget unsets every delivered secret from this process's environment and +// reports how many there were. +// +// It is what a cold suspend asks for. The VM is about to be terminated with +// no memory image written anywhere, so this is belt and braces rather than +// the guarantee — but it is the half this process can make, and it is free: +// a sessiond that is frozen instead of terminated (a host that sent `cold` +// and then changed its mind) has no secret left in its environment for +// whatever reads it next. +// +// It cannot reach a child that already inherited them. That is stated rather +// than papered over: the agent has the values, by design, and the thing that +// takes them away is the VM ending. +func (b *bootstrapper) forget() int { + b.mu.Lock() + names := b.delivered + b.delivered = nil + // `attempted` is deliberately NOT cleared. The token this session was + // handed is spent whether or not this process still holds its values, + // and a resume brings a new one — so forgetting which token was already + // presented would only make the next connection re-present a dead one. + b.mu.Unlock() + for _, n := range names { + _ = os.Unsetenv(n) + } + return len(names) +} + +// applyBootConfig puts a boot configuration into this process's environment, +// in the shape the boot chain already reads (bootEnvFromOS). +// +// Translating into the environment block rather than threading the struct +// through is deliberate: everything downstream — prepareBoot, the agent +// sync, the exec runner's inherited environment, the child the agent becomes +// — already reads exactly these variables on the Docker path, and a second +// way of saying the same thing is a second way for the two paths to diverge. +// The scripts arrive as plain strings on the wire and are base64-encoded +// here for the same reason they are on the other path: that is what the +// reader expects. +// +// RAINIER_DIAL is deliberately NOT set. There is nothing to dial. +func applyBootConfig(cfg runner.BootConfig, secrets map[string]string) error { + set := func(k, v string) error { + if v == "" { + return nil + } + return os.Setenv(k, v) + } + b64 := func(s string) string { + if s == "" { + return "" + } + return base64.StdEncoding.EncodeToString([]byte(s)) + } + + // The secrets go FIRST, so that the configuration below wins over them. + // That is the same precedence the control plane already applies on the + // Docker path, where the agent-home keys are launch invariants a + // workspace's own configuration may not replace; a delivered value + // spelled like one of them must not redirect credential custody here + // either. (The plane also drops such a name from SecretNames, so this is + // the second of two fences rather than the only one.) + for k, v := range secrets { + if err := os.Setenv(k, v); err != nil { + return fmt.Errorf("applying this session's environment: %w", err) + } + } + + if err := set("RAINIER_SESSION", cfg.SessionID); err != nil { + return err + } + if cfg.ProxyURL != "" { + for _, k := range []string{"HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy"} { + if err := set(k, cfg.ProxyURL); err != nil { + return err + } + } + for _, k := range []string{"NO_PROXY", "no_proxy"} { + if err := set(k, cfg.NoProxy); err != nil { + return err + } + } + } + if cfg.Setup != "" { + if err := set("RAINIER_SETUP_B64", b64(cfg.Setup)); err != nil { + return err + } + if err := set("RAINIER_SETUP_TIMEOUT", strconv.Itoa(cfg.SetupTimeoutSec)); err != nil { + return err + } + } + if cfg.Init != "" { + if err := set("RAINIER_INIT_B64", b64(cfg.Init)); err != nil { + return err + } + if err := set("RAINIER_INIT_TIMEOUT", strconv.Itoa(cfg.InitTimeoutSec)); err != nil { + return err + } + } + if len(cfg.Repos) > 0 { + blob, err := json.Marshal(cfg.Repos) + if err != nil { + return fmt.Errorf("encoding this session's repository list: %w", err) + } + if err := set("RAINIER_REPOS_B64", base64.StdEncoding.EncodeToString(blob)); err != nil { + return err + } + } + if err := set("RAINIER_GIT_AUTHOR_NAME", cfg.GitAuthorName); err != nil { + return err + } + if err := set("RAINIER_GIT_AUTHOR_EMAIL", cfg.GitAuthorEmail); err != nil { + return err + } + for k, v := range cfg.Env { + if err := os.Setenv(k, v); err != nil { + return fmt.Errorf("applying this session's configuration: %w", err) + } + } + return nil +} + +// bootOverVsock is the whole microVM boot preamble, run once before this +// process has a session at all: dial, read the configuration, exchange the +// token, apply both. +// +// It returns the connection it bootstrapped on, which dialLoop then serves +// its relay over — the SAME conn, because the boot configuration and the +// terminal stream and the session RPC all ride one channel by design, and +// because a second dial would be a second guest for a socket that serves one. +// +// A failed exchange is returned as a *bootFailure and NOT as a dial error: +// the connection is good, the session is real, and what has to happen is +// that the boot chain fails loudly enough for a person to see why. The +// caller turns it into a failing stage. +func bootOverVsock(ctx context.Context, dial dialSession, b *bootstrapper) (relay.Conn, runner.BootConfig, *bootFailure, error) { + conn, err := dialBoot(ctx, dial) + if err != nil { + return nil, runner.BootConfig{}, nil, err + } + cfg, err := readBootConfig(ctx, conn) + if err != nil { + _ = conn.Close() + return nil, runner.BootConfig{}, nil, err + } + secrets, xerr := b.exchange(ctx, conn, cfg) + var failure *bootFailure + if xerr != nil { + bf, ok := xerr.(*bootFailure) + if !ok { + _ = conn.Close() + return nil, cfg, nil, xerr + } + failure = bf + // The conn is NOT handed back on a failed exchange. It may be + // perfectly good (a refusal) or it may already be gone: a timeout + // expires the context this exchange bounded its read and write + // with, and relay.NetConn answers an expired context by CLOSING the + // conn — deliberately, and the same answer *websocket.Conn gives + // (netconn.go). It is a close, not a poisoned deadline, so there is + // nothing here that could be cleared and reused. The boot cannot + // tell the two cases apart from the error, so it closes and dialLoop + // dials a fresh one. The cost is one extra connection on a session + // that is about to fail its boot chain anyway; the alternative is + // serving a conn that may already be dead and discovering it one + // frame later. + // + // The re-dial is safe precisely because the token is recorded as + // ATTEMPTED: the preamble on the new connection reads the + // configuration again and asks for nothing. + _ = conn.Close() + conn = nil + } + if err := applyBootConfig(cfg, secrets); err != nil { + if conn != nil { + _ = conn.Close() + } + return nil, cfg, nil, err + } + b.remember(namesOf(secrets)) + log.Printf("sessiond booted as %s over vsock with %d declared secret(s), %d delivered", + cfg.SessionID, len(cfg.SecretNames), len(secrets)) + return conn, cfg, failure, nil +} + +// reBootstrap is the preamble on every LATER connection: read the +// configuration again, and exchange again when it brings a token this +// session has not spent — which is what a cold resume brings and a redial +// does not. +// +// The two failures it can meet are answered differently, and the difference +// is the difference between a session that recovers and one that vanishes. +// +// A configuration that never arrived means this conn is not usable, so the +// error is returned and dialLoop backs off and dials another. +// +// A refused EXCHANGE does not: the session is already running, its agent +// already has whatever it was given, and it has already reported a failed +// boot chain if it was given nothing. Tearing the conn down there would take +// a session that was merely missing its secrets off the air completely — no +// terminal, no attach, no events — for as long as the refusal persisted, +// which for a spent token is forever. So it is logged and the connection is +// served. +func reBootstrap(ctx context.Context, conn relay.Conn, b *bootstrapper) error { + cfg, err := readBootConfig(ctx, conn) + if err != nil { + return err + } + secrets, xerr := b.exchange(ctx, conn, cfg) + if xerr != nil { + log.Printf("this session's secrets were not re-delivered on a new connection (%v); "+ + "serving it anyway — the session is running and has already reported what it was given", xerr) + return nil + } + if len(secrets) == 0 { + return nil + } + if err := applyBootConfig(cfg, secrets); err != nil { + return err + } + b.remember(namesOf(secrets)) + log.Printf("sessiond re-applied %d environment secret(s) after a resume", len(secrets)) + return nil +} + +// dialBoot is the first dial, with the retry every later one already has. +// +// A failure here is fatal to the session, which is why it is the one dial +// that must not give up on the first refusal: a guest whose /dev/vsock came +// up a moment after this process did, or a host whose accept loop was a +// scheduling quantum behind, is a condition that resolves itself in +// milliseconds. A guest that genuinely has no channel still fails, and still +// fails quickly. +func dialBoot(ctx context.Context, dial dialSession) (relay.Conn, error) { + var err error + for attempt := 1; ; attempt++ { + var conn relay.Conn + conn, err = dial(ctx) + if err == nil { + return conn, nil + } + if attempt >= bootDialAttempts || ctx.Err() != nil { + return nil, fmt.Errorf("after %d attempt(s): %w", attempt, err) + } + log.Printf("the host control channel is not answering yet (%v); retrying", err) + select { + case <-time.After(bootDialBackoff): + case <-ctx.Done(): + return nil, ctx.Err() + } + } +} + +// namesOf returns a map's keys. Names, which are not values — the same +// distinction the launch-material resolver and the create both make. +func namesOf(m map[string]string) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + slices.Sort(out) + return out +} diff --git a/cmd/sessiond/bootconfig_test.go b/cmd/sessiond/bootconfig_test.go new file mode 100644 index 00000000..f358adf4 --- /dev/null +++ b/cmd/sessiond/bootconfig_test.go @@ -0,0 +1,697 @@ +package main + +import ( + "context" + "encoding/base64" + "encoding/json" + "net" + "os" + "strings" + "testing" + "time" + + "github.com/tokencanopy/rainier/internal/relay" + "github.com/tokencanopy/rainier/protocol/runner" +) + +// The microVM boot, over a net.Pipe standing in for the vsock conn. +// +// There is no VM and no KVM here, and that is the point of the seam: the +// transport is a func returning a relay.Conn, so everything above it — the +// boot configuration, the exchange, what lands in this process's environment +// — is exercised end to end against a fake host. What only a real host can +// answer is that Firecracker forwards _1024 to the socket and that +// the guest kernel has /dev/vsock; see the PR body. + +// fakeHost is the runner's end of a guest's control channel: it writes the +// boot configuration as the first frame and answers (or refuses) the one +// exchange the guest makes. +// +// It READS the connection continuously, from the moment it exists, and that +// is load-bearing rather than tidy. net.Pipe is synchronous: a guest that +// writes a request nobody is reading blocks in Write until its own deadline +// expires, and relay.NetConn answers an expired write context by closing the +// conn. A host that only looked for the request afterwards would therefore +// find a broken connection and read it as silence — which is how an +// "exactly once" assertion passes for a guest that asked twice. With a +// reader running the whole time, a request the guest makes is read off the +// wire as it is written, and is still here to be counted when the assertion +// runs. +type fakeHost struct { + conn relay.Conn + t *testing.T + + // reqs carries every control REQUEST the guest sent, in order. + reqs chan relay.ControlEvent + // readErr carries the one error that ended the drain, which is what lets + // an assertion tell "the guest sent nothing" from "the connection died". + readErr chan error +} + +func newFakeHost(t *testing.T, conn relay.Conn) *fakeHost { + h := &fakeHost{ + conn: conn, t: t, + reqs: make(chan relay.ControlEvent, 8), + readErr: make(chan error, 1), + } + go h.drain() + return h +} + +// drain is the reader. It runs until the connection ends, and it never +// touches *testing.T: a failure reported from this goroutine would be +// reported after the test that owns it may have finished. +func (h *fakeHost) drain() { + for { + raw, err := h.conn.Read(context.Background()) + if err != nil { + h.readErr <- err + return + } + f, derr := relay.Decode(raw) + if derr != nil || f.Type != relay.FrameControl { + continue + } + var ev relay.ControlEvent + if json.Unmarshal(f.Payload, &ev) != nil { + continue + } + if strings.HasPrefix(ev.Kind, "req:") { + h.reqs <- ev + } + } +} + +func (h *fakeHost) sendBootConfig(cfg runner.BootConfig) { + h.t.Helper() + body, err := json.Marshal(cfg) + if err != nil { + h.t.Fatal(err) + } + h.sendControl(relay.ControlEvent{Kind: relay.KindBootConfig, Payload: body}) +} + +func (h *fakeHost) sendControl(ev relay.ControlEvent) { + h.t.Helper() + payload, err := json.Marshal(ev) + if err != nil { + h.t.Fatal(err) + } + frame, err := relay.Encode(relay.Frame{Type: relay.FrameControl, Payload: payload}) + if err != nil { + h.t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := h.conn.Write(ctx, frame); err != nil { + h.t.Fatalf("write to the guest: %v", err) + } +} + +// nextRequest returns the next control REQUEST the guest sent, failing the +// test if none arrives or if the connection ends first. +func (h *fakeHost) nextRequest() relay.ControlEvent { + h.t.Helper() + select { + case ev := <-h.reqs: + return ev + case err := <-h.readErr: + h.t.Fatalf("the guest's connection ended before it asked for anything: %v", err) + case <-time.After(5 * time.Second): + h.t.Fatal("the guest made no request within 5s") + } + return relay.ControlEvent{} +} + +// nothingMore asserts the guest made no further request within a short +// window. It is how "exactly once" is checked, and it fails the test itself +// so that the reason is the assertion's own sentence. +// +// The three ways the window can end are three different facts and only one +// of them is silence: +// +// - a request arrives: the guest asked for something it must not have +// asked for, and `what` says why that matters; +// - the window expires with nothing on the conn: silence, which is what +// the caller is claiming; +// - the connection ends: NOT silence by itself. It is only evidence +// because the drain has been reading since this host existed, so a +// request the guest did write is already in h.reqs and is checked for +// once more before this returns. Without that reader, a guest whose +// write timed out on an undrained pipe would break its own connection +// and be reported as quiet — which is exactly the assertion this +// replaces. +func (h *fakeHost) nothingMore(within time.Duration, what string) { + h.t.Helper() + select { + case ev := <-h.reqs: + h.t.Fatalf("%s: the guest sent %q", what, ev.Kind) + case err := <-h.readErr: + select { + case ev := <-h.reqs: + h.t.Fatalf("%s: the guest sent %q", what, ev.Kind) + default: + } + h.t.Logf("the guest's connection ended within the window (%v) having asked for nothing", err) + case <-time.After(within): + } +} + +// waitClosed returns the error that ended the drain — how a test observes +// the guest hanging up. +func (h *fakeHost) waitClosed(within time.Duration) error { + h.t.Helper() + select { + case err := <-h.readErr: + return err + case <-time.After(within): + h.t.Fatalf("the guest's connection was still open after %s", within) + return nil + } +} + +func (h *fakeHost) answer(id uint64, env map[string]string) { + h.t.Helper() + body, err := json.Marshal(struct { + Env map[string]string `json:"env"` + }{env}) + if err != nil { + h.t.Fatal(err) + } + h.sendControl(relay.ControlEvent{Kind: "resp", ID: id, OK: true, Payload: body}) +} + +func (h *fakeHost) refuse(id uint64, reason string) { + h.t.Helper() + body, err := json.Marshal(struct { + Error string `json:"error"` + }{reason}) + if err != nil { + h.t.Fatal(err) + } + h.sendControl(relay.ControlEvent{Kind: "resp", ID: id, Payload: body}) +} + +// fakeTransport returns a dialer that hands out one pipe per dial and the +// host ends of those pipes, in order. +func fakeTransport(t *testing.T) (dialSession, <-chan *fakeHost) { + hosts := make(chan *fakeHost, 4) + return func(context.Context) (relay.Conn, error) { + guest, host := net.Pipe() + t.Cleanup(func() { _ = guest.Close(); _ = host.Close() }) + hosts <- newFakeHost(t, relay.NetConn(host)) + return relay.NetConn(guest), nil + }, hosts +} + +// cleanEnv unsets every variable the boot might set, before and after, so a +// case reads what THIS boot applied and not what the last one left. +func cleanEnv(t *testing.T) { + t.Helper() + names := []string{ + "RAINIER_SESSION", "RAINIER_DIAL", + "HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy", "NO_PROXY", "no_proxy", + "RAINIER_SETUP_B64", "RAINIER_SETUP_TIMEOUT", "RAINIER_INIT_B64", "RAINIER_INIT_TIMEOUT", + "RAINIER_REPOS_B64", "RAINIER_GIT_AUTHOR_NAME", "RAINIER_GIT_AUTHOR_EMAIL", + "CLAUDE_CONFIG_DIR", "RAINIER_AGENTS_B64", "DEPLOY_KEY", "NPM_TOKEN", + } + unset := func() { + for _, n := range names { + _ = os.Unsetenv(n) + } + } + unset() + t.Cleanup(unset) +} + +// TestBootOverVsockAppliesTheConfigurationAndTheSecrets is the whole boot in +// one case: the guest reads a configuration off its first frame, exchanges +// its token exactly once, and ends up with an environment block in the shape +// the boot chain already reads — which is what makes everything downstream +// (prepareBoot, the agent sync, the exec runner, the agent itself) identical +// to the Docker path. +func TestBootOverVsockAppliesTheConfigurationAndTheSecrets(t *testing.T) { + cleanEnv(t) + dial, hosts := fakeTransport(t) + boots := &bootstrapper{} + + cfg := runner.BootConfig{ + Protocol: runner.SessionBootstrapProtocolVersion, + SessionID: "sess_example", + Cmd: []string{"claude"}, + ProxyURL: "http://sess_example:rainier@proxy.invalid:3128", + NoProxy: "127.0.0.1,localhost", + Setup: "npm ci\n", + SetupTimeoutSec: 600, + Init: "make dev\n", + InitTimeoutSec: 180, + Repos: []runner.RepoSpec{{Owner: "acme", Name: "app", BaseBranch: "main", + SessionBranch: "rainier/work", Dir: "app"}}, + GitAuthorName: "example", + GitAuthorEmail: "42+example@users.noreply.github.com", + Env: map[string]string{"CLAUDE_CONFIG_DIR": "/rainier/agents/claude"}, + SecretNames: []string{"DEPLOY_KEY", "NPM_TOKEN"}, + BootstrapToken: "token_example", + } + + done := make(chan struct{}) + var ( + conn relay.Conn + failure *bootFailure + bootErr error + ) + go func() { + defer close(done) + conn, _, failure, bootErr = bootOverVsock(context.Background(), dial, boots) + }() + + host := <-hosts + host.sendBootConfig(cfg) + req := host.nextRequest() + if req.Kind != "req:"+runner.MethodFetchSessionSecrets { + t.Fatalf("the guest asked %q, want the secret fetch", req.Kind) + } + var body struct { + Protocol int `json:"protocol"` + Token string `json:"token"` + } + if err := json.Unmarshal(req.Payload, &body); err != nil { + t.Fatal(err) + } + if body.Protocol != runner.SessionBootstrapProtocolVersion || body.Token != "token_example" { + t.Fatalf("the request carried %+v", body) + } + host.answer(req.ID, map[string]string{"DEPLOY_KEY": "value_example", "NPM_TOKEN": "other_value_example"}) + + <-done + if bootErr != nil { + t.Fatalf("boot: %v", bootErr) + } + if failure != nil { + t.Fatalf("the boot reported a failure: %v", failure) + } + if conn == nil { + t.Fatal("the boot returned no connection to serve") + } + + // The configuration, in the shape bootEnvFromOS reads. + env := bootEnvFromOS() + if env.SetupB64 != base64.StdEncoding.EncodeToString([]byte("npm ci\n")) || env.SetupTimeout != "600" { + t.Errorf("setup = %q/%q", env.SetupB64, env.SetupTimeout) + } + if env.InitB64 != base64.StdEncoding.EncodeToString([]byte("make dev\n")) || env.InitTimeout != "180" { + t.Errorf("init = %q/%q", env.InitB64, env.InitTimeout) + } + if env.GitAuthorName != "example" || env.GitAuthorEmail != "42+example@users.noreply.github.com" { + t.Errorf("git identity = %q <%q>", env.GitAuthorName, env.GitAuthorEmail) + } + repos, err := decodeRepos(env.ReposB64) + if err != nil || len(repos) != 1 || repos[0].Dir != "app" { + t.Errorf("repos = %+v (%v)", repos, err) + } + if os.Getenv("RAINIER_SESSION") != "sess_example" { + t.Errorf("RAINIER_SESSION = %q", os.Getenv("RAINIER_SESSION")) + } + if os.Getenv("CLAUDE_CONFIG_DIR") != "/rainier/agents/claude" { + t.Errorf("the agent home configuration was not applied") + } + for _, k := range []string{"HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy"} { + if os.Getenv(k) != cfg.ProxyURL { + t.Errorf("%s = %q, want the proxy", k, os.Getenv(k)) + } + } + if os.Getenv("NO_PROXY") != "127.0.0.1,localhost" { + t.Errorf("NO_PROXY = %q", os.Getenv("NO_PROXY")) + } + // There is nothing to dial, and the guest must not think there is. + if v := os.Getenv("RAINIER_DIAL"); v != "" { + t.Errorf("RAINIER_DIAL = %q on a session with no URL to dial", v) + } + // And the secrets, which arrived over the exchange and from nowhere else. + if os.Getenv("DEPLOY_KEY") != "value_example" || os.Getenv("NPM_TOKEN") != "other_value_example" { + t.Error("the delivered secrets did not reach this process's environment") + } + + // Exactly once: the token is single-use, so a second exchange on the same + // token would be refused and would fail a boot that had already worked. + host.nothingMore(200*time.Millisecond, "the guest made a second request on one boot") +} + +// TestBootOverVsockWithNoSecretsDeclaredIsACleanBoot pins the difference +// between "none declared" and "declared and never arrived": the first asks +// for nothing at all. +func TestBootOverVsockWithNoSecretsDeclaredIsACleanBoot(t *testing.T) { + cleanEnv(t) + dial, hosts := fakeTransport(t) + boots := &bootstrapper{} + + done := make(chan struct{}) + var failure *bootFailure + var bootErr error + go func() { + defer close(done) + _, _, failure, bootErr = bootOverVsock(context.Background(), dial, boots) + }() + + host := <-hosts + host.sendBootConfig(runner.BootConfig{ + Protocol: runner.SessionBootstrapProtocolVersion, SessionID: "sess_example", + BootstrapToken: "token_example", + }) + <-done + if bootErr != nil || failure != nil { + t.Fatalf("a session declaring no secrets did not boot cleanly: %v / %v", bootErr, failure) + } + host.nothingMore(200*time.Millisecond, "a session declaring no secrets asked for some anyway") +} + +// TestARefusedExchangeFailsTheBootChain is the compatibility table's last +// row, and the one that decides what a user sees. A refusal must not boot an +// agent into an environment that promised a credential it does not have; it +// must fail the boot chain, as the existing stage_failed, naming the COUNT +// of undelivered names and none of their values. +func TestARefusedExchangeFailsTheBootChain(t *testing.T) { + cleanEnv(t) + dial, hosts := fakeTransport(t) + boots := &bootstrapper{} + + done := make(chan struct{}) + var ( + conn relay.Conn + failure *bootFailure + bootErr error + ) + go func() { + defer close(done) + conn, _, failure, bootErr = bootOverVsock(context.Background(), dial, boots) + }() + + cfg := runner.BootConfig{ + Protocol: runner.SessionBootstrapProtocolVersion, SessionID: "sess_example", + SecretNames: []string{"DEPLOY_KEY", "NPM_TOKEN"}, BootstrapToken: "token_example", + } + host := <-hosts + host.sendBootConfig(cfg) + req := host.nextRequest() + host.refuse(req.ID, "this session's bootstrap token has already been exchanged") + + <-done + if bootErr != nil { + t.Fatalf("a refusal was reported as a boot error rather than a failed chain: %v", bootErr) + } + if failure == nil { + t.Fatal("a refused exchange did not fail the boot") + } + if conn != nil { + t.Fatal("a failed boot handed back a connection it may already have broken") + } + // The count, the plane's own condition, and no value or name. + if !strings.Contains(failure.Error(), "2 secret(s)") { + t.Errorf("the failure does not name the count: %q", failure) + } + if !strings.Contains(failure.Error(), "already been exchanged") { + t.Errorf("the failure drops the control plane's reason: %q", failure) + } + for _, leaked := range []string{"DEPLOY_KEY", "NPM_TOKEN", "token_example"} { + if strings.Contains(failure.Error(), leaked) { + t.Errorf("the failure carries %q: %q", leaked, failure) + } + } + + // And it becomes a STAGE: the first one, which only fails, so the agent + // is never exec'd and the watcher reports stage_failed with this tail. + dir := t.TempDir() + stages, _, err := prepareBoot(dir, "/workspace", bootEnv{SecretsFailure: failure.Error(), SetupB64: "ZWNobyBoaQo="}) + if err != nil { + t.Fatal(err) + } + if len(stages) != 2 || stages[0].Name != stageSecrets { + t.Fatalf("stages = %v, want the secrets stage first", stageNames(stages)) + } + script, err := os.ReadFile(stages[0].ScriptPath) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(script), "exit 1") { + t.Fatalf("the secrets stage does not fail:\n%s", script) + } + + // And the session still comes up. This is the half that decides whether + // a user can see WHY: the boot chain has failed and said so, and the + // session now has to register so that the failure, the terminal and an + // attach are reachable at all. + // + // The token is spent — the control plane consumes it before it resolves + // anything — so the connection that carries the session must not + // re-present it. A sessiond that retried would be refused on every + // redial, `connect` would drop every connection, and a session that was + // merely missing its secrets would be off the air completely. + redial, err := dial(context.Background()) + if err != nil { + t.Fatal(err) + } + redialHost := <-hosts + redialDone := make(chan error, 1) + go func() { redialDone <- reBootstrap(context.Background(), redial, boots) }() + redialHost.sendBootConfig(cfg) + // Asserted BEFORE the preamble is waited on, and that ordering is the + // test: a guest that re-presented the token would be sitting in + // exchange() waiting out its whole 30-second answer budget, so waiting + // on reBootstrap first would turn a bug into a slow pass. + redialHost.nothingMore(200*time.Millisecond, + "the redial re-presented a token the control plane had already spent") + if err := <-redialDone; err != nil { + t.Fatalf("the connection after a failed boot was refused: %v", err) + } +} + +// TestTheBootExchangeIsNumberedOutOfTheDispatchersSpace pins the id the boot +// exchange uses, which is the one request this process makes before its RPC +// dispatcher exists. +// +// It cannot be 1. The dispatcher counts from 1, so a boot exchange that +// timed out and was answered late would have its `{"env": …}` matched to the +// dispatcher's first call — an agent credential fetch, typically — which +// would decode it as an empty credential set and report a shape it cannot +// read for a request that was never answered. +// +// And it cannot carry the high bit, which runnerd reserves for its own +// requests and refuses a sandbox for using. +func TestTheBootExchangeIsNumberedOutOfTheDispatchersSpace(t *testing.T) { + cleanEnv(t) + dial, hosts := fakeTransport(t) + boots := &bootstrapper{} + + done := make(chan struct{}) + go func() { + defer close(done) + _, _, _, _ = bootOverVsock(context.Background(), dial, boots) + }() + host := <-hosts + host.sendBootConfig(runner.BootConfig{ + Protocol: runner.SessionBootstrapProtocolVersion, SessionID: "sess_example", + SecretNames: []string{"DEPLOY_KEY"}, BootstrapToken: "token_example", + }) + req := host.nextRequest() + host.answer(req.ID, map[string]string{"DEPLOY_KEY": "value_example"}) + <-done + + if req.ID == 0 { + t.Fatal("the boot exchange carried no id") + } + // The dispatcher's own first id, which this must not collide with. + d := newRPCDispatcher() + d.online(&recordingSender{}) + firstDispatcherID := d.seq.Add(1) + if req.ID == firstDispatcherID { + t.Fatalf("the boot exchange uses id %d, which is the dispatcher's first", req.ID) + } + if req.ID&(1<<63) != 0 { + t.Fatalf("the boot exchange uses id %d, which is in the runner's reserved space", req.ID) + } +} + +// TestAMissingTokenWithDeclaredSecretsFailsTheBootChain is the same row from +// the other direction: an older control plane that declared nothing and +// withheld nothing sends no token, and a guest whose configuration names +// secrets it was given no way to fetch must not start either. +func TestAMissingTokenWithDeclaredSecretsFailsTheBootChain(t *testing.T) { + cleanEnv(t) + dial, hosts := fakeTransport(t) + boots := &bootstrapper{} + + done := make(chan struct{}) + var failure *bootFailure + go func() { + defer close(done) + _, _, failure, _ = bootOverVsock(context.Background(), dial, boots) + }() + + host := <-hosts + host.sendBootConfig(runner.BootConfig{ + Protocol: runner.SessionBootstrapProtocolVersion, SessionID: "sess_example", + SecretNames: []string{"DEPLOY_KEY"}, + }) + <-done + if failure == nil { + t.Fatal("a configuration declaring secrets with no token booted anyway") + } + if !strings.Contains(failure.Error(), "1 secret(s)") || !strings.Contains(failure.Error(), "no bootstrap token") { + t.Errorf("the failure = %q", failure) + } + host.nothingMore(200*time.Millisecond, "the guest asked for its secrets with no token to ask with") +} + +// TestAResumeReExchangesAndARedialDoesNot is the rule that makes single use +// workable. +// +// A bootstrap token is spent by its first exchange. A sessiond that +// re-exchanged on every connection would be refused on the first redial +// inside a live VM and stay refused forever; one that never re-exchanged +// would come back from a cold resume with no secrets. What tells the two +// apart is the token itself: a resume brings a new one, a redial brings the +// one already spent. +func TestAResumeReExchangesAndARedialDoesNot(t *testing.T) { + cleanEnv(t) + dial, hosts := fakeTransport(t) + boots := &bootstrapper{} + + // Boot. + done := make(chan struct{}) + go func() { + defer close(done) + _, _, _, _ = bootOverVsock(context.Background(), dial, boots) + }() + host := <-hosts + cfg := runner.BootConfig{ + Protocol: runner.SessionBootstrapProtocolVersion, SessionID: "sess_example", + SecretNames: []string{"DEPLOY_KEY"}, BootstrapToken: "token_from_the_create", + } + host.sendBootConfig(cfg) + first := host.nextRequest() + host.answer(first.ID, map[string]string{"DEPLOY_KEY": "value_example"}) + <-done + // Exactly one on the boot connection, so the redial below is being + // compared against a known number and not against "at least one". + host.nothingMore(200*time.Millisecond, "the boot exchanged more than once") + + // A REDIAL inside the same VM: the same configuration, the same token. + // Nothing is asked for, because the token has been spent and the values + // are already in this process. + redialConn, err := dial(context.Background()) + if err != nil { + t.Fatal(err) + } + redialHost := <-hosts + redialDone := make(chan error, 1) + go func() { redialDone <- reBootstrap(context.Background(), redialConn, boots) }() + redialHost.sendBootConfig(cfg) + // Before the wait, for the same reason as above: a guest that + // re-exchanged would be blocked on an answer, not finished. + redialHost.nothingMore(200*time.Millisecond, + "a redial re-exchanged a spent token, which would be refused forever") + if err := <-redialDone; err != nil { + t.Fatalf("a redial failed: %v", err) + } + + // A RESUME: a new VM, a new socket, a new token. The secrets are fetched + // again, because the process on this side may be a completely new one + // and the values it holds may be from a boot that no longer exists. + _ = os.Unsetenv("DEPLOY_KEY") + resumeConn, err := dial(context.Background()) + if err != nil { + t.Fatal(err) + } + resumeHost := <-hosts + resumeDone := make(chan error, 1) + go func() { resumeDone <- reBootstrap(context.Background(), resumeConn, boots) }() + resumed := cfg + resumed.BootstrapToken = "token_from_the_resume" + resumeHost.sendBootConfig(resumed) + req := resumeHost.nextRequest() + var body struct { + Token string `json:"token"` + } + if err := json.Unmarshal(req.Payload, &body); err != nil { + t.Fatal(err) + } + if body.Token != "token_from_the_resume" { + t.Fatalf("the resume exchanged %q, want the freshly minted token", body.Token) + } + resumeHost.answer(req.ID, map[string]string{"DEPLOY_KEY": "value_after_the_resume"}) + if err := <-resumeDone; err != nil { + t.Fatalf("a resume failed: %v", err) + } + if os.Getenv("DEPLOY_KEY") != "value_after_the_resume" { + t.Errorf("the resumed session's secret = %q", os.Getenv("DEPLOY_KEY")) + } +} + +// TestForgetUnsetsEveryDeliveredSecret is the cold suspend's third act. The +// VM is about to end with no memory image anywhere, so this is belt and +// braces — but it is the half this process can make, and a sessiond that +// was frozen instead of terminated has nothing left in its environment. +func TestForgetUnsetsEveryDeliveredSecret(t *testing.T) { + cleanEnv(t) + boots := &bootstrapper{} + cfg := runner.BootConfig{Protocol: runner.SessionBootstrapProtocolVersion, SessionID: "sess_example"} + if err := applyBootConfig(cfg, map[string]string{"DEPLOY_KEY": "value_example", "NPM_TOKEN": "other"}); err != nil { + t.Fatal(err) + } + boots.remember([]string{"DEPLOY_KEY", "NPM_TOKEN"}) + if os.Getenv("DEPLOY_KEY") == "" { + t.Fatal("the fixture did not apply anything, so forgetting it proves nothing") + } + if n := boots.forget(); n != 2 { + t.Fatalf("forgot %d secret(s), want 2", n) + } + for _, k := range []string{"DEPLOY_KEY", "NPM_TOKEN"} { + if v, set := os.LookupEnv(k); set { + t.Errorf("%s survived the cold suspend as %q", k, v) + } + } + // And the session id, which is configuration and not a secret, is still + // there: forgetting is by name and not a blanket wipe. + if os.Getenv("RAINIER_SESSION") != "sess_example" { + t.Error("forgetting the secrets also dropped the session's own configuration") + } +} + +// TestAConfigurationWinsOverADeliveredSecret pins the precedence the control +// plane already applies on the Docker path: the agent-home keys are launch +// invariants, and a delivered value spelled like one of them must not +// redirect credential custody. The plane also drops such a name from +// SecretNames, so this is the second of two fences. +func TestAConfigurationWinsOverADeliveredSecret(t *testing.T) { + cleanEnv(t) + cfg := runner.BootConfig{ + Protocol: runner.SessionBootstrapProtocolVersion, SessionID: "sess_example", + Env: map[string]string{"CLAUDE_CONFIG_DIR": "/rainier/agents/claude"}, + } + if err := applyBootConfig(cfg, map[string]string{"CLAUDE_CONFIG_DIR": "/workspace/attacker"}); err != nil { + t.Fatal(err) + } + if got := os.Getenv("CLAUDE_CONFIG_DIR"); got != "/rainier/agents/claude" { + t.Fatalf("CLAUDE_CONFIG_DIR = %q; a delivered value replaced the agent home", got) + } +} + +// TestAMisversionedBootConfigIsRefused pins that a guest booted from an +// image older than the host's configuration protocol says so rather than +// running on a configuration it has half understood. +func TestAMisversionedBootConfigIsRefused(t *testing.T) { + cleanEnv(t) + dial, hosts := fakeTransport(t) + conn, err := dial(context.Background()) + if err != nil { + t.Fatal(err) + } + host := <-hosts + go host.sendBootConfig(runner.BootConfig{Protocol: 99, SessionID: "sess_example"}) + + if _, err := readBootConfig(context.Background(), conn); err == nil { + t.Fatal("a configuration from an unknown protocol was accepted") + } else if !strings.Contains(err.Error(), "must be replaced") { + t.Errorf("error = %q, want it to say which side has to change", err) + } +} diff --git a/cmd/sessiond/coldsuspend_test.go b/cmd/sessiond/coldsuspend_test.go new file mode 100644 index 00000000..3a0a95b8 --- /dev/null +++ b/cmd/sessiond/coldsuspend_test.go @@ -0,0 +1,97 @@ +package main + +import ( + "encoding/json" + "os" + "testing" + + "github.com/tokencanopy/rainier/internal/relay" + "github.com/tokencanopy/rainier/protocol/runner" +) + +// The cold half of the suspend handshake, wired the way main wires it, so +// what is exercised is the frame's whole journey — OnControl's kind +// dispatch, the `cold` flag, the handler, and the two answers — rather than +// quiesceCold called directly. +func coldWired(execs execKiller, boots *bootstrapper) (*rpcDispatcher, *recordingSender) { + d := newRPCDispatcher() + sender := &recordingSender{} + d.online(sender) + d.RegisterEventHandler(relay.KindSuspending, func(ev relay.ControlEvent) { + if ev.Cold { + quiesceCold(execs, d, nil, boots, ev.ID) + return + } + quiesceExecs(execs, d, ev.ID) + }) + return d, sender +} + +func suspendingFrame(t *testing.T, cold bool, nonce uint64) []byte { + t.Helper() + b, err := json.Marshal(relay.ControlEvent{Kind: relay.KindSuspending, ID: nonce, Cold: cold}) + if err != nil { + t.Fatal(err) + } + return b +} + +// TestAColdSuspendForgetsTheDeliveredSecrets is §4.4 from inside the guest: +// a cold notice is not a freeze, so the sandbox ends its execs AND forgets +// every secret the bootstrap exchange delivered before it answers ready. +func TestAColdSuspendForgetsTheDeliveredSecrets(t *testing.T) { + cleanEnv(t) + boots := &bootstrapper{} + if err := applyBootConfig( + runner.BootConfig{Protocol: runner.SessionBootstrapProtocolVersion, SessionID: "sess_example"}, + map[string]string{"DEPLOY_KEY": "value_example"}); err != nil { + t.Fatal(err) + } + boots.remember([]string{"DEPLOY_KEY"}) + + execs := &recordingExecs{} + d, sender := coldWired(execs, boots) + d.OnControl(suspendingFrame(t, true, 91)) + + if _, set := os.LookupEnv("DEPLOY_KEY"); set { + t.Error("a cold suspend left a delivered secret in this process's environment") + } + if calls, budget := execs.state(); calls != 1 || budget != execQuiesceBudget { + t.Fatalf("the cold suspend killed the execs %d time(s) with budget %s", calls, budget) + } + got := sender.events() + if len(got) != 2 || got[0].Kind != relay.KindSuspendAck || got[1].Kind != relay.KindSuspendReady { + t.Fatalf("the sandbox answered %+v, want an ack and then a ready", got) + } + for _, ev := range got { + if ev.ID != 91 { + t.Fatalf("an answer carries nonce %d, want the notice's 91", ev.ID) + } + } +} + +// TestAWarmSuspendKeepsTheDeliveredSecrets is the other side of the same +// flag, and the reason it exists: a freeze is not the end of this VM, so the +// session comes back with the same process, the same agent, and the same +// environment. Forgetting there would break a warm resume for nothing. +func TestAWarmSuspendKeepsTheDeliveredSecrets(t *testing.T) { + cleanEnv(t) + boots := &bootstrapper{} + if err := applyBootConfig( + runner.BootConfig{Protocol: runner.SessionBootstrapProtocolVersion, SessionID: "sess_example"}, + map[string]string{"DEPLOY_KEY": "value_example"}); err != nil { + t.Fatal(err) + } + boots.remember([]string{"DEPLOY_KEY"}) + + execs := &recordingExecs{} + d, sender := coldWired(execs, boots) + d.OnControl(suspendingFrame(t, false, 92)) + + if os.Getenv("DEPLOY_KEY") != "value_example" { + t.Error("a warm suspend forgot a delivered secret the resumed session still needs") + } + if got := sender.events(); len(got) != 2 { + t.Fatalf("the sandbox answered %+v, want an ack and then a ready", got) + } +} diff --git a/cmd/sessiond/gitchain.go b/cmd/sessiond/gitchain.go index fabbab1c..c4931ade 100644 --- a/cmd/sessiond/gitchain.go +++ b/cmd/sessiond/gitchain.go @@ -44,19 +44,28 @@ import ( // The stage names. They are wire-visible: they travel in ControlEvent.Stage // and controld composes a session's error text out of them. const ( - stageSetup = "setup" - stageClone = "clone" - stageAgents = "agents" - stageInit = "init" + // stageSecrets is FIRST in the chain and exists only to fail. A microVM + // session whose environment declared secrets it could not be given is + // not the environment it was created from, so the agent must not start — + // and the way a boot chain refuses to start the agent is a stage that + // exits non-zero, which the watcher then reports as stage_failed with a + // tail a person can read. See bootconfig.go. + stageSecrets = "secrets" + stageSetup = "setup" + stageClone = "clone" + stageAgents = "agents" + stageInit = "init" ) // The chain's files, all in the session's own .rainier directory beside the // setup script Plan 4 put there. const ( - clonesScriptName = "clones.sh" - cloneRCName = "clone.rc" - agentsScriptName = "agents.sh" - agentsRCName = "agents.rc" + secretsScriptName = "secrets.sh" + secretsRCName = "secrets.rc" + clonesScriptName = "clones.sh" + cloneRCName = "clone.rc" + agentsScriptName = "agents.sh" + agentsRCName = "agents.rc" // agentsDoneName is the marker sessiond writes when the agent homes are as // ready as they are going to get, and the file the agents stage waits for. // It lives beside the rc files, on the same persistent volume, which is why @@ -147,6 +156,20 @@ type bootEnv struct { GitAuthorName string GitAuthorEmail string + + // SecretsFailure is the one field that does not come from the + // environment block: it is set by the microVM boot when the values this + // session's create promised could not be delivered (bootconfig.go). + // + // It becomes the first stage of the chain, and that stage only fails. + // Putting it here rather than handling it in main is what makes it a + // STAGE — reported as stage_failed with a tail, with the agent never + // exec'd — through exactly the machinery a failed setup or a failed + // clone already goes through, rather than through a second path that + // would have to be kept in step with it. + // + // It names a count and never a value; see undeliveredSecrets. + SecretsFailure string } // bootEnvFromOS reads the variables the driver injects (internal/driver @@ -175,7 +198,8 @@ func bootEnvFromOS() bootEnv { // design's intent: a home nobody waited for is a login that lands after the // agent already read its configuration. func (e bootEnv) any() bool { - return e.SetupB64 != "" || e.ReposB64 != "" || e.InitB64 != "" || e.AgentsB64 != "" + return e.SetupB64 != "" || e.ReposB64 != "" || e.InitB64 != "" || e.AgentsB64 != "" || + e.SecretsFailure != "" } // git reports whether git will run in this session — the clone stage, or an @@ -337,6 +361,23 @@ func prepareBoot(dir, root string, env bootEnv) ([]bootStage, []envVar, error) { } var stages []bootStage + // First, and only when the microVM boot could not get what this + // session's create promised it. Everything after it is skipped by the + // chain's own `exit $rc`, which is the point: an environment missing its + // credentials must not run a setup script, clone a repository, or start + // an agent that will fail at whatever it reaches for first. + if env.SecretsFailure != "" { + if err := writeStageScript(dir, secretsScriptName, secretsRCName, + []byte(failingScript(env.SecretsFailure))); err != nil { + return nil, nil, err + } + stages = append(stages, bootStage{ + Name: stageSecrets, + ScriptPath: dir + "/" + secretsScriptName, + RCPath: dir + "/" + secretsRCName, + Timeout: stageTimeout(""), + }) + } if env.SetupB64 != "" { if err := prepareSetup(dir, env.SetupB64); err != nil { return nil, nil, err diff --git a/cmd/sessiond/main.go b/cmd/sessiond/main.go index 4929c72d..c7414779 100644 --- a/cmd/sessiond/main.go +++ b/cmd/sessiond/main.go @@ -18,8 +18,6 @@ import ( "syscall" "time" - "github.com/coder/websocket" - "github.com/tokencanopy/rainier/internal/eventlog" "github.com/tokencanopy/rainier/internal/reap" "github.com/tokencanopy/rainier/internal/relay" @@ -60,6 +58,8 @@ func main() { rows := flag.Int("rows", 32, "initial rows") dial := flag.String("dial", "", "runnerd URL to dial and register with (relay mode)") sessionID := flag.String("session", "", "session id to register as (relay mode)") + transport := flag.String("transport", envOr("RAINIER_TRANSPORT", transportWebSocket), + "how to reach the runner: websocket (the default, and every Docker session) or vsock (a microVM guest, which receives its whole configuration over the channel rather than in its environment)") flag.Parse() argv := flag.Args() if len(argv) == 0 { @@ -82,6 +82,55 @@ func main() { } } + // The microVM boot, before anything reads the environment block — + // because on this path there IS no environment block until this runs. + // The guest opens its vsock channel, reads its whole configuration off + // the first frame, exchanges its bootstrap token for the secrets its + // create deliberately withheld, and puts both into this process's + // environment in the shape everything below already reads. + // + // The connection it bootstrapped on is kept and served: the + // configuration, the terminal stream and the session RPC all ride one + // channel by design, and a second dial would be a second guest for a + // socket that serves one. + var ( + dialer dialSession + preamble func(context.Context, relay.Conn) error + firstConn relay.Conn + // overVsock is what puts this process in relay mode on the microVM + // path, and it is NOT "firstConn != nil": a boot whose exchange was + // refused closes its connection and dials again, and such a session + // is still very much in relay mode — it has a boot chain to fail + // loudly through. + overVsock bool + boots = &bootstrapper{} + // secretsFailure is a boot that must not start an agent: the + // environment declared credentials this session could not be given. + // It becomes the chain's first stage, which only fails. + secretsFailure string + ) + if *transport == transportVsock { + dialer = vsockTransport() + preamble = func(ctx context.Context, c relay.Conn) error { return reBootstrap(ctx, c, boots) } + conn, cfg, failure, err := bootOverVsock(context.Background(), dialer, boots) + if err != nil { + // Deliberately fatal, and the same judgement prepareBoot's own + // failure gets: a guest that could not read its configuration + // does not know what session it is, what to run, or where its + // egress goes. Dying is what makes the runner notice. + log.Fatalf("microvm boot: %v", err) + } + firstConn, overVsock = conn, true + if *sessionID == "" { + *sessionID = cfg.SessionID + } + if failure != nil { + secretsFailure = failure.Error() + } + } else if *transport != transportWebSocket { + log.Fatalf("unknown --transport %q (valid: %s, %s)", *transport, transportWebSocket, transportVsock) + } + // The boot chain (design §4.3). An environment's setup script (Plan 4), the // repositories controld resolved, and the environment's per-boot init hook // all arrive as base64 in the environment block, injected by the driver. @@ -97,8 +146,13 @@ func main() { // exists on a dialed conn, the credential the clone stage needs can only be // minted over it, and the local dev listener has no runnerd to reach. bootEnvironment := bootEnvFromOS() + bootEnvironment.SecretsFailure = secretsFailure + // A vsock session is in relay mode by construction: it HAS a connection, + // it just did not get it from a URL. Everything gated on relay mode + // below reads this rather than the flag. + relayMode := *dial != "" || overVsock var stages []bootStage - if bootEnvironment.any() && *dial == "" { + if bootEnvironment.any() && !relayMode { // Can't happen from the driver (it injects RAINIER_DIAL alongside), but // a human running sessiond by hand with the vars set would otherwise // get an environment silently missing its setup and its repositories. @@ -134,7 +188,7 @@ func main() { // carried no manifest — a session with no creator, or an older controld — // and then nothing in agents.go runs at all. var agents *agentSync - if *dial != "" { + if relayMode { rpc = newRPCDispatcher() startAgentSocket(context.Background(), agentSocketPath, rpc, events) // The workspace-inspection methods controld drives INTO this sandbox: @@ -234,6 +288,14 @@ func main() { // the path users actually take. if rpc != nil { rpc.RegisterEventHandler(relay.KindSuspending, func(ev relay.ControlEvent) { + if ev.Cold { + // Not a freeze: this VM is ending, with no memory image + // written anywhere. Everything that has to survive has to be + // on a disk by the time this answers, and everything that + // must not survive has to be gone. + quiesceCold(execs, rpc, agents, boots, ev.ID) + return + } quiesceExecs(execs, rpc, ev.ID) }) } @@ -278,11 +340,16 @@ func main() { os.Exit(0) }() - if *dial != "" { + if relayMode { if len(stages) > 0 { startStageWatcher(stageCtx, s.Stop, stages, *logPath, events) } - dialLoop(context.Background(), *dial, *sessionID, s, events, execCounts, rpc, execs) + if dialer == nil { + dialer = websocketTransport(*dial, *sessionID) + } + dialLoop(context.Background(), sessionTransport{ + dial: dialer, preamble: preamble, first: firstConn, name: *sessionID, + }, s, events, execCounts, rpc, execs) return } @@ -317,15 +384,53 @@ func main() { // conn's sender for as long as it lives. The asymmetry with events is // deliberate — an event queues across a reconnect, a request does not (see // rpcConn). -func dialLoop(ctx context.Context, dial, sessionID string, s *session.Session, events <-chan []byte, +// sessionTransport is how one dialLoop reaches its runner: the dialer, the +// per-connection preamble (nil on the WebSocket path), the connection the +// microVM boot already established, and the session's name for the log. +// +// It is a struct rather than four parameters because the four are one +// decision — which transport this session has — and a caller that could +// supply a vsock preamble with a WebSocket dialer would be describing a +// session that does not exist. +type sessionTransport struct { + dial dialSession + preamble func(context.Context, relay.Conn) error + first relay.Conn + name string +} + +// connect returns the next connection to serve, running the preamble on it. +// The connection the boot established is used once, first, and then dropped: +// the boot already read its configuration and made its exchange, and running +// the preamble over it again would read a second configuration that is not +// coming. +func (t *sessionTransport) connect(ctx context.Context) (relay.Conn, error) { + if t.first != nil { + c := t.first + t.first = nil + return c, nil + } + c, err := t.dial(ctx) + if err != nil { + return nil, err + } + if t.preamble != nil { + if err := t.preamble(ctx, c); err != nil { + _ = c.Close() + return nil, err + } + } + return c, nil +} + +func dialLoop(ctx context.Context, tr sessionTransport, s *session.Session, events <-chan []byte, execCounts *execCountMailbox, rpc *rpcDispatcher, execs *sandboxexec.Runner) { backoff := time.Second var pending [][]byte // control payloads no connection has accepted yet for { - c, _, err := websocket.Dial(ctx, dial+"?session="+sessionID, nil) + conn, err := tr.connect(ctx) if err == nil { - c.SetReadLimit(16 << 20) - log.Printf("sessiond registered with runnerd as %s", sessionID) + log.Printf("sessiond registered with runnerd as %s", tr.name) backoff = time.Second // WithControl, not plain ServeSession: the returned sender shares // the relay's single writer, so a control event and a terminal @@ -345,7 +450,7 @@ func dialLoop(ctx context.Context, dial, sessionID string, s *session.Session, e // cannot arrive, and a handler that outlives the conn answers over // the (now dead) conn its request came in on rather than over a // later one — see rpc.go. - sender, errc := relay.ServeSessionWithExec(ctx, relay.WSConn(c), s, rpc.OnControl, execs) + sender, errc := relay.ServeSessionWithExec(ctx, conn, s, rpc.OnControl, execs) rpc.online(sender) var relayErr error pending, relayErr = serveConn(sender, errc, events, execCounts.c(), execs, pending) @@ -463,6 +568,52 @@ func quiesceExecs(execs execKiller, notifier eventNotifier, nonce uint64) { } } +// agentHomeMount is where the agent homes are mounted inside a session. It +// is controlapp.HomeMountPath, spelled here because this process must not +// import the control plane's application package — the two ends of the same +// mount, like every other wire word this file shares with it. +const agentHomeMount = "/rainier/agents" + +// quiesceCold is the cold half of the suspend handshake: this VM is ending, +// with no memory image written anywhere (ADR-0003 §2.2), so a resume is a +// fresh boot from disks and nothing else survives. +// +// Three things, in the order they have to happen: +// +// 1. FLUSH. The last thing an agent wrote is usually the thing worth +// keeping — a login completed seconds ago — and the sync's own tick must +// not be what decides whether it makes it into custody. +// 2. END the execs, which is what the warm path does too and for the same +// reason: a detached command's lifetime is its session's. +// 3. UNMOUNT the agent home and FORGET the delivered secrets. The unmount +// is what makes the home's filesystem consistent before the block device +// goes away under it; forgetting is belt and braces beside a VM that is +// about to cease to exist, and it costs nothing. +// +// Every step is best effort and the answer goes out regardless, for the same +// reason the warm path's does: the host terminates the VM when it hears +// nothing, so staying silent would only make the stop slower and put the +// reason nowhere. +func quiesceCold(execs execKiller, notifier eventNotifier, agents *agentSync, boots *bootstrapper, nonce uint64) { + if err := notifier.Notify(relay.ControlEvent{Kind: relay.KindSuspendAck, ID: nonce}); err != nil { + log.Printf("acknowledging the cold suspend notice: %v", err) + } + if agents != nil { + agents.flush() + } + if n := execs.KillAllAndWait(execQuiesceBudget); n > 0 { + log.Printf("%d exec(s) had not ended %s after the cold suspend notice; answering anyway", + n, execQuiesceBudget) + } + unmountAgentHome(agentHomeMount) + if n := boots.forget(); n > 0 { + log.Printf("forgot %d delivered environment secret(s) before this VM ends", n) + } + if err := notifier.Notify(relay.ControlEvent{Kind: relay.KindSuspendReady, ID: nonce}); err != nil { + log.Printf("reporting the cold suspend ready: %v", err) + } +} + // execReporter is the one method serveConn needs from the exec runner, named // as an interface so the delivery rules can be tested without a sandbox to // have processes in. diff --git a/cmd/sessiond/transport.go b/cmd/sessiond/transport.go new file mode 100644 index 00000000..1f72e59e --- /dev/null +++ b/cmd/sessiond/transport.go @@ -0,0 +1,93 @@ +package main + +import ( + "context" + "fmt" + "os" + + "github.com/coder/websocket" + + "github.com/tokencanopy/rainier/internal/relay" +) + +// The transport seam: how this sessiond reaches the runner holding it. +// +// There are two, and everything above them is identical. A Docker session +// dials a WebSocket over the container network and asserts its session id in +// a query parameter; a microVM session opens an AF_VSOCK connection to the +// host, which Firecracker forwards to a socket inside that VM's own +// directory — so the id is the host's statement rather than the guest's +// claim, and the channel works before the guest's network does. +// +// relay.ServeSessionWithExec, the RPC dispatcher and the agent sync are +// untouched by the difference, because every one of them already takes a +// relay.Conn. + +const ( + // transportWebSocket is the default and the compatibility floor: every + // session that exists today, and every Docker session there will ever be. + transportWebSocket = "websocket" + // transportVsock is the microVM path. It is a flag (or RAINIER_TRANSPORT) + // rather than something inferred, and it comes from the session IMAGE + // rather than from a create: a microVM guest receives no environment at + // all until its boot configuration arrives, so there is nothing per-session + // for it to be inferred from. Which transport a guest has is a property of + // the rootfs it booted, which is exactly where the flag is set. + transportVsock = "vsock" +) + +// dialSession opens one connection to the runner. It is the whole seam: a +// function of a context, because that is all either transport needs. +type dialSession func(context.Context) (relay.Conn, error) + +// websocketTransport is the Docker path, byte for byte what dialLoop did +// before the seam existed: the same URL, the same nil options, the same read +// limit. A session keeps the sessiond it booted with for life, so this hop +// is a contract with images months old and is not a place to tidy anything. +func websocketTransport(dial, sessionID string) dialSession { + return func(ctx context.Context) (relay.Conn, error) { + c, _, err := websocket.Dial(ctx, dial+"?session="+sessionID, nil) + if err != nil { + return nil, err + } + c.SetReadLimit(16 << 20) + return relay.WSConn(c), nil + } +} + +// vsockTransport is the microVM path: AF_VSOCK to (HOST_CID, 1024). +// +// One port, because internal/relay already multiplexes the configuration, +// the terminal stream, the session RPC and the lifecycle handshake over one +// conn. The host never dials in; the guest opens this and everything rides +// it. +func vsockTransport() dialSession { + return func(ctx context.Context) (relay.Conn, error) { + c, err := dialVsock(ctx, vsockHostCID, vsockControlPort) + if err != nil { + return nil, fmt.Errorf("dial the host control channel over vsock: %w", err) + } + return relay.NetConn(c), nil + } +} + +// envOr reads a variable, falling back to def when it is unset or empty. +// It is how --transport takes its default from the session IMAGE: a microVM +// guest has no per-session environment to be configured from, but it does +// have the rootfs it booted, and that is the right place for a fact about +// which transport this guest has. +func envOr(name, def string) string { + if v := os.Getenv(name); v != "" { + return v + } + return def +} + +const ( + // vsockHostCID is HOST_CID: the host's context id is always 2. + vsockHostCID = 2 + // vsockControlPort is the host port the runner listens on. It matches + // the driver's guestControlPort, and the two are the same wire fact + // spelled in two binaries that ship in different artifacts. + vsockControlPort = 1024 +) diff --git a/cmd/sessiond/transport_test.go b/cmd/sessiond/transport_test.go new file mode 100644 index 00000000..0dce3499 --- /dev/null +++ b/cmd/sessiond/transport_test.go @@ -0,0 +1,124 @@ +package main + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/coder/websocket" + + "github.com/tokencanopy/rainier/internal/relay" +) + +// TestTheWebSocketTransportDialsTheURLItAlwaysHas is the compatibility floor +// of this whole change, pinned at the one hop that moved. +// +// dialLoop used to call websocket.Dial inline; it now calls a dialSession. +// A session keeps the sessiond it booted with for life, so the URL that hop +// produces is a contract with a runnerd that may be months newer — and the +// seam must have changed where the call lives and nothing about what it +// sends. +func TestTheWebSocketTransportDialsTheURLItAlwaysHas(t *testing.T) { + got := make(chan string, 1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got <- r.URL.RequestURI() + c, err := websocket.Accept(w, r, nil) + if err != nil { + return + } + defer c.CloseNow() + <-r.Context().Done() + })) + defer srv.Close() + + base := strings.Replace(srv.URL, "http", "ws", 1) + "/register" + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + conn, err := websocketTransport(base, "sess_example")(ctx) + if err != nil { + t.Fatalf("dial: %v", err) + } + defer conn.Close() + + select { + case uri := <-got: + if uri != "/register?session=sess_example" { + t.Fatalf("the transport dialed %q, want /register?session=sess_example", uri) + } + case <-time.After(5 * time.Second): + t.Fatal("the transport never reached the server") + } +} + +// TestTheBootConnectionIsServedOnceAndThenTheDialerIs pins the one piece of +// state sessionTransport holds: the connection the microVM boot already +// established is served first and then dropped, because its configuration +// has been read and its exchange made. Running the preamble over it again +// would wait for a second configuration that is not coming. +func TestTheBootConnectionIsServedOnceAndThenTheDialerIs(t *testing.T) { + dial, _ := fakeTransport(t) + first, _ := fakeTransport(t) + boot, err := first(context.Background()) + if err != nil { + t.Fatal(err) + } + + preambles := 0 + tr := sessionTransport{ + dial: dial, + first: boot, + preamble: func(context.Context, relay.Conn) error { preambles++; return nil }, + } + + got, err := tr.connect(context.Background()) + if err != nil { + t.Fatal(err) + } + if got != boot { + t.Fatal("the first connect did not serve the connection the boot established") + } + if preambles != 0 { + t.Fatalf("the boot connection was put through the preamble %d time(s)", preambles) + } + + next, err := tr.connect(context.Background()) + if err != nil { + t.Fatal(err) + } + if next == boot { + t.Fatal("the boot connection was served twice") + } + if preambles != 1 { + t.Fatalf("a redial ran the preamble %d time(s), want exactly once", preambles) + } +} + +// TestAFailedPreambleClosesTheConnection: a connection whose configuration +// could not be read is not a connection to serve, and leaving it open would +// leak one per redial for the life of the session. +func TestAFailedPreambleClosesTheConnection(t *testing.T) { + dial, hosts := fakeTransport(t) + tr := sessionTransport{ + dial: dial, + preamble: func(context.Context, relay.Conn) error { return errPreambleFailed }, + } + if _, err := tr.connect(context.Background()); err != errPreambleFailed { + t.Fatalf("connect = %v, want the preamble's own error", err) + } + host := <-hosts + // The guest end is closed, so the host's drain ends rather than + // blocking. It is read through the host's own reader rather than off the + // conn here: two readers on one net.Pipe end would race. + if err := host.waitClosed(2 * time.Second); err == nil { + t.Fatal("a connection whose preamble failed was left open") + } +} + +var errPreambleFailed = errStub("the boot configuration never arrived") + +type errStub string + +func (e errStub) Error() string { return string(e) } diff --git a/cmd/sessiond/unmount_linux.go b/cmd/sessiond/unmount_linux.go new file mode 100644 index 00000000..112a6112 --- /dev/null +++ b/cmd/sessiond/unmount_linux.go @@ -0,0 +1,59 @@ +//go:build linux + +package main + +import ( + "errors" + "log" + "os" + + "golang.org/x/sys/unix" +) + +// unmountAgentHome unmounts the agent home before this VM ends. +// +// It is the one filesystem act a cold suspend needs from inside the guest: +// the home is a block device of its own (ADR-0003 §4.1 keeps it out of the +// workspace disk and its checkpoint), and a device detached with dirty pages +// above it is a home that comes back needing a fsck for a credential set +// somebody logged in for once. +// +// MNT_DETACH is deliberate, and the sync(2) in front of it is what makes it +// safe. A lazy unmount detaches the tree immediately and lets the kernel +// finish when the last reference goes, which is what turns an ordinary +// unmount into EBUSY when a process is still holding a file under it — and +// the alternative to detaching is refusing to answer the host, which ends the +// VM anyway with nothing flushed at all. But "finish when the last reference +// goes" is precisely a writeback this VM may not live to see: the host +// terminates it on the ready answer, so a detach on its own is NOT a promise +// that the device is clean. sync() before it is: it returns once every dirty +// page on every mounted filesystem has been handed to its device, so what the +// lazy detach has left to do afterwards is teardown rather than data. +// +// (sync() is whole-machine rather than per-mount, which inside a microVM +// whose only writable devices are this session's own is exactly the right +// scope — the workspace disk wants flushing before this VM ends for the same +// reason. syncfs(2) on a descriptor under the mount would be narrower and +// buys nothing here.) +// +// Every failure is a log line and not an error. A session with no home +// mounted (no creator, or an older control plane) is the ordinary case and +// reports ENOENT or EINVAL; a failure here must not be what stops this +// process answering the host. +func unmountAgentHome(path string) { + if _, err := os.Stat(path); err != nil { + return + } + // Before the detach: see above. It cannot fail and returns nothing. + unix.Sync() + if err := unix.Unmount(path, unix.MNT_DETACH); err != nil { + if errors.Is(err, unix.EINVAL) || errors.Is(err, unix.ENOENT) { + // Not a mount point: nothing was mounted here, which is what a + // session with no agent home looks like from inside. + return + } + log.Printf("unmounting the agent home at %s before this VM ends: %v", path, err) + return + } + log.Printf("unmounted the agent home at %s", path) +} diff --git a/cmd/sessiond/unmount_other.go b/cmd/sessiond/unmount_other.go new file mode 100644 index 00000000..d9a41433 --- /dev/null +++ b/cmd/sessiond/unmount_other.go @@ -0,0 +1,8 @@ +//go:build !linux + +package main + +// unmountAgentHome on anything but Linux. A cold suspend only ever happens +// to a microVM guest, which is Linux by construction; this exists so the +// package builds on a developer's machine. +func unmountAgentHome(string) {} diff --git a/cmd/sessiond/vsock_linux.go b/cmd/sessiond/vsock_linux.go new file mode 100644 index 00000000..643fd0b0 --- /dev/null +++ b/cmd/sessiond/vsock_linux.go @@ -0,0 +1,56 @@ +//go:build linux + +package main + +import ( + "context" + "fmt" + "net" + "os" + + "golang.org/x/sys/unix" +) + +// dialVsock opens an AF_VSOCK stream to (cid, port). +// +// There is no net.Dial("vsock", …): the address family is not in the +// standard library, so the socket is made by hand and handed to net.FileConn, +// which gives back an ordinary net.Conn with the runtime's poller behind it +// — deadlines, cancellation and all, which relay.NetConn relies on. +// +// The guest needs CONFIG_VIRTIO_VSOCKETS and /dev/vsock for this to work at +// all, and whether the session image's kernel has them is one of the things +// only a real host can answer (design note §7). +func dialVsock(ctx context.Context, cid, port uint32) (net.Conn, error) { + fd, err := unix.Socket(unix.AF_VSOCK, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0) + if err != nil { + return nil, fmt.Errorf("vsock socket: %w", err) + } + // From here on the fd is owned by this function until os.NewFile takes + // it: every failure below closes it, or a guest that retries its dial + // leaks one per attempt for the life of the session. + if err := unix.Connect(fd, &unix.SockaddrVM{CID: cid, Port: port}); err != nil { + _ = unix.Close(fd) + return nil, fmt.Errorf("vsock connect to (%d,%d): %w", cid, port, err) + } + f := os.NewFile(uintptr(fd), fmt.Sprintf("vsock:%d:%d", cid, port)) + if f == nil { + _ = unix.Close(fd) + return nil, fmt.Errorf("vsock: fd %d is not a file", fd) + } + // FileConn DUPLICATES the descriptor, so the original is closed here + // whether it succeeded or not. + conn, err := net.FileConn(f) + _ = f.Close() + if err != nil { + return nil, fmt.Errorf("vsock: adopting the connection: %w", err) + } + // A context that is already done must not leave a live connection + // behind; there is nothing to cancel mid-connect, because unix.Connect + // on a blocking socket does not take one. + if err := ctx.Err(); err != nil { + _ = conn.Close() + return nil, err + } + return conn, nil +} diff --git a/cmd/sessiond/vsock_other.go b/cmd/sessiond/vsock_other.go new file mode 100644 index 00000000..65622ebb --- /dev/null +++ b/cmd/sessiond/vsock_other.go @@ -0,0 +1,16 @@ +//go:build !linux + +package main + +import ( + "context" + "errors" + "net" +) + +// dialVsock on anything but Linux. A microVM guest is Linux by construction +// — Firecracker boots a Linux kernel — so this exists only so the package +// builds on a developer's machine, and it refuses rather than pretending. +func dialVsock(context.Context, uint32, uint32) (net.Conn, error) { + return nil, errors.New("vsock is a Linux facility; this sessiond was built for another platform") +} diff --git a/control/bootstrap.go b/control/bootstrap.go new file mode 100644 index 00000000..5adc3d7e --- /dev/null +++ b/control/bootstrap.go @@ -0,0 +1,101 @@ +package control + +import ( + "context" + "errors" + "time" +) + +// SessionBootstrap is the control plane's whole memory of one minted session +// bootstrap token: the hash of the token, the placement it was minted under, +// and when it stops being one. The token itself is put on the wire once and +// never stored — a store that held it would be a store that could hand an +// environment's secrets to anyone who could read a row. +// +// There is no "consumed" field here because a caller never writes one: the +// spend is the store's own atomic step (ConsumeSessionBootstrap), and a +// record a caller could mark spent is a record two callers could both spend. +type SessionBootstrap struct { + // Hash is the hex-encoded SHA-256 of the token. Hex rather than raw + // bytes so a durable store keeps one comparable column and no encoding + // decision of its own — the same shape the bearer-token hash already has. + Hash string + // PlacementGeneration is the generation the sandbox this token was minted + // for belongs to. It is the fence: a session re-placed onto another + // runner has moved past it, and the token minted for the sandbox that no + // longer exists stops being an answer. + PlacementGeneration uint64 + // ExpiresAt is when the token stops being one, whatever else is true of + // it. Absolute rather than a TTL because the store is the authority on + // "now" for every replica reading the row. + ExpiresAt time.Time +} + +// The four ways a bootstrap exchange is refused. They are four sentinels +// rather than one because each names a different fact about the fleet, and a +// single "no" would make a routine cold resume (fenced) and a replay attempt +// (spent) indistinguishable in an operator's log — while the sandbox still +// gets the same closed answer either way. +// +// None of them, and nothing wrapping them, may carry a token or a secret +// value: a refusal names a condition and a session, never a credential. +var ( + // ErrBootstrapUnknown reports that the session has no minted token whose + // hash matches the one presented. It is also the answer for ANOTHER + // session's token, because the lookup is keyed by the session the + // placement guard read and never by anything in the request. + ErrBootstrapUnknown = errors.New("control: bootstrap token unknown") + + // ErrBootstrapSpent reports a token that matched and has already been + // exchanged. Single-use is the design's decision: a second boot inside + // one VM asks for a second mint rather than replaying the first token. + ErrBootstrapSpent = errors.New("control: bootstrap token already used") + + // ErrBootstrapExpired reports a token whose ExpiresAt has passed. + ErrBootstrapExpired = errors.New("control: bootstrap token expired") + + // ErrBootstrapFenced reports a token whose PlacementGeneration is not the + // row's current one: the session has been placed again since, so the + // sandbox this token was minted for is not the sandbox asking. + ErrBootstrapFenced = errors.New("control: bootstrap token superseded by a newer placement") +) + +// SessionBootstrapStore is the persistence behind the one-shot token a +// microVM session exchanges for its environment's decrypted secrets. It is +// its own port, sized to two methods, for the reason every other port here is +// sized to its job: minting and spending a capability has nothing to do with +// a session's lifecycle, and a repository that could do both would put "hand +// out this workspace's secrets" one method away from "list sessions". +type SessionBootstrapStore interface { + // PutSessionBootstrap records b as id's ONLY acceptable token, replacing + // whatever was recorded before — a fresh mint invalidates its + // predecessor, which is what makes a cold resume's new token the only one + // that works. ErrInvalid on an empty workspace, session, or hash. + PutSessionBootstrap(ctx context.Context, ws WorkspaceID, id SessionID, b SessionBootstrap) error + + // ConsumeSessionBootstrap spends id's token, atomically with respect to + // every other caller, and reports which of the four refusals applies when + // it cannot. gen is the session row's CURRENT placement generation, read + // by the caller under the same guard that authorized the request, and now + // is the caller's clock. + // + // The atomicity is the whole contract: two exchanges racing on one token + // must produce exactly one success, or single-use means nothing. An + // implementation that reads the record and then writes it back is not + // this method, whatever its tests say on an idle machine. + // + // It returns nothing but an error on purpose. The VALUES the token buys + // are not the store's to know — they are resolved above it, from the + // environment, by the one component that holds the secrets key. + // + // When more than one refusal applies, they are reported in this order: + // Unknown, Fenced, Expired, Spent. The order is part of the contract + // rather than each store's own choice, because it is what an operator + // reads: two implementations that disagreed about whether a replayed, + // expired token is "expired" or "already exchanged" would make the same + // fleet describe the same failure two ways. It runs from the most + // structural fact to the most transient, and it puts the hash first so + // that a caller holding no valid token learns nothing at all about the + // state of the one that exists. + ConsumeSessionBootstrap(ctx context.Context, ws WorkspaceID, id SessionID, hash string, gen uint64, now time.Time) error +} diff --git a/controlapp/agents_test.go b/controlapp/agents_test.go index 558f03a3..55b09ec4 100644 --- a/controlapp/agents_test.go +++ b/controlapp/agents_test.go @@ -146,7 +146,7 @@ func TestCreateSpecCarriesTheHome(t *testing.T) { row := control.Session{ID: "sess_example", WorkspaceID: "ws_example", CreatorID: "user_example", Spec: control.PortableSpec{Image: "registry.example.invalid/base@sha256:0000"}} - spec, fail := fx.service.createSpec(fleetCtx, row, nil) + spec, fail := fx.service.createSpec(fleetCtx, row, nil, nil, 0) if fail != "" { t.Fatalf("createSpec failed: %s", fail) } @@ -216,7 +216,7 @@ func TestCreateSpecCarriesTheHome(t *testing.T) { // a login that cannot happen. anon := row anon.CreatorID = "" - spec, fail = fx.service.createSpec(fleetCtx, anon, nil) + spec, fail = fx.service.createSpec(fleetCtx, anon, nil, nil, 0) if fail != "" { t.Fatalf("createSpec failed: %s", fail) } @@ -247,7 +247,7 @@ func TestCreateSpecCarriesTheHome(t *testing.T) { agentsEnvVar: "untrusted-manifest", "USER_VALUE": "preserved", }}}) - spec, fail = fx.service.createSpec(fleetCtx, row, nil) + spec, fail = fx.service.createSpec(fleetCtx, row, nil, nil, 0) if fail != "" { t.Fatalf("createSpec failed: %s", fail) } diff --git a/controlapp/bootstrap.go b/controlapp/bootstrap.go new file mode 100644 index 00000000..ef673c0f --- /dev/null +++ b/controlapp/bootstrap.go @@ -0,0 +1,117 @@ +package controlapp + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "errors" + "time" + + "github.com/tokencanopy/rainier/control" +) + +// The session bootstrap token: what a microVM session presents, once, to be +// given the environment secrets its Spec.Env deliberately does not carry. +// See docs/design/2026-09-20-microvm-bootstrap-token-and-vsock.md §3. +// +// It is opaque to every hop that carries it — not a JWT, carrying no claims — +// because the control plane holds the state. What a hop can do with it is +// therefore exactly nothing except hand it on. + +// SessionBootstrapTTL is how long a minted token lives. It must cover a base +// microVM claim, a guest boot and one round trip; the note's open question 1 +// is whether 120 seconds is the right number, and until Phase 1 measures it +// this constant is the one place to change it. +const SessionBootstrapTTL = 120 * time.Second + +// sessionBootstrapTokenBytes is the token's entropy: 32 bytes from +// crypto/rand. Base64url-encoded it is 43 characters with no padding, and it +// is never truncated, prefixed or decorated — a token with a readable prefix +// is a token somebody grep's a log for. +const sessionBootstrapTokenBytes = 32 + +// HashSessionBootstrapToken is the one-way function the store keeps a token +// under: hex-encoded SHA-256, the same shape the bearer-token hash already +// has. It is exported because the mint and the exchange are answered in two +// different packages and must agree byte for byte; a second spelling would be +// a token that can never be redeemed. +func HashSessionBootstrapToken(token string) string { + sum := sha256.Sum256([]byte(token)) + return hex.EncodeToString(sum[:]) +} + +// SessionBootstrapMinter mints and records one session's bootstrap token. +// +// It is a value rather than a service because it holds no policy: the TTL is +// a constant above it, the fence is the caller's placement generation, and +// the only decision it makes is that the plaintext leaves here and is stored +// nowhere. Two callers share it — the scheduler, at create, and the host's +// answer to a runner's mint_session_bootstrap on a cold resume — and they +// share it precisely so a resume's token cannot end up minted differently +// from a create's. +type SessionBootstrapMinter struct { + Store control.SessionBootstrapStore + Clock control.Clock +} + +// Mint returns a fresh token for id and records its hash against gen with an +// expiry SessionBootstrapTTL from now, replacing any token id already had. +// +// The plaintext is returned and never retained: it exists in this process +// only long enough to be copied into the message that carries it, which is +// the same rule the git credential mint follows. +// +// A store that cannot record the hash is a hard failure, not a token minted +// anyway. The alternative — handing a sandbox a capability nothing can ever +// verify — is a create that boots and then cannot get its secrets, reported +// as a healthy session. +func (m SessionBootstrapMinter) Mint(ctx context.Context, ws control.WorkspaceID, id control.SessionID, gen uint64) (string, error) { + if m.Store == nil || m.Clock == nil { + return "", errors.New("controlapp: this control plane cannot mint a session bootstrap token") + } + b := make([]byte, sessionBootstrapTokenBytes) + if _, err := rand.Read(b); err != nil { + // A broken entropy source has no safe fallback: every caller needs an + // unguessable capability, and a predictable one is worse than none. + return "", errors.New("controlapp: the session bootstrap token could not be minted") + } + token := base64.RawURLEncoding.EncodeToString(b) + if err := m.Store.PutSessionBootstrap(ctx, ws, id, control.SessionBootstrap{ + Hash: HashSessionBootstrapToken(token), + PlacementGeneration: gen, + ExpiresAt: m.Clock.Now().Add(SessionBootstrapTTL), + }); err != nil { + return "", err + } + return token, nil +} + +// SessionBootstrapRefusal renders one of control's four bootstrap sentinels +// as the sentence a sandbox is given, and reports whether err was one of +// them at all. +// +// The four sentences are deliberately different from each other and +// deliberately identical in what they withhold: none names a token, a secret, +// a name, or another session. They say which condition holds so that a person +// reading a failed boot can tell "this token has already been used" from +// "this session was placed again while it was booting" — two facts with +// completely different remedies — and nothing more. +// +// Anything that is not one of the four is NOT rendered here: the caller +// answers with its own flat sentence, because an error nobody wrote to be +// shown may quote a row, a column, or a value. +func SessionBootstrapRefusal(err error) (string, bool) { + switch { + case errors.Is(err, control.ErrBootstrapUnknown): + return "this session has no bootstrap token matching the one presented", true + case errors.Is(err, control.ErrBootstrapSpent): + return "this session's bootstrap token has already been exchanged", true + case errors.Is(err, control.ErrBootstrapExpired): + return "this session's bootstrap token has expired", true + case errors.Is(err, control.ErrBootstrapFenced): + return "this session has been placed again since its bootstrap token was minted", true + } + return "", false +} diff --git a/controlapp/bootstrap_stub_external_test.go b/controlapp/bootstrap_stub_external_test.go new file mode 100644 index 00000000..059dbb63 --- /dev/null +++ b/controlapp/bootstrap_stub_external_test.go @@ -0,0 +1,24 @@ +package controlapp_test + +import ( + "context" + "time" + + "github.com/tokencanopy/rainier/control" +) + +// extBootstrapStub is the external mirror of bootstrapStub: controlapp_test +// is a separate package, exactly as a Rainier Cloud module would be, so it +// supplies its own bootstrap store rather than borrowing the internal one — +// which is itself the proof that the port is satisfiable from outside. +var _ control.SessionBootstrapStore = extBootstrapStub{} + +type extBootstrapStub struct{} + +func (extBootstrapStub) PutSessionBootstrap(context.Context, control.WorkspaceID, control.SessionID, control.SessionBootstrap) error { + return nil +} + +func (extBootstrapStub) ConsumeSessionBootstrap(context.Context, control.WorkspaceID, control.SessionID, string, uint64, time.Time) error { + return control.ErrBootstrapUnknown +} diff --git a/controlapp/bootstrap_stub_test.go b/controlapp/bootstrap_stub_test.go new file mode 100644 index 00000000..186605c5 --- /dev/null +++ b/controlapp/bootstrap_stub_test.go @@ -0,0 +1,83 @@ +package controlapp + +import ( + "context" + "sync" + "time" + + "github.com/tokencanopy/rainier/control" +) + +// bootstrapStub is control.SessionBootstrapStore for every fixture in this +// package: it records what a mint wrote and answers a consume with the same +// four sentinels a real store does. +// +// It is a real (if small) implementation rather than a recorder that always +// says yes, because single use is a rule about the STORE — a fixture that +// accepted every token would let a change that spends one twice pass every +// test here that is not specifically about the exchange. +var _ control.SessionBootstrapStore = (*bootstrapStub)(nil) + +type bootstrapStub struct { + mu sync.Mutex + rows map[control.SessionID]*stubBootstrapRow + // putErr, when set, fails every mint. It is how a test drives the one + // path createSpec must fail closed on. + putErr error +} + +type stubBootstrapRow struct { + hash string + gen uint64 + expiresAt time.Time + spent bool +} + +func newBootstrapStub() *bootstrapStub { + return &bootstrapStub{rows: map[control.SessionID]*stubBootstrapRow{}} +} + +func (b *bootstrapStub) PutSessionBootstrap(_ context.Context, ws control.WorkspaceID, id control.SessionID, rec control.SessionBootstrap) error { + if ws == "" || id == "" || rec.Hash == "" { + return control.ErrInvalid + } + b.mu.Lock() + defer b.mu.Unlock() + if b.putErr != nil { + return b.putErr + } + b.rows[id] = &stubBootstrapRow{hash: rec.Hash, gen: rec.PlacementGeneration, expiresAt: rec.ExpiresAt} + return nil +} + +func (b *bootstrapStub) ConsumeSessionBootstrap(_ context.Context, ws control.WorkspaceID, id control.SessionID, hash string, gen uint64, now time.Time) error { + if ws == "" || id == "" || hash == "" { + return control.ErrInvalid + } + b.mu.Lock() + defer b.mu.Unlock() + row, ok := b.rows[id] + switch { + case !ok || row.hash != hash: + return control.ErrBootstrapUnknown + case row.gen != gen: + return control.ErrBootstrapFenced + case !now.Before(row.expiresAt): + return control.ErrBootstrapExpired + case row.spent: + return control.ErrBootstrapSpent + } + row.spent = true + return nil +} + +// minted returns the record held for id, if any. +func (b *bootstrapStub) minted(id control.SessionID) (stubBootstrapRow, bool) { + b.mu.Lock() + defer b.mu.Unlock() + row, ok := b.rows[id] + if !ok { + return stubBootstrapRow{}, false + } + return *row, true +} diff --git a/controlapp/bootstrap_test.go b/controlapp/bootstrap_test.go new file mode 100644 index 00000000..2fd0c03d --- /dev/null +++ b/controlapp/bootstrap_test.go @@ -0,0 +1,321 @@ +package controlapp + +import ( + "context" + "encoding/base64" + "errors" + "slices" + "strings" + "testing" + "time" + + "github.com/tokencanopy/rainier/control" + "github.com/tokencanopy/rainier/protocol/runner" +) + +// bootstrapRow is the session every case here dispatches: a creator (so the +// agent-home block runs and Spec.Env is non-empty even with nothing secret in +// it), an environment, and an image. +func bootstrapRow() control.Session { + return control.Session{ + ID: "sess_example", WorkspaceID: "ws_example", CreatorID: "user_example", + EnvironmentID: "env_example", + Spec: control.PortableSpec{Image: "registry.example.invalid/base@sha256:0000"}, + } +} + +// TestCreateSpecWithholdsOnlyForMicrovm is the whole of §3's "either the +// secrets or the token, never both", over the matrix of capabilities a +// placement's runner can announce. +// +// The Docker rows are the compatibility floor and are asserted as strictly as +// the microVM one: a runner that does not announce microvm.v1 — which is +// every runner in the fleet today — must be dispatched the values in +// Spec.Env, with no token and no names, byte for byte what it has always +// received. +func TestCreateSpecWithholdsOnlyForMicrovm(t *testing.T) { + for _, tc := range []struct { + name string + caps []string + withheld bool + }{ + {"a runner announcing nothing", nil, false}, + {"a runner announcing exec.v1", []string{runner.CapabilityExecV1}, false}, + {"a runner announcing microvm.v1", []string{runner.CapabilityMicrovmV1}, true}, + {"a runner announcing both", []string{runner.CapabilityExecV1, runner.CapabilityMicrovmV1}, true}, + {"a runner announcing something that merely looks like it", []string{"microvm"}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + fx := newFleetFixture(t) + fx.resolver.material = LaunchMaterial{Environment: map[string]string{ + "DEPLOY_KEY": "value_must_not_travel", + "NPM_TOKEN": "value_must_not_travel_either", + }} + + spec, fail := fx.service.createSpec(fleetCtx, bootstrapRow(), nil, tc.caps, 7) + if fail != "" { + t.Fatalf("createSpec failed: %s", fail) + } + + if !tc.withheld { + if spec.BootstrapToken != "" || len(spec.SecretNames) != 0 { + t.Fatalf("a runner that announced no microvm.v1 was sent a token or names: %+v", spec) + } + if spec.Env["DEPLOY_KEY"] != "value_must_not_travel" || + spec.Env["NPM_TOKEN"] != "value_must_not_travel_either" { + t.Fatalf("the docker path lost its secret values: %v", slices.Sorted(mapKeys(spec.Env))) + } + if _, minted := fx.bootstraps.minted("sess_example"); minted { + t.Fatal("a docker create minted a bootstrap token") + } + return + } + + // Withheld: the names travel, the values do not, and a token does. + if !slices.Equal(spec.SecretNames, []string{"DEPLOY_KEY", "NPM_TOKEN"}) { + t.Fatalf("secret names = %v, want both, sorted", spec.SecretNames) + } + for _, name := range spec.SecretNames { + if v, present := spec.Env[name]; present { + t.Fatalf("a withheld create carried %s = %q in Spec.Env", name, v) + } + } + if spec.BootstrapToken == "" { + t.Fatal("a withheld create carries no bootstrap token; its guest can never get its secrets") + } + // The agent home is configuration and stays, which is what makes + // the boot config a useful message at all. + for k, v := range AgentsEnv(AgentProviders()) { + if spec.Env[k] != v { + t.Fatalf("a withheld create lost the agent-home key %s: %v", + k, slices.Sorted(mapKeys(spec.Env))) + } + } + + // The token is 32 bytes of base64url, and what was RECORDED is its + // hash against this placement — never the token. + raw, err := base64.RawURLEncoding.DecodeString(spec.BootstrapToken) + if err != nil || len(raw) != 32 { + t.Fatalf("token decodes to %d bytes (%v), want 32 base64url bytes", len(raw), err) + } + rec, minted := fx.bootstraps.minted("sess_example") + if !minted { + t.Fatal("nothing was recorded for the token that was handed out") + } + if rec.hash != HashSessionBootstrapToken(spec.BootstrapToken) { + t.Fatal("the recorded hash is not this token's") + } + if rec.hash == spec.BootstrapToken || strings.Contains(rec.hash, spec.BootstrapToken) { + t.Fatal("the plaintext token reached the store") + } + if rec.gen != 7 { + t.Fatalf("recorded placement generation = %d, want the create's 7", rec.gen) + } + want := fx.clock.Now().Add(SessionBootstrapTTL) + if !rec.expiresAt.Equal(want) { + t.Fatalf("expiry = %s, want %s (%s from the mint)", rec.expiresAt, want, SessionBootstrapTTL) + } + }) + } +} + +// TestCreateSpecMintsEvenWithNoSecrets pins the rule that makes the driver's +// own refusal safe: a microVM create ALWAYS carries a token, even when the +// environment declares no secret_refs at all. +// +// The alternative — mint only when there is something to fetch — looks +// tidier and breaks the compatibility row it is supposed to serve. A session +// with a creator and no secrets still has a non-empty Spec.Env (the agent +// manifest), and the microVM driver refuses a create with values and no +// token; so "mint only when needed" would refuse every scratch session on a +// microVM runner. +func TestCreateSpecMintsEvenWithNoSecrets(t *testing.T) { + fx := newFleetFixture(t) + fx.resolver.material = LaunchMaterial{} + + spec, fail := fx.service.createSpec(fleetCtx, bootstrapRow(), nil, []string{runner.CapabilityMicrovmV1}, 3) + if fail != "" { + t.Fatalf("createSpec failed: %s", fail) + } + if spec.BootstrapToken == "" { + t.Fatal("a microVM create with no secrets carries no token, so its driver will refuse it") + } + if len(spec.SecretNames) != 0 { + t.Fatalf("secret names = %v, want none declared", spec.SecretNames) + } + if len(spec.Env) == 0 { + t.Fatal("the create lost the agent manifest, which is configuration and must stay") + } +} + +// TestCreateSpecFailsClosedWhenTheTokenCannotBeRecorded is the fail-closed +// half. A store that will not record the hash leaves nothing able to verify +// the token, so the alternatives are refusing the create or dispatching the +// secrets after all — and the second is the exposure this design removes. +func TestCreateSpecFailsClosedWhenTheTokenCannotBeRecorded(t *testing.T) { + fx := newFleetFixture(t) + fx.resolver.material = LaunchMaterial{Environment: map[string]string{"DEPLOY_KEY": "value_must_not_travel"}} + fx.bootstraps.putErr = errors.New("the store is down") + + spec, fail := fx.service.createSpec(fleetCtx, bootstrapRow(), nil, []string{runner.CapabilityMicrovmV1}, 1) + if fail == "" { + t.Fatalf("createSpec succeeded with an unrecordable token: %+v", spec) + } + if spec != nil { + t.Fatalf("a failed create still produced a spec: %+v", spec) + } + if strings.Contains(fail, "value_must_not_travel") || strings.Contains(fail, "DEPLOY_KEY") { + t.Fatalf("the failure reason quotes the material: %q", fail) + } +} + +// TestAnUnreadablePlacementFailsAWithholdingCreateOnly is finding 6 of the +// branch's own review, and the reason placedGeneration now reports whether +// it could read anything. +// +// Zero has always been "not carried", and on an EVENT it fences nothing. The +// same number is now also the generation a bootstrap token is minted +// against, and there it is not inert: a token recorded at 0 against a row at +// 3 is refused on its one and only exchange, as superseded by the very +// placement that minted it — a store blip becoming a dead session with a +// misleading reason and no way back, because a guest cannot re-mint. +// +// So a withholding create refuses, and every other create is dispatched +// exactly as it always was. Both halves are asserted, because failing the +// second would be a regression for the whole fleet. +func TestAnUnreadablePlacementFailsAWithholdingCreateOnly(t *testing.T) { + // The row is deliberately never seeded, so the read-back GetSession + // makes cannot answer — the same shape as a store that is down. + row := control.Session{ + ID: "sess_unplaced", WorkspaceID: "ws_example", State: control.StateCreating, + PoolID: "pool_example", RunnerID: "vm1", + Spec: control.PortableSpec{Image: "img:latest"}, + } + + t.Run("a withholding create refuses", func(t *testing.T) { + fx := newFleetFixture(t) + fx.st.seedRunner(fleetSeededRunner("vm1", 2, 0, true)) + fx.service.dispatchCreate(fleetCtx, "pool_example", row, "vm1", + []string{runner.CapabilityMicrovmV1}, nil) + + if got := fx.transport.dispatchedCommands(); len(got) != 0 { + t.Fatalf("dispatched %d command(s) with a placement it could not read: %+v", len(got), got) + } + if _, minted := fx.bootstraps.minted("sess_unplaced"); minted { + t.Fatal("a token was minted against a placement generation nobody could read") + } + }) + + t.Run("every other create is dispatched", func(t *testing.T) { + fx := newFleetFixture(t) + fx.st.seedRunner(fleetSeededRunner("vm1", 2, 0, true)) + fx.service.dispatchCreate(fleetCtx, "pool_example", row, "vm1", nil, nil) + + got := fx.transport.dispatchedCommands() + if len(got) != 1 { + t.Fatalf("dispatched %d command(s), want 1 — an unreadable placement fences nothing here", len(got)) + } + if got[0].PlacementGeneration != 0 { + t.Fatalf("placement generation = %d, want 0 (not carried)", got[0].PlacementGeneration) + } + }) +} + +// TestWithholdableNamesRespectsTheAgentHomeReservation pins that a +// secret_ref spelled like an agent-home variable is dropped on BOTH paths. +// +// On the Docker path the reservation already wins — AgentsEnv's keys are +// launch invariants a workspace's configuration may not replace — and a +// microVM guest that was handed the same name as a name to fetch would apply +// the workspace's value over its own credential-custody path after boot, +// which is the reservation defeated one hop later. +func TestWithholdableNamesRespectsTheAgentHomeReservation(t *testing.T) { + fx := newFleetFixture(t) + reserved := AgentsEnv(AgentProviders()) + var anyReserved string + for k := range reserved { + if anyReserved == "" || k < anyReserved { + anyReserved = k + } + } + fx.resolver.material = LaunchMaterial{Environment: map[string]string{ + anyReserved: "value_must_not_travel", + "DEPLOY_KEY": "value_must_not_travel_either", + }} + + spec, fail := fx.service.createSpec(fleetCtx, bootstrapRow(), nil, []string{runner.CapabilityMicrovmV1}, 1) + if fail != "" { + t.Fatalf("createSpec failed: %s", fail) + } + if slices.Contains(spec.SecretNames, anyReserved) { + t.Fatalf("the reserved key %q was promised to the guest as a secret name: %v", anyReserved, spec.SecretNames) + } + if !slices.Contains(spec.SecretNames, "DEPLOY_KEY") { + t.Fatalf("the ordinary secret name was dropped too: %v", spec.SecretNames) + } +} + +// TestSessionBootstrapMinterRecordsAndNeverRepeats pins the two properties of +// the mint that no store can supply: the token is fresh every time, and what +// leaves the minter is the only copy. +func TestSessionBootstrapMinterRecordsAndNeverRepeats(t *testing.T) { + store := newBootstrapStub() + clock := &fleetFakeClock{now: time.Unix(1_700_000_000, 0)} + m := SessionBootstrapMinter{Store: store, Clock: clock} + + seen := map[string]struct{}{} + for i := 0; i < 64; i++ { + tok, err := m.Mint(context.Background(), "ws_example", "sess_example", uint64(i)) + if err != nil { + t.Fatalf("mint %d: %v", i, err) + } + if _, dup := seen[tok]; dup { + t.Fatalf("mint %d repeated a token", i) + } + seen[tok] = struct{}{} + } + + // And a minter with no store refuses rather than handing out a + // capability nothing can ever check. + if _, err := (SessionBootstrapMinter{Clock: clock}).Mint(context.Background(), "ws_example", "sess_example", 1); err == nil { + t.Fatal("a minter with no store handed out a token") + } +} + +// TestSessionBootstrapRefusalNamesTheConditionAndNothingElse pins §15.1 at +// the one place these sentences are written: four distinguishable answers, +// none of which carries a token, a value, or another session's id. +func TestSessionBootstrapRefusalNamesTheConditionAndNothingElse(t *testing.T) { + seen := map[string]struct{}{} + for _, err := range []error{ + control.ErrBootstrapUnknown, control.ErrBootstrapSpent, + control.ErrBootstrapExpired, control.ErrBootstrapFenced, + } { + sentence, ok := SessionBootstrapRefusal(err) + if !ok || sentence == "" { + t.Fatalf("%v produced no sentence", err) + } + if _, dup := seen[sentence]; dup { + t.Fatalf("%v repeats another refusal's sentence: %q", err, sentence) + } + seen[sentence] = struct{}{} + } + if _, ok := SessionBootstrapRefusal(errors.New("the store is down")); ok { + t.Fatal("an error nobody wrote to be shown was rendered as a refusal") + } + if _, ok := SessionBootstrapRefusal(nil); ok { + t.Fatal("a nil error was rendered as a refusal") + } +} + +// mapKeys is slices.Sorted's input for a map whose keys a failure message +// names. It exists so a failure prints key names and never values. +func mapKeys(m map[string]string) func(func(string) bool) { + return func(yield func(string) bool) { + for k := range m { + if !yield(k) { + return + } + } + } +} diff --git a/controlapp/fleet.go b/controlapp/fleet.go index 773504af..0f4eed40 100644 --- a/controlapp/fleet.go +++ b/controlapp/fleet.go @@ -52,6 +52,15 @@ type FleetOptions struct { // elsewhere; nothing in this task asks it yet. Checkpoints control.CheckpointLocator + // Bootstraps records the single-use token a microVM session exchanges for + // its environment's secrets. It is required rather than optional, like + // every other port here, and for a sharper reason than symmetry: a + // dispatch that could not mint must REFUSE, and a service with no store + // behind it would have to choose between refusing every microVM create + // and quietly falling back to putting the secrets in Spec.Env — which is + // the exposure this whole design removes. + Bootstraps control.SessionBootstrapStore + // DefaultEgress replaces the built-in developer egress baseline // (DefaultDeveloperEgressHosts) that every dispatched session's allowlist // is unioned with. It is a POINTER because "leave it alone" and "make it @@ -96,6 +105,11 @@ type FleetService struct { uow control.UnitOfWork checkpoints control.CheckpointLocator + // bootstraps mints and records a microVM session's bootstrap token + // (FleetOptions.Bootstraps), composed once here over the same clock every + // other decision in this service reads. + bootstraps SessionBootstrapMinter + // defaultEgress is the host's developer egress baseline, resolved once at // construction (FleetOptions.DefaultEgress). Held as a plain slice because // by this point "unset" has already been answered. @@ -126,7 +140,7 @@ func NewFleetService(opts FleetOptions) (*FleetService, error) { if opts.LaunchMaterial == nil { return nil, control.ErrInvalid } - if opts.UnitOfWork == nil || opts.Checkpoints == nil { + if opts.UnitOfWork == nil || opts.Checkpoints == nil || opts.Bootstraps == nil { return nil, control.ErrInvalid } return &FleetService{ @@ -145,6 +159,7 @@ func NewFleetService(opts FleetOptions) (*FleetService, error) { defaultInitTimeout: opts.DefaultInitTimeoutSec, uow: opts.UnitOfWork, checkpoints: opts.Checkpoints, + bootstraps: SessionBootstrapMinter{Store: opts.Bootstraps, Clock: opts.Clock}, defaultEgress: resolveDefaultEgress(opts.DefaultEgress), wake: make(chan control.PoolID, 64), known: make(map[control.PoolID]struct{}), diff --git a/controlapp/fleet_external_test.go b/controlapp/fleet_external_test.go index 3884358e..5e3f27f2 100644 --- a/controlapp/fleet_external_test.go +++ b/controlapp/fleet_external_test.go @@ -31,6 +31,7 @@ func fleetExternalOptions() controlapp.FleetOptions { LaunchMaterial: fleetExtResolver{}, UnitOfWork: extDirectUOW{}, Checkpoints: extLocatorStub{}, + Bootstraps: extBootstrapStub{}, } } diff --git a/controlapp/fleet_test.go b/controlapp/fleet_test.go index 0f668813..3a29e0e5 100644 --- a/controlapp/fleet_test.go +++ b/controlapp/fleet_test.go @@ -567,6 +567,9 @@ type fleetFixture struct { // checkpoints is the fixture's locator. A test sets its answer BEFORE // starting the scheduler loop, which is the only reader. checkpoints *locatorStub + // bootstraps is the fixture's session bootstrap store: what a microVM + // create minted, and the four refusals an exchange can meet. + bootstraps *bootstrapStub } // fleetTestNoBaseline switches the developer egress baseline OFF for the @@ -602,6 +605,7 @@ func newFleetFixtureWith(t *testing.T, resolver LaunchMaterialResolver, defaultE clock := &fleetFakeClock{now: time.Unix(1_700_000_000, 0)} ids := &fleetFakeIDs{} ckpts := &locatorStub{} + boots := newBootstrapStub() r := resolver if r == nil { r = &fleetFakeResolver{} @@ -624,6 +628,7 @@ func newFleetFixtureWith(t *testing.T, resolver LaunchMaterialResolver, defaultE LaunchMaterial: r, UnitOfWork: directUOW{}, Checkpoints: ckpts, + Bootstraps: boots, DefaultEgress: defaultEgress, }) if err != nil { @@ -632,7 +637,7 @@ func newFleetFixtureWith(t *testing.T, resolver LaunchMaterialResolver, defaultE return &fleetFixture{ service: svc, auth: auth, sessions: sessions, envs: envs, fleet: fleet, pools: pools, transport: transport, events: events, clock: clock, ids: ids, st: st, - resolver: fr, checkpoints: ckpts, + resolver: fr, checkpoints: ckpts, bootstraps: boots, } } @@ -695,6 +700,7 @@ func TestNewFleetServiceRequiresEveryPort(t *testing.T) { LaunchMaterial: &fleetFakeResolver{}, UnitOfWork: directUOW{}, Checkpoints: locatorStub{}, + Bootstraps: newBootstrapStub(), } } if _, err := NewFleetService(base()); err != nil { @@ -713,6 +719,7 @@ func TestNewFleetServiceRequiresEveryPort(t *testing.T) { "launch material": func(o *FleetOptions) { o.LaunchMaterial = nil }, "unit of work": func(o *FleetOptions) { o.UnitOfWork = nil }, "checkpoints": func(o *FleetOptions) { o.Checkpoints = nil }, + "bootstraps": func(o *FleetOptions) { o.Bootstraps = nil }, } { o := base() zero(&o) diff --git a/controlapp/repotest/bootstrap.go b/controlapp/repotest/bootstrap.go new file mode 100644 index 00000000..a67472ef --- /dev/null +++ b/controlapp/repotest/bootstrap.go @@ -0,0 +1,155 @@ +package repotest + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/tokencanopy/rainier/control" +) + +// The bootstrap-token half of the contract. It is short because the port is +// two methods, and it is here rather than in either store's own tests because +// it is the one place where "the in-memory store and the Postgres store agree" +// is checked rather than assumed — and because a hosted cell's store has to +// pass exactly the same cases. +// +// Every case creates its session first: a durable store may (and this repo's +// does) key the token to the session row, so a token minted against a session +// that does not exist is not a case the contract has an opinion about. + +// bootstrapSession is the session every case below mints against, placed on a +// runner at a known generation so the fence has something to fence. +func bootstrapSession(t *testing.T, s Stores, id control.SessionID) control.Session { + t.Helper() + return mustCreate(t, s, Alpha, control.Session{ + ID: id, CreatorID: "act_a", State: control.StateQueued, PoolID: PoolA, + }) +} + +// caseSessionBootstrap (B1) is the whole of single-use and the whole of the +// fence, in the five outcomes the design note's test plan names: one success +// and four refusals, each distinguishable from the others. +func caseSessionBootstrap(t *testing.T, s Stores) { + ctx := context.Background() + bootstrapSession(t, s, "sess_boot") + bootstrapSession(t, s, "sess_other") + + now := baseTime() + const hash = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + rec := control.SessionBootstrap{Hash: hash, PlacementGeneration: 4, ExpiresAt: now.Add(120 * time.Second)} + if err := s.Bootstraps.PutSessionBootstrap(ctx, Alpha, "sess_boot", rec); err != nil { + t.Fatalf("put: %v", err) + } + + // A token nobody minted — including, and especially, ANOTHER session's — + // is unknown. The lookup is keyed by the session, so "sess_other presents + // sess_boot's token" and "sess_boot presents a token out of thin air" are + // deliberately the same answer: neither caller learns anything about the + // token that does exist. + if err := s.Bootstraps.ConsumeSessionBootstrap(ctx, Alpha, "sess_other", hash, 4, now); !errors.Is(err, control.ErrBootstrapUnknown) { + t.Fatalf("another session's token: err = %v, want ErrBootstrapUnknown", err) + } + const otherHash = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + if err := s.Bootstraps.ConsumeSessionBootstrap(ctx, Alpha, "sess_boot", otherHash, 4, now); !errors.Is(err, control.ErrBootstrapUnknown) { + t.Fatalf("a token that was never minted: err = %v, want ErrBootstrapUnknown", err) + } + + // A placement that has moved past the one the token was minted under. + if err := s.Bootstraps.ConsumeSessionBootstrap(ctx, Alpha, "sess_boot", hash, 5, now); !errors.Is(err, control.ErrBootstrapFenced) { + t.Fatalf("a superseded generation: err = %v, want ErrBootstrapFenced", err) + } + + // Expiry is checked at the boundary: at exactly ExpiresAt the token is + // already gone, so a 120-second TTL is 120 seconds and not 121. + if err := s.Bootstraps.ConsumeSessionBootstrap(ctx, Alpha, "sess_boot", hash, 4, rec.ExpiresAt); !errors.Is(err, control.ErrBootstrapExpired) { + t.Fatalf("at the expiry instant: err = %v, want ErrBootstrapExpired", err) + } + + // The one success, and then the replay. + if err := s.Bootstraps.ConsumeSessionBootstrap(ctx, Alpha, "sess_boot", hash, 4, now); err != nil { + t.Fatalf("a fresh token: err = %v, want success", err) + } + if err := s.Bootstraps.ConsumeSessionBootstrap(ctx, Alpha, "sess_boot", hash, 4, now); !errors.Is(err, control.ErrBootstrapSpent) { + t.Fatalf("a replayed token: err = %v, want ErrBootstrapSpent", err) + } + + // And the token is not the other workspace's to spend, whatever it holds. + if err := s.Bootstraps.ConsumeSessionBootstrap(ctx, Beta, "sess_boot", hash, 4, now); !errors.Is(err, control.ErrBootstrapUnknown) { + t.Fatalf("another workspace: err = %v, want ErrBootstrapUnknown", err) + } +} + +// caseSessionBootstrapRemint (B2) is what makes a cold resume safe: the token +// runnerd fetches before it boots a new VM is the only one that works, and the +// one the previous boot was handed stops being an answer — including when that +// one was never spent. +func caseSessionBootstrapRemint(t *testing.T, s Stores) { + ctx := context.Background() + bootstrapSession(t, s, "sess_remint") + now := baseTime() + const first = "1111111111111111111111111111111111111111111111111111111111111111" + const second = "2222222222222222222222222222222222222222222222222222222222222222" + + if err := s.Bootstraps.PutSessionBootstrap(ctx, Alpha, "sess_remint", control.SessionBootstrap{ + Hash: first, PlacementGeneration: 1, ExpiresAt: now.Add(time.Minute)}); err != nil { + t.Fatalf("first put: %v", err) + } + if err := s.Bootstraps.PutSessionBootstrap(ctx, Alpha, "sess_remint", control.SessionBootstrap{ + Hash: second, PlacementGeneration: 2, ExpiresAt: now.Add(time.Minute)}); err != nil { + t.Fatalf("second put: %v", err) + } + if err := s.Bootstraps.ConsumeSessionBootstrap(ctx, Alpha, "sess_remint", first, 1, now); !errors.Is(err, control.ErrBootstrapUnknown) { + t.Fatalf("the retired token: err = %v, want ErrBootstrapUnknown", err) + } + if err := s.Bootstraps.ConsumeSessionBootstrap(ctx, Alpha, "sess_remint", second, 2, now); err != nil { + t.Fatalf("the current token: err = %v, want success", err) + } + + // A re-mint after a spend is unspent again: that is the resume path, and + // a store that kept the consumed mark would refuse every session that + // was ever cold-parked twice. + if err := s.Bootstraps.PutSessionBootstrap(ctx, Alpha, "sess_remint", control.SessionBootstrap{ + Hash: first, PlacementGeneration: 3, ExpiresAt: now.Add(time.Minute)}); err != nil { + t.Fatalf("third put: %v", err) + } + if err := s.Bootstraps.ConsumeSessionBootstrap(ctx, Alpha, "sess_remint", first, 3, now); err != nil { + t.Fatalf("a re-minted token: err = %v, want success", err) + } +} + +// caseSessionBootstrapEmpty (B3) pins that neither method accepts an unscoped +// call, the same rule every other port here keeps. +func caseSessionBootstrapEmpty(t *testing.T, s Stores) { + ctx := context.Background() + now := baseTime() + const hash = "3333333333333333333333333333333333333333333333333333333333333333" + for _, tc := range []struct { + name string + call func() error + }{ + {"Put with no workspace", func() error { + return s.Bootstraps.PutSessionBootstrap(ctx, "", "sess_example", control.SessionBootstrap{Hash: hash, ExpiresAt: now}) + }}, + {"Put with no session", func() error { + return s.Bootstraps.PutSessionBootstrap(ctx, Alpha, "", control.SessionBootstrap{Hash: hash, ExpiresAt: now}) + }}, + {"Put with no hash", func() error { + return s.Bootstraps.PutSessionBootstrap(ctx, Alpha, "sess_example", control.SessionBootstrap{ExpiresAt: now}) + }}, + {"Consume with no workspace", func() error { + return s.Bootstraps.ConsumeSessionBootstrap(ctx, "", "sess_example", hash, 1, now) + }}, + {"Consume with no session", func() error { + return s.Bootstraps.ConsumeSessionBootstrap(ctx, Alpha, "", hash, 1, now) + }}, + {"Consume with no hash", func() error { + return s.Bootstraps.ConsumeSessionBootstrap(ctx, Alpha, "sess_example", "", 1, now) + }}, + } { + if err := tc.call(); !errors.Is(err, control.ErrInvalid) { + t.Errorf("%s: err = %v, want ErrInvalid", tc.name, err) + } + } +} diff --git a/controlapp/repotest/repotest.go b/controlapp/repotest/repotest.go index f670dbcc..dfd825f1 100644 --- a/controlapp/repotest/repotest.go +++ b/controlapp/repotest/repotest.go @@ -28,7 +28,12 @@ type Stores struct { Sessions control.SessionRepository Environments control.EnvironmentRepository Fleet control.FleetRepository - Provision func(ctx context.Context, ws control.WorkspaceID) error + // Bootstraps is the session bootstrap tokens, over the SAME backing + // store: a token minted against a session created through Sessions must + // be consumable here, and a durable store that keys the two differently + // fails the suite rather than a microVM session's boot. + Bootstraps control.SessionBootstrapStore + Provision func(ctx context.Context, ws control.WorkspaceID) error } // Run drives the contract. open is called once per case and must return an @@ -88,6 +93,10 @@ func cases() []suiteCase { {"F5 sessions on a runner", caseSessionsOnRunner}, {"F6 oldest queued", caseOldestQueued}, {"F7 an empty pool is invalid on every fleet method", caseFleetEmptyPool}, + + {"B1 a bootstrap token is single-use and fenced", caseSessionBootstrap}, + {"B2 a fresh mint retires its predecessor", caseSessionBootstrapRemint}, + {"B3 an empty workspace, session or hash is invalid", caseSessionBootstrapEmpty}, } } diff --git a/controlapp/scheduler.go b/controlapp/scheduler.go index bbf19418..b2b9145a 100644 --- a/controlapp/scheduler.go +++ b/controlapp/scheduler.go @@ -113,7 +113,11 @@ func (s *FleetService) drainPool(ctx context.Context, pool control.PoolID) { } return } - go s.dispatchCreate(ctx, pool, row, runnerID, env) + // The runner's announced capabilities travel with the placement + // rather than being re-read at dispatch: they are the claims this + // pass placed ON, and a second read could disagree with the decision + // already made. createSpec keys its withholding on them. + go s.dispatchCreate(ctx, pool, row, runnerID, capabilitiesOf(views, runnerID), env) } } @@ -244,6 +248,20 @@ func overridesEnvironmentImage(row control.Session, env control.Environment) boo return row.Spec.Image != env.Image && row.Spec.Image != env.Snapshot.Ref } +// capabilitiesOf returns the capabilities the placement pass saw on id, or +// nil when the view has gone (a runner that disconnected between the pick and +// the dispatch). Nil is the safe answer: a runner whose claims this pass +// cannot state is a runner nothing is withheld from, which dispatches today's +// Spec — the same thing an older runner gets. +func capabilitiesOf(views []runnerView, id control.RunnerID) []string { + for _, v := range views { + if v.id == id { + return v.caps + } + } + return nil +} + func hasAllCapabilities(caps, reqs []string) bool { for _, want := range reqs { if !slices.Contains(caps, want) { @@ -313,8 +331,21 @@ func cloneSession(s control.Session) control.Session { // dispatchCreate builds the create spec, pins setup provenance, dispatches, // and settles the uncertain-delivery outcome without ever duplicating a // delivered create. -func (s *FleetService) dispatchCreate(ctx context.Context, pool control.PoolID, row control.Session, runnerID control.RunnerID, env *control.Environment) { - spec, fail := s.createSpec(ctx, row, env) +func (s *FleetService) dispatchCreate(ctx context.Context, pool control.PoolID, row control.Session, runnerID control.RunnerID, runnerCaps []string, env *control.Environment) { + // Read BEFORE the spec is built, not after, because a microVM create's + // bootstrap token is FENCED by this number: the hash recorded against the + // session has to name the same placement the create carries, or the first + // exchange is refused as superseded by the very placement that minted it. + gen, known := s.placedGeneration(ctx, row) + if !known && slices.Contains(runnerCaps, runner.CapabilityMicrovmV1) { + // Only for a withholding placement, and only because the token's + // fence is this number: everywhere else an unknown generation is + // inert, and failing a create over a transient read would be a + // regression for every session in the fleet. + s.failCreate(ctx, row, "could not read this session's placement to mint its bootstrap token") + return + } + spec, fail := s.createSpec(ctx, row, env, runnerCaps, gen) if fail != "" { s.failCreate(ctx, row, fail) return @@ -326,7 +357,7 @@ func (s *FleetService) dispatchCreate(ctx context.Context, pool control.PoolID, Type: "create", Session: string(row.ID), Spec: spec, - PlacementGeneration: s.placedGeneration(ctx, row), + PlacementGeneration: gen, }) switch { case err != nil: @@ -358,18 +389,34 @@ func (s *FleetService) dispatchCreate(ctx context.Context, pool control.PoolID, // A read that cannot be made carries nothing rather than the value it knows // to be stale: zero is "not carried" on the wire and fences nothing, while a // wrong number would fence every event this sandbox ever sends. -func (s *FleetService) placedGeneration(ctx context.Context, row control.Session) uint64 { +// +// The bool is the same fact said out loud, and it exists because that +// reasoning stopped being universally true. Zero fences nothing on an EVENT, +// which is what this was written for — but the same number is now also the +// generation a microVM session's bootstrap token is minted against, and +// there a wrong value is not inert: a token recorded at 0 against a row at 3 +// is refused on its one and only exchange, as superseded by the very +// placement that minted it. So the caller is told, and refuses the create +// rather than dispatching a session that cannot get its secrets. +func (s *FleetService) placedGeneration(ctx context.Context, row control.Session) (uint64, bool) { placed, err := s.sessions.GetSession(ctx, row.WorkspaceID, row.ID) if err != nil { - return 0 + return 0, false } - return placed.PlacementGeneration + return placed.PlacementGeneration, true } // createSpec builds the runner create spec from the session and its current // environment, resolving sensitive launch material only here and never // storing it. -func (s *FleetService) createSpec(ctx context.Context, row control.Session, env *control.Environment) (*runner.Spec, string) { +// +// runnerCaps are the capabilities the placement's runner announced, and gen +// the placement generation the create carries. Together they are the whole of +// what this function needs to decide the one question the microVM bootstrap +// design added to it: whether this session's environment secrets travel in +// Spec.Env, as they always have, or stay behind a single-use token the guest +// exchanges for them after it boots (see §3 of the design note). +func (s *FleetService) createSpec(ctx context.Context, row control.Session, env *control.Environment, runnerCaps []string, gen uint64) (*runner.Spec, string) { spec := runner.Spec{ Name: row.Name, Image: row.Spec.Image, @@ -402,7 +449,17 @@ func (s *FleetService) createSpec(ctx context.Context, row control.Session, env spec.Repos = slices.Clone(material.Repos) spec.GitAuthorName = material.GitAuthorName spec.GitAuthorEmail = material.GitAuthorEmail - spec.Env = cloneMap(material.Environment) + // The one branch in this function that is about WHERE the session will + // run rather than what it is. A runner announcing microvm.v1 boots each + // session in its own VM on a shared regional host, where a driver cannot + // hand a value to a daemon through an argv and every channel it does have + // ends in a file; so the values stay here and the create carries their + // NAMES and a token instead (ADR-0003 §2.7 item 1). Every other runner — + // which is every runner today — is dispatched exactly what it always was. + withheld := slices.Contains(runnerCaps, runner.CapabilityMicrovmV1) + if !withheld { + spec.Env = cloneMap(material.Environment) + } // The agent home is the creator's, in this workspace: every session they // start there mounts the same volume, and that sameness is the whole of // "log in once". A session with no creator gets none of this — a home @@ -427,6 +484,43 @@ func (s *FleetService) createSpec(ctx context.Context, row control.Session, env } spec.Env = agentEnv } + // The names, and then the token, in that order: the names are derived + // from the material this function already holds, and they are computed + // AFTER the agent-home block so that the one rule that block states — + // agent paths and the manifest are launch invariants a workspace's own + // configuration may not replace — holds identically on both paths. A + // secret_ref that happens to be spelled CLAUDE_CONFIG_DIR is dropped for + // a Docker session and must not be smuggled back in as a name a microVM + // guest then applies over its own agent home. + if withheld { + spec.SecretNames = withholdableNames(material.Environment, spec.Env) + token, err := s.bootstraps.Mint(ctx, row.WorkspaceID, row.ID, gen) + if err != nil { + // Fail closed. The alternative to refusing here is a session + // dispatched with neither its secrets nor a way to ask for them, + // which boots, reports healthy, and fails at whatever the first + // credential-shaped thing it does is. + return nil, "could not mint this session's bootstrap token" + } + spec.BootstrapToken = token + } + // Either the values or the token, never both — stated as a check rather + // than as a comment, because it is the whole security claim of §3 and it + // is one careless merge away from being false. + // + // It is a BACKSTOP and not an independent proof: withholdableNames + // already excludes every name spec.Env holds, so as this function stands + // it cannot fire. What it is for is the edit that adds a third writer of + // spec.Env below the withholding branch, or reorders the two — at which + // point a failed create is a much better answer than a secret on a + // shared host's disk. + if spec.BootstrapToken != "" { + for _, name := range spec.SecretNames { + if _, both := spec.Env[name]; both { + return nil, "could not resolve launch material" + } + } + } // The session row stores only the egress its caller or environment // declared; the hosts the resolved material needs are the resolver's // knowledge and are added here, at dispatch, so the row and the view a @@ -502,6 +596,33 @@ func boundOr(declared, fallback int) int { return fallback } +// withholdableNames are the names a withheld create promises its guest: every +// key of the resolved secret environment that the spec's own configuration +// does not already reserve, sorted. +// +// Sorted because the list is on the wire and in a guest's failure message, +// and a set rendered in map order would make two identical creates look +// different. Nil when there is nothing to promise, so a session with no +// secret_refs puts no `secret_names` on the wire at all and its guest reads +// the honest "none declared" rather than "declared and never arrived". +func withholdableNames(material, reserved map[string]string) []string { + if len(material) == 0 { + return nil + } + out := make([]string, 0, len(material)) + for name := range material { + if _, taken := reserved[name]; taken { + continue + } + out = append(out, name) + } + if len(out) == 0 { + return nil + } + slices.Sort(out) + return out +} + func cloneMap(m map[string]string) map[string]string { if m == nil { return nil diff --git a/controlapp/scheduler_test.go b/controlapp/scheduler_test.go index a0d03a4c..04584ac8 100644 --- a/controlapp/scheduler_test.go +++ b/controlapp/scheduler_test.go @@ -418,7 +418,7 @@ func TestDispatchCreateFailureAndUncertainDelivery(t *testing.T) { fx.transport.dispatchErr = errors.New("connection closed") fx.transport.connectedOverrides["pool_example/vm1"] = false - fx.service.dispatchCreate(context.Background(), "pool_example", row, "vm1", nil) + fx.service.dispatchCreate(context.Background(), "pool_example", row, "vm1", nil, nil) got := fleetGetSessionState(t, fx, "ws_example", "sess_conn") if got.State != control.StateQueued || got.RunnerID != "" { @@ -438,7 +438,7 @@ func TestDispatchCreateFailureAndUncertainDelivery(t *testing.T) { fx.transport.dispatchErr = errors.New("no result before timeout") // Connected stays true: the command was delivered. - fx.service.dispatchCreate(context.Background(), "pool_example", row, "vm1", nil) + fx.service.dispatchCreate(context.Background(), "pool_example", row, "vm1", nil, nil) got := fleetGetSessionState(t, fx, "ws_example", "sess_timeout") if got.State != control.StateCreating || got.RunnerID != "vm1" { @@ -706,7 +706,7 @@ func TestCreateSpecBoundsOnlyTheHooksThatRun(t *testing.T) { if tc.rowImage != "" { r.Spec.Image = tc.rowImage // a row created against a current snapshot boots it } - spec, fail := fx.service.createSpec(fleetCtx, r, &env) + spec, fail := fx.service.createSpec(fleetCtx, r, &env, nil, 0) if fail != "" { t.Fatalf("createSpec failed: %s", fail) } @@ -741,7 +741,7 @@ func TestCreateSpecSendsSetupUnlessTheRowBootsTheSnapshot(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { row := control.Session{ID: "sess_example", WorkspaceID: "ws_example", EnvironmentID: "env_example", Spec: control.PortableSpec{Image: tc.rowImage}} - spec, fail := fx.service.createSpec(fleetCtx, row, &env) + spec, fail := fx.service.createSpec(fleetCtx, row, &env, nil, 0) if fail != "" { t.Fatalf("createSpec failed: %s", fail) } @@ -774,7 +774,7 @@ func TestDispatchCreateCarriesThePlacementGeneration(t *testing.T) { // BEFORE the placement, one generation behind the store. stale := row stale.PlacementGeneration = 2 - fx.service.dispatchCreate(context.Background(), "pool_example", stale, "vm1", nil) + fx.service.dispatchCreate(context.Background(), "pool_example", stale, "vm1", nil, nil) dispatched := fx.transport.dispatchedCommands() if len(dispatched) != 1 { diff --git a/controlapp/uow_test.go b/controlapp/uow_test.go index beb702da..c46db7ca 100644 --- a/controlapp/uow_test.go +++ b/controlapp/uow_test.go @@ -490,6 +490,7 @@ func newUOWFleetFixture(t *testing.T) *uowFleetFixture { LaunchMaterial: &fleetFakeResolver{}, UnitOfWork: fx.uow, Checkpoints: locatorStub{}, + Bootstraps: newBootstrapStub(), }) if err != nil { t.Fatalf("NewFleetService: %v", err) diff --git a/docs/design/2026-09-20-microvm-bootstrap-token-and-vsock.md b/docs/design/2026-09-20-microvm-bootstrap-token-and-vsock.md index 78cac9e3..64863ad2 100644 --- a/docs/design/2026-09-20-microvm-bootstrap-token-and-vsock.md +++ b/docs/design/2026-09-20-microvm-bootstrap-token-and-vsock.md @@ -257,7 +257,14 @@ socket is inside one VM's jail directory, so the connection's identity is its pa 4. The guest boots. `sessiond` opens `AF_VSOCK` to (2, 1024). 5. Firecracker forwards it to `_1024`; runnerd accepts and builds the relay hub with `relay.NewHubWithControl`, as `register` does today, keyed by the session the - socket path names. + socket path names. **Exactly one** connection is served per boot generation, and every + later one on that socket is closed having received nothing at all: `/dev/vsock` is + world-accessible inside an ordinary guest, so any process in the sandbox can dial + (2, 1024), and step 6's frame is the session's whole configuration plus a live token. + Each accepted connection is handled on a goroutine of its own and that frame's write is + bounded, so a peer that connects and never reads holds nothing but its own connection. + A sessiond that crashes and restarts inside a live VM is therefore NOT re-served: it is + a new boot generation, with a new socket and a fresh mint (open question 2). 6. runnerd's first frame is `FrameControl{Kind: "boot_config"}`: session id, command, proxy URL, egress allowlist, agent manifest, git author, repos, setup and init scripts with their bounds, `SecretNames`, and the token. @@ -271,8 +278,15 @@ socket is inside one VM's jail directory, so the connection's identity is its pa Cold suspend reuses the handshake that exists, with one additive field: `ControlEvent` gains ``Cold bool `json:"cold,omitempty"` `` on `KindSuspending`, meaning "this is not a freeze — flush, unmount `/rainier/agents`, forget every delivered secret". `sessiond` -answers `KindSuspendAck` at once and `KindSuspendReady` when done, under the budgets that -already exist (2s and 12s). runnerd then terminates the microVM — no memory image, per +answers `KindSuspendAck` at once — before any of that work, so the ack budget is the warm +path's **2s**, unchanged — and `KindSuspendReady` when done, under a cold ready budget of +**30s** rather than the warm 12s. The warm 12s covers one thing: the sandbox's own 10s +exec kill (`execQuiesceBudget`), with two to spare. The cold path wraps a synchronous +agent-home flush and an unmount (which syncs the device before it detaches it) around +that same 10s kill, and this VM ends with no memory image — so work the budget cuts short +is lost rather than deferred, where a warm freeze merely postpones it. 30s is the kill +with a factor of three over it. A sandbox that answers spends none of it; one that +predates `Cold` answers as fast as it always did. runnerd then terminates the microVM — no memory image, per ADR §2.2 — detaches the disk, drops the token, and produces the checkpoint behind the §4.4 barrier. A sessiond predating `Cold` reads a plain `suspending`, quiesces its execs, and the host-side unmount still happens; the guest simply did not help. diff --git a/internal/controld/controld.go b/internal/controld/controld.go index 3ca56a3b..894431b1 100644 --- a/internal/controld/controld.go +++ b/internal/controld/controld.go @@ -303,6 +303,11 @@ func (s *Server) compose() error { DefaultInitTimeoutSec: defaultInitTimeoutSec, UnitOfWork: uow, Checkpoints: ckpts, + // The same store, through the port that mints and spends a microVM + // session's bootstrap token. srpc.go reads it through the same + // accessor to answer the exchange, so the mint and the redemption + // are one table and not two. + Bootstraps: s.st.Bootstraps(), }) if err != nil { return fmt.Errorf("controld: composing the fleet service: %w", err) diff --git a/internal/controld/memstore.go b/internal/controld/memstore.go index 8b0550aa..4e8615a2 100644 --- a/internal/controld/memstore.go +++ b/internal/controld/memstore.go @@ -61,6 +61,11 @@ type memStore struct { // credential set per (user, provider), keyed the same way and holding no // workspace, exactly as the schema does. agentCredentials map[credKey]*AgentCredential + + // bootstraps is the one-shot token a microVM session exchanges for its + // environment's secrets, keyed like every other session row and holding + // only the HASH — the plaintext left the mint and was never stored. + bootstraps map[sessionKey]*bootstrapRow } var _ MemStore = (*memStore)(nil) @@ -116,6 +121,7 @@ func NewMemStore() MemStore { credentials: map[credKey]*Credential{}, agentCredentials: map[credKey]*AgentCredential{}, + bootstraps: map[sessionKey]*bootstrapRow{}, } } @@ -127,12 +133,75 @@ func (m *memStore) Environments() control.EnvironmentRepository { return memEnvi func (m *memStore) Fleet() control.FleetRepository { return memFleet{m} } +// Bootstraps is the fourth: the session bootstrap tokens, a view over this +// store's own rows exactly as the other three are. +func (m *memStore) Bootstraps() control.SessionBootstrapStore { return memBootstraps{m} } + var ( _ control.SessionRepository = memSessions{} _ control.EnvironmentRepository = memEnvironments{} _ control.FleetRepository = memFleet{} + _ control.SessionBootstrapStore = memBootstraps{} ) +// bootstrapRow is one session's minted token as this store keeps it: the +// hash, its fence, its expiry, and whether it has been spent. `spent` is the +// store's own column and no caller's — the whole of single-use is that the +// check and the mark are one step under this store's mutex. +type bootstrapRow struct { + hash string + gen uint64 + expiresAt time.Time + spent bool +} + +// memBootstraps is the in-memory SessionBootstrapStore. +type memBootstraps struct{ m *memStore } + +func (b memBootstraps) PutSessionBootstrap(_ context.Context, ws control.WorkspaceID, id control.SessionID, rec control.SessionBootstrap) error { + if ws == "" || id == "" || rec.Hash == "" { + return control.ErrInvalid + } + b.m.mu.Lock() + defer b.m.mu.Unlock() + // A fresh mint REPLACES its predecessor rather than joining it, which is + // what makes a cold resume's token the only one that works and retires + // the one the previous boot was handed. + b.m.bootstraps[sessionKey{ws: ws, id: id}] = &bootstrapRow{ + hash: rec.Hash, gen: rec.PlacementGeneration, expiresAt: rec.ExpiresAt, + } + return nil +} + +// ConsumeSessionBootstrap checks and spends under one lock. +// +// The order of the four refusals is deliberate: the hash first, so that a +// caller holding no valid token learns nothing about the state of the one +// that exists; then the fence, the expiry, and the spend, from the most +// structural fact to the most transient. +func (b memBootstraps) ConsumeSessionBootstrap(_ context.Context, ws control.WorkspaceID, id control.SessionID, hash string, gen uint64, now time.Time) error { + if ws == "" || id == "" || hash == "" { + return control.ErrInvalid + } + b.m.mu.Lock() + defer b.m.mu.Unlock() + row, ok := b.m.bootstraps[sessionKey{ws: ws, id: id}] + if !ok || row.hash != hash { + return control.ErrBootstrapUnknown + } + if row.gen != gen { + return control.ErrBootstrapFenced + } + if !now.Before(row.expiresAt) { + return control.ErrBootstrapExpired + } + if row.spent { + return control.ErrBootstrapSpent + } + row.spent = true + return nil +} + // --------------------------------------------------------------------------- // clones // --------------------------------------------------------------------------- diff --git a/internal/controld/memstore_test.go b/internal/controld/memstore_test.go index 9912daf8..401bcdf2 100644 --- a/internal/controld/memstore_test.go +++ b/internal/controld/memstore_test.go @@ -22,6 +22,7 @@ func TestMemStoreRepositories(t *testing.T) { Sessions: st.Sessions(), Environments: st.Environments(), Fleet: st.Fleet(), + Bootstraps: st.Bootstraps(), Provision: st.EnsureWorkspace, } }) diff --git a/internal/controld/pgstore/bootstrap.go b/internal/controld/pgstore/bootstrap.go new file mode 100644 index 00000000..8ab63178 --- /dev/null +++ b/internal/controld/pgstore/bootstrap.go @@ -0,0 +1,112 @@ +// internal/controld/pgstore/bootstrap.go +package pgstore + +import ( + "context" + "errors" + "time" + + "github.com/jackc/pgx/v5" + + "github.com/tokencanopy/rainier/control" +) + +// pgBootstraps is the durable half of the session bootstrap token: one row +// per session, holding a hash and never a token. +type pgBootstraps struct{ s *Store } + +var _ control.SessionBootstrapStore = pgBootstraps{} + +// Bootstraps is the fourth control repository port, a view over this store's +// own rows exactly as the other three are. +func (s *Store) Bootstraps() control.SessionBootstrapStore { return pgBootstraps{s} } + +// PutSessionBootstrap records rec as id's only acceptable token. +// +// It is an upsert onto the session's own primary key, which is what makes a +// fresh mint retire its predecessor: consumed_at resets to NULL along with +// the hash, so the new token is unspent and the old one is simply not there +// to be presented any more. +func (r pgBootstraps) PutSessionBootstrap(ctx context.Context, ws control.WorkspaceID, id control.SessionID, rec control.SessionBootstrap) error { + if ws == "" || id == "" || rec.Hash == "" { + return control.ErrInvalid + } + _, err := r.s.q(ctx).Exec(ctx, ` + INSERT INTO session_bootstraps + (session_id, workspace_id, token_hash, placement_generation, expires_at, consumed_at) + VALUES ($1, $2, $3, $4, $5, NULL) + ON CONFLICT (workspace_id, session_id) DO UPDATE SET + workspace_id = EXCLUDED.workspace_id, + token_hash = EXCLUDED.token_hash, + placement_generation = EXCLUDED.placement_generation, + expires_at = EXCLUDED.expires_at, + consumed_at = NULL, + created_at = now()`, + string(id), string(ws), rec.Hash, int64(rec.PlacementGeneration), rec.ExpiresAt) + if err != nil { + // The error text is dropped rather than wrapped: a constraint + // violation here quotes the row, and this row's business is a + // capability. The caller fails the create either way. + return unavailable("put session bootstrap", err) + } + return nil +} + +// ConsumeSessionBootstrap spends the token in ONE predicated statement. +// +// The UPDATE is the whole of single-use: every condition a success requires +// is in its WHERE clause, so two exchanges racing on one token produce +// exactly one row affected. A read-then-write would produce two, which is +// the bug this method exists not to have. +// +// The second query runs only on the miss, and only to say WHY. It reads a row +// that no longer has a race to lose — nothing can turn an unspent token into +// a spendable one — so a diagnosis taken after the fact is as true as one +// taken during. It reports nothing it did not already know about a token the +// caller presented. +func (r pgBootstraps) ConsumeSessionBootstrap(ctx context.Context, ws control.WorkspaceID, id control.SessionID, hash string, gen uint64, now time.Time) error { + if ws == "" || id == "" || hash == "" { + return control.ErrInvalid + } + ct, err := r.s.q(ctx).Exec(ctx, ` + UPDATE session_bootstraps SET consumed_at = $1 + WHERE workspace_id = $2 AND session_id = $3 AND token_hash = $4 + AND placement_generation = $5 AND expires_at > $1 AND consumed_at IS NULL`, + now, string(ws), string(id), hash, int64(gen)) + if err != nil { + return unavailable("consume session bootstrap", err) + } + if ct.RowsAffected() == 1 { + return nil + } + + var ( + rowGen int64 + expiresAt time.Time + consumed *time.Time + ) + qerr := r.s.q(ctx).QueryRow(ctx, ` + SELECT placement_generation, expires_at, consumed_at FROM session_bootstraps + WHERE workspace_id = $1 AND session_id = $2 AND token_hash = $3`, + string(ws), string(id), hash).Scan(&rowGen, &expiresAt, &consumed) + switch { + case errors.Is(qerr, pgx.ErrNoRows): + // No row, or a row under a different hash: one answer, because a + // caller holding no valid token must not learn the state of the one + // that exists. + return control.ErrBootstrapUnknown + case qerr != nil: + return unavailable("read session bootstrap", qerr) + case uint64(rowGen) != gen: + return control.ErrBootstrapFenced + case !now.Before(expiresAt): + return control.ErrBootstrapExpired + case consumed != nil: + return control.ErrBootstrapSpent + } + // The row satisfies every condition the UPDATE tested and the UPDATE + // still matched nothing: a concurrent exchange spent it between the two + // statements, which is exactly the race single-use exists to settle, and + // this caller is the one that lost. + return control.ErrBootstrapSpent +} diff --git a/internal/controld/pgstore/migrations/0015_session_bootstrap.sql b/internal/controld/pgstore/migrations/0015_session_bootstrap.sql new file mode 100644 index 00000000..da4f00b2 --- /dev/null +++ b/internal/controld/pgstore/migrations/0015_session_bootstrap.sql @@ -0,0 +1,35 @@ +-- 0015_session_bootstrap.sql — the single-use bootstrap token a microVM +-- session exchanges for its environment's decrypted secrets +-- (docs/design/2026-09-20-microvm-bootstrap-token-and-vsock.md §3). +-- +-- One row per session and no history: a fresh mint REPLACES its predecessor, +-- because a cold resume's token must be the only one that works and the one +-- the previous boot was handed must stop being an answer. That is why the +-- primary key is the session rather than the token. +-- +-- token_hash is the hex-encoded SHA-256 of the token; the plaintext left the +-- mint and is stored nowhere, so a reader of this table learns which sessions +-- have a live token and nothing that could redeem one. +-- +-- placement_generation is the fence: a session re-placed onto another runner +-- has moved past the sandbox the token was minted for. expires_at is +-- absolute so every replica agrees without agreeing on a TTL. consumed_at is +-- the spend, and single-use is enforced by the predicated UPDATE that sets it +-- — never by a read followed by a write. +-- +-- ON DELETE CASCADE because a token outliving the session it names is a row +-- nothing will ever consume and nothing will ever clean up. +-- sessions has been keyed by (workspace_id, id) since migration 0007, so the +-- token row is keyed and referenced the same way: a session id alone is not +-- unique across workspaces and Postgres rightly refuses a foreign key to it. +CREATE TABLE IF NOT EXISTS session_bootstraps ( + workspace_id text NOT NULL, + session_id text NOT NULL, + token_hash text NOT NULL, + placement_generation bigint NOT NULL, + expires_at timestamptz NOT NULL, + consumed_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, session_id), + FOREIGN KEY (workspace_id, session_id) REFERENCES sessions(workspace_id, id) ON DELETE CASCADE +); diff --git a/internal/controld/pgstore/pgstore_test.go b/internal/controld/pgstore/pgstore_test.go index 9c193c8d..7bb056e2 100644 --- a/internal/controld/pgstore/pgstore_test.go +++ b/internal/controld/pgstore/pgstore_test.go @@ -272,13 +272,14 @@ func TestMigrate0003To0004AddsColumnsToLegacyRows(t *testing.T) { if want := embeddedMigrationVersions(t); !slices.Equal(applied, want) { t.Fatalf("schema_migrations = %v, want every embedded migration in order %v", applied, want) } - // This release's head is 14: a database that stopped at 0003 runs the + // This release's head is 15: a database that stopped at 0003 runs the // expand step (0007), the contract step (0008), the events table // (0009), the agent credentials table (0010), the tombstone (0011), the - // durable revoke fence (0012), the controller lease (0013), and the exec - // event's command name (0014) in the same start. - if head := applied[len(applied)-1]; head != 14 { - t.Fatalf("head migration = %d, want 14", head) + // durable revoke fence (0012), the controller lease (0013), the exec + // event's command name (0014), and the session bootstrap token (0015) in + // the same start. + if head := applied[len(applied)-1]; head != 15 { + t.Fatalf("head migration = %d, want 15", head) } // The legacy session survived, and its new columns read as "never exited" @@ -529,6 +530,7 @@ func TestPGStoreRepositories(t *testing.T) { Sessions: st.Sessions(), Environments: st.Environments(), Fleet: st.Fleet(), + Bootstraps: st.Bootstraps(), Provision: st.EnsureWorkspace, } }) diff --git a/internal/controld/srpc.go b/internal/controld/srpc.go index 7b257936..8e0f2aa5 100644 --- a/internal/controld/srpc.go +++ b/internal/controld/srpc.go @@ -68,6 +68,10 @@ func (s *Server) handleSessionRequest(ctx context.Context, runnerName string, ro return s.answerFetchAgentCredentials(ctx, runnerName, row, env) case runner.MethodPutAgentCredentials: return s.answerPutAgentCredentials(ctx, runnerName, row, env) + case runner.MethodFetchSessionSecrets: + return s.answerFetchSessionSecrets(ctx, runnerName, row, env) + case runner.MethodMintSessionBootstrap: + return s.answerMintSessionBootstrap(ctx, runnerName, row, env) default: log.Printf("controld: runner %s: session %s asked for unknown method %q", runnerName, row.ID, clip(env.Method)) diff --git a/internal/controld/srpc_bootstrap.go b/internal/controld/srpc_bootstrap.go new file mode 100644 index 00000000..b6b2b368 --- /dev/null +++ b/internal/controld/srpc_bootstrap.go @@ -0,0 +1,230 @@ +// internal/controld/srpc_bootstrap.go +package controld + +import ( + "context" + "encoding/json" + "errors" + "log" + + "github.com/tokencanopy/rainier/control" + "github.com/tokencanopy/rainier/controlapp" + "github.com/tokencanopy/rainier/protocol/runner" +) + +// The two methods the microVM bootstrap token rides, answered here beside the +// three that were already on this channel and under the same discipline: the +// placement guard above has established which session asked, every answer is +// derived from the ROW that guard read, and the value appears in exactly one +// place — the payload. +// +// Neither method is gated on the runner having announced microvm.v1, and that +// is deliberate rather than an omission. A runner that did not announce it was +// dispatched the secret values in Spec.Env already, so nothing here is a +// privilege it does not have; adding a capability check would be a second, +// weaker fence in front of the one that actually holds (the placement guard, +// the token hash, the generation, and single use), and two fences that can +// disagree are worse than one that cannot. +// +// See docs/design/2026-09-20-microvm-bootstrap-token-and-vsock.md §3 and §5. + +// sessionBootstrapRequest is the body of both requests: the fetch sends the +// protocol and the token, the mint sends the protocol alone. Decoding both +// through one type is the same economy the agent credential pair makes, and +// the field a mint does not send is simply absent. +type sessionBootstrapRequest struct { + Protocol uint64 `json:"protocol"` + Token string `json:"token"` +} + +// sessionSecretsAnswer and sessionBootstrapAnswer are the two success bodies, +// named types for the same reason mintAnswer is one: sessiond and runnerd read +// these exact keys, so a renamed field here would silently stop a microVM +// session from ever getting its environment rather than failing a build. +type sessionSecretsAnswer struct { + Env map[string]string `json:"env"` +} + +type sessionBootstrapAnswer struct { + Token string `json:"token"` + ExpiresInSec int `json:"expires_in_sec"` +} + +// sessionBootstrapRequestMaxBytes bounds a request body before it is decoded. +// Both bodies are a version and at most one 43-character token; a kilobyte is +// slack rather than a working limit, and bounding before the decode is what +// keeps a misbehaving peer from choosing how much memory this process spends. +const sessionBootstrapRequestMaxBytes = 4 << 10 + +// answerFetchSessionSecrets answers one sandbox's boot-time exchange: +// {"protocol", "token"} → {"env": {name: value}}. +// +// This is the only path in this installation that hands an environment's +// decrypted secret_refs to anything, and every refusal below is a closed one. +// The secrets are re-resolved HERE, from the environment as it stands now, +// rather than remembered from the create: the control plane deliberately never +// held them between the two, which is the same rule that keeps them out of a +// session row. +// +// The value appears in exactly one place: the payload. Not in the log line, +// which names the session, the runner, and a COUNT; not in an error, which is +// why a decode failure is never relayed; and not in a refusal, which names a +// condition. §15.1 of the tenancy specification, pinned by a test that greps +// this package's own log output for its fixture secret. +func (s *Server) answerFetchSessionSecrets(ctx context.Context, runnerName string, row control.Session, env runner.RPCEnvelope) runner.RPCEnvelope { + req, bad := decodeSessionBootstrapRequest(env) + if bad != nil { + log.Printf("controld: runner %s: session %s sent an unusable %s request", + runnerName, row.ID, clip(env.Method)) + return *bad + } + if req.Token == "" { + log.Printf("controld: runner %s: session %s asked for its secrets with no token", runnerName, row.ID) + return rpcRefusal(env.ID, "this request carried no bootstrap token") + } + + // The spend comes BEFORE the secrets are resolved, and single use means + // single use even when what follows fails: a token that was presented has + // been presented, and re-spendability on a failed resolve would turn one + // refused exchange into an unbounded number of attempts. + // + // The generation is the ROW's, read by the guard that authorized this + // request, never anything in the request. + err := s.st.Bootstraps().ConsumeSessionBootstrap(ctx, installWorkspace, row.ID, + controlapp.HashSessionBootstrapToken(req.Token), row.PlacementGeneration, s.clock.Now()) + if err != nil { + sentence, known := controlapp.SessionBootstrapRefusal(err) + if !known { + log.Printf("controld: session %s: spending a bootstrap token on runner %s: %v", + row.ID, runnerName, err) + return rpcRefusal(env.ID, "the bootstrap token could not be checked") + } + log.Printf("controld: session %s: refused a secret fetch on runner %s: %s", + row.ID, runnerName, sentence) + return rpcRefusal(env.ID, sentence) + } + + vars, err := s.sessionSecrets(ctx, row) + if err != nil { + // The resolver's own sentence may name a secret REFERENCE, which is a + // name and not a value, and which a person cannot fix without. It is + // logged and relayed for exactly the reason secretEnvironment says: + // a dangling reference is unfixable without its name. + log.Printf("controld: session %s: resolving the environment's secrets on runner %s: %v", + row.ID, runnerName, err) + return rpcRefusal(env.ID, err.Error()) + } + if vars == nil { + // The key is always present, even when it is empty: the guest reads + // "env", and a missing key would make "this environment declares no + // secrets" and "a malformed answer" look alike on the far side. + vars = map[string]string{} + } + body, err := json.Marshal(sessionSecretsAnswer{Env: vars}) + if err != nil { + // Logged WITHOUT the error: json's own message quotes the value it + // failed on, and that value is the secret. + log.Printf("controld: session %s: encoding the session secrets failed", row.ID) + return rpcRefusal(env.ID, "the session secrets could not be encoded") + } + log.Printf("controld: session %s: delivered %d environment secret(s) to runner %s", + row.ID, len(vars), runnerName) + return runner.RPCEnvelope{ID: env.ID, Method: "resp", OK: true, Payload: body} +} + +// answerMintSessionBootstrap answers a RUNNER's request for a fresh token on +// a cold resume: {"protocol"} → {"token", "expires_in_sec"}. +// +// It is the one method on this channel a sandbox does not originate, and it +// needs no new authorization for it: the request names no session, so the +// session it mints for is FromRunner.Session — the id the placement guard +// already checked — and a runner holding session A cannot mint for B because +// there is nowhere in the message to say B. +// +// The new token retires whatever this session had, which is what makes a +// resume's token the only one that works. The token appears in exactly one +// place: the payload. +func (s *Server) answerMintSessionBootstrap(ctx context.Context, runnerName string, row control.Session, env runner.RPCEnvelope) runner.RPCEnvelope { + if _, bad := decodeSessionBootstrapRequest(env); bad != nil { + log.Printf("controld: runner %s: session %s sent an unusable %s request", + runnerName, row.ID, clip(env.Method)) + return *bad + } + minter := controlapp.SessionBootstrapMinter{Store: s.st.Bootstraps(), Clock: s.clock} + token, err := minter.Mint(ctx, installWorkspace, row.ID, row.PlacementGeneration) + if err != nil { + log.Printf("controld: session %s: minting a bootstrap token for runner %s: %v", + row.ID, runnerName, err) + return rpcRefusal(env.ID, "a bootstrap token could not be minted for this session") + } + body, err := json.Marshal(sessionBootstrapAnswer{ + Token: token, + ExpiresInSec: int(controlapp.SessionBootstrapTTL.Seconds()), + }) + if err != nil { + // Unreachable (a string and an int always marshal) and logged WITHOUT + // the error, which is the one error here whose text could quote the + // value it failed on. + log.Printf("controld: session %s: encoding a bootstrap token failed", row.ID) + return rpcRefusal(env.ID, "the bootstrap token could not be encoded") + } + log.Printf("controld: session %s: minted a bootstrap token for runner %s at placement generation %d", + row.ID, runnerName, row.PlacementGeneration) + return runner.RPCEnvelope{ID: env.ID, Method: "resp", OK: true, Payload: body} +} + +// sessionSecrets resolves the environment secrets row's create would have +// carried, through the same resolver the scheduler dispatches with. +// +// It is the resolver and not a second decryption path on purpose: launchMaterial +// is the only place in the self-hosted adapter set that holds the secrets key, +// and a second one would be a second thing to get wrong. A session with no +// environment, or one whose environment declares no secret_refs, resolves to +// nothing and gets an empty map — a truthful answer, not a refusal. +func (s *Server) sessionSecrets(ctx context.Context, row control.Session) (map[string]string, error) { + env, err := s.sessionEnvironment(ctx, row) + if err != nil { + return nil, err + } + return launchMaterial{st: s.st, key: s.cfg.SecretsKey}.secretEnvironment(ctx, env) +} + +// sessionEnvironment reads the environment row came from, or nil for a +// scratch session. An environment that has been deleted since the create is +// nil too: the session is still running, it simply has no declared secrets to +// be given, and refusing its boot over a row somebody removed would be the +// control plane taking a live session down. +func (s *Server) sessionEnvironment(ctx context.Context, row control.Session) (*control.Environment, error) { + if row.EnvironmentID == "" { + return nil, nil + } + env, err := s.st.Environments().GetEnvironment(ctx, installWorkspace, row.EnvironmentID) + switch { + case errors.Is(err, control.ErrNotFound): + return nil, nil + case err != nil: + return nil, errors.New("this session's environment could not be read") + } + return &env, nil +} + +// decodeSessionBootstrapRequest bounds and decodes one request body, +// returning the refusal to send when it cannot. The decode error is never +// relayed and never logged: a JSON error quotes the bytes it choked on, and +// on this path those bytes are a capability. +func decodeSessionBootstrapRequest(env runner.RPCEnvelope) (sessionBootstrapRequest, *runner.RPCEnvelope) { + var req sessionBootstrapRequest + if len(env.Payload) > sessionBootstrapRequestMaxBytes { + refusal := rpcRefusal(env.ID, "the bootstrap request is too large") + return req, &refusal + } + if err := json.Unmarshal(env.Payload, &req); err != nil { + refusal := rpcRefusal(env.ID, "the bootstrap request could not be decoded") + return req, &refusal + } + if req.Protocol != runner.SessionBootstrapProtocolVersion { + refusal := rpcRefusal(env.ID, "this session must be replaced before it can exchange a bootstrap token") + return req, &refusal + } + return req, nil +} diff --git a/internal/controld/srpc_bootstrap_test.go b/internal/controld/srpc_bootstrap_test.go new file mode 100644 index 00000000..f71e2ec7 --- /dev/null +++ b/internal/controld/srpc_bootstrap_test.go @@ -0,0 +1,447 @@ +// internal/controld/srpc_bootstrap_test.go +package controld + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "log" + "strings" + "sync" + "testing" + "time" + + "github.com/tokencanopy/rainier/control" + "github.com/tokencanopy/rainier/controlapp" +) + +// The two bootstrap methods, driven end to end through the runner plane: a +// scripted fake runner sends the session_req exactly as runnerd forwards one +// out of a sandbox, and the assertions are on the session_rpc that comes back +// down. +// +// The fixture secret is spelled "value_must_not_appear_anywhere" on purpose. +// It is the string the log test greps for, and having every case use it means +// any case that starts leaking it fails that one too. + +const bootstrapFixtureSecret = "value_must_not_appear_anywhere" + +// seedBootstrapSession seeds a running session on runner, with an environment +// declaring one secret ref whose sealed value is in the vault, and returns the +// session id. It is the state a microVM create leaves behind, minus the token. +func seedBootstrapSession(t *testing.T, st MemStore, id, runnerName string) string { + t.Helper() + ctx := context.Background() + ciphertext, nonce, err := Seal(testSecretsKey, []byte(bootstrapFixtureSecret)) + if err != nil { + t.Fatalf("Seal: %v", err) + } + if err := st.PutSecret(ctx, "DEPLOY_KEY", ciphertext, nonce); err != nil { + t.Fatalf("PutSecret: %v", err) + } + if _, err := st.Environments().CreateEnvironment(ctx, installWorkspace, control.Environment{ + ID: "env_example", WorkspaceID: installWorkspace, Name: "example", + SecretRefs: []string{"DEPLOY_KEY"}, + }); err != nil { + t.Fatalf("CreateEnvironment: %v", err) + } + seedSession(t, st, control.Session{ + ID: control.SessionID(id), State: control.StateRunning, + RunnerID: control.RunnerID(runnerName), EnvironmentID: "env_example", + }) + return id +} + +// mintFor records a token for id at the placement generation the row +// currently holds, the way createSpec does, and returns the plaintext. +func mintFor(t *testing.T, s *Server, st MemStore, id string) string { + t.Helper() + row, err := st.Sessions().GetSession(context.Background(), installWorkspace, control.SessionID(id)) + if err != nil { + t.Fatalf("read %s: %v", id, err) + } + minter := controlapp.SessionBootstrapMinter{Store: st.Bootstraps(), Clock: s.clock} + tok, err := minter.Mint(context.Background(), installWorkspace, control.SessionID(id), row.PlacementGeneration) + if err != nil { + t.Fatalf("mint for %s: %v", id, err) + } + return tok +} + +// bootstrapRequest is the body a sandbox sends: the protocol and the token. +func bootstrapRequest(token string) string { + b, _ := json.Marshal(map[string]any{"protocol": 1, "token": token}) + return string(b) +} + +// refusalText reads the {"error": ...} sentence off an ok:false answer, +// failing the test when the answer was a success. +func refusalText(t *testing.T, payload json.RawMessage) string { + t.Helper() + var body struct { + Error string `json:"error"` + } + if err := json.Unmarshal(payload, &body); err != nil { + t.Fatalf("decoding a refusal payload %s: %v", payload, err) + } + return body.Error +} + +// TestFetchSessionSecretsTable is the exchange in its five outcomes: one +// success and four refusals with distinct reasons, over the live plane. +// +// Each row drives the same method against a differently prepared session, so +// what distinguishes them is only ever the state of the token — which is the +// claim §3's "three independent refusals, all in the control plane" makes. +func TestFetchSessionSecretsTable(t *testing.T) { + for _, tc := range []struct { + name string + // prepare returns the token the sandbox will present. + prepare func(t *testing.T, s *Server, st MemStore, id string) string + wantOK bool + // wantReason is a distinguishing fragment of the refusal sentence. + wantReason string + }{ + { + name: "a fresh token is exchanged", + prepare: func(t *testing.T, s *Server, st MemStore, id string) string { + return mintFor(t, s, st, id) + }, + wantOK: true, + }, + { + name: "a replayed token is refused", + prepare: func(t *testing.T, s *Server, st MemStore, id string) string { + tok := mintFor(t, s, st, id) + row, _ := st.Sessions().GetSession(context.Background(), installWorkspace, control.SessionID(id)) + if err := st.Bootstraps().ConsumeSessionBootstrap(context.Background(), installWorkspace, + control.SessionID(id), controlapp.HashSessionBootstrapToken(tok), + row.PlacementGeneration, s.clock.Now()); err != nil { + t.Fatalf("first spend: %v", err) + } + return tok + }, + wantReason: "already been exchanged", + }, + { + name: "an expired token is refused", + prepare: func(t *testing.T, s *Server, st MemStore, id string) string { + row, _ := st.Sessions().GetSession(context.Background(), installWorkspace, control.SessionID(id)) + // Minted as if two hours ago: the mint's own clock is this + // replica's, so an expiry in the past is written directly. + tok := "expired_token_example" + if err := st.Bootstraps().PutSessionBootstrap(context.Background(), installWorkspace, + control.SessionID(id), control.SessionBootstrap{ + Hash: controlapp.HashSessionBootstrapToken(tok), + PlacementGeneration: row.PlacementGeneration, + ExpiresAt: s.clock.Now().Add(-2 * time.Hour), + }); err != nil { + t.Fatalf("put an expired token: %v", err) + } + return tok + }, + wantReason: "expired", + }, + { + name: "a token from a superseded placement is refused", + prepare: func(t *testing.T, s *Server, st MemStore, id string) string { + tok := mintFor(t, s, st, id) + // The session is placed again, which is exactly what a + // re-placement onto another runner does to the row — and the + // token minted for the sandbox that no longer exists must + // stop being an answer. It is placed back onto the SAME + // runner so the placement guard still admits the request and + // the fence is what refuses it. + holder := control.RunnerID("vm1") + if err := st.Sessions().Transition(context.Background(), installWorkspace, control.SessionID(id), + control.NonTerminal, control.StateRunning, + control.TransitionOpts{RunnerID: &holder}); err != nil { + t.Fatalf("re-place the session: %v", err) + } + return tok + }, + wantReason: "placed again", + }, + { + name: "another session's token is refused", + prepare: func(t *testing.T, s *Server, st MemStore, id string) string { + seedSession(t, st, control.Session{ID: "sess_neighbour", State: control.StateRunning, + RunnerID: "vm1", EnvironmentID: "env_example"}) + return mintFor(t, s, st, "sess_neighbour") + }, + wantReason: "no bootstrap token matching", + }, + } { + t.Run(tc.name, func(t *testing.T) { + s, st, ts := newTestControld(t) + f := joinRunner(t, s, ts, runnerScript{Name: "vm1"}) + id := seedBootstrapSession(t, st, "sess_bootstrap", "vm1") + token := tc.prepare(t, s, st, id) + + f.sandboxRequest(t, id, 11, "fetch_session_secrets", bootstrapRequest(token)) + + cmd := nextSessionRPC(t, f) + if cmd.Session != id || cmd.RPC.ID != 11 || cmd.RPC.Method != "resp" { + t.Fatalf("answer = %+v, want a resp for id 11 on %s", cmd.RPC, id) + } + if !tc.wantOK { + if cmd.RPC.OK { + t.Fatalf("a %s was accepted: %s", tc.name, cmd.RPC.Payload) + } + reason := refusalText(t, cmd.RPC.Payload) + if !strings.Contains(reason, tc.wantReason) { + t.Fatalf("refusal = %q, want it to name %q", reason, tc.wantReason) + } + if strings.Contains(reason, bootstrapFixtureSecret) || strings.Contains(reason, token) { + t.Fatalf("a refusal carried the secret or the token: %q", reason) + } + return + } + if !cmd.RPC.OK { + t.Fatalf("a fresh token was refused: %q", refusalText(t, cmd.RPC.Payload)) + } + var answer struct { + Env map[string]string `json:"env"` + } + if err := json.Unmarshal(cmd.RPC.Payload, &answer); err != nil { + t.Fatalf("decoding the answer: %v", err) + } + if answer.Env["DEPLOY_KEY"] != bootstrapFixtureSecret { + t.Fatalf("the answer delivered %d name(s) and not the environment's secret", len(answer.Env)) + } + }) + } +} + +// TestFetchSessionSecretsIsSingleUseOverTheWire is the replay row again, but +// driven the way a replay actually happens: the same token presented twice +// over the plane, with nothing else touching the store in between. +func TestFetchSessionSecretsIsSingleUseOverTheWire(t *testing.T) { + s, st, ts := newTestControld(t) + f := joinRunner(t, s, ts, runnerScript{Name: "vm1"}) + id := seedBootstrapSession(t, st, "sess_single_use", "vm1") + token := mintFor(t, s, st, id) + + f.sandboxRequest(t, id, 1, "fetch_session_secrets", bootstrapRequest(token)) + if first := nextSessionRPC(t, f); !first.RPC.OK { + t.Fatalf("the first exchange was refused: %q", refusalText(t, first.RPC.Payload)) + } + + f.sandboxRequest(t, id, 2, "fetch_session_secrets", bootstrapRequest(token)) + second := nextSessionRPC(t, f) + if second.RPC.OK { + t.Fatalf("the second exchange succeeded: %s", second.RPC.Payload) + } + if reason := refusalText(t, second.RPC.Payload); !strings.Contains(reason, "already been exchanged") { + t.Fatalf("the replay's refusal = %q", reason) + } +} + +// TestFetchSessionSecretsRefusesAnEmptyOrMisversionedRequest pins the two +// pre-checks that run before the store is touched at all. +func TestFetchSessionSecretsRefusesAnEmptyOrMisversionedRequest(t *testing.T) { + s, st, ts := newTestControld(t) + f := joinRunner(t, s, ts, runnerScript{Name: "vm1"}) + id := seedBootstrapSession(t, st, "sess_prechecks", "vm1") + mintFor(t, s, st, id) + + for i, tc := range []struct { + body string + wantReason string + }{ + {`{"protocol":1}`, "carried no bootstrap token"}, + {`{"protocol":2,"token":"x"}`, "must be replaced"}, + {`{"protocol":"one"}`, "could not be decoded"}, + } { + f.sandboxRequest(t, id, uint64(20+i), "fetch_session_secrets", tc.body) + cmd := nextSessionRPC(t, f) + if cmd.RPC.OK { + t.Fatalf("%q was accepted", tc.body) + } + if reason := refusalText(t, cmd.RPC.Payload); !strings.Contains(reason, tc.wantReason) { + t.Fatalf("%q refused with %q, want it to name %q", tc.body, reason, tc.wantReason) + } + } +} + +// TestMintSessionBootstrapAnswersARunner is the cold-resume half: runnerd +// originates the request on a session the guard has placed on it, and gets a +// token that works exactly once. +func TestMintSessionBootstrapAnswersARunner(t *testing.T) { + s, st, ts := newTestControld(t) + f := joinRunner(t, s, ts, runnerScript{Name: "vm1"}) + id := seedBootstrapSession(t, st, "sess_mint", "vm1") + + f.sandboxRequest(t, id, 31, "mint_session_bootstrap", `{"protocol":1}`) + + cmd := nextSessionRPC(t, f) + if cmd.RPC.ID != 31 || !cmd.RPC.OK { + t.Fatalf("mint answer = %+v", cmd.RPC) + } + var answer struct { + Token string `json:"token"` + ExpiresInSec int `json:"expires_in_sec"` + } + if err := json.Unmarshal(cmd.RPC.Payload, &answer); err != nil { + t.Fatalf("decoding the mint answer: %v", err) + } + if answer.Token == "" { + t.Fatal("the mint answered with no token") + } + if answer.ExpiresInSec != int(controlapp.SessionBootstrapTTL.Seconds()) { + t.Fatalf("expires_in_sec = %d, want %d", answer.ExpiresInSec, int(controlapp.SessionBootstrapTTL.Seconds())) + } + + // And the token it minted is the one that works. + f.sandboxRequest(t, id, 32, "fetch_session_secrets", bootstrapRequest(answer.Token)) + used := nextSessionRPC(t, f) + if !used.RPC.OK { + t.Fatalf("the freshly minted token was refused: %q", refusalText(t, used.RPC.Payload)) + } +} + +// TestMintSessionBootstrapRetiresThePreviousToken is what makes a cold resume +// safe: the token the last boot was handed stops working the moment a new one +// is minted, whether or not it was ever spent. +func TestMintSessionBootstrapRetiresThePreviousToken(t *testing.T) { + s, st, ts := newTestControld(t) + f := joinRunner(t, s, ts, runnerScript{Name: "vm1"}) + id := seedBootstrapSession(t, st, "sess_remint", "vm1") + stale := mintFor(t, s, st, id) + + f.sandboxRequest(t, id, 41, "mint_session_bootstrap", `{"protocol":1}`) + if cmd := nextSessionRPC(t, f); !cmd.RPC.OK { + t.Fatalf("the re-mint was refused: %q", refusalText(t, cmd.RPC.Payload)) + } + + f.sandboxRequest(t, id, 42, "fetch_session_secrets", bootstrapRequest(stale)) + cmd := nextSessionRPC(t, f) + if cmd.RPC.OK { + t.Fatalf("the retired token still worked: %s", cmd.RPC.Payload) + } + if reason := refusalText(t, cmd.RPC.Payload); !strings.Contains(reason, "no bootstrap token matching") { + t.Fatalf("the retired token's refusal = %q", reason) + } +} + +// TestBootstrapMethodsAreRefusedForAnotherRunnersSession pins that the guard +// above these arms is the one that decides who may ask. It is the same guard +// the three existing methods keep, and it is what makes "no session id in the +// mint request" a design and not an omission: the id is the one the store +// placed on the asking runner. +func TestBootstrapMethodsAreRefusedForAnotherRunnersSession(t *testing.T) { + s, st, ts := newTestControld(t) + f := joinRunner(t, s, ts, runnerScript{Name: "vm1"}) + id := seedBootstrapSession(t, st, "sess_elsewhere", "vm2") + // A token really does exist for it, so the refusal below is the guard's + // and not an accident of there being nothing to find. + token := mintFor(t, s, st, id) + + for i, method := range []string{"fetch_session_secrets", "mint_session_bootstrap"} { + f.sandboxRequest(t, id, uint64(50+i), method, bootstrapRequest(token)) + cmd := nextSessionRPC(t, f) + if cmd.RPC.OK { + t.Fatalf("%s was answered for a session placed on another runner: %s", method, cmd.RPC.Payload) + } + if reason := refusalText(t, cmd.RPC.Payload); !strings.Contains(reason, "not placed on the runner that asked") { + t.Fatalf("%s refused with %q, want the placement guard's sentence", method, reason) + } + } +} + +// TestTheBootstrapExchangeLeavesNothingInTheLog is §15.1, checked rather than +// claimed: this test captures the package's own log output across a whole +// successful exchange — a mint, a fetch, and a refusal — and greps it for the +// fixture secret and for every token that crossed the wire. +// +// A log line is the easiest place for a value to end up and the hardest place +// to get it back out of, which is why this is a test and not a review note. +// syncBuffer is a log sink that can be READ while something is still writing +// to it. +// +// log.Logger serializes its own writers under its own mutex, so concurrent +// log.Printf calls are safe against each other — but the test's read of the +// buffer is not one of those writers, and this package's runner-plane +// goroutines log on their own schedule. A plain bytes.Buffer here is a data +// race that `go test -race` catches in roughly one run in three, which is +// the worst frequency a race can have. +type syncBuffer struct { + mu sync.Mutex + b bytes.Buffer +} + +func (s *syncBuffer) Write(p []byte) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.b.Write(p) +} + +func (s *syncBuffer) String() string { + s.mu.Lock() + defer s.mu.Unlock() + return s.b.String() +} + +func TestTheBootstrapExchangeLeavesNothingInTheLog(t *testing.T) { + captured := &syncBuffer{} + prevOut, prevFlags := log.Writer(), log.Flags() + log.SetOutput(captured) + log.SetFlags(0) + t.Cleanup(func() { log.SetOutput(prevOut); log.SetFlags(prevFlags) }) + + s, st, ts := newTestControld(t) + f := joinRunner(t, s, ts, runnerScript{Name: "vm1"}) + id := seedBootstrapSession(t, st, "sess_log", "vm1") + + // A mint, whose answer carries a token. + f.sandboxRequest(t, id, 61, "mint_session_bootstrap", `{"protocol":1}`) + mintCmd := nextSessionRPC(t, f) + var minted struct { + Token string `json:"token"` + } + if err := json.Unmarshal(mintCmd.RPC.Payload, &minted); err != nil { + t.Fatalf("decoding the mint answer: %v", err) + } + + // A successful fetch, which delivers the fixture secret... + f.sandboxRequest(t, id, 62, "fetch_session_secrets", bootstrapRequest(minted.Token)) + if cmd := nextSessionRPC(t, f); !cmd.RPC.OK { + t.Fatalf("the fetch was refused: %q", refusalText(t, cmd.RPC.Payload)) + } + // ...and a replay, which is refused. + f.sandboxRequest(t, id, 63, "fetch_session_secrets", bootstrapRequest(minted.Token)) + if cmd := nextSessionRPC(t, f); cmd.RPC.OK { + t.Fatal("the replay succeeded") + } + + // Everything this test set in motion is stopped before the log is read. + // The runner plane holds a goroutine per connected runner and it logs — + // so a grep taken while one is still running is a grep over a buffer + // that is still being written, and the question "did anything leak + // across this whole exchange" has no answer until the exchange's own + // machinery has finished saying things. (syncBuffer makes reading it + // safe either way; this makes the answer complete.) + f.close() + eventually(t, 3*time.Second, func() error { + if s.runnerConnected("vm1") { + return errors.New("controld has not noticed the runner leaving yet") + } + return nil + }) + + out := captured.String() + for _, forbidden := range []string{bootstrapFixtureSecret, minted.Token} { + if forbidden == "" { + t.Fatal("the test's own fixture is empty; it would grep for nothing") + } + if strings.Contains(out, forbidden) { + t.Fatalf("the log carries a value it must never carry:\n%s", out) + } + } + // And the log DID say something about this session, so the grep above is + // over real output rather than over an empty buffer. + if !strings.Contains(out, string(id)) { + t.Fatalf("no log line mentions the session at all; the grep proves nothing:\n%s", out) + } +} diff --git a/internal/controld/store.go b/internal/controld/store.go index 5d9f87c3..e858b4c0 100644 --- a/internal/controld/store.go +++ b/internal/controld/store.go @@ -206,6 +206,12 @@ type Repositories interface { Sessions() control.SessionRepository Environments() control.EnvironmentRepository Fleet() control.FleetRepository + // Bootstraps is the session bootstrap tokens: the hash, the placement it + // was minted under, its expiry, and the atomic spend. It is a port of its + // own rather than four methods on Sessions because minting and spending a + // capability has nothing to do with a session's lifecycle — see + // control.SessionBootstrapStore. + Bootstraps() control.SessionBootstrapStore } // MemStore is the shape the in-memory store has: Store itself, plus the one diff --git a/internal/driver/contract.go b/internal/driver/contract.go index 9940fca3..e32a0582 100644 --- a/internal/driver/contract.go +++ b/internal/driver/contract.go @@ -36,7 +36,10 @@ func cleanupSnapshotRef(d Driver, ref string) { // rather than carrying anything at all. Docker's semantics are // strip-to-empty — the key survives the commit set to "" — which is why an // empty value passes here and not only an absent one. -func assertStrippedFromImage(t *testing.T, d Driver, ref, value string, stripped []string) { +// survivor is a key the create set that the caller did NOT strip, or "" when +// there is none to name. It is what makes the microVM arm an assertion rather +// than a statement about an empty list — see below. +func assertStrippedFromImage(t *testing.T, d Driver, ref, value string, stripped []string, survivor string) { t.Helper() switch dd := d.(type) { case *Docker: @@ -68,11 +71,23 @@ func assertStrippedFromImage(t *testing.T, d Driver, ref, value string, stripped if !ok { t.Fatalf("no snapshot manifest for ref %q: nothing to assert the strip against", ref) } + // The manifest has to actually DESCRIBE the create, or the strip + // assertion below is a statement about an empty list. survivor is a + // key the create set and the caller did not strip: if the driver + // stopped recording what a session was configured with, this fires + // first and the strip checks stop being vacuous silently. + if survivor != "" && !slices.Contains(keys, survivor) { + t.Fatalf("the committed manifest does not describe the create at all "+ + "(no %s among %v), so the strip below would assert nothing", survivor, keys) + } for _, k := range stripped { if slices.Contains(keys, k) { t.Fatalf("stripped key %s survived into the committed manifest: %v", k, keys) } } + // Keys and no values, checked against the raw bytes rather than the + // decoded keys: the manifest type has no field a value could live in + // today, and this is what would notice if one were added. if raw := dd.snapshotManifestBytes(ref); strings.Contains(string(raw), value) { t.Fatalf("the committed manifest carries a stripped value:\n%s", raw) } @@ -113,6 +128,20 @@ func workspaceExists(t *testing.T, d Driver, sessionID string) bool { } } +// driverCapabilities names the portable capabilities a driver's runner +// announces on its behalf, so a shared subtest can ask "is this a driver the +// control plane withholds secrets from?" without a per-driver branch. +// +// It asks through the same optional interface the runner asks through +// (CapabilityDriver), so the subtest below and the announce a runner actually +// makes cannot come to different conclusions about the same driver. +func driverCapabilities(d Driver) []string { + if cd, ok := d.(CapabilityDriver); ok { + return cd.Capabilities() + } + return nil +} + func RunContract(t *testing.T, newDriver func(t *testing.T) (Driver, func())) { t.Run("create-inspect-destroy", func(t *testing.T) { d, cleanup := newDriver(t) @@ -253,7 +282,28 @@ func RunContract(t *testing.T, newDriver func(t *testing.T) (Driver, func())) { h, err := d.Create(ctx, Spec{ Name: "t9", Image: "", SessionID: "s9", DialURL: "ws://x", Setup: "true", - Env: map[string]string{"CONTRACT_SECRET": "must-not-survive"}, + // CONTRACT_KEPT stands for the configuration a create legitimately + // carries — an agent-home path, a manifest — and is what proves + // the committed configuration describes this create at all. + // CONTRACT_SECRET is the one being stripped. + Env: map[string]string{ + "CONTRACT_SECRET": "must-not-survive", + "CONTRACT_KEPT": "configuration-not-a-credential", + }, + // The token is here so this subtest can still be written for a + // driver that refuses secret VALUES without one — which is the + // twelfth subtest below, and is the microVM driver. The Docker + // driver ignores the field entirely (nothing in runArgs reads + // it), so its create is the one it has always been. + // + // The PAIRING is synthetic and worth saying so: the control plane + // never sends a secret value together with a token — that is the + // whole of §3, and createSpec checks it. What it does send with a + // token is the configuration CONTRACT_KEPT stands for. This + // subtest keeps the secret-shaped value because stripping is what + // it is about, and the rule the plane actually keeps is pinned by + // the twelfth subtest below and by controlapp's own matrix. + BootstrapToken: "contract-token-example", }) if err != nil { t.Fatal(err) @@ -270,7 +320,85 @@ func RunContract(t *testing.T, newDriver func(t *testing.T) (Driver, func())) { if snap.Ref != ref { t.Fatalf("snapshot ref = %q, want %q verbatim", snap.Ref, ref) } - assertStrippedFromImage(t, d, ref, "must-not-survive", strip) + assertStrippedFromImage(t, d, ref, "must-not-survive", strip, "CONTRACT_KEPT") + }) + + t.Run("a create carrying secret values and no token is refused", func(t *testing.T) { + // The twelfth subtest, and the one that makes the fifth honest for a + // driver that never accepts secret values at all. + // + // The compatibility table's "old controld + new runner" row: a + // control plane that predates the session bootstrap exchange + // dispatches an environment's decrypted secrets in Spec.Env with no + // token. Whether that is acceptable is a per-driver question with + // two different right answers, which is why it is asserted here + // rather than at either driver: + // + // - Docker puts them in a `docker run` argv and nothing reaches the + // host filesystem. It has always accepted them and must keep + // accepting them, or a rollback of the control plane would take + // every self-hosted session down. + // - a microVM host has no argv, and every channel it does have ends + // in a file that outlives the session on a machine shared with + // other tenants. It must refuse, loudly, naming what the control + // plane has to be — a refusal costs one session, and a quiet + // write is the exposure ADR-0003 §2.7 was written against. + // + // The driver announcing microvm.v1 is the one required to refuse, + // because that capability is exactly the claim the control plane + // withholds on: a driver that announces it and then accepts the + // values has told the plane it will do something it does not do. + d, cleanup := newDriver(t) + defer cleanup() + ctx := context.Background() + + spec := Spec{ + Name: "t10", Image: "", SessionID: "s10", DialURL: "ws://x", + Env: map[string]string{"CONTRACT_SECRET": "must-not-be-accepted"}, + } + h, err := d.Create(ctx, spec) + if err == nil { + defer d.Destroy(ctx, h.ID) + } + if !slices.Contains(driverCapabilities(d), "microvm.v1") { + // Not a withholding driver: accepting is the contract, and a + // refusal here would be the rollback failure described above. + if err != nil { + t.Fatalf("a create carrying env values was refused by a driver that does not withhold: %v", err) + } + return + } + if err == nil { + t.Fatal("a driver announcing microvm.v1 accepted a create carrying secret values with no bootstrap token") + } + // The refusal has to say what the operator must change, which is the + // control plane's version and not anything about this host. + for _, want := range []string{"bootstrap token", "microvm.v1"} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("the refusal = %q, want it to name %q", err, want) + } + } + if strings.Contains(err.Error(), "must-not-be-accepted") { + t.Fatalf("the refusal quotes the value it refused: %q", err) + } + // And nothing was half-created: a refused create leaves no session. + listed, lerr := d.List(ctx) + if lerr != nil { + t.Fatal(lerr) + } + for _, l := range listed { + if l.SessionID == "s10" { + t.Fatalf("a refused create left a session behind: %+v", l) + } + } + // The same create WITH a token is accepted, which is what makes the + // refusal about the token rather than about the env block. + spec.BootstrapToken = "contract-token-example" + withToken, err := d.Create(ctx, spec) + if err != nil { + t.Fatalf("a withheld create carrying a token was refused: %v", err) + } + d.Destroy(ctx, withToken.ID) }) t.Run("capacity", func(t *testing.T) { diff --git a/internal/driver/driver.go b/internal/driver/driver.go index 1c795668..7751adc7 100644 --- a/internal/driver/driver.go +++ b/internal/driver/driver.go @@ -74,6 +74,25 @@ type Spec struct { // no creator and for every create a control plane older than the field // sent. The session's agents then simply ask for a login. Home *HomeMount + // BootstrapToken and SecretNames are the microVM bootstrap exchange + // (runner.Spec's two fields of the same names, carried across this + // boundary unchanged). For a session placed on a runner that announced + // microvm.v1 the control plane leaves the environment's decrypted + // secret_refs OUT of Env and puts these here instead: the names the + // guest should expect, and the single-use capability it exchanges for + // their values once it has booted. + // + // Both are empty on every Docker create and on every create from a + // control plane older than them, which is exactly why the Docker driver + // ignores them and Env keeps its meaning. The microVM driver is the only + // one that reads them, and the only one that refuses a create carrying + // values in Env with no token — see (*Microvm).Create. + // + // Neither is ever written to host disk, logged, or put in a snapshot + // manifest. The token's whole lifetime on a host is the hop from this + // struct into the guest's boot configuration. + BootstrapToken string + SecretNames []string } // HomeMount names the agent home volume and where it lands in the container. @@ -180,6 +199,28 @@ type Listed struct { Handle Handle } +// CapabilityDriver is a driver that names portable capability tokens the +// runner announces on its behalf. +// +// It is an optional interface rather than a method on Driver because most +// drivers have nothing to say: a capability is a claim the control plane +// SCHEDULES on, and inventing one per driver would make the fleet's +// vocabulary an implementation detail of whichever sandbox a runner happens +// to use. The microVM driver is the one that has something to claim +// (microvm.v1), and what it claims is the fence the control plane withholds +// an environment's secret values on — so the announcement and the driver +// that must honour it are the same fact, read from one place. +type CapabilityDriver interface { + Capabilities() []string +} + +// HostedDriver is a driver that needs the runner above it: see MicrovmHost +// for the two things, and why they cannot be constructor arguments (the +// runner is composed OVER the driver, so the driver exists first). +type HostedDriver interface { + SetHost(MicrovmHost) +} + type Driver interface { Create(ctx context.Context, spec Spec) (Handle, error) Suspend(ctx context.Context, id string, warm bool) error // warm=pause, cold=stop diff --git a/internal/driver/microvm.go b/internal/driver/microvm.go index 562bc73a..968acbca 100644 --- a/internal/driver/microvm.go +++ b/internal/driver/microvm.go @@ -3,16 +3,22 @@ // The microVM driver: one Firecracker microVM per session, in place of the // docker driver's one container per session. // -// This is the second step of that driver, and it is deliberately not a -// working end-to-end path yet. What it establishes is the seam and the -// invariants: a production `--driver=microvm` either has hardware +// This is the third step of that driver, and it is deliberately not a +// working end-to-end path yet. What the second step established is the seam +// and the invariants: a production `--driver=microvm` either has hardware // virtualization, a kernel, a rootfs, a formatter and the Firecracker binary, // or it refuses to start; nothing decrypted is written to host disk; a // cold-parked session reads as suspended rather than gone; and the simulated // engine behind the tests is reachable only by explicitly injecting it. -// Guest configuration delivery (vsock), the jailer, per-VM network slots, -// nftables, metering, and real image and snapshot work by digest each land in -// their own change — see the TODO markers below and ADR-0003. +// +// What this step adds is the host-to-guest channel those invariants were +// waiting for: virtio-vsock, carrying a session's whole configuration and +// its bootstrap token, in place of a staged file and in place of MMDS. See +// microvm_vsock.go, and docs/design/2026-09-20-microvm-bootstrap-token-and-vsock.md. +// +// The jailer, per-VM network slots, nftables, metering, and real image and +// snapshot work by digest each land in their own change — see the TODO +// markers below and ADR-0003. package driver import ( @@ -37,6 +43,8 @@ import ( "sync/atomic" "syscall" "time" + + "github.com/tokencanopy/rainier/protocol/runner" ) const ( @@ -121,19 +129,27 @@ const ( // enforcement, not a convention — every structure this driver persists either // embeds VMMConfig or is derived from it. type VMMConfig struct { - ID string `json:"id"` - SessionID string `json:"session_id"` - VCPU int `json:"vcpu"` - MemoryMiB int `json:"memory_mib"` - KernelPath string `json:"kernel_path"` - RootfsPath string `json:"rootfs_path"` - WorkspaceDiskPath string `json:"workspace_disk_path"` - HomeDiskPath string `json:"home_disk_path"` - Cmd []string `json:"cmd"` - DialURL string `json:"dial_url"` - ProxyURL string `json:"proxy_url"` - TapDevice string `json:"tap_device"` - Env map[string]string `json:"-"` + ID string `json:"id"` + SessionID string `json:"session_id"` + VCPU int `json:"vcpu"` + MemoryMiB int `json:"memory_mib"` + KernelPath string `json:"kernel_path"` + RootfsPath string `json:"rootfs_path"` + WorkspaceDiskPath string `json:"workspace_disk_path"` + HomeDiskPath string `json:"home_disk_path"` + Cmd []string `json:"cmd"` + DialURL string `json:"dial_url"` + ProxyURL string `json:"proxy_url"` + TapDevice string `json:"tap_device"` + // VsockUDSPath is the host path Firecracker is told to serve this VM's + // virtio-vsock device on: the guest's connections to host port N are + // forwarded to "_N", and 1024 is the only port this design + // uses. It is a path and not a value — recorded like every other path + // here, and per BOOT rather than per instance, because the documentation + // warns that one uds_path cannot be multiplexed across VMs and a cold + // resume is a new VM. + VsockUDSPath string `json:"vsock_uds_path"` + Env map[string]string `json:"-"` // EgressAllow is the session's allowlist. It is carried and recorded, and // nothing enforces it. @@ -186,12 +202,36 @@ type instanceRecord struct { PID int `json:"pid"` Cfg VMMConfig `json:"cfg"` - // envLive marks a record whose Cfg.Env is the live environment THIS - // process built in Create. It is unexported and therefore never - // serialized, which is the point: a record recovered from disk after a - // runnerd restart has no environment behind it, and a cold resume must - // say so rather than boot a guest with a silently empty one. - envLive bool + // boot is the configuration the guest is handed over vsock, and channel + // is the host end of that conn. Both are unexported and therefore never + // serialized, which is the point: the configuration carries the session's + // bootstrap token and ADR-0003 §2.7 item 1 keeps every part of it off a + // shared host's disk. + // + // bootLive says the pair is this process's own. A record recovered from + // disk after a runnerd restart has no configuration behind it, and a cold + // resume must say so rather than boot a guest that will never be told + // what it is. (The bootstrap token would survive such a restart — the + // runner can mint a fresh one — but the rest of the configuration would + // not, and persisting it is precisely what this design does not do.) + boot runner.BootConfig + channel *guestChannel + bootLive bool + + // boots counts launches of this instance, so each one gets a vsock socket + // path of its own. A cold resume is a new VM and Firecracker's own + // documentation warns that one uds_path cannot be multiplexed across two. + // + // It is read AND advanced under the driver mutex by the one resume that + // claimed the instance (see resuming), so no two boots of one instance + // can ever be handed the same path. + boots int + + // resuming is the claim one Resume holds over this instance while it is + // in flight. A second Resume for the same id is refused rather than run + // beside it: see Resume for what two concurrent cold ones would do to + // each other's socket. + resuming bool // epoch counts mutations of this record, and exists because Inspect and // List ask the hypervisor with the driver mutex RELEASED (its answer is @@ -224,32 +264,24 @@ type snapshotManifest struct { CreatedAt time.Time `json:"created_at"` } -// guestSessionConfig is the structured configuration staged on the host for a -// session, ready for the channel that will carry it. +// A session's configuration is no longer staged on the host at all. // -// It carries no environment. What a session needs in order to identify itself -// and reach the relay is not secret; what an environment resolved out of its -// secret refs is, and ADR-0003 §2.7 item 1 has those arriving in the guest -// from cell-gateway against a short-lived bootstrap token, never from a file -// on the host. +// It used to be written to "/instances//session.json", against +// the day something would carry it into the guest. That day is this change, +// and what carries it is virtio-vsock: the configuration is composed in +// memory (bootConfigFor), handed to the guest as the first control frame on +// the conn it opens, and never written anywhere. A file is not needed, and a +// file on a shared host that describes one tenant's session — and, once the +// bootstrap token exists, carries a capability — is exactly what ADR-0003 +// §2.7 asks for there not to be. // -// TODO(PR 2): nothing reads this yet — it is staged on the host and there is -// no path into the guest. The channel is virtio-vsock (ADR-0003 §2.7 item 2), -// which also carries the bootstrap token. -// -// It is deliberately NOT delivered through the session's own workspace. A -// file inside a volume the agent can write is a file the agent can rewrite, -// and sessiond reading a session id or a dial URL back out of one would let a -// session re-register as another or point its relay somewhere else. That is -// why sessiond has no loader for this file: the guest must receive it from -// the host, over a channel the guest cannot write. -type guestSessionConfig struct { - SessionID string `json:"session_id"` - DialURL string `json:"dial_url"` - ProxyURL string `json:"proxy_url"` - Cmd []string `json:"cmd"` - EgressAllow []string `json:"egress_allow"` -} +// It was also never delivered through the session's own workspace, for a +// reason that still holds and is worth keeping written down: a file inside a +// volume the agent can write is a file the agent can rewrite, and a sessiond +// reading its session id or its proxy back out of one would let a session +// re-register as another or point its egress somewhere else. Over vsock the +// configuration arrives on a socket inside one VM's own directory, which the +// guest cannot write and cannot forge. // Microvm implements driver.Driver for hardware-isolated microVMs. type Microvm struct { @@ -264,6 +296,11 @@ type Microvm struct { instances map[string]*instanceRecord pulls []string strips [][]string + + // host is the runner above this driver: where a guest's control + // connection goes, and who asks the control plane for a fresh bootstrap + // token on a cold resume. nil is a real state — see MicrovmHost. + host MicrovmHost } // NewMicrovm creates a new microVM driver, or fails. @@ -400,13 +437,17 @@ func (m *Microvm) instanceMetaPath(id string) string { return filepath.Join(m.instanceDir(id), "instance.json") } -// persistable copies a record for writing to disk. The copy drops Cfg.Env — -// which `json:"-"` would drop anyway — so that a record handed to a goroutine -// outside the driver mutex carries no reference to the live map either. +// persistable copies a record for writing to disk. The copy drops Cfg.Env and +// the guest's boot configuration — which being unexported, or `json:"-"`, +// would drop anyway — so that a record handed to a goroutine outside the +// driver mutex carries no reference to the live map, the live configuration, +// or the bootstrap token in it either. func persistable(rec *instanceRecord) instanceRecord { cp := *rec cp.Cfg.Env = nil - cp.envLive = false + cp.boot = runner.BootConfig{} + cp.channel = nil + cp.bootLive = false return cp } @@ -710,9 +751,19 @@ func (m *Microvm) resolveRootfs(rootfsRef string) (string, error) { return m.locateRootfs(rootfsRef) } -// buildGuestEnv translates the full Spec into the environment map passed into -// the microVM guest. The result is held in memory for the lifetime of this +// buildGuestEnv is the driver's own record of the CONFIGURATION variables a +// session was created with. It is held in memory for the lifetime of this // process and never written to disk; see VMMConfig.Env. +// +// It no longer copies spec.Env, and that deletion is the point of this +// change. Spec.Env is where an environment's decrypted secret_refs live on +// the Docker path, and on this one they are not dispatched at all — the +// create carries their names and a token instead, and the values reach the +// guest from the control plane over the exchange the boot configuration +// starts. What a guest actually runs on arrives over vsock +// (bootConfigFor), not from this map, which is why nothing here is a +// delivery channel any more: it is what a snapshot's manifest names as +// having been configured, and nothing else reads it. func buildGuestEnv(spec Spec) map[string]string { env := make(map[string]string) @@ -751,26 +802,9 @@ func buildGuestEnv(spec Spec) map[string]string { if spec.GitAuthorEmail != "" { env["RAINIER_GIT_AUTHOR_EMAIL"] = spec.GitAuthorEmail } - for k, v := range spec.Env { - env[k] = v - } return env } -// stageGuestSessionConfig writes the non-secret half of a session's -// configuration for sessiond. -func (m *Microvm) stageGuestSessionConfig(id string, cfg guestSessionConfig) error { - dir := m.instanceDir(id) - if err := os.MkdirAll(dir, microvmDirMode); err != nil { - return err - } - data, err := json.MarshalIndent(cfg, "", " ") - if err != nil { - return err - } - return os.WriteFile(filepath.Join(dir, "session.json"), data, microvmFileMode) -} - // reserveSlot takes the capacity decision and an id under the mutex, so that // the rest of Create — disks, mkfs, the rootfs lookup, the TAP, the launch — // runs with the mutex RELEASED. Holding it across engine.Launch froze @@ -805,6 +839,12 @@ func (m *Microvm) Create(ctx context.Context, spec Spec) (Handle, error) { if err := checkHome(spec.Home); err != nil { return Handle{}, err } + // Before anything with a side effect, and before a slot is even + // reserved: a create this host must not perform is refused rather than + // half-performed. See refuseUnwithheldEnv. + if err := refuseUnwithheldEnv(spec); err != nil { + return Handle{}, err + } id, err := m.reserveSlot() if err != nil { @@ -876,17 +916,23 @@ func (m *Microvm) launch(ctx context.Context, id string, spec Spec) (*instanceRe cmd = []string{"/bin/bash"} } - if err := m.stageGuestSessionConfig(id, guestSessionConfig{ - SessionID: spec.SessionID, - DialURL: spec.DialURL, - ProxyURL: spec.ProxyURL, - Cmd: slices.Clone(cmd), - EgressAllow: slices.Clone(spec.EgressAllow), - }); err != nil { - return nil, fmt.Errorf("stage guest session config: %w", err) - } undo = append(undo, func() { m.deleteInstanceRecord(id) }) + // The guest's control channel, opened BEFORE the VM starts: the guest + // dials host port 1024 as soon as its sessiond is up, and a listener + // created after InstanceStart is a race whose loser is a session that + // boots and is never configured. + bootCfg := bootConfigFor(spec) + udsPath, listenPath, err := m.vsockPaths(id, 1) + if err != nil { + return nil, err + } + channel, err := m.openGuestChannel(spec.SessionID, udsPath, listenPath, bootCfg) + if err != nil { + return nil, err + } + undo = append(undo, channel.close) + tapDevice := "tap-" + id if err := m.tap.Allocate(tapDevice, m.opts.Network); err != nil { return nil, fmt.Errorf("allocate tap device %s: %w", tapDevice, err) @@ -907,6 +953,7 @@ func (m *Microvm) launch(ctx context.Context, id string, spec Spec) (*instanceRe DialURL: spec.DialURL, ProxyURL: spec.ProxyURL, TapDevice: tapDevice, + VsockUDSPath: udsPath, Env: buildGuestEnv(spec), } @@ -922,7 +969,10 @@ func (m *Microvm) launch(ctx context.Context, id string, spec Spec) (*instanceRe Volume: workspaceVolume(spec.SessionID), PID: m.engine.PID(id), Cfg: cfg, - envLive: true, + boot: bootCfg, + channel: channel, + bootLive: true, + boots: 1, } if err := m.saveRecord(persistable(rec)); err != nil { return nil, fmt.Errorf("save instance metadata %s: %w", id, err) @@ -967,6 +1017,15 @@ func (m *Microvm) Suspend(ctx context.Context, id string, warm bool) error { inst.Cold = !warm if !warm { inst.PID = 0 + // The VM is gone, so its vsock socket is a path nothing serves. It + // is closed and removed here rather than left for the resume to + // overwrite: a stale socket on a shared host is one more file + // describing a session that is not running, and the resume gets a + // path of its own anyway. + if inst.channel != nil { + inst.channel.close() + inst.channel = nil + } } inst.bump() rec := persistable(inst) @@ -986,8 +1045,8 @@ func (m *Microvm) Resume(ctx context.Context, id string) (bool, error) { return false, fmt.Errorf("no such id %s", id) } running := inst.State == StateRunning - cold, envLive, cfg := inst.Cold, inst.envLive, inst.Cfg - m.mu.Unlock() + cold, bootLive, cfg := inst.Cold, inst.bootLive, inst.Cfg + bootCfg, sessionID := inst.boot, inst.SessionID // Resuming a session that is already running restarts nothing, and says // so without touching the hypervisor (driver.go's Resume contract). The @@ -996,31 +1055,90 @@ func (m *Microvm) Resume(ctx context.Context, id string) (bool, error) { // resume turned "nothing to do" into a failed Resume — and runnerd reads // a failed Resume as a session it could not bring back. if running { + m.mu.Unlock() return false, nil } + // One resume at a time per instance, claimed under the same lock that + // reads the state it decided on. + // + // Two concurrent cold resumes would otherwise both see the same state, + // both compute the same next boot number, and both open a channel on the + // same socket path — and openGuestChannel UNLINKS before it listens, so + // the second would remove the first's live socket out from under a VM + // that had already been told about it. That session then boots and is + // never configured, which is the one outcome this whole path exists to + // prevent. The pair of them would also mint two tokens and launch twice. + if inst.resuming { + m.mu.Unlock() + return false, fmt.Errorf("resume of %s: a resume is already in flight for this instance", id) + } + inst.resuming = true + // The boot number is taken HERE, under the same lock, and it advances + // even for an attempt that fails: a failed launch may have left a socket + // behind at that path, and an attempt that reuses a number is an attempt + // that inherits it. The counter is only ever a source of distinct paths. + boots := inst.boots + if cold { + inst.boots++ + boots = inst.boots + } + m.mu.Unlock() + defer func() { + m.mu.Lock() + if e, ok := m.instances[id]; ok { + e.resuming = false + } + m.mu.Unlock() + }() + restarted := false + var channel *guestChannel if cold { // A cold resume is a fresh boot, and a fresh boot needs the session's - // environment. This driver holds that in memory only (ADR-0003 §2.7 - // item 1), so a record recovered from disk after a runnerd restart - // has none — including the case where the original session carried no - // Spec.Env at all, which the driver deliberately cannot tell apart, - // having refused to write the evidence down. + // whole configuration. This driver holds that in memory only + // (ADR-0003 §2.7 item 1), so a record recovered from disk after a + // runnerd restart has none — including the case where the original + // session carried nothing secret at all, which the driver + // deliberately cannot tell apart, having refused to write the + // evidence down. // - // The honest answer is to refuse. Relaunching would boot a guest with - // a silently empty environment: no relay dial, no proxy, no - // credentials, and an agent that reports itself healthy. + // The honest answer is to refuse. Relaunching would boot a guest that + // is never told what it is: no session id, no proxy, no boot chain, + // no secrets, and an agent that reports itself healthy. // - // TODO(PR 2): the fix is not to persist the environment. It is the - // bootstrap token over vsock — sessiond asks cell-gateway for fresh - // short-lived credentials at boot and after every resume (ADR-0003 - // §2.7 item 1, §4.3) — after which a cold resume needs nothing from - // the host but the token. - if !envLive { - return false, fmt.Errorf("cold resume of %s: this session's guest environment was held in memory only and did not survive a runnerd restart; a clean relaunch needs the bootstrap token over vsock (ADR-0003 §2.7 item 1), which is not implemented yet", id) + // The bootstrap token is not what is missing here — the runner can + // mint a fresh one, and does, three lines below. What is missing is + // everything else, and persisting THAT is what this design rules out. + // Rebuilding it from the control plane on a resume is the follow-up + // (a create-shaped resume, ADR-0003 §2.3's portable checkpoint). + if !bootLive { + return false, fmt.Errorf("cold resume of %s: this session's guest configuration was held in memory only (ADR-0003 §2.7 item 1) and did not survive a runnerd restart; a clean relaunch needs the control plane to re-resolve it, which is the portable-checkpoint work and not this change", id) + } + // A new VM gets a new token and a new socket. The token because the + // old one is single-use and fenced by a placement generation the + // plane may have moved past; the socket because Firecracker's own + // documentation warns that one uds_path cannot be multiplexed across + // two VMs. + host := m.currentHost() + if host == nil { + return false, fmt.Errorf("cold resume of %s: this driver has no runner above it to mint a bootstrap token", id) } + token, err := host.MintSessionBootstrap(ctx, sessionID) + if err != nil { + return false, fmt.Errorf("cold resume of %s: minting a bootstrap token: %w", id, err) + } + bootCfg.BootstrapToken = token + udsPath, listenPath, err := m.vsockPaths(id, boots) + if err != nil { + return false, err + } + if channel, err = m.openGuestChannel(sessionID, udsPath, listenPath, bootCfg); err != nil { + return false, err + } + cfg.VsockUDSPath = udsPath if err := m.engine.Launch(ctx, cfg); err != nil { + channel.close() return false, fmt.Errorf("relaunch cold microvm %s: %w", id, err) } restarted = true @@ -1032,12 +1150,23 @@ func (m *Microvm) Resume(ctx context.Context, id string) (bool, error) { inst, ok = m.instances[id] if !ok { m.mu.Unlock() + if channel != nil { + channel.close() + } return restarted, fmt.Errorf("no such id %s", id) } inst.State = StateRunning inst.Cold = false if restarted { inst.PID = m.engine.PID(id) + if inst.channel != nil { + inst.channel.close() + } + inst.channel = channel + inst.boot = bootCfg + inst.Cfg.VsockUDSPath = cfg.VsockUDSPath + // inst.boots was advanced when this resume claimed the instance, so + // that the path it opened could not collide with a concurrent one. } inst.bump() rec := persistable(inst) @@ -1068,8 +1197,23 @@ func (m *Microvm) Snapshot(ctx context.Context, id, ref string, stripEnv []strin m.snapSeq.Add(1) ref = fmt.Sprintf("rainier-mvm:%s-%d", id, m.snapSeq.Load()) } - survivingKeys := make([]string, 0, len(inst.Cfg.Env)) + // What this session was CONFIGURED with, from both places a key can come + // from: the driver's own injection (buildGuestEnv) and the configuration + // block the guest was handed over vsock. Both, because the strip list is + // a promise about the committed image and a key that survived through + // the channel the driver does not happen to be looking at is a key that + // survived. + // + // Keys, never values: see snapshotManifest. + surviving := map[string]struct{}{} for k := range inst.Cfg.Env { + surviving[k] = struct{}{} + } + for k := range inst.boot.Env { + surviving[k] = struct{}{} + } + survivingKeys := make([]string, 0, len(surviving)) + for k := range surviving { if !slices.Contains(stripEnv, k) { survivingKeys = append(survivingKeys, k) } @@ -1155,7 +1299,14 @@ func (m *Microvm) DestroyContainer(ctx context.Context, id string) error { return nil } tapDevice := inst.Cfg.TapDevice + channel := inst.channel + inst.channel = nil m.mu.Unlock() + // Closed before the VM is signalled, so nothing can dial a control + // channel for a session that is being torn down. + if channel != nil { + channel.close() + } if err := m.engine.Stop(ctx, id); err != nil { st, stateErr := m.engine.State(ctx, id) @@ -1553,17 +1704,36 @@ func (f *FirecrackerEngine) Launch(ctx context.Context, cfg VMMConfig) error { } } - // There is deliberately no MMDS configuration here. MMDS answers at - // 169.254.169.254 — the exact address ADR-0003 §4.3 requires the host to - // drop on every TAP — so the config channel and the metadata-denial rule - // could never both hold. The two PUTs also discarded their errors, and - // /mmds/config without network_interfaces is rejected outright, so what - // the guest actually received was nothing. + // 5b. virtio-vsock: the single host-to-guest control channel (ADR-0003 + // §2.7 item 2). It carries the boot configuration, the bootstrap token, + // the sessiond-to-runnerd stream that used to ride a WebSocket through + // the TAP device, the lifecycle handshake, and the credential fetch — all + // over one conn, because internal/relay already multiplexes exactly those + // things over one conn. + // + // The guest's connections to host port N arrive on "_N", and + // the driver is already listening on _1024 by the time this runs. The + // host never sends CONNECT, so there is no host-initiated direction to + // configure. // - // TODO(PR 2): virtio-vsock is the single host-to-guest control channel - // (ADR-0003 §2.7 item 2), carrying the boot configuration and the - // bootstrap token. Until it lands, a guest booted by this engine receives - // no session configuration at all. + // There is deliberately still no MMDS configuration. MMDS answers + // unauthenticated HTTP at 169.254.169.254 — the exact address ADR-0003 + // §4.3 requires the host to drop on every TAP — so a configuration + // channel that needed it would be at war with the rule that exists to + // block it. vsock is invisible to guest routing and to the TAP firewall + // alike, which is why the metadata denial can stay unconditional. + if cfg.VsockUDSPath != "" { + if err := fcClient.putJSON(ctx, "/vsock", map[string]any{ + "guest_cid": guestCID, + "uds_path": cfg.VsockUDSPath, + }); err != nil { + // Not a discarded error, unlike the two MMDS PUTs this replaces: + // a guest with no control channel is a guest that can never be + // configured, and a silently config-less VM reporting itself + // healthy is the failure that made this whole design necessary. + return fmt.Errorf("configure the guest control channel (vsock): %w", err) + } + } // 6. Start the microVM instance if err := fcClient.putJSON(ctx, "/actions", map[string]any{ diff --git a/internal/driver/microvm_sim_test.go b/internal/driver/microvm_sim_test.go index dfdd57c8..a86c558b 100644 --- a/internal/driver/microvm_sim_test.go +++ b/internal/driver/microvm_sim_test.go @@ -78,6 +78,7 @@ type SimulatedEngine struct { states map[string]VMMState configs map[string]VMMConfig failOnStop map[string]error + failLaunch error } func NewSimulatedEngine() *SimulatedEngine { @@ -158,9 +159,21 @@ func (s *SimulatedEngine) FailOnStop(id string, err error) { s.failOnStop[id] = err } +// FailLaunch makes every Launch fail with err, standing in for a VMM that +// will not start — the one case that leaves a half-configured boot behind. +// A nil err clears it. +func (s *SimulatedEngine) FailLaunch(err error) { + s.mu.Lock() + defer s.mu.Unlock() + s.failLaunch = err +} + func (s *SimulatedEngine) Launch(_ context.Context, cfg VMMConfig) error { s.mu.Lock() defer s.mu.Unlock() + if s.failLaunch != nil { + return s.failLaunch + } s.states[cfg.ID] = VMMStateRunning s.configs[cfg.ID] = cfg s.saveState(cfg.ID, VMMStateRunning) diff --git a/internal/driver/microvm_test.go b/internal/driver/microvm_test.go index 0b8a149b..5dcf08d3 100644 --- a/internal/driver/microvm_test.go +++ b/internal/driver/microvm_test.go @@ -3,7 +3,6 @@ package driver import ( "context" - "encoding/json" "errors" "fmt" "io" @@ -21,6 +20,8 @@ import ( "syscall" "testing" "time" + + "github.com/tokencanopy/rainier/internal/relay" ) // testMicrovm builds a driver over a fresh state directory with the whole @@ -33,7 +34,7 @@ import ( func testMicrovm(t *testing.T, opts MicrovmOpts) (*Microvm, *SimulatedEngine) { t.Helper() if opts.StateDir == "" { - opts.StateDir = t.TempDir() + opts.StateDir = shortTempDir(t) } if opts.BaseRootfs == "" { opts.BaseRootfs = writeFakeRootfs(t, opts.StateDir) @@ -56,6 +57,23 @@ func testMicrovm(t *testing.T, opts MicrovmOpts) (*Microvm, *SimulatedEngine) { return m, sim } +// shortTempDir is t.TempDir with a name short enough to hang a unix socket +// off. t.TempDir composes the TEST's name into the path, and the guest +// control socket lands at "/instances//v1.sock_1024" — which for a +// test called TestMicrovmSomethingDescriptive is over the 108-byte sun_path +// limit before the driver has done anything wrong. A production state +// directory is nowhere near it; this is a fixture concern and not a +// behaviour one. +func shortTempDir(t *testing.T) string { + t.Helper() + dir, err := os.MkdirTemp("", "mvm") + if err != nil { + t.Fatalf("temp dir: %v", err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + return dir +} + // writeFakeRootfs puts a file where a base rootfs image would be. It is not a // filesystem and nothing boots it; it is there so the driver's "this host has // the image" check has something true to find. @@ -71,10 +89,66 @@ func writeFakeRootfs(t *testing.T, dir string) string { func TestMicrovmSatisfiesContract(t *testing.T) { RunContract(t, func(t *testing.T) (Driver, func()) { d, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4}) + // A microVM driver in production always has a runner above it + // (runnerd.New installs itself), and a cold resume asks it for a + // fresh bootstrap token. The contract drives cold resumes, so the + // fixture supplies the smallest thing that can answer one. + d.SetHost(&stubMicrovmHost{}) return d, func() {} }) } +// stubMicrovmHost is a runner that mints a token and drops every guest +// connection. It is what the contract needs and nothing more: the contract is +// about the driver's own behaviour, and a guest that connects to a fixture +// has nowhere to go. +type stubMicrovmHost struct { + mu sync.Mutex + mints int + conns []string + // mintErr, when set, is what a cold resume's mint reports — the state a + // runner with no control connection is in. + mintErr error +} + +func (h *stubMicrovmHost) GuestConnected(sessionID string, conn relay.Conn) { + h.mu.Lock() + h.conns = append(h.conns, sessionID) + h.mu.Unlock() + _ = conn.Close() +} + +func (h *stubMicrovmHost) MintSessionBootstrap(_ context.Context, sessionID string) (string, error) { + h.mu.Lock() + defer h.mu.Unlock() + if h.mintErr != nil { + return "", h.mintErr + } + h.mints++ + return fmt.Sprintf("token_example_%s_%d", sessionID, h.mints), nil +} + +// connCount is how many guest connections the runner was handed. One per +// BOOT is the rule the driver enforces; anything more is a second process in +// the sandbox having been served a session's configuration. +func (h *stubMicrovmHost) connCount() int { + h.mu.Lock() + defer h.mu.Unlock() + return len(h.conns) +} + +func (h *stubMicrovmHost) mintCount() int { + h.mu.Lock() + defer h.mu.Unlock() + return h.mints +} + +func (h *stubMicrovmHost) connected() []string { + h.mu.Lock() + defer h.mu.Unlock() + return slices.Clone(h.conns) +} + // TestMicrovmRefusesToStartWithoutAHost is the fail-closed half: there is no // combination of missing pieces that yields a working-looking driver. func TestMicrovmRefusesToStartWithoutAHost(t *testing.T) { @@ -137,7 +211,7 @@ func TestMicrovmRefusesToStartWithoutAHost(t *testing.T) { // drop it, and runnerd.Recover would forget a session whose files are sitting // right there on disk. func TestMicrovmColdSuspendedSessionIsNotGone(t *testing.T) { - stateDir := t.TempDir() + stateDir := shortTempDir(t) m, _ := testMicrovm(t, MicrovmOpts{ TotalSlots: 4, StateDir: stateDir, @@ -206,7 +280,7 @@ func listedState(t *testing.T, m *Microvm, sessionID string) State { } func TestMicrovmRestartRecovery(t *testing.T) { - stateDir := t.TempDir() + stateDir := shortTempDir(t) m1, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4, StateDir: stateDir}) ctx := context.Background() @@ -254,6 +328,7 @@ func TestMicrovmRestartRecovery(t *testing.T) { func TestMicrovmWorkspaceFilesSurviveColdPark(t *testing.T) { m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4}) + m.SetHost(&stubMicrovmHost{}) ctx := context.Background() h, err := m.Create(ctx, Spec{SessionID: "sess-persist"}) @@ -321,7 +396,12 @@ func TestMicrovmGuestEnvTranslationAndBootstrap(t *testing.T) { GitAuthorName: "Test Author", GitAuthorEmail: "author@example.invalid", Cmd: []string{"claude", "--model", "haiku"}, + // Env plus a token is what a create from a control plane that + // withholds looks like: the values that remain are configuration + // (the agent home, the manifest), and this driver refuses a create + // carrying values with no token at all. Env: map[string]string{"APP_ENV": "production"}, + BootstrapToken: "token_example", } h, err := m.Create(ctx, spec) @@ -340,7 +420,6 @@ func TestMicrovmGuestEnvTranslationAndBootstrap(t *testing.T) { "RAINIER_SESSION": "sess-trans", "RAINIER_GIT_AUTHOR_NAME": "Test Author", "RAINIER_GIT_AUTHOR_EMAIL": "author@example.invalid", - "APP_ENV": "production", "RAINIER_SETUP_TIMEOUT": "600", "RAINIER_INIT_TIMEOUT": "180", } @@ -355,25 +434,22 @@ func TestMicrovmGuestEnvTranslationAndBootstrap(t *testing.T) { if cfg.TapDevice == "" { t.Errorf("TapDevice was not allocated on VMMConfig") } - - // The staged session config is the non-secret half, and only that half. - jsonPath := filepath.Join(m.instanceDir(h.ID), "session.json") - data, err := os.ReadFile(jsonPath) - if err != nil { - t.Fatalf("read session.json: %v", err) - } - var gsc guestSessionConfig - if err := json.Unmarshal(data, &gsc); err != nil { - t.Fatalf("unmarshal session.json: %v", err) - } - if gsc.SessionID != "sess-trans" { - t.Errorf("gsc.SessionID = %q, want sess-trans", gsc.SessionID) + // Spec.Env is no longer copied in. This map is the driver's record of + // what a session was CONFIGURED with, read by nothing but a snapshot + // manifest; what the guest actually runs on arrives over vsock, so a + // create's own env block has no business being duplicated here. + if _, copied := cfg.Env["APP_ENV"]; copied { + t.Errorf("the create's env block was copied into the driver's own map: %+v", cfg.Env) } - if !reflect.DeepEqual(gsc.Cmd, []string{"claude", "--model", "haiku"}) { - t.Errorf("gsc.Cmd = %v, want claude --model haiku", gsc.Cmd) + if cfg.VsockUDSPath == "" { + t.Error("no vsock uds_path was configured; the guest has no control channel") } - if strings.Contains(string(data), "APP_ENV") || strings.Contains(string(data), "production") { - t.Errorf("session.json carries the session environment:\n%s", data) + + // And nothing is staged on the host for the guest to read. session.json + // is gone: the configuration goes over the vsock conn and is written + // nowhere. + if _, err := os.Stat(filepath.Join(m.instanceDir(h.ID), "session.json")); !os.IsNotExist(err) { + t.Errorf("the driver staged a session config on the host: %v", err) } } @@ -383,7 +459,7 @@ func TestMicrovmGuestEnvTranslationAndBootstrap(t *testing.T) { // write. func TestMicrovmWritesNoDecryptedEnvironment(t *testing.T) { const secret = "must-not-reach-host-disk" - stateDir := t.TempDir() + stateDir := shortTempDir(t) m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4, StateDir: stateDir}) ctx := context.Background() @@ -391,7 +467,14 @@ func TestMicrovmWritesNoDecryptedEnvironment(t *testing.T) { SessionID: "sess-secret", DialURL: "ws://runner.example.com:8080", Setup: "echo setting up", - Env: map[string]string{"DEPLOY_TOKEN": secret}, + // A withheld create's own env block is CONFIGURATION, and this one + // is a fixture standing in for it. The value below must reach the + // guest over vsock and no file, which is what the walk asserts; the + // token is what makes this an accepted create at all (a create with + // values and no token is refused, and asserted by the twelfth + // contract subtest). + Env: map[string]string{"DEPLOY_TOKEN": secret}, + BootstrapToken: "token_must_not_reach_host_disk", }) if err != nil { t.Fatal(err) @@ -441,18 +524,43 @@ func TestMicrovmWritesNoDecryptedEnvironment(t *testing.T) { t.Fatalf("walk state dir: %v", err) } - // The instance record and the staged guest config carry not even the - // KEYS: the first is the file a runnerd restart reads back, the second is - // bound for the guest, and neither has any business describing an - // environment the driver was told to keep in memory. - for _, name := range []string{"instance.json", "session.json"} { - data, err := os.ReadFile(filepath.Join(m.instanceDir(h.ID), name)) - if err != nil { - t.Fatalf("read %s: %v", name, err) + // The bootstrap token is under the same rule as a value and is checked + // separately, because it is a capability and not just a string: it + // travels from the create into the guest's boot configuration over the + // vsock conn, and a copy of it in any host file would be a credential + // somebody with a shell on a shared host could spend. + err = filepath.WalkDir(stateDir, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err } - if strings.Contains(string(data), "DEPLOY_TOKEN") { - t.Errorf("%s names a key from the session's decrypted environment:\n%s", name, data) + info, ierr := d.Info() + if ierr != nil || info.Size() > 1<<20 { + return ierr } + data, rerr := os.ReadFile(path) + if rerr != nil { + return rerr + } + if strings.Contains(string(data), "token_must_not_reach_host_disk") { + t.Errorf("%s carries the session's bootstrap token", path) + } + return nil + }) + if err != nil { + t.Fatalf("walk state dir for the token: %v", err) + } + + // The instance record carries not even the KEYS: it is the file a runnerd + // restart reads back, and it has no business describing an environment + // the driver was told to keep in memory. There is no staged guest config + // beside it any more — the configuration goes over vsock and is written + // nowhere at all. + data, err := os.ReadFile(filepath.Join(m.instanceDir(h.ID), "instance.json")) + if err != nil { + t.Fatalf("read instance.json: %v", err) + } + if strings.Contains(string(data), "DEPLOY_TOKEN") { + t.Errorf("instance.json names a key from the session's environment:\n%s", data) } } @@ -460,11 +568,11 @@ func TestMicrovmWritesNoDecryptedEnvironment(t *testing.T) { // decision: having refused to write the environment down, the driver says so // rather than boot a guest with an empty one. func TestMicrovmColdResumeAfterRestartRefuses(t *testing.T) { - stateDir := t.TempDir() + stateDir := shortTempDir(t) m1, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4, StateDir: stateDir}) ctx := context.Background() - h, err := m1.Create(ctx, Spec{SessionID: "sess-resume", Env: map[string]string{"TOKEN": "v"}}) + h, err := m1.Create(ctx, Spec{SessionID: "sess-resume", Env: map[string]string{"TOKEN": "v"}, BootstrapToken: "token_example"}) if err != nil { t.Fatal(err) } @@ -473,10 +581,11 @@ func TestMicrovmColdResumeAfterRestartRefuses(t *testing.T) { } m2, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4, StateDir: stateDir}) + m2.SetHost(&stubMicrovmHost{}) if _, err := m2.Resume(ctx, h.ID); err == nil { - t.Fatal("cold resume across a restart succeeded; it would have booted a guest with no environment at all") - } else if !strings.Contains(err.Error(), "vsock") { - t.Errorf("error = %q, want it to name the follow-up work (vsock bootstrap token)", err) + t.Fatal("cold resume across a restart succeeded; it would have booted a guest that is never told what it is") + } else if !strings.Contains(err.Error(), "held in memory only") { + t.Errorf("error = %q, want it to name why the configuration is gone", err) } // And the session is still there to be resumed once that lands. if st := listedState(t, m2, "sess-resume"); st != StateSuspended { @@ -553,6 +662,7 @@ func TestMicrovmSnapshotRefAssociationAndStrip(t *testing.T) { "SECRET_KEY": "super_secret_value", "PUBLIC_KEY": "public_value", }, + BootstrapToken: "token_example", }) if err != nil { t.Fatal(err) @@ -796,7 +906,7 @@ func TestMicrovmStateReconciliation(t *testing.T) { // TestMicrovmPrepullNeverFabricatesAnImage is finding 6's second half: a ref // this host cannot boot is an error, not an empty file and a recorded pull. func TestMicrovmPrepullNeverFabricatesAnImage(t *testing.T) { - stateDir := t.TempDir() + stateDir := shortTempDir(t) m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 2, StateDir: stateDir}) ctx := context.Background() @@ -844,7 +954,7 @@ func TestMicrovmCreateRefusesAnAbsentImage(t *testing.T) { func TestMicrovmRecordsSnapshotStrips(t *testing.T) { m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 2}) ctx := context.Background() - h, err := m.Create(ctx, Spec{SessionID: "mvm-sess-s", Env: map[string]string{"TOKEN": "v"}}) + h, err := m.Create(ctx, Spec{SessionID: "mvm-sess-s", Env: map[string]string{"TOKEN": "v"}, BootstrapToken: "token_example"}) if err != nil { t.Fatal(err) } @@ -927,7 +1037,7 @@ func mustSanitizeRef(t *testing.T, ref string) string { // not produce a silly path, it produces a real one outside the state // directory that RemoveWorkspace would then delete. func TestMicrovmHostileNamesNeverBecomePaths(t *testing.T) { - stateDir := t.TempDir() + stateDir := shortTempDir(t) m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4, StateDir: stateDir}) ctx := context.Background() @@ -981,7 +1091,7 @@ func TestMicrovmHostileNamesNeverBecomePaths(t *testing.T) { // ".." exactly as they are, so a ref of ".." used to name the parent of the // refs directory and a commit wrote its manifest one level up. func TestMicrovmHostileSnapshotRefsAreRefused(t *testing.T) { - stateDir := t.TempDir() + stateDir := shortTempDir(t) m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4, StateDir: stateDir}) ctx := context.Background() @@ -1077,7 +1187,7 @@ func (c *countingResumeEngine) Resume(ctx context.Context, id string) error { // which would leak a slot and send the next Resume down the warm path into a // VM that is no longer there. func TestMicrovmReconcileDoesNotClobberAConcurrentSuspend(t *testing.T) { - stateDir := t.TempDir() + stateDir := shortTempDir(t) gate := make(chan struct{}) m, _ := testMicrovm(t, MicrovmOpts{ TotalSlots: 4, diff --git a/internal/driver/microvm_vsock.go b/internal/driver/microvm_vsock.go new file mode 100644 index 00000000..eab8948b --- /dev/null +++ b/internal/driver/microvm_vsock.go @@ -0,0 +1,486 @@ +// internal/driver/microvm_vsock.go +// +// virtio-vsock: the single host-to-guest control channel a microVM session +// has, and the whole of how one is configured (ADR-0003 §2.7 item 2, and +// docs/design/2026-09-20-microvm-bootstrap-token-and-vsock.md §4). +// +// It replaces two things the driver spike had and this driver deliberately +// does not: MMDS, which answers unauthenticated HTTP at the one address the +// host firewall exists to drop, and a host-side session.json, which put a +// tenant's configuration on a shared host's disk. Nothing this file writes to +// disk is a value: the only file it creates is a unix socket. +// +// This is the one place the driver layer depends on protocol/runner, and it +// is deliberate rather than a leak of the control-plane vocabulary this +// package otherwise keeps out (see driver.Spec and driver.RepoSpec, which +// have their own twins for exactly that reason). runner.BootConfig is not a +// runner-plane message: it is the host-to-GUEST schema, and the design note +// pins it beside runner.Spec because every field of it is a field of the +// create it came from. Somebody has to depend on it, and a second spelling +// of it here would be a session configured with something the control plane +// never dispatched. +package driver + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log" + "net" + "os" + "path/filepath" + "slices" + "strconv" + "sync" + "time" + + "github.com/tokencanopy/rainier/internal/relay" + "github.com/tokencanopy/rainier/protocol/runner" +) + +const ( + // guestCID is the guest's context id. 3 is the value Firecracker's own + // documentation uses; the host is always HOST_CID, which is 2. + guestCID = 3 + // guestControlPort is the ONE port this design uses, and there is not a + // second one. internal/relay already multiplexes configuration, the + // terminal stream, the session RPC and the lifecycle handshake over a + // single conn — that is what the package is for — so three ports would be + // three sockets carrying what one already carries, with three teardowns + // to get wrong. + // + // A guest connection to host port N is forwarded by Firecracker to an + // AF_UNIX socket at "_N", which is why the host listens on a + // path and never calls connect: the host never sends CONNECT. + guestControlPort = 1024 + // unixPathMax is the bound on a sockaddr_un path: Linux's sun_path is 108 + // bytes including its NUL terminator. It is checked here rather than left + // to bind(2) because the path is composed from an operator's + // --microvm-state-dir, and "invalid argument" on a socket is not an error + // anybody can act on, where naming the limit and the path is. + unixPathMax = 107 + // bootConfigWriteTimeout bounds the ONE write this driver makes onto a + // guest connection, and it exists because the thing written is large and + // the peer is untrusted. + // + // A boot configuration carries two scripts of up to MaxSetupBytes each + // (driver.go), so it can reach well over a megabyte — far past any socket + // buffer. A guest that connects and never reads would park an unbounded + // write forever, which is a session's control channel held open by + // whoever dialled it first. Ten seconds is orders of magnitude more than + // a guest that IS reading needs, and it is finite. + // + // relay.NetConn answers an expired write context by CLOSING the conn, + // which is exactly the right answer here: a guest that will not take its + // own configuration has nothing further to say on this connection. + bootConfigWriteTimeout = 10 * time.Second +) + +// MicrovmHost is what the microVM driver needs from the runner above it. +// +// There are exactly two things, and both are things only the runner can do. +// A guest's connection has to become a relay Hub keyed by the session — which +// means the registry, the boot epoch, the control routing and the event +// callbacks, all of which live in runnerd and none of which belong in a +// driver. And a cold resume needs a fresh bootstrap token, which means a +// session RPC upward to the control plane on the runner's own connection. +// +// nil is a real state and not an unconfigured one: the local dev surface and +// the driver contract suite have no runner above them, and a driver with no +// host simply has no guest channel and refuses a cold resume that would need +// one, rather than inventing either. +type MicrovmHost interface { + // GuestConnected hands over one guest's control connection, already + // carrying its boot configuration as the first frame it will read. The + // implementation builds the session's relay hub over it exactly as the + // WebSocket register path does. + // + // It must not block: the driver calls it from its accept loop. + GuestConnected(sessionID string, conn relay.Conn) + // MintSessionBootstrap asks the control plane for a fresh single-use + // token for sessionID, which is what a cold resume boots with. The + // session id is the runner's own — the control plane answers from the row + // its placement guard read, never from anything in the request. + MintSessionBootstrap(ctx context.Context, sessionID string) (string, error) +} + +var ( + _ CapabilityDriver = (*Microvm)(nil) + _ HostedDriver = (*Microvm)(nil) +) + +// Capabilities is what the runner announces on this driver's behalf, and it +// is the fence rather than a hint: the control plane withholds an +// environment's decrypted secret values from a placement whose runner +// announced microvm.v1, and this driver refuses a create that carries them +// anyway. The claim and the refusal are two halves of one promise, so they +// are read from one place. +func (m *Microvm) Capabilities() []string { return []string{runner.CapabilityMicrovmV1} } + +// SetHost installs the runner above this driver. It is a setter rather than a +// field on MicrovmOpts because the runner is composed OVER the driver +// (runnerd.New takes one), so the two cannot both be constructed first. +// +// Called once, before the driver serves anything. +func (m *Microvm) SetHost(h MicrovmHost) { + m.mu.Lock() + defer m.mu.Unlock() + m.host = h +} + +func (m *Microvm) currentHost() MicrovmHost { + m.mu.Lock() + defer m.mu.Unlock() + return m.host +} + +// vsockPaths returns the Firecracker uds_path for one boot of an instance and +// the socket the guest's port-1024 connections are forwarded to. +// +// The path is per BOOT, not per instance: Firecracker's own documentation +// warns that one uds_path cannot be multiplexed across VMs, and a cold resume +// is a new VM. boot is the instance's launch counter. +func (m *Microvm) vsockPaths(id string, boot int) (udsPath, listenPath string, err error) { + if err := checkPathSegment("instance id", id); err != nil { + return "", "", err + } + udsPath = filepath.Join(m.instanceDir(id), "v"+strconv.Itoa(boot)+".sock") + listenPath = udsPath + "_" + strconv.Itoa(guestControlPort) + if len(listenPath) > unixPathMax { + return "", "", fmt.Errorf( + "microvm: the vsock socket path %q is %d bytes, over the %d-byte kernel limit; "+ + "a shorter --microvm-state-dir is the fix, because a truncated path is a guest that boots and is never configured", + listenPath, len(listenPath), unixPathMax) + } + return udsPath, listenPath, nil +} + +// guestChannel is one instance's host end of the vsock control channel: the +// listener Firecracker forwards the guest's port-1024 connections to, and the +// boot configuration whoever connects is handed as their first frame. +// +// The configuration is held HERE, in memory, and nowhere else. It carries the +// bootstrap token, and §2.7 item 1's whole point is that no part of a +// session's configuration reaches a shared host's disk — which is why this +// struct has no persisted twin and why a runnerd restart loses it (see +// (*Microvm).Resume). +type guestChannel struct { + listener net.Listener + // listenPath is "_1024", the socket this end serves; udsPath is + // the one FIRECRACKER binds when it is told about the device. Both are + // held because both have to be removed: the second is not this process's + // to create, but a VM that died leaves it behind, and a later boot that + // found it would fail its PUT /vsock with "address already in use" for a + // path nothing is serving. + listenPath string + udsPath string + // boot is the configuration whoever connects is handed. It is written + // once, at construction, and never again: a cold resume mints a new + // token and gets a new CHANNEL, because the socket path is per-boot too. + boot runner.BootConfig + + mu sync.Mutex + // closed makes the accept loop's exit quiet: a listener closed by + // teardown reports an error like any other, and logging that as a + // failure would put a line in an operator's log for every session that + // ends normally. + closed bool + // served records that this boot generation's ONE guest connection has + // been taken. /dev/vsock is world-accessible inside an ordinary guest, + // so every process in the sandbox can dial (2, 1024) — and what the + // first frame carries is the session's whole configuration and a LIVE + // bootstrap token. Serving every connection would hand that to whoever + // asked, as many times as they asked, and let the last one become the + // session's hub. + // + // So the model is the design note's: one guest-initiated connection per + // boot (§4). The first is sessiond; every later one is refused and closed + // having received nothing at all. A sessiond that crashed and came back + // is a NEW boot generation — a new VM, a new socket, a new mint (open + // question 2) — and not something to re-serve this token to. + served bool + // conns are the connections this channel is still responsible for. It + // holds at most the one served guest, and it exists so close() can end + // it: a Destroy that closed only the listener would leave a wedged + // boot-config write (a guest that never drains) holding a goroutine and + // a socket for as long as its deadline, with nothing able to interrupt + // it. + conns map[net.Conn]struct{} +} + +// claim reserves this boot generation's one guest connection for c and takes +// responsibility for closing it. It reports false for a second connection and +// for one that arrived after teardown — in both cases the caller closes c +// without writing a byte to it. +func (g *guestChannel) claim(c net.Conn) bool { + g.mu.Lock() + defer g.mu.Unlock() + if g.closed || g.served { + return false + } + g.served = true + if g.conns == nil { + g.conns = map[net.Conn]struct{}{} + } + g.conns[c] = struct{}{} + return true +} + +// drop closes a claimed connection this channel will not be handing on, and +// stops tracking it. The claim is NOT released: a boot generation gets one +// attempt, and a guest that could not be configured needs a fresh boot and a +// fresh token rather than a second go at a spent one. +func (g *guestChannel) drop(c net.Conn) { + g.mu.Lock() + delete(g.conns, c) + g.mu.Unlock() + _ = c.Close() +} + +func (g *guestChannel) close() { + g.mu.Lock() + g.closed = true + conns := make([]net.Conn, 0, len(g.conns)) + for c := range g.conns { + conns = append(conns, c) + } + g.conns = nil + g.mu.Unlock() + if g.listener != nil { + _ = g.listener.Close() + } + // The accepted connection goes too, and it is what makes Destroy + // unblockable: a boot-config write into a guest that is not draining is + // only interrupted by the conn dying. + for _, c := range conns { + _ = c.Close() + } + // Both paths, and an absent one is not an error: a teardown that runs + // twice is ordinary, and a VM that never started leaves only one of them. + _ = os.Remove(g.listenPath) + _ = os.Remove(g.udsPath) +} + +func (g *guestChannel) isClosed() bool { + g.mu.Lock() + defer g.mu.Unlock() + return g.closed +} + +// openGuestChannel creates the host end of the channel before the VM starts. +// +// Before, deliberately: the guest dials port 1024 as soon as its sessiond is +// up, and Firecracker refuses a guest connection whose "_1024" does +// not exist. A listener created after InstanceStart is a race whose loser is +// a session that boots and is never configured. +func (m *Microvm) openGuestChannel(sessionID, udsPath, listenPath string, cfg runner.BootConfig) (*guestChannel, error) { + if err := os.MkdirAll(filepath.Dir(listenPath), microvmDirMode); err != nil { + return nil, fmt.Errorf("create the guest channel directory: %w", err) + } + // Sockets left behind by a VM that is gone would make this Listen, or + // Firecracker's own bind at PUT /vsock, fail with "address already in + // use" for paths nothing is serving. A launch that failed after the + // device was configured leaves exactly that. A resume no longer reuses + // the failed attempt's number (Resume advances the counter under the + // driver mutex, so no two boots of one instance share a path) — but a + // create is always boot 1, and a record recovered after a runnerd + // restart counts from whatever it was, so neither path gets to assume + // the path it was handed is unused. + _ = os.Remove(listenPath) + _ = os.Remove(udsPath) + ln, err := net.Listen("unix", listenPath) + if err != nil { + return nil, fmt.Errorf("listen on the guest control socket %s: %w", listenPath, err) + } + // 0600: the socket is one tenant's control channel on a shared host, and + // anything that can connect to it can be that session's sandbox. Listen + // creates it with the process umask, which is not a policy this driver + // gets to inherit. + if err := os.Chmod(listenPath, microvmFileMode); err != nil { + _ = ln.Close() + _ = os.Remove(listenPath) + return nil, fmt.Errorf("restrict the guest control socket %s: %w", listenPath, err) + } + g := &guestChannel{listener: ln, listenPath: listenPath, udsPath: udsPath, boot: cfg} + go m.acceptGuests(sessionID, g) + return g, nil +} + +// acceptGuests keeps accepting for the life of the instance. +// +// It keeps accepting even though exactly one connection is ever SERVED, +// because the alternative is worse in both directions: an accept loop that +// stopped would leave later dials queued in the kernel with nobody to refuse +// them, and a loop that served inline would be wedged for good by the first +// guest that connected and did not read (the boot config can exceed a +// megabyte — see bootConfigWriteTimeout). So each connection gets a goroutine +// of its own, and all but the first are refused in it. +func (m *Microvm) acceptGuests(sessionID string, g *guestChannel) { + for { + c, err := g.listener.Accept() + if err != nil { + if !g.isClosed() { + log.Printf("microvm: session %s: the guest control socket stopped accepting: %v", sessionID, err) + } + return + } + go m.serveGuest(sessionID, g, c) + } +} + +// serveGuest hands ONE guest connection its configuration and then hands the +// connection to the runner. +// +// The boot configuration is the FIRST frame on the conn, written here before +// anything else can write to it, which is what makes "nothing has to predate +// the stream" true: the guest reads its whole configuration off the same +// connection it will then serve its terminal and its RPC over. +// +// Every connection after the first is closed having received nothing — not +// the configuration, not the token, not a byte. See guestChannel.served: the +// socket is reachable by every process in the sandbox, and this is the one +// place that decides the first dial is the session's and no other is. +// +// A host that has not been installed closes the connection rather than +// holding it. There is nothing to attach it to, and the claim stays spent: +// this boot has had its one connection. +func (m *Microvm) serveGuest(sessionID string, g *guestChannel, c net.Conn) { + if !g.claim(c) { + // Not an error the operator can act on and not a rarity worth a + // line per occurrence — but it IS somebody in the guest dialling a + // socket that is not theirs, so it is said once per attempt and + // names nothing about the session but its id. + log.Printf("microvm: session %s: refusing a second connection on a control channel that serves one guest per boot", sessionID) + _ = c.Close() + return + } + conn := relay.NetConn(c) + ctx, cancel := context.WithTimeout(context.Background(), bootConfigWriteTimeout) + defer cancel() + if err := writeBootConfig(ctx, conn, g.boot); err != nil { + log.Printf("microvm: session %s: sending the boot configuration: %v", sessionID, err) + g.drop(c) + return + } + host := m.currentHost() + if host == nil { + log.Printf("microvm: session %s: a guest connected but this driver has no runner above it to serve it", sessionID) + g.drop(c) + return + } + host.GuestConnected(sessionID, conn) +} + +// writeBootConfig sends one boot configuration as an ordinary FrameControl — +// the same frame shape every event, request and response on this channel +// uses, so the guest needs no special reader for it. +// +// The context is the caller's own bounded one rather than a create's: this is +// the one write whose failure means the guest is unconfigurable, so +// inheriting a create's context would let a create that has just returned +// cancel the configuration of the guest it started — but leaving it unbounded +// would let a guest that never reads hold the write open forever. See +// bootConfigWriteTimeout. +func writeBootConfig(ctx context.Context, conn relay.Conn, cfg runner.BootConfig) error { + body, err := json.Marshal(cfg) + if err != nil { + // Unreachable — every field is a string, an int, or a slice of them — + // and logged without the error, because a json message quotes what it + // choked on and this value carries the bootstrap token. + return errors.New("the boot configuration could not be encoded") + } + payload, err := json.Marshal(relay.ControlEvent{Kind: relay.KindBootConfig, Payload: body}) + if err != nil { + return errors.New("the boot configuration frame could not be encoded") + } + frame, err := relay.Encode(relay.Frame{Type: relay.FrameControl, Payload: payload}) + if err != nil { + return errors.New("the boot configuration frame could not be encoded") + } + return conn.Write(ctx, frame) +} + +// bootConfigFor composes what a guest is told about itself. +// +// Every field is one the create resolved, carried across unchanged. There are +// two things deliberately NOT in it: any environment secret value (they are +// exactly what the token buys, and they never reach this host), and a dial +// URL (there is nothing to dial — the connection this rides on IS the +// channel, which is why createWithID leaves Spec.DialURL empty on this +// driver). +// +// The proxy URL carries the session's identity as URL userinfo, composed here +// with the same helper the Docker driver uses, so egressd reads one thing +// whichever driver the session is on. +func bootConfigFor(spec Spec) runner.BootConfig { + cfg := runner.BootConfig{ + Protocol: runner.SessionBootstrapProtocolVersion, + SessionID: spec.SessionID, + Cmd: slices.Clone(spec.Cmd), + EgressAllow: slices.Clone(spec.EgressAllow), + Setup: spec.Setup, + SetupTimeoutSec: spec.SetupTimeoutSec, + Init: spec.Init, + InitTimeoutSec: spec.InitTimeoutSec, + GitAuthorName: spec.GitAuthorName, + GitAuthorEmail: spec.GitAuthorEmail, + SecretNames: slices.Clone(spec.SecretNames), + BootstrapToken: spec.BootstrapToken, + } + if len(cfg.Cmd) == 0 { + cfg.Cmd = []string{"/bin/bash"} + } + if spec.ProxyURL != "" { + cfg.ProxyURL = withSessionUserinfo(spec.ProxyURL, spec.SessionID) + // No dial host to exempt, because there is no dial: NO_PROXY is the + // base list and nothing else. + cfg.NoProxy = noProxyFor("") + } + for _, r := range spec.Repos { + cfg.Repos = append(cfg.Repos, runner.RepoSpec{ + Owner: r.Owner, Name: r.Name, BaseBranch: r.BaseBranch, + SessionBranch: r.SessionBranch, Dir: r.Dir, + }) + } + // Env is the NON-SECRET configuration block, which for a microVM create + // is the agent-home path variables and the agent manifest — paths and + // names. Create has already refused any create whose Env carries values + // with no token, and a create WITH a token is one the control plane + // withheld the values from, so what is left here is configuration by + // construction. + if len(spec.Env) > 0 { + cfg.Env = make(map[string]string, len(spec.Env)) + for k, v := range spec.Env { + cfg.Env[k] = v + } + } + return cfg +} + +// refuseUnwithheldEnv is the microVM driver's half of the compatibility +// table's "old controld + new runner" row. +// +// A control plane that predates the bootstrap token dispatches an +// environment's decrypted secrets in Spec.Env and no token. On the Docker +// path that is fine and always has been — the values reach a `docker run` +// argv and no file. On this one it is not: this driver has no argv to hand +// them to, and every channel it does have ends in something that outlives the +// session on a host shared with other tenants. +// +// So it refuses, loudly, naming what the control plane has to be. A refusal +// costs one session; the alternative — writing them somewhere and booting +// anyway — is the exposure §2.7 was written against, and it would be silent. +func refuseUnwithheldEnv(spec Spec) error { + if len(spec.Env) == 0 || spec.BootstrapToken != "" { + return nil + } + return fmt.Errorf( + "microvm: refusing to create session %s: its create carries %d environment value(s) and no bootstrap token, "+ + "which means a control plane older than the session bootstrap exchange "+ + "(docs/design/2026-09-20-microvm-bootstrap-token-and-vsock.md). Not all of those values are necessarily "+ + "secret — a session with a creator carries its agent-home configuration in the same block — but this host "+ + "cannot tell them apart, has no argv to hand any of them to, and will not write one to disk. Upgrade the "+ + "control plane to one that withholds them for a runner announcing %q", + spec.SessionID, len(spec.Env), runner.CapabilityMicrovmV1) +} diff --git a/internal/driver/microvm_vsock_test.go b/internal/driver/microvm_vsock_test.go new file mode 100644 index 00000000..05099751 --- /dev/null +++ b/internal/driver/microvm_vsock_test.go @@ -0,0 +1,641 @@ +// internal/driver/microvm_vsock_test.go +package driver + +import ( + "context" + "encoding/json" + "errors" + "io" + "net" + "os" + "strings" + "testing" + "time" + + "github.com/tokencanopy/rainier/internal/relay" + "github.com/tokencanopy/rainier/protocol/runner" +) + +// dialGuest connects to the host end of a session's vsock control channel the +// way Firecracker's forwarding does: an AF_UNIX connection to +// "_1024". No VM and no KVM are involved — the socket is the part +// of this that runs anywhere, and the part a host harness has to prove is +// that Firecracker forwards to it. +func dialGuest(t *testing.T, m *Microvm, instanceID string, boot int) relay.Conn { + t.Helper() + _, listenPath, err := m.vsockPaths(instanceID, boot) + if err != nil { + t.Fatalf("vsock paths: %v", err) + } + // The driver opens the listener before it launches, so it is there by the + // time Create returns; a short retry covers nothing but scheduling. + deadline := time.Now().Add(2 * time.Second) + for { + c, err := net.Dial("unix", listenPath) + if err == nil { + t.Cleanup(func() { _ = c.Close() }) + return relay.NetConn(c) + } + if time.Now().After(deadline) { + t.Fatalf("dial the guest control socket %s: %v", listenPath, err) + } + time.Sleep(5 * time.Millisecond) + } +} + +// readBootConfig reads the first frame off a guest connection and requires it +// to be the boot configuration. +func readBootConfig(t *testing.T, conn relay.Conn) runner.BootConfig { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + raw, err := conn.Read(ctx) + if err != nil { + t.Fatalf("read the first frame: %v", err) + } + frame, err := relay.Decode(raw) + if err != nil { + t.Fatalf("decode the first frame: %v", err) + } + if frame.Type != relay.FrameControl || frame.AttachID != 0 { + t.Fatalf("the first frame is %+v, want a control frame on attachment 0", frame) + } + var ev relay.ControlEvent + if err := json.Unmarshal(frame.Payload, &ev); err != nil { + t.Fatalf("decode the control event: %v", err) + } + if ev.Kind != relay.KindBootConfig { + t.Fatalf("the first control frame is %q, want %q", ev.Kind, relay.KindBootConfig) + } + var cfg runner.BootConfig + if err := json.Unmarshal(ev.Payload, &cfg); err != nil { + t.Fatalf("decode the boot configuration: %v", err) + } + return cfg +} + +// TestMicrovmSendsTheBootConfigFirst is the whole configuration path in one +// test: a guest connects to the socket Firecracker forwards port 1024 to, and +// the first thing it reads is everything it needs to be a session. +// +// The assertions are on what the GUEST sees, not on what the driver stored, +// because the guest is the only consumer and a field it does not receive is a +// session that boots wrong however neatly the host recorded it. +func TestMicrovmSendsTheBootConfigFirst(t *testing.T) { + m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4}) + host := &stubMicrovmHost{} + m.SetHost(host) + ctx := context.Background() + + h, err := m.Create(ctx, Spec{ + SessionID: "sess-boot", + ProxyURL: "http://proxy.example.invalid:3128", + EgressAllow: []string{"registry.npmjs.org"}, + Setup: "npm ci\n", + SetupTimeoutSec: 600, + Init: "make dev\n", + InitTimeoutSec: 180, + Repos: []RepoSpec{{Owner: "acme", Name: "app", BaseBranch: "main", SessionBranch: "rainier/work", Dir: "app"}}, + GitAuthorName: "example", + GitAuthorEmail: "42+example@users.noreply.github.com", + Cmd: []string{"claude"}, + Env: map[string]string{"CLAUDE_CONFIG_DIR": "/rainier/agents/claude"}, + SecretNames: []string{"DEPLOY_KEY"}, + BootstrapToken: "token_example", + }) + if err != nil { + t.Fatal(err) + } + defer m.Destroy(ctx, h.ID) + + cfg := readBootConfig(t, dialGuest(t, m, h.ID, 1)) + + if cfg.Protocol != runner.SessionBootstrapProtocolVersion { + t.Errorf("protocol = %d, want %d", cfg.Protocol, runner.SessionBootstrapProtocolVersion) + } + // The session id is the HOST's statement, taken from the socket path the + // connection arrived on, and not something the guest asserted. + if cfg.SessionID != "sess-boot" { + t.Errorf("session id = %q, want sess-boot", cfg.SessionID) + } + if len(cfg.Cmd) != 1 || cfg.Cmd[0] != "claude" { + t.Errorf("cmd = %v", cfg.Cmd) + } + if cfg.Setup != "npm ci\n" || cfg.SetupTimeoutSec != 600 || cfg.Init != "make dev\n" || cfg.InitTimeoutSec != 180 { + t.Errorf("the boot chain did not survive: %+v", cfg) + } + if len(cfg.Repos) != 1 || cfg.Repos[0].Dir != "app" || cfg.Repos[0].SessionBranch != "rainier/work" { + t.Errorf("repos = %+v", cfg.Repos) + } + if cfg.GitAuthorName != "example" || cfg.GitAuthorEmail != "42+example@users.noreply.github.com" { + t.Errorf("git identity = %q <%q>", cfg.GitAuthorName, cfg.GitAuthorEmail) + } + if cfg.Env["CLAUDE_CONFIG_DIR"] != "/rainier/agents/claude" { + t.Errorf("the agent-home configuration did not reach the guest: %v", cfg.Env) + } + if len(cfg.SecretNames) != 1 || cfg.SecretNames[0] != "DEPLOY_KEY" || cfg.BootstrapToken != "token_example" { + t.Errorf("the bootstrap pair did not reach the guest: names=%v", cfg.SecretNames) + } + // The proxy carries the session's identity as URL userinfo, exactly as + // the Docker driver composes it, so egressd reads one thing on both + // paths. And NO_PROXY has no dial host to exempt, because there is no + // dial: the conn this rode in on IS the channel. + if !strings.Contains(cfg.ProxyURL, "sess-boot:") { + t.Errorf("proxy url = %q, want the session's identity as userinfo", cfg.ProxyURL) + } + if cfg.NoProxy == "" || strings.Contains(cfg.NoProxy, "runner") { + t.Errorf("no_proxy = %q, want the base list with no dial host in it", cfg.NoProxy) + } + + // And the connection was handed to the runner above the driver, which is + // what turns it into the session's relay hub. + waitFor(t, func() bool { + for _, s := range host.connected() { + if s == "sess-boot" { + return true + } + } + return false + }, "the guest connection was never handed to the runner") +} + +// TestMicrovmGuestChannelIsPrivateToItsSession pins the mode on the one file +// this path creates. The socket is a session's control channel on a host +// shared with other tenants, and anything that can connect to it can be that +// session's sandbox. +func TestMicrovmGuestChannelIsPrivateToItsSession(t *testing.T) { + m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4}) + m.SetHost(&stubMicrovmHost{}) + ctx := context.Background() + + h, err := m.Create(ctx, Spec{SessionID: "sess-mode"}) + if err != nil { + t.Fatal(err) + } + defer m.Destroy(ctx, h.ID) + + _, listenPath, err := m.vsockPaths(h.ID, 1) + if err != nil { + t.Fatal(err) + } + info, err := os.Stat(listenPath) + if err != nil { + t.Fatalf("stat the guest control socket: %v", err) + } + if info.Mode().Perm() != microvmFileMode { + t.Errorf("the guest control socket has mode %04o, want %04o", info.Mode().Perm(), microvmFileMode) + } +} + +// TestMicrovmColdResumeMintsAFreshTokenAndSocket is §4.4's resume, in the two +// things that make it safe: the token the previous boot was handed is +// single-use and fenced, so a new VM gets a new one; and Firecracker's own +// documentation warns that one uds_path cannot be multiplexed across VMs, so +// a new VM gets a new socket. +func TestMicrovmColdResumeMintsAFreshTokenAndSocket(t *testing.T) { + m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4}) + host := &stubMicrovmHost{} + m.SetHost(host) + ctx := context.Background() + + h, err := m.Create(ctx, Spec{SessionID: "sess-resume", BootstrapToken: "token_from_the_create"}) + if err != nil { + t.Fatal(err) + } + defer m.Destroy(ctx, h.ID) + if got := readBootConfig(t, dialGuest(t, m, h.ID, 1)).BootstrapToken; got != "token_from_the_create" { + t.Fatalf("the first boot got token %q, want the create's", got) + } + + if err := m.Suspend(ctx, h.ID, false); err != nil { + t.Fatal(err) + } + // The cold-parked VM's socket goes with it: a path nothing serves is one + // more file on a shared host describing a session that is not running. + _, firstPath, _ := m.vsockPaths(h.ID, 1) + if _, err := os.Stat(firstPath); !os.IsNotExist(err) { + t.Errorf("the cold-parked session left its control socket behind: %v", err) + } + + restarted, err := m.Resume(ctx, h.ID) + if err != nil { + t.Fatalf("cold resume: %v", err) + } + if !restarted { + t.Error("a cold resume reported no restart") + } + if host.mintCount() != 1 { + t.Fatalf("the cold resume made %d mint(s), want exactly 1", host.mintCount()) + } + + // A new socket, and the new token on it. + cfg := readBootConfig(t, dialGuest(t, m, h.ID, 2)) + if cfg.BootstrapToken == "token_from_the_create" { + t.Fatal("the resumed guest was handed the previous boot's token, which is spent and fenced") + } + if cfg.BootstrapToken == "" { + t.Fatal("the resumed guest was handed no token at all") + } + if cfg.SessionID != "sess-resume" { + t.Errorf("the resumed guest's session id = %q", cfg.SessionID) + } +} + +// TestAFailedBootLeavesNoSocketBehind is finding 5 of this branch's own +// review, and it is the difference between a session that can be resumed +// again and one that cannot. +// +// Firecracker binds "" itself at PUT /vsock; the driver binds +// "_1024". A launch that fails after the device was configured +// leaves the first behind, on a path a later boot can meet again — a create +// is always boot 1, and a record recovered after a runnerd restart counts +// from whatever it was — and Firecracker's bind would then fail EADDRINUSE +// on a socket nothing is serving, forever. So a failed attempt takes both +// paths with it. +func TestAFailedBootLeavesNoSocketBehind(t *testing.T) { + m, sim := testMicrovm(t, MicrovmOpts{TotalSlots: 4}) + host := &stubMicrovmHost{} + m.SetHost(host) + ctx := context.Background() + + h, err := m.Create(ctx, Spec{SessionID: "sess-stale", BootstrapToken: "token_example"}) + if err != nil { + t.Fatal(err) + } + defer m.Destroy(ctx, h.ID) + udsPath, listenPath, err := m.vsockPaths(h.ID, 1) + if err != nil { + t.Fatal(err) + } + // Stand in for what Firecracker leaves at PUT /vsock: the simulated + // engine does not bind anything, so the file is created here and the + // assertion is about whether the DRIVER cleans it up. + if err := os.WriteFile(udsPath, nil, microvmFileMode); err != nil { + t.Fatal(err) + } + if err := m.Suspend(ctx, h.ID, false); err != nil { + t.Fatal(err) + } + for _, p := range []string{udsPath, listenPath} { + if _, err := os.Stat(p); !os.IsNotExist(err) { + t.Errorf("a cold park left %s behind: %v", p, err) + } + } + + // And a resume whose launch fails must not poison the path it was going + // to use either: the next one recomputes the same one. + sim.FailLaunch(errors.New("the VMM refused to start")) + if _, err := m.Resume(ctx, h.ID); err == nil { + t.Fatal("a resume whose launch fails reported success") + } + _, secondListen, err := m.vsockPaths(h.ID, 2) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(secondListen); !os.IsNotExist(err) { + t.Errorf("a failed resume left %s behind", secondListen) + } + sim.FailLaunch(nil) + if _, err := m.Resume(ctx, h.ID); err != nil { + t.Fatalf("the resume after a failed one: %v", err) + } +} + +// TestMicrovmColdResumeFailsWhenTheTokenCannotBeMinted is the fail-closed +// half: a resume that cannot get a token reports that, rather than booting a +// guest that will ask for its secrets and be refused. +func TestMicrovmColdResumeFailsWhenTheTokenCannotBeMinted(t *testing.T) { + m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4}) + host := &stubMicrovmHost{mintErr: errors.New("this runner has no controld connection")} + m.SetHost(host) + ctx := context.Background() + + h, err := m.Create(ctx, Spec{SessionID: "sess-nomint", BootstrapToken: "token_example"}) + if err != nil { + t.Fatal(err) + } + defer m.Destroy(ctx, h.ID) + if err := m.Suspend(ctx, h.ID, false); err != nil { + t.Fatal(err) + } + + if _, err := m.Resume(ctx, h.ID); err == nil { + t.Fatal("a cold resume with no token succeeded") + } else if !strings.Contains(err.Error(), "no controld connection") { + t.Errorf("error = %q, want the mint's own reason", err) + } + // And the session is still parked, so a later resume can try again. + if st := listedState(t, m, "sess-nomint"); st != StateSuspended { + t.Errorf("a refused resume changed the session's state to %q", st) + } +} + +// TestMicrovmRefusesSecretValuesWithNoToken is the driver's own half of the +// compatibility table's "old controld + new runner" row. The shared contract +// asserts the same refusal for any driver announcing microvm.v1; this one +// asserts the message says what an operator has to change. +func TestMicrovmRefusesSecretValuesWithNoToken(t *testing.T) { + m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4}) + m.SetHost(&stubMicrovmHost{}) + ctx := context.Background() + + _, err := m.Create(ctx, Spec{ + SessionID: "sess-unwithheld", + Env: map[string]string{"DEPLOY_KEY": "value_must_not_be_accepted"}, + }) + if err == nil { + t.Fatal("a create carrying secret values with no token was accepted") + } + for _, want := range []string{"bootstrap token", "microvm.v1", "sess-unwithheld"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the refusal = %q, want it to name %q", err, want) + } + } + if strings.Contains(err.Error(), "value_must_not_be_accepted") { + t.Errorf("the refusal quotes the value it refused: %q", err) + } + // Nothing was reserved, created, or left behind: the check runs before + // the slot is taken. + if used, _, _ := m.Capacity(ctx); used != 0 { + t.Errorf("a refused create left %d slot(s) occupied", used) + } + // And an empty env with no token is a perfectly ordinary create: the + // refusal is about VALUES, not about the field existing. + if _, err := m.Create(ctx, Spec{SessionID: "sess-plain"}); err != nil { + t.Fatalf("a create with no env and no token was refused: %v", err) + } +} + +// TestMicrovmVsockPathRefusesAnOverlongStateDir pins the one configuration +// mistake that would otherwise surface as "invalid argument" from bind(2). +func TestMicrovmVsockPathRefusesAnOverlongStateDir(t *testing.T) { + long := "/" + strings.Repeat("d", 120) + m := &Microvm{opts: MicrovmOpts{StateDir: long}} + if _, _, err := m.vsockPaths("mvm-1", 1); err == nil { + t.Fatal("an overlong state directory produced a socket path") + } else if !strings.Contains(err.Error(), "--microvm-state-dir") { + t.Errorf("error = %q, want it to name the flag an operator can change", err) + } +} + +// waitFor polls cond until it holds or the test gives up. It exists for the +// one assertion here that crosses a goroutine — the driver hands a guest +// connection to the runner from its accept loop. +func waitFor(t *testing.T, cond func() bool, msg string) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for { + if cond() { + return + } + if time.Now().After(deadline) { + t.Fatal(msg) + } + time.Sleep(5 * time.Millisecond) + } +} + +// dialRawGuest is dialGuest without the relay framing: the tests below care +// about BYTES on the socket — whether any arrive at all, and whether reading +// them is what the far end is waiting for — which a relay.Conn's reader +// hides. +func dialRawGuest(t *testing.T, m *Microvm, instanceID string, boot int) net.Conn { + t.Helper() + _, listenPath, err := m.vsockPaths(instanceID, boot) + if err != nil { + t.Fatalf("vsock paths: %v", err) + } + deadline := time.Now().Add(2 * time.Second) + for { + c, err := net.Dial("unix", listenPath) + if err == nil { + t.Cleanup(func() { _ = c.Close() }) + return c + } + if time.Now().After(deadline) { + t.Fatalf("dial the guest control socket %s: %v", listenPath, err) + } + time.Sleep(5 * time.Millisecond) + } +} + +// oversizedSpec is a create whose boot configuration cannot fit in a socket +// buffer: two scripts at the driver's own cap, so the JSON the host writes is +// comfortably past a megabyte. A guest that does not read one of these is a +// guest the write is genuinely parked on, which is the condition the two +// tests below are about. +func oversizedSpec(sessionID string) Spec { + script := strings.Repeat("x", MaxSetupBytes) + return Spec{ + SessionID: sessionID, + Setup: script, + Init: script, + SecretNames: []string{"DEPLOY_KEY"}, + BootstrapToken: "token_example", + } +} + +// TestASilentGuestDoesNotWedgeTheControlChannel is review round 2, finding 1. +// +// The accept loop used to write the boot configuration inline. That +// configuration can exceed a megabyte (two scripts at MaxSetupBytes), so the +// first process in the guest to connect and then NOT read parked the loop for +// good: every later connection sat in the kernel's backlog accepted by nobody, +// the real sessiond's 30-second wait for its configuration expired, and the +// session died — with no way for a Destroy to break the write, because the +// channel tracked no connections at all. +// +// So: a silent guest holds nothing but its own connection. A second +// connection is still ANSWERED (refused, per the one-guest-per-boot rule +// below — but answered, which is the part that proves the loop is alive), and +// Destroy returns. +func TestASilentGuestDoesNotWedgeTheControlChannel(t *testing.T) { + m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4}) + m.SetHost(&stubMicrovmHost{}) + ctx := context.Background() + + h, err := m.Create(ctx, oversizedSpec("sess-silent")) + if err != nil { + t.Fatal(err) + } + defer m.Destroy(ctx, h.ID) + + // Connect, take ONE byte, and then stop reading. That byte is what makes + // this deterministic rather than a race with the dial below: it proves + // this connection is the one the channel claimed and that its + // configuration is already on its way. The remaining megabyte-odd has + // nowhere to go, so the host's write is now parked for good. + silent := dialRawGuest(t, m, h.ID, 1) + defer silent.Close() + if err := silent.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil { + t.Fatal(err) + } + var one [1]byte + if _, err := io.ReadFull(silent, one[:]); err != nil { + t.Fatalf("the first guest was sent nothing at all: %v", err) + } + + // The accept loop must still be serving. A second connection is closed + // with nothing on it, and — this is the assertion — it gets that answer + // promptly rather than after the silent guest's write deadline. + second := dialRawGuest(t, m, h.ID, 1) + if err := second.SetReadDeadline(time.Now().Add(3 * time.Second)); err != nil { + t.Fatal(err) + } + var buf [1]byte + n, err := second.Read(buf[:]) + if n != 0 { + t.Fatalf("a second connection was sent %d byte(s); it must receive nothing at all", n) + } + if !errors.Is(err, io.EOF) { + t.Fatalf("a second connection ended with %v, want EOF — it must be closed, not left waiting", err) + } + + // And the session can still be torn down with that write still parked. + done := make(chan error, 1) + go func() { done <- m.Destroy(ctx, h.ID) }() + select { + case err := <-done: + if err != nil { + t.Fatalf("Destroy: %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("Destroy never returned while a guest held the control channel open") + } +} + +// TestOnlyOneGuestIsServedPerBoot is review round 2, finding 2. +// +// /dev/vsock is world-accessible inside an ordinary guest, so every process +// in the sandbox can dial (2, 1024). The first frame on that connection is +// the session's whole configuration AND a live, single-use bootstrap token — +// so a host that served every connection handed both to whoever asked, as +// often as they asked, and let the last one become the session's hub. +// +// The design's model is one guest-initiated connection per boot (§4). This +// pins it: the first connection is served, every later one receives NOTHING +// and is closed, the hub the first produced is not displaced, and a Destroy +// afterwards still cleans the socket up. +func TestOnlyOneGuestIsServedPerBoot(t *testing.T) { + m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4}) + host := &stubMicrovmHost{} + m.SetHost(host) + ctx := context.Background() + + h, err := m.Create(ctx, Spec{ + SessionID: "sess-once", SecretNames: []string{"DEPLOY_KEY"}, + BootstrapToken: "token_example", + }) + if err != nil { + t.Fatal(err) + } + defer m.Destroy(ctx, h.ID) + + // The first connection is sessiond, and it gets everything. + if got := readBootConfig(t, dialGuest(t, m, h.ID, 1)).BootstrapToken; got != "token_example" { + t.Fatalf("the first guest was handed token %q, want the create's", got) + } + waitFor(t, func() bool { return host.connCount() == 1 }, + "the first guest connection never reached the runner") + + // Every later one is a process inside the sandbox dialling a socket that + // is not its to dial. It reads EOF, having been sent nothing. + for i := 0; i < 2; i++ { + other := dialRawGuest(t, m, h.ID, 1) + if err := other.SetReadDeadline(time.Now().Add(3 * time.Second)); err != nil { + t.Fatal(err) + } + b, err := io.ReadAll(other) + if len(b) != 0 { + t.Fatalf("connection %d received %d byte(s) of a session's configuration", i+2, len(b)) + } + if err != nil { + t.Fatalf("connection %d was left open rather than closed: %v", i+2, err) + } + } + + // And it did not become the session's hub: the runner was handed exactly + // one connection, the first. + if n := host.connCount(); n != 1 { + t.Fatalf("the runner was handed %d guest connection(s) for one boot, want 1", n) + } + + // A Destroy after all that still tears the channel down. + if err := m.Destroy(ctx, h.ID); err != nil { + t.Fatalf("Destroy: %v", err) + } + _, listenPath, err := m.vsockPaths(h.ID, 1) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(listenPath); !os.IsNotExist(err) { + t.Errorf("Destroy left the guest control socket behind: %v", err) + } +} + +// TestTwoConcurrentColdResumesDoNotShareASocket is review round 2, finding 7. +// +// The boot counter used to be read with the driver mutex released, so two +// cold resumes of one instance could compute the same next boot number — and +// openGuestChannel unlinks before it listens, so the second would remove the +// first's live socket out from under a VM that had already been told about +// it. That session boots and is never configured, which is the one outcome +// this whole path exists to prevent. +// +// One resume wins; the other is refused; the winner's socket is there +// afterwards. +func TestTwoConcurrentColdResumesDoNotShareASocket(t *testing.T) { + m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4}) + m.SetHost(&stubMicrovmHost{}) + ctx := context.Background() + + h, err := m.Create(ctx, Spec{SessionID: "sess-race", BootstrapToken: "token_example"}) + if err != nil { + t.Fatal(err) + } + defer m.Destroy(ctx, h.ID) + if err := m.Suspend(ctx, h.ID, false); err != nil { + t.Fatal(err) + } + + type outcome struct { + restarted bool + err error + } + out := make(chan outcome, 2) + start := make(chan struct{}) + for i := 0; i < 2; i++ { + go func() { + <-start + restarted, err := m.Resume(ctx, h.ID) + out <- outcome{restarted, err} + }() + } + close(start) + + restarts, refusals := 0, 0 + for i := 0; i < 2; i++ { + o := <-out + switch { + case o.err == nil && o.restarted: + restarts++ + case o.err != nil && strings.Contains(o.err.Error(), "already in flight"): + refusals++ + default: + t.Fatalf("a concurrent resume returned (%v, %v), want a restart or an in-flight refusal", o.restarted, o.err) + } + } + if restarts != 1 || refusals != 1 { + t.Fatalf("%d restart(s) and %d refusal(s) from two concurrent resumes, want 1 and 1", restarts, refusals) + } + + // The winner's guest channel is live: a guest can still be configured, + // which is what the losing unlink used to take away. + _, _, err = m.vsockPaths(h.ID, 2) + if err != nil { + t.Fatal(err) + } + if got := readBootConfig(t, dialGuest(t, m, h.ID, 2)).SessionID; got != "sess-race" { + t.Fatalf("the resumed guest read session %q off its channel", got) + } +} diff --git a/internal/relay/bootconfig_test.go b/internal/relay/bootconfig_test.go new file mode 100644 index 00000000..d606810a --- /dev/null +++ b/internal/relay/bootconfig_test.go @@ -0,0 +1,88 @@ +package relay + +import ( + "bytes" + "encoding/json" + "strings" + "testing" +) + +// TestColdIsAdditiveOnTheSuspendNotice is the compatibility promise for the +// one field the cold-suspend handshake adds. runnerd runs on the host and +// sessiond ships inside the session image, and a session keeps the sessiond +// it booted with for life — so the suspend notice a sandbox already reads +// must be byte-identical after this, and a sandbox that never learns the +// field must read a plain "suspending". +func TestColdIsAdditiveOnTheSuspendNotice(t *testing.T) { + warm, err := json.Marshal(ControlEvent{Kind: KindSuspending, ID: 4}) + if err != nil { + t.Fatal(err) + } + if string(warm) != `{"kind":"suspending","id":4}` { + t.Fatalf("a warm suspend notice = %s", warm) + } + if strings.Contains(string(warm), `"cold"`) { + t.Fatalf("a warm suspend notice leaked a cold key: %s", warm) + } + + cold, err := json.Marshal(ControlEvent{Kind: KindSuspending, ID: 5, Cold: true}) + if err != nil { + t.Fatal(err) + } + if string(cold) != `{"kind":"suspending","id":5,"cold":true}` { + t.Fatalf("a cold suspend notice = %s", cold) + } + + // An older sandbox decodes it as the plain notice it has always answered: + // unknown JSON keys are dropped, so it quiesces its execs and acks, and + // the host-side unmount happens without its help. + var old struct { + Kind string `json:"kind"` + ID uint64 `json:"id,omitempty"` + } + if err := json.Unmarshal(cold, &old); err != nil { + t.Fatalf("an old build could not decode a cold notice at all: %v", err) + } + if old.Kind != KindSuspending || old.ID != 5 { + t.Fatalf("an old build decoded a cold notice as %+v", old) + } + + var back ControlEvent + if err := json.Unmarshal(cold, &back); err != nil { + t.Fatal(err) + } + if !back.Cold || back.ID != 5 { + t.Fatalf("cold round trip mangled: %+v", back) + } +} + +// TestBootConfigKindRidesAnOrdinaryControlFrame pins that the boot +// configuration needs no new frame type and no new envelope: it is a +// FrameControl with AttachID 0 carrying a ControlEvent whose payload relay +// does not read, exactly like every request and response already on this +// channel. +func TestBootConfigKindRidesAnOrdinaryControlFrame(t *testing.T) { + if KindBootConfig != "boot_config" { + t.Fatalf("boot config kind = %q", KindBootConfig) + } + ev, err := json.Marshal(ControlEvent{Kind: KindBootConfig, + Payload: json.RawMessage(`{"protocol":1,"session_id":"sess_example"}`)}) + if err != nil { + t.Fatal(err) + } + const wantEvent = `{"kind":"boot_config","payload":{"protocol":1,"session_id":"sess_example"}}` + if string(ev) != wantEvent { + t.Fatalf("a boot config event = %s\nwant %s", ev, wantEvent) + } + raw, err := Encode(Frame{Type: FrameControl, Payload: ev}) + if err != nil { + t.Fatal(err) + } + var f Frame + if err := json.Unmarshal(raw, &f); err != nil { + t.Fatal(err) + } + if f.Type != FrameControl || f.AttachID != 0 || !bytes.Equal(f.Payload, ev) { + t.Fatalf("boot config frame round trip mangled: %+v", f) + } +} diff --git a/internal/relay/frame.go b/internal/relay/frame.go index b02f7341..e540840a 100644 --- a/internal/relay/frame.go +++ b/internal/relay/frame.go @@ -143,6 +143,20 @@ const ( // // See docs/design/exec-idle-stop.md. KindExecCount = "exec_count" + // KindBootConfig is the FIRST control frame a microVM host sends down the + // vsock channel the guest has just opened: the session's whole + // configuration, carried as a runner.BootConfig in Payload. + // + // It travels downward only, and only on that path. A Docker session is + // configured through its container's environment block before sessiond + // starts and never sees this kind at all; a sessiond that predates it + // logs one unknown frame and carries on, which is the honest outcome for + // a guest that was configured some other way. + // + // Its schema is runner.BootConfig, in protocol/runner beside Spec, and + // not here: every field of it is a field of the create this session came + // from, and relay interprets no payload it carries. + KindBootConfig = "boot_config" ) type ControlEvent struct { @@ -199,6 +213,18 @@ type ControlEvent struct { // does not speak this event. Live int `json:"live,omitempty"` Seq uint64 `json:"seq,omitempty"` + // Cold qualifies a KindSuspending: this is not a freeze, it is the end of + // this VM. A cold-suspended microVM session is TERMINATED — ADR-0003 §2.2 + // keeps no memory image anywhere — so the sandbox is being asked to flush + // what it has, unmount the agent home, and forget every secret the + // bootstrap exchange delivered, before it answers KindSuspendReady. + // + // It is omitempty, and false is both the zero value and today's whole + // meaning: a warm pause carries no `cold`, so the suspend notice a Plan 6 + // sandbox has always read is byte-identical. A sessiond that predates the + // field reads a plain "suspending", quiesces its execs, and answers; the + // host-side unmount still happens, the guest simply did not help. + Cold bool `json:"cold,omitempty"` } func Encode(f Frame) ([]byte, error) { return json.Marshal(f) } diff --git a/internal/relay/netconn.go b/internal/relay/netconn.go new file mode 100644 index 00000000..1168ef23 --- /dev/null +++ b/internal/relay/netconn.go @@ -0,0 +1,150 @@ +// internal/relay/netconn.go +package relay + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "net" + "sync" +) + +// NetConn adapts a byte STREAM to Conn, which is a message interface. +// +// It exists for virtio-vsock, the single host-to-guest control channel a +// microVM session has (ADR-0003 §2.7 item 2): a vsock connection is an +// ordinary AF_VSOCK stream on the guest side and an AF_UNIX stream on the +// host side, with no framing of its own, where the WebSocket this package +// grew up on delivered whole messages. Everything above Conn — the hub, the +// session side, the control channel, the RPC dispatcher — is unchanged by +// this, which is the whole reason the seam is at Conn. +// +// The framing is one JSON value per LINE. It is chosen rather than a length +// prefix because every message on this conn is already a JSON object produced +// by encoding/json, which never emits a bare newline inside one: a framing +// that needs no escaping and no second encoder cannot disagree with the +// encoder above it. maxFrameBytes bounds a line, so a peer cannot make this +// end allocate without limit by never sending one. +// +// Writes are serialized. That is not belt-and-braces: a Hub writes from the +// goroutine of every attached client AND from SendControl, and two concurrent +// Write calls on a stream interleave the bytes of two frames and corrupt it +// permanently for the reader. A *websocket.Conn serializes internally and a +// net.Conn does not, so the discipline has to be here. +func NetConn(c net.Conn) Conn { + return &netConn{ + c: c, + // The same 16 MiB bound both WebSocket ends set with SetReadLimit, so + // a frame that crosses this transport and a frame that crosses that + // one are the same size of thing. + r: bufio.NewReaderSize(c, netConnReadBuffer), + } +} + +const ( + // netConnReadBuffer is the reader's buffer, and it is FIXED — + // bufio.Reader never grows one. A frame larger than it is assembled a + // bufferful at a time by the ErrBufferFull loop in Read, which is what + // keeps a 64 KiB buffer from being a 64 KiB frame limit. A control frame + // is a few hundred bytes and a terminal frame a few KiB, so the common + // case takes one pass. + netConnReadBuffer = 64 << 10 + // maxFrameBytes is the largest single message this transport will + // assemble, matching the WebSocket ends' SetReadLimit. + maxFrameBytes = 16 << 20 +) + +type netConn struct { + c net.Conn + r *bufio.Reader + + // wmu serializes writers — see NetConn's doc comment. + wmu sync.Mutex +} + +// ErrFrameTooLarge is returned when a peer sends a line longer than this +// transport will assemble. It ends the connection rather than skipping the +// frame: a reader that resynchronized mid-stream would hand the decoder the +// tail of one message as though it were a whole one. +var ErrFrameTooLarge = errors.New("relay: frame over the transport's size limit") + +func (n *netConn) Read(ctx context.Context) ([]byte, error) { + // A cancelled context CLOSES the conn, which is the only thing that can + // interrupt a blocking stream read — and it is deliberately the same + // answer *websocket.Conn gives, rather than merely poisoning a deadline. + // + // The difference matters at one caller. connWriter.writeWithin bounds an + // exec frame's write and documents the transport's answer to an expired + // write context as closing the conn, "because closing it is what makes + // sessiond redial". A transport that instead left the conn readable and + // unwritable would leave a session whose output and RPC were silently + // dead upward while both ends still believed the conn was live — which + // is worse than either failing or working. + // + // It follows that no caller may bound a read or a write on a conn it + // wants to keep. cmd/sessiond's boot preamble is the one that tries, and + // it closes and re-dials rather than handing on a conn it may have + // broken. + if ctx.Done() != nil { + stop := context.AfterFunc(ctx, func() { _ = n.c.Close() }) + defer stop() + } + var line []byte + for { + chunk, err := n.r.ReadSlice('\n') + // The terminator is not part of the message, so a frame of exactly + // maxFrameBytes is readable: the budget is the message's, and the + // one byte of framing is this transport's own. + if len(line)+len(chunk) > maxFrameBytes+1 { + return nil, ErrFrameTooLarge + } + line = append(line, chunk...) + if err == nil { + break + } + if errors.Is(err, bufio.ErrBufferFull) { + continue + } + // Anything else ends the read, whatever was accumulated: a line with + // no terminator is not a message, and handing half of one to the + // decoder would be worse than reporting the conn's own error. + return nil, err + } + // A JSON value never contains a bare newline or carriage return, so + // trimming them is free and makes the framing tolerant of a peer that + // writes CRLF. + return bytes.TrimRight(line[:len(line)-1], "\r"), nil +} + +func (n *netConn) Write(ctx context.Context, b []byte) error { + if len(b) > maxFrameBytes { + return fmt.Errorf("%w: %d bytes", ErrFrameTooLarge, len(b)) + } + if bytes.ContainsRune(b, '\n') { + // Unreachable for anything encoding/json produced, and a hard error + // rather than an escape: a message with a newline in it would frame + // as two, and the second would be garbage the peer could not decode + // and could not resynchronize from. + return errors.New("relay: a frame containing a newline cannot be framed by line") + } + n.wmu.Lock() + defer n.wmu.Unlock() + // Same answer as Read's, and for the same reason: a cancelled write + // context closes the conn, which is what *websocket.Conn does and what + // connWriter.writeWithin's own doc comment relies on. + if ctx.Done() != nil { + stop := context.AfterFunc(ctx, func() { _ = n.c.Close() }) + defer stop() + } + // One Write call, not two: a frame and its terminator must not be + // separable by a concurrent writer or by a short write in between. + out := make([]byte, 0, len(b)+1) + out = append(out, b...) + out = append(out, '\n') + _, err := n.c.Write(out) + return err +} + +func (n *netConn) Close() error { return n.c.Close() } diff --git a/internal/relay/netconn_test.go b/internal/relay/netconn_test.go new file mode 100644 index 00000000..22d86e46 --- /dev/null +++ b/internal/relay/netconn_test.go @@ -0,0 +1,138 @@ +package relay + +import ( + "bytes" + "context" + "errors" + "net" + "strings" + "sync" + "testing" +) + +// TestNetConnRoundTripsFrames pins that a stream transport delivers the same +// MESSAGES a WebSocket one does: what goes in as one Write comes out as one +// Read, whatever the stream does with the boundaries in between. +func TestNetConnRoundTripsFrames(t *testing.T) { + a, b := net.Pipe() + defer a.Close() + defer b.Close() + left, right := NetConn(a), NetConn(b) + ctx := context.Background() + + frames := [][]byte{ + []byte(`{"t":4,"a":0,"p":"eyJraW5kIjoiYm9vdF9jb25maWcifQ=="}`), + []byte(`{"t":3,"a":7,"p":"` + strings.Repeat("A", 200<<10) + `"}`), + []byte(`{"t":1,"a":7}`), + } + go func() { + for _, f := range frames { + if err := left.Write(ctx, f); err != nil { + return + } + } + }() + for i, want := range frames { + got, err := right.Read(ctx) + if err != nil { + t.Fatalf("frame %d: %v", i, err) + } + if !bytes.Equal(got, want) { + t.Fatalf("frame %d: got %d bytes, want %d", i, len(got), len(want)) + } + } +} + +// TestNetConnSerializesConcurrentWrites is the property a raw stream needs +// and a WebSocket supplies for free. A Hub writes from the goroutine of every +// attached client and from SendControl; two interleaved writes would corrupt +// the stream permanently for the reader, so every frame must arrive whole. +func TestNetConnSerializesConcurrentWrites(t *testing.T) { + a, b := net.Pipe() + defer a.Close() + defer b.Close() + left, right := NetConn(a), NetConn(b) + ctx := context.Background() + + const writers, each = 8, 16 + var wg sync.WaitGroup + for w := 0; w < writers; w++ { + wg.Add(1) + go func(w int) { + defer wg.Done() + payload := []byte(strings.Repeat(string(rune('a'+w)), 32<<10)) + for i := 0; i < each; i++ { + if err := left.Write(ctx, payload); err != nil { + return + } + } + }(w) + } + go func() { wg.Wait(); a.Close() }() + + for n := 0; n < writers*each; n++ { + got, err := right.Read(ctx) + if err != nil { + t.Fatalf("frame %d: %v", n, err) + } + if len(got) != 32<<10 { + t.Fatalf("frame %d is %d bytes: two writes interleaved", n, len(got)) + } + if bytes.Count(got, got[:1]) != len(got) { + t.Fatalf("frame %d mixes two writers' bytes", n) + } + } +} + +// TestNetConnRefusesAnUnframeableWrite pins the one shape this framing cannot +// carry. It is unreachable for anything encoding/json produces, and it is an +// error rather than an escape because a silently split frame would leave the +// peer decoding garbage it could never resynchronize from. +func TestNetConnRefusesAnUnframeableWrite(t *testing.T) { + a, b := net.Pipe() + defer a.Close() + defer b.Close() + if err := NetConn(a).Write(context.Background(), []byte("one\ntwo")); err == nil { + t.Fatal("a frame containing a newline was accepted") + } + _ = b +} + +// TestNetConnReadEndsOnACancelledContext pins that a blocking stream read is +// interruptible, which is what a Hub whose context ends relies on. +func TestNetConnReadEndsOnACancelledContext(t *testing.T) { + a, b := net.Pipe() + defer a.Close() + defer b.Close() + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + _, err := NetConn(a).Read(ctx) + done <- err + }() + cancel() + if err := <-done; err == nil { + t.Fatal("a read on a cancelled context returned no error") + } +} + +// TestNetConnRefusesAnOversizedFrame pins the bound: a peer cannot make this +// end assemble an unbounded line. +func TestNetConnRefusesAnOversizedFrame(t *testing.T) { + a, b := net.Pipe() + defer a.Close() + defer b.Close() + go func() { + // Written straight to the stream, bypassing the writer's own bound, + // because it is the READER's bound this pins. + chunk := bytes.Repeat([]byte("x"), 1<<20) + for i := 0; i < 20; i++ { + if _, err := a.Write(chunk); err != nil { + return + } + } + }() + if _, err := NetConn(b).Read(context.Background()); !errors.Is(err, ErrFrameTooLarge) { + t.Fatalf("an oversized frame gave %v, want ErrFrameTooLarge", err) + } +} diff --git a/internal/runnerd/agent.go b/internal/runnerd/agent.go index 773a112c..b28e5580 100644 --- a/internal/runnerd/agent.go +++ b/internal/runnerd/agent.go @@ -332,7 +332,7 @@ func (s *Server) agentSession(ctx context.Context, cfg AgentConfig) (established active, idleExited := s.reg.counts() ann := runner.FromRunner{Type: "announce", Proto: runner.ProtocolVersion, Runner: cfg.RunnerName, Sessions: s.Announce(), Used: used, Total: total, Active: active, IdleExited: idleExited, - Capabilities: buildCapabilities(cfg.Capabilities)} + Capabilities: buildCapabilities(cfg.Capabilities, s.driverCapabilities()...)} if err := wsjson.Write(connCtx, c, ann); err != nil { return false, err // nothing can have been accepted before the announce } @@ -449,6 +449,14 @@ func (s *Server) execute(ctx context.Context, m runner.ToRunner, send func(runne Init: m.Spec.Init, InitTimeoutSec: m.Spec.InitTimeoutSec, GitAuthorName: m.Spec.GitAuthorName, GitAuthorEmail: m.Spec.GitAuthorEmail, Home: driverHome(m.Spec.Home), + // The microVM bootstrap pair, carried through like + // everything else. This runner does not read the token — it + // goes into the guest's boot configuration and is dropped — + // and it does not check the names against Env. The DRIVER + // decides what an unwithheld create means, because the answer + // is different for each one: Docker has always accepted the + // values and still does. + BootstrapToken: m.Spec.BootstrapToken, SecretNames: m.Spec.SecretNames, } allow = m.Spec.EgressAllow } @@ -598,6 +606,24 @@ func (s *Server) forwardSessionRPC(m runner.ToRunner, send func(runner.FromRunne log.Printf("agent: session_rpc for %s carried no id or method; ignoring", m.Session) return } + // A response to a request this RUNNER originated stops here: the runner + // is a pure forwarder for everything a sandbox asked, and the one thing + // it asks for itself (a bootstrap token on a cold resume) is answered to + // a caller inside this process, not to the guest. The id spaces are + // disjoint so the two can share one connection — see + // runnerOriginatedIDBase. + // + // The SESSION is part of the match, not just the id: an answer naming a + // different session than the call was made for is not that call's answer, + // and delivering it would hand a cold resume of one session a token + // minted against another's row. + if env.Method == "resp" && isRunnerOriginated(env.ID) { + if !s.runnerRPC.deliver(m.Session, env) { + log.Printf("agent: an answer for this runner's own request %d on session %s has no caller waiting (timed out, or a different session than it was made for); dropping", + env.ID, m.Session) + } + return + } err := s.sendSessionRPC(m.Session, env) if err == nil { return @@ -738,19 +764,45 @@ func (s *Server) dialAttachBack(ctx context.Context, m runner.ToRunner, cfg Agen // far worse failure than the 501 this append exists to avoid for what is a // cheap pre-check and not the fence. The sandbox's own `exec_started` is the // fence, so the only cost of dropping the claim is one wasted round trip. -func buildCapabilities(declared []string) []string { +// `microvm.v1` is the second such fact, and the one place this comment needs +// a caveat: it is not merely a pre-check. The control plane WITHHOLDS an +// environment's decrypted secret values from a runner that announces it, so +// announcing it is what makes the withholding happen and failing to announce +// it is what gets the values dispatched to a driver that will refuse them. +// It comes from the driver this runner was started with rather than from a +// flag, for the same reason exec.v1 does: it is already decided by +// --driver=microvm, and a second flag beside it is a second thing to get +// wrong. See (*Server).driverCapabilities. +func buildCapabilities(declared []string, fromBuild ...string) []string { + out := declared + for _, c := range append([]string{runner.CapabilityExecV1}, fromBuild...) { + out = appendCapability(out, c) + } + return out +} + +// appendCapability adds one build fact to an operator's list, unless it is +// already there or there is no room for it. +// +// The no-room case is why this is not one line. runnerplane refuses a WHOLE +// registration whose claim carries more than MaxCapabilities, so an operator +// already passing the maximum would announce one too many after this rolls +// and never reconnect — a working runner out of the fleet permanently, which +// is far worse than the capability being absent. The operator's list is +// otherwise left exactly as given, order included: it is a claim the control +// plane decides whether to schedule on, not something to tidy. +func appendCapability(declared []string, capability string) []string { for _, c := range declared { - if c == runner.CapabilityExecV1 { + if c == capability { return declared } } if len(declared) >= runnerplane.MaxCapabilities { log.Printf("agent: %d capabilities declared, which is runnerplane's maximum; "+ - "announcing without %s (exec is still fenced by the sandbox's own handshake)", - len(declared), runner.CapabilityExecV1) + "announcing without %s", len(declared), capability) return declared } - return append(append([]string(nil), declared...), runner.CapabilityExecV1) + return append(append([]string(nil), declared...), capability) } // attachDialTimeout bounds one attach-back handshake. It sits below diff --git a/internal/runnerd/microvm.go b/internal/runnerd/microvm.go new file mode 100644 index 00000000..32ae7689 --- /dev/null +++ b/internal/runnerd/microvm.go @@ -0,0 +1,290 @@ +// internal/runnerd/microvm.go +// +// The runner's half of the microVM driver's two needs: where a guest's vsock +// connection goes, and where a cold resume's bootstrap token comes from. +// Both are implementations of driver.MicrovmHost, and both are here rather +// than in the driver because both are things only the runner can do — one +// needs the session registry, the other needs the control connection. +package runnerd + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log" + "slices" + "sync" + "sync/atomic" + "time" + + "github.com/tokencanopy/rainier/internal/driver" + "github.com/tokencanopy/rainier/internal/relay" + "github.com/tokencanopy/rainier/protocol/runner" +) + +var _ driver.MicrovmHost = (*Server)(nil) + +// GuestConnected serves one microVM guest's control connection, exactly as +// the /register handler serves a container's WebSocket. +// +// On its own goroutine because the driver calls this from its accept loop and +// serveSessionConn blocks for the life of the conn — a loop that waited would +// never accept the guest's next dial. +// +// The context is this runner's own and not a request's, which is the one +// difference from the WebSocket door: there is no request. The hub's life is +// then bounded by the conn and by hub.Close(), which is what +// serveSessionConn's own teardown already relies on — `register` blocks on +// hub.Done() exactly because r.Context() does not reflect the socket dying. +// +// The session id is the DRIVER's, taken from the socket path Firecracker +// forwarded the connection to, and not anything the guest said. That is the +// whole difference between this door and the WebSocket one, where `register` +// believes a query parameter with no authentication on the hop at all. +func (s *Server) GuestConnected(sessionID string, conn relay.Conn) { + if _, ok := s.reg.get(sessionID); !ok { + // A guest for a session this runner no longer holds: a destroy that + // raced the boot. Closing it is what tells the guest to stop; leaving + // it open would leak the conn and its goroutine with no registry + // entry that could ever reap them. + log.Printf("microvm guest connected for unknown session %s; closing", sessionID) + _ = conn.Close() + return + } + go s.serveSessionConn(context.Background(), sessionID, conn) +} + +// runnerOriginatedIDBase separates the request ids this RUNNER assigns from +// the ones a sandbox assigns. +// +// Both travel up the same connection as a "session_req" for the same session, +// and the control plane echoes whichever id it was given, so the two spaces +// must not collide: a response to the runner's mint carrying id 1 would +// otherwise be delivered into a sandbox that is waiting on its own id 1. The +// high bit is the cheapest disjoint space there is, and a sandbox counting +// from 1 will not reach it. +const runnerOriginatedIDBase uint64 = 1 << 63 + +// isRunnerOriginated reports whether an id came from this runner's own +// counter. forwardSessionRPC asks before it routes a response into a sandbox. +func isRunnerOriginated(id uint64) bool { return id&runnerOriginatedIDBase != 0 } + +// refuseSandboxOrigin reports why an upward message must not be forwarded, +// or "" when it may be. It is the fence on the one door an untrusted peer +// has into the control plane's method table. +// +// It guards RESPONSES as well as requests, and that is not a formality: an +// id space is closed at both ends or it is not closed. A sandbox that may +// not send `req:` numbered 1<<63|n but may send `resp` numbered 1<<63|n has +// put an id from the runner's space on the wire either way — and the `resp` +// travels further, because the runner forwards it to the control plane +// rather than answering it there. `method` is the envelope's method, which +// is "resp" for a response, so both arms consult the same fence. +// +// Two things are refused, and only this hop can refuse either of them. +// +// A sandbox may not mint its own bootstrap token. The design's whole point +// is that the token is SINGLE-USE and lives 120 seconds: a guest that could +// ask for a fresh one whenever it liked would hold an unbounded, self- +// renewing capability to re-read its environment's current secrets, which is +// the property being removed rather than added. The method exists for a COLD +// RESUME, which is a thing the runner does and the guest cannot observe, and +// controld cannot tell the two apart — a "session_req" proves only that some +// runner sent it, and which end of the runner originated it is a fact only +// the runner has. (This applies to a Docker sandbox too, which is why it is +// not conditional on the driver.) +// +// And a sandbox may not use an id from the runner's own space. The high bit +// is how a response is routed back to a caller inside this process rather +// than into the sandbox (see forwardSessionRPC), and an id space the +// untrusted end can write into is not a space: a guest choosing +// 1<<63|n could have controld's echo delivered to a cold resume that is +// waiting on that number. +func refuseSandboxOrigin(method string, id uint64) string { + switch { + case method == runner.MethodMintSessionBootstrap: + return "a sandbox may not mint its own bootstrap token; a fresh one is minted by the runner on a cold resume" + case isRunnerOriginated(id): + return "this id is reserved for the runner's own requests" + } + return "" +} + +// runnerRPCTable is the pending table for requests this runner originated. +// +// It is deliberately tiny and deliberately separate from the sandbox's: the +// runner asks the control plane exactly one thing (a bootstrap token, on a +// cold resume), and a table shared with the forwarding path would make the +// forwarder a participant in the conversation it is supposed to be a pipe +// for. +type runnerRPCTable struct { + seq atomic.Uint64 + mu sync.Mutex + // waiting is id → the one call waiting on it. Every caller removes its + // own entry, so nothing sweeps this map. + waiting map[uint64]pendingRunnerCall +} + +// pendingRunnerCall is one in-flight request this runner made, and it records +// the SESSION it was made for as well as the channel to answer on. +// +// The session is half of the correlation, not decoration. An id alone says +// "some call is waiting on this number"; a mint for session A answered by a +// message naming session B would otherwise be delivered to A's caller, which +// would boot A's new VM with a token minted against B's row. The id spaces +// are per-process, the sessions are not, so the pair is what identifies a +// call. +type pendingRunnerCall struct { + session string + ch chan runner.RPCEnvelope +} + +func newRunnerRPCTable() *runnerRPCTable { + return &runnerRPCTable{waiting: map[uint64]pendingRunnerCall{}} +} + +func (t *runnerRPCTable) begin(session string) (uint64, chan runner.RPCEnvelope) { + id := runnerOriginatedIDBase | t.seq.Add(1) + ch := make(chan runner.RPCEnvelope, 1) + t.mu.Lock() + t.waiting[id] = pendingRunnerCall{session: session, ch: ch} + t.mu.Unlock() + return id, ch +} + +func (t *runnerRPCTable) end(id uint64) { + t.mu.Lock() + delete(t.waiting, id) + t.mu.Unlock() +} + +// deliver hands a response to the call waiting on its id AND its session, +// reporting whether one was. The channel is buffered by one and every caller +// removes its own entry, so this never blocks the reader that called it. +// +// A mismatched session is not delivered anywhere: the caller learns nothing +// arrived and times out, which is the honest outcome for an answer that does +// not belong to the question. +func (t *runnerRPCTable) deliver(session string, env runner.RPCEnvelope) bool { + t.mu.Lock() + p, ok := t.waiting[env.ID] + t.mu.Unlock() + if !ok || p.session != session { + return false + } + ch := p.ch + select { + case ch <- env: + default: + } + return true +} + +// mintBootstrapTimeout bounds one upward mint. It sits well under a cold +// resume's own patience and well over a round trip to a control plane that is +// answering: a resume that waits longer than this is waiting on a plane that +// is not going to answer, and reporting that is better than holding the +// session's resume open. +const mintBootstrapTimeout = 10 * time.Second + +// MintSessionBootstrap asks the control plane for a fresh single-use token +// for sessionID, on the runner's own control connection. +// +// It is the one request this runner originates on the session RPC. The +// message carries no session id of its own — FromRunner.Session is the id, +// and the control plane answers from the row its placement guard read — which +// is what stops a runner holding session A from minting for session B. +// +// A runner with no control connection fails immediately rather than waiting: +// there is no queue behind this channel by design, and a cold resume that +// cannot get a token has to report that rather than boot a guest that will +// ask for its secrets and be refused. +func (s *Server) MintSessionBootstrap(ctx context.Context, sessionID string) (string, error) { + body, err := json.Marshal(struct { + Protocol int `json:"protocol"` + }{runner.SessionBootstrapProtocolVersion}) + if err != nil { + return "", fmt.Errorf("encoding the bootstrap mint request: %w", err) + } + + id, ch := s.runnerRPC.begin(sessionID) + defer s.runnerRPC.end(id) + + if !s.fireSessionRPC(sessionID, runner.RPCEnvelope{ + ID: id, Method: runner.MethodMintSessionBootstrap, Payload: body}) { + return "", errors.New("this runner has no controld connection to mint a bootstrap token on") + } + + timer := time.NewTimer(mintBootstrapTimeout) + defer timer.Stop() + var answer runner.RPCEnvelope + select { + case answer = <-ch: + case <-ctx.Done(): + return "", fmt.Errorf("minting a bootstrap token for %s: %w", sessionID, ctx.Err()) + case <-timer.C: + return "", fmt.Errorf("minting a bootstrap token for %s: no answer within %s", sessionID, mintBootstrapTimeout) + } + if !answer.OK { + // The control plane's own sentence, relayed: it names the condition + // a person can act on, and it never carries a token or a value. + return "", errors.New(rpcErrorText(answer.Payload)) + } + var minted struct { + Token string `json:"token"` + } + if err := json.Unmarshal(answer.Payload, &minted); err != nil { + // Logged without the error and without the payload: a json message + // quotes what it choked on, and what it choked on is the token. + return "", errors.New("the control plane's bootstrap token could not be decoded") + } + if minted.Token == "" { + return "", errors.New("the control plane answered a bootstrap mint with no token") + } + return minted.Token, nil +} + +// rpcErrorText reads the {"error": ...} sentence a failed response carries, +// falling back to a flat one so a refusal with no body is still a reason. +func rpcErrorText(payload json.RawMessage) string { + var body struct { + Error string `json:"error"` + } + if len(payload) > 0 && json.Unmarshal(payload, &body) == nil && body.Error != "" { + return body.Error + } + return "the control plane refused a bootstrap mint without a reason" +} + +// driverCapabilities are the capability tokens that are facts about which +// DRIVER this runner was started with, appended to the operator's own claims +// exactly as exec.v1 is and for the same reason: whether a session on this +// runner is a microVM is decided by a flag the operator already passed +// (--driver=microvm), not by a second flag they have to remember beside it. +// +// It is the fence, not merely a pre-check, which is the one way it differs +// from exec.v1: the control plane WITHHOLDS an environment's secret values +// from a runner that announces microvm.v1, so a runner that announced it +// falsely would be dispatched creates it cannot fulfil, and one that failed +// to announce it would be dispatched the values it must not accept — which +// its driver then refuses (driver.refuseUnwithheldEnv). +func (s *Server) driverCapabilities() []string { + if cd, ok := s.drv.(driver.CapabilityDriver); ok { + return cd.Capabilities() + } + return nil +} + +// withholdsSecrets reports whether this runner's driver is one the control +// plane withholds an environment's secret values from. +// +// It is read off the announced capability rather than off the driver's +// concrete type, and that is the point: the two places this runner behaves +// differently for a microVM session — a create carries no dial URL, and a +// cold suspend tells the sandbox it is cold — are consequences of the same +// claim the control plane acts on. Keying them on a second fact would let +// the announcement and the behaviour disagree. +func (s *Server) withholdsSecrets() bool { + return slices.Contains(s.driverCapabilities(), runner.CapabilityMicrovmV1) +} diff --git a/internal/runnerd/microvm_test.go b/internal/runnerd/microvm_test.go new file mode 100644 index 00000000..be214a85 --- /dev/null +++ b/internal/runnerd/microvm_test.go @@ -0,0 +1,609 @@ +// internal/runnerd/microvm_test.go +package runnerd + +import ( + "context" + "encoding/json" + "errors" + "net" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/tokencanopy/rainier/internal/driver" + "github.com/tokencanopy/rainier/internal/relay" + "github.com/tokencanopy/rainier/protocol/runner" +) + +// withholdingDriver is a driver that announces microvm.v1 and takes a host, +// over the ordinary fake. +// +// It is NOT a *driver.Microvm, and it deliberately cannot be: the microVM +// driver's simulated engine lives in a _test.go file of its own package +// precisely so no code outside it can construct a simulated hypervisor, and +// that rule is worth more than the convenience of reusing it here. What this +// file tests is the RUNNER's half — what it announces, where a guest's +// connection goes, what a create carries, what a cold suspend says — and +// every one of those is keyed on the announced capability rather than on a +// concrete type, so a driver that announces it is exactly the right fixture. +// The driver's own half is tested in internal/driver. +type withholdingDriver struct { + *driver.Fake + mu sync.Mutex + host driver.MicrovmHost +} + +func (d *withholdingDriver) Capabilities() []string { return []string{runner.CapabilityMicrovmV1} } + +func (d *withholdingDriver) SetHost(h driver.MicrovmHost) { + d.mu.Lock() + defer d.mu.Unlock() + d.host = h +} + +func (d *withholdingDriver) installedHost() driver.MicrovmHost { + d.mu.Lock() + defer d.mu.Unlock() + return d.host +} + +// testMicrovmServer builds a runner whose driver withholds, with the dial +// base and proxy a real one is configured with. +func testMicrovmServer(t *testing.T) (*Server, *withholdingDriver) { + t.Helper() + d := &withholdingDriver{Fake: driver.NewFake(4)} + return New(d, "ws://runner.example.invalid:8080", "", "http://proxy.example.invalid:3128"), d +} + +// TestMicrovmRunnerAnnouncesTheCapability is the fence the control plane keys +// its withholding on. It is a fact about which driver this runner was started +// with, not a flag an operator has to remember beside --driver=microvm, and a +// runner that failed to announce it would be dispatched the secret values its +// own driver then refuses. +func TestMicrovmRunnerAnnouncesTheCapability(t *testing.T) { + s, _ := testMicrovmServer(t) + caps := buildCapabilities(nil, s.driverCapabilities()...) + if !slices.Contains(caps, runner.CapabilityMicrovmV1) { + t.Fatalf("a microVM runner announces %v, want %s among them", caps, runner.CapabilityMicrovmV1) + } + if !slices.Contains(caps, runner.CapabilityExecV1) { + t.Fatalf("announcing microvm.v1 dropped exec.v1: %v", caps) + } + + // And a Docker runner announces neither it nor anything new: the whole + // compatibility floor is that a runner without this capability is + // dispatched exactly what it always was. + docker := New(driver.NewFake(4), "", "", "") + if got := buildCapabilities(nil, docker.driverCapabilities()...); !slices.Equal(got, []string{runner.CapabilityExecV1}) { + t.Fatalf("a non-microVM runner announces %v, want only exec.v1", got) + } + + // The operator's own claims are kept, in order, and not reordered by the + // build facts appended after them. + declared := []string{"gpu", "docker.rootless"} + got := buildCapabilities(declared, s.driverCapabilities()...) + if len(got) != 4 || got[0] != "gpu" || got[1] != "docker.rootless" { + t.Fatalf("buildCapabilities(%v) = %v; it reordered the operator's list", declared, got) + } +} + +// TestMicrovmCreateLeavesNoDialURL is §4's step 2. There is no URL to dial: +// the guest's control channel is the vsock conn IT opens, so a dial URL would +// be a promise of an endpoint the guest has no route to — and an empty one is +// what makes NO_PROXY inside the guest the base list with no dial host in it. +func TestMicrovmCreateLeavesNoDialURL(t *testing.T) { + s, d := testMicrovmServer(t) + if err := s.CreateWithID(context.Background(), "sess-nodial", driver.Spec{}, nil); err != nil { + t.Fatal(err) + } + got := d.LastSpec() + if got.DialURL != "" { + t.Fatalf("a withholding driver's create carries dial URL %q; there is nothing to dial", got.DialURL) + } + // Everything else a create carries is unchanged: the id and the proxy + // are still this runner's own concerns and still set here. + if got.SessionID != "sess-nodial" || got.ProxyURL != "http://proxy.example.invalid:3128" { + t.Fatalf("the create lost something else: %+v", got) + } + + // The Docker path keeps its dial URL, byte for byte: that is the floor + // this whole change promises not to move. + dockerRunner := New(driver.NewFake(4), "ws://runner.example.invalid:8080", "", "") + if err := dockerRunner.CreateWithID(context.Background(), "sess-docker", driver.Spec{}, nil); err != nil { + t.Fatal(err) + } + fake := dockerRunner.drv.(*driver.Fake) + if got := fake.LastSpec().DialURL; got != "ws://runner.example.invalid:8080/register" { + t.Fatalf("the docker path's dial URL = %q", got) + } +} + +// TestGuestConnectedBecomesTheSessionsHub is the vsock half of `register`: +// the connection Firecracker forwarded becomes the session's relay hub, +// keyed by the session the SOCKET PATH named, and the session reports +// running. +func TestGuestConnectedBecomesTheSessionsHub(t *testing.T) { + s, _ := testMicrovmServer(t) + events := make(chan string, 8) + s.SetOnEvent(func(id, state, _ string) { events <- id + ":" + state }) + + s.reg.put("sess-guest", &sessionEntry{id: "sess-guest", state: "running"}) + + guest, host := net.Pipe() + defer guest.Close() + s.GuestConnected("sess-guest", relay.NetConn(host)) + + select { + case got := <-events: + if got != "sess-guest:running" { + t.Fatalf("event = %q, want sess-guest:running", got) + } + case <-time.After(3 * time.Second): + t.Fatal("a guest connection produced no running event") + } + if _, ok := s.reg.hub("sess-guest"); !ok { + t.Fatal("a guest connection did not become the session's hub") + } + + // A guest for a session this runner does not hold is closed rather than + // served: a destroy that raced the boot must not leave a conn and a + // goroutine with no entry that could ever reap them. + other, otherHost := net.Pipe() + defer other.Close() + s.GuestConnected("sess-unknown", relay.NetConn(otherHost)) + if _, err := other.Read(make([]byte, 1)); err == nil { + t.Fatal("a guest for an unknown session was left connected") + } +} + +// TestMintSessionBootstrapRidesTheControlConnection is the cold-resume mint, +// end to end over a fake control plane: the runner originates the request, +// the plane answers it, and the answer reaches the caller inside this process +// rather than being forwarded into the sandbox. +func TestMintSessionBootstrapRidesTheControlConnection(t *testing.T) { + s, _ := testMicrovmServer(t) + s.reg.put("sess-mint", &sessionEntry{id: "sess-mint", state: "running"}) + // A real sandbox on the far side, so "the answer did not reach the + // guest" is something this test can actually observe rather than + // something the absence of a hub makes true by accident. + guest, host := net.Pipe() + defer guest.Close() + s.GuestConnected("sess-mint", relay.NetConn(host)) + waitForHub(t, s, "sess-mint") + + fc := newFakeControld(t, testToken) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go s.RunAgent(ctx, AgentConfig{ControldURL: fc.wsURL(), Token: testToken, RunnerName: "vm1"}) + conn := fc.nextConn(t) + conn.readAnnounce(t) + + minted := make(chan string, 1) + mintErr := make(chan error, 1) + go func() { + tok, err := s.MintSessionBootstrap(context.Background(), "sess-mint") + if err != nil { + mintErr <- err + return + } + minted <- tok + }() + + req := conn.readMsg(t) + if req.Type != "session_req" || req.Session != "sess-mint" || req.RPC == nil { + t.Fatalf("the runner sent %+v, want a session_req for sess-mint", req) + } + if req.RPC.Method != runner.MethodMintSessionBootstrap { + t.Fatalf("method = %q, want %q", req.RPC.Method, runner.MethodMintSessionBootstrap) + } + // The id is in the runner's own space, so it cannot collide with a + // sandbox's — both travel up this one connection for this one session. + if !isRunnerOriginated(req.RPC.ID) { + t.Fatalf("the runner's own request carries id %d, which is in the sandbox's space", req.RPC.ID) + } + var body struct { + Protocol int `json:"protocol"` + } + if err := json.Unmarshal(req.RPC.Payload, &body); err != nil || body.Protocol != runner.SessionBootstrapProtocolVersion { + t.Fatalf("request body = %s (%v)", req.RPC.Payload, err) + } + + answer, _ := json.Marshal(map[string]any{"token": "token_example", "expires_in_sec": 120}) + conn.send(t, runner.ToRunner{Type: "session_rpc", Session: "sess-mint", + RPC: &runner.RPCEnvelope{ID: req.RPC.ID, Method: "resp", OK: true, Payload: answer}}) + + select { + case tok := <-minted: + if tok != "token_example" { + t.Fatalf("minted token = %q", tok) + } + case err := <-mintErr: + t.Fatalf("mint: %v", err) + case <-time.After(5 * time.Second): + t.Fatal("the mint never returned") + } + + // And the answer did NOT reach the sandbox. The runner is a pure + // forwarder for everything the sandbox asked and a participant only in + // this one, so a response on its own id must be consumed here — if it + // were forwarded, the guest would receive a `resp` for a request it + // never made and its dispatcher would log an unknown id. + _ = guest.SetReadDeadline(time.Now().Add(300 * time.Millisecond)) + if n, err := guest.Read(make([]byte, 256)); err == nil { + t.Fatalf("the runner forwarded its own answer into the sandbox (%d bytes)", n) + } +} + +// TestASandboxMayNotMintItsOwnBootstrapToken is the fence on the one door an +// untrusted peer has into the control plane's method table. +// +// The token is single-use and lives 120 seconds, and both of those mean +// nothing if a guest can ask for a fresh one whenever it likes: it would hold +// an unbounded, self-renewing capability to re-read its environment's current +// secrets. controld cannot make this check — a session_req proves only that +// SOME runner sent it — so the runner refuses it here, and the refusal is not +// conditional on the driver, because a Docker sandbox is on the same channel. +func TestASandboxMayNotMintItsOwnBootstrapToken(t *testing.T) { + s, _ := testMicrovmServer(t) + s.reg.put("sess-guard", &sessionEntry{id: "sess-guard", state: "running"}) + guest, host := net.Pipe() + defer guest.Close() + s.GuestConnected("sess-guard", relay.NetConn(host)) + waitForHub(t, s, "sess-guard") + + // Nothing may be forwarded upstream, so the sink records what was. + forwarded := make(chan runner.RPCEnvelope, 4) + s.SetOnSessionRPC(func(_ string, env runner.RPCEnvelope) { forwarded <- env }) + defer s.SetOnSessionRPC(nil) + + sandbox := relay.NetConn(guest) + for _, tc := range []struct { + name string + ev relay.ControlEvent + wantReason string + }{ + { + name: "the mint method itself", + ev: relay.ControlEvent{Kind: "req:" + runner.MethodMintSessionBootstrap, ID: 7, Payload: []byte(`{"protocol":1}`)}, + wantReason: "may not mint its own bootstrap token", + }, + { + name: "an id from the runner's own space", + ev: relay.ControlEvent{Kind: "req:" + runner.MethodFetchSessionSecrets, + ID: runnerOriginatedIDBase | 9, Payload: []byte(`{"protocol":1,"token":"x"}`)}, + wantReason: "reserved for the runner's own requests", + }, + } { + t.Run(tc.name, func(t *testing.T) { + payload, err := json.Marshal(tc.ev) + if err != nil { + t.Fatal(err) + } + frame, err := relay.Encode(relay.Frame{Type: relay.FrameControl, Payload: payload}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := sandbox.Write(ctx, frame); err != nil { + t.Fatal(err) + } + + answer := readControlEvent(t, sandbox) + if answer.Kind != "resp" || answer.ID != tc.ev.ID || answer.OK { + t.Fatalf("the sandbox got %+v, want an ok:false resp for its own id", answer) + } + if reason := rpcErrorText(answer.Payload); !strings.Contains(reason, tc.wantReason) { + t.Fatalf("refusal = %q, want it to name %q", reason, tc.wantReason) + } + select { + case env := <-forwarded: + t.Fatalf("the request was forwarded to the control plane anyway: %+v", env) + default: + } + }) + } + + // And the methods a sandbox IS allowed to originate still go up + // untouched — the guard is two names and an id range, not a new + // allowlist that every future method has to be added to. + allowed := relay.ControlEvent{Kind: "req:" + runner.MethodFetchSessionSecrets, ID: 11, + Payload: []byte(`{"protocol":1,"token":"x"}`)} + payload, err := json.Marshal(allowed) + if err != nil { + t.Fatal(err) + } + frame, err := relay.Encode(relay.Frame{Type: relay.FrameControl, Payload: payload}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := sandbox.Write(ctx, frame); err != nil { + t.Fatal(err) + } + select { + case env := <-forwarded: + if env.ID != 11 || env.Method != runner.MethodFetchSessionSecrets { + t.Fatalf("the forwarded request = %+v", env) + } + case <-time.After(3 * time.Second): + t.Fatal("an ordinary sandbox request was not forwarded") + } +} + +// TestMintSessionBootstrapFailsClosed pins the two ways a mint can go wrong, +// both of which a cold resume has to hear about rather than boot through. +func TestMintSessionBootstrapFailsClosed(t *testing.T) { + t.Run("no control connection", func(t *testing.T) { + s, _ := testMicrovmServer(t) + if _, err := s.MintSessionBootstrap(context.Background(), "sess-x"); err == nil { + t.Fatal("a mint with no controld connection succeeded") + } else if !strings.Contains(err.Error(), "no controld connection") { + t.Fatalf("error = %q", err) + } + }) + + t.Run("a refusal is relayed verbatim", func(t *testing.T) { + s, _ := testMicrovmServer(t) + s.reg.put("sess-refused", &sessionEntry{id: "sess-refused", state: "running"}) + fc := newFakeControld(t, testToken) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go s.RunAgent(ctx, AgentConfig{ControldURL: fc.wsURL(), Token: testToken, RunnerName: "vm1"}) + conn := fc.nextConn(t) + conn.readAnnounce(t) + + out := make(chan error, 1) + go func() { + _, err := s.MintSessionBootstrap(context.Background(), "sess-refused") + out <- err + }() + req := conn.readMsg(t) + refusal, _ := json.Marshal(map[string]string{"error": "this session is not placed on the runner that asked"}) + conn.send(t, runner.ToRunner{Type: "session_rpc", Session: "sess-refused", + RPC: &runner.RPCEnvelope{ID: req.RPC.ID, Method: "resp", Payload: refusal}}) + + select { + case err := <-out: + if err == nil || !strings.Contains(err.Error(), "not placed on the runner that asked") { + t.Fatalf("error = %v, want the control plane's own sentence", err) + } + case <-time.After(5 * time.Second): + t.Fatal("the mint never returned") + } + }) + + t.Run("a cancelled context ends the wait", func(t *testing.T) { + s, _ := testMicrovmServer(t) + s.reg.put("sess-cancel", &sessionEntry{id: "sess-cancel", state: "running"}) + fc := newFakeControld(t, testToken) + agentCtx, stopAgent := context.WithCancel(context.Background()) + defer stopAgent() + go s.RunAgent(agentCtx, AgentConfig{ControldURL: fc.wsURL(), Token: testToken, RunnerName: "vm1"}) + fc.nextConn(t).readAnnounce(t) + + ctx, cancel := context.WithCancel(context.Background()) + out := make(chan error, 1) + go func() { + _, err := s.MintSessionBootstrap(ctx, "sess-cancel") + out <- err + }() + cancel() + select { + case err := <-out: + if !errors.Is(err, context.Canceled) { + t.Fatalf("error = %v, want a cancellation", err) + } + case <-time.After(5 * time.Second): + t.Fatal("a cancelled mint never returned") + } + }) +} + +// TestColdSuspendTellsAMicrovmSandboxItIsCold is §4.4's handshake. A cold +// microVM suspend terminates the VM with no memory image anywhere, so the +// sandbox is told first and told that it is cold — flush, unmount, forget the +// delivered secrets — where a warm one is a freeze and says nothing new. +func TestColdSuspendTellsAMicrovmSandboxItIsCold(t *testing.T) { + s, d := testMicrovmServer(t) + if d.installedHost() == nil { + t.Fatal("New did not install the runner as the driver's host") + } + s.suspendAckWait = 200 * time.Millisecond + s.suspendReadyWait = 200 * time.Millisecond + // The cold path has a budget of its own, and this test's sandbox never + // answers — so without shortening it too, this test would sit out the + // production thirty seconds. + s.coldSuspendReadyWait = 200 * time.Millisecond + ctx := context.Background() + if err := s.CreateWithID(ctx, "sess-cold", driver.Spec{}, nil); err != nil { + t.Fatal(err) + } + + // A sandbox on the far side of the session's conn, reading control + // frames. The vsock transport is the same relay.Conn the WebSocket one + // is, so a pipe stands in for it exactly. + guest, host := net.Pipe() + defer guest.Close() + s.GuestConnected("sess-cold", relay.NetConn(host)) + waitForHub(t, s, "sess-cold") + sandbox := relay.NetConn(guest) + + done := make(chan error, 1) + go func() { done <- s.Op(ctx, "sess-cold", "suspend", false) }() + + ev := readControlEvent(t, sandbox) + if ev.Kind != relay.KindSuspending { + t.Fatalf("the sandbox was sent %q, want %q", ev.Kind, relay.KindSuspending) + } + if !ev.Cold { + t.Fatal("a cold microVM suspend was announced as a freeze; the guest would not flush or unmount") + } + if ev.ID == 0 { + t.Fatal("the suspend notice carries no nonce") + } + if err := <-done; err != nil { + t.Fatalf("cold suspend: %v", err) + } +} + +// TestColdSuspendOnDockerIsUnchanged is the other half, and the one that +// matters most: the Docker path sends no notice on a cold stop, exactly as it +// never has. `docker stop` delivers the SIGTERM sessiond's own handler +// answers, and a frame added here would be a behaviour change on the one path +// this design promises to leave alone. +func TestColdSuspendOnDockerIsUnchanged(t *testing.T) { + fd := driver.NewFake(4) + s := New(fd, "", "", "") + ctx := context.Background() + if err := s.CreateWithID(ctx, "sess-docker", driver.Spec{}, nil); err != nil { + t.Fatal(err) + } + guest, host := net.Pipe() + defer guest.Close() + go s.serveSessionConn(context.Background(), "sess-docker", relay.NetConn(host)) + waitForHub(t, s, "sess-docker") + + if err := s.Op(ctx, "sess-docker", "suspend", false); err != nil { + t.Fatalf("cold suspend: %v", err) + } + // Nothing was written to the sandbox at all. A read with a deadline is + // how "no frame" is asserted; the conn is live throughout. + _ = guest.SetReadDeadline(time.Now().Add(200 * time.Millisecond)) + if n, err := guest.Read(make([]byte, 64)); err == nil { + t.Fatalf("the docker path sent the sandbox %d bytes on a cold stop", n) + } +} + +// readControlEvent reads one frame off a sandbox-side conn and decodes the +// control event in it. +func readControlEvent(t *testing.T, conn relay.Conn) relay.ControlEvent { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + raw, err := conn.Read(ctx) + if err != nil { + t.Fatalf("read a frame: %v", err) + } + f, err := relay.Decode(raw) + if err != nil { + t.Fatalf("decode a frame: %v", err) + } + if f.Type != relay.FrameControl { + t.Fatalf("frame type = %d, want a control frame", f.Type) + } + var ev relay.ControlEvent + if err := json.Unmarshal(f.Payload, &ev); err != nil { + t.Fatalf("decode a control event: %v", err) + } + return ev +} + +// TestASandboxMayNotAnswerWithARunnerSpaceID is review round 2, finding 6. +// +// The guard used to read only the `req:` arm, so a sandbox could not ASK with +// an id out of the runner's reserved space but could ANSWER with one — and an +// answer travels further than a request does, because the runner forwards it +// to the control plane rather than refusing it locally. An id space the +// untrusted end can write into is not a space, and it is not a space at +// either end of it. +func TestASandboxMayNotAnswerWithARunnerSpaceID(t *testing.T) { + s, _ := testMicrovmServer(t) + s.reg.put("sess-resp", &sessionEntry{id: "sess-resp", state: "running"}) + guest, host := net.Pipe() + defer guest.Close() + s.GuestConnected("sess-resp", relay.NetConn(host)) + waitForHub(t, s, "sess-resp") + + forwarded := make(chan runner.RPCEnvelope, 4) + s.SetOnSessionRPC(func(_ string, env runner.RPCEnvelope) { forwarded <- env }) + defer s.SetOnSessionRPC(nil) + + sandbox := relay.NetConn(guest) + send := func(ev relay.ControlEvent) { + t.Helper() + payload, err := json.Marshal(ev) + if err != nil { + t.Fatal(err) + } + frame, err := relay.Encode(relay.Frame{Type: relay.FrameControl, Payload: payload}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := sandbox.Write(ctx, frame); err != nil { + t.Fatal(err) + } + } + + // A response numbered out of the runner's own space: dropped, not + // forwarded. It is not refused back down either — answering an answer is + // meaningless, and there was never a caller for this one. + send(relay.ControlEvent{Kind: "resp", ID: runnerOriginatedIDBase | 5, OK: true, + Payload: []byte(`{"token":"token_the_sandbox_chose"}`)}) + + // An ordinary response after it, to prove the channel still works and + // that the drop above is about the id and not about `resp` at all. It + // arriving is also what makes the "nothing was forwarded" check below + // read over real traffic rather than over an empty channel. + send(relay.ControlEvent{Kind: "resp", ID: 12, OK: true, Payload: []byte(`{"ok":true}`)}) + + select { + case env := <-forwarded: + if isRunnerOriginated(env.ID) { + t.Fatalf("a sandbox put a runner-space id on the wire: %+v", env) + } + if env.ID != 12 { + t.Fatalf("the forwarded response = %+v, want the sandbox's own id 12", env) + } + case <-time.After(3 * time.Second): + t.Fatal("no response reached the control plane at all") + } + select { + case env := <-forwarded: + t.Fatalf("a second response was forwarded: %+v", env) + default: + } +} + +// TestARunnerAnswerForAnotherSessionIsNotDelivered is the other half of +// review round 2, finding 6: the runner's pending table used to match on the +// id alone and ignore the session the answer named. +// +// The id space is per-process and the session is not, so a mint for one +// session answered by a message naming another would have been delivered to +// the first's caller — which is a cold resume booting a VM with a token +// minted against a different session's row. +func TestARunnerAnswerForAnotherSessionIsNotDelivered(t *testing.T) { + tbl := newRunnerRPCTable() + id, ch := tbl.begin("sess-a") + defer tbl.end(id) + + answer, err := json.Marshal(map[string]any{"token": "token_for_b"}) + if err != nil { + t.Fatal(err) + } + env := runner.RPCEnvelope{ID: id, Method: "resp", OK: true, Payload: answer} + + if tbl.deliver("sess-b", env) { + t.Fatal("an answer naming another session was delivered to this call") + } + select { + case got := <-ch: + t.Fatalf("the caller was woken by another session's answer: %+v", got) + default: + } + + // The same envelope on the right session is the same call's answer, so + // the check above is the session and not a blanket refusal. + if !tbl.deliver("sess-a", env) { + t.Fatal("the call's own answer was not delivered") + } + if got := <-ch; got.ID != id { + t.Fatalf("delivered %+v, want the call's own id", got) + } +} diff --git a/internal/runnerd/registry.go b/internal/runnerd/registry.go index 93752456..b687de3b 100644 --- a/internal/runnerd/registry.go +++ b/internal/runnerd/registry.go @@ -341,15 +341,29 @@ func (r *registry) list() []sessionEntry { // still booting/dialing in) — the caller has a live hub with a running // readLoop goroutine that will now never be found through the registry, so // it must close that hub itself instead of leaking it. -func (r *registry) setHub(id string, h *relay.Hub) bool { +// +// It also RETURNS the hub it displaced, when there was one and it is not the +// same hub again, because a displaced hub is nobody's otherwise: nothing in +// the registry points at it any more, so if its conn never dies its readLoop +// never errors and its conn, its fd and its attachments are held by a +// structure nothing can reach. That is not hypothetical — the peer on the +// other end of a displaced conn is a sandbox, and a sandbox can simply +// decline to hang up. It is returned rather than closed here because the +// registry is not where a hub's lifetime is decided and because a displaced +// hub is not immediately useless: see retireDisplacedHub. +func (r *registry) setHub(id string, h *relay.Hub) (displaced *relay.Hub, ok bool) { r.mu.Lock() defer r.mu.Unlock() e, ok := r.items[id] if !ok { - return false + return nil, false } + displaced = e.hub e.hub = h - return true + if displaced == h { + displaced = nil + } + return displaced, true } // hub reads an entry's hub field under the registry lock. attach's diff --git a/internal/runnerd/registry_test.go b/internal/runnerd/registry_test.go index 283c97f1..7064e18a 100644 --- a/internal/runnerd/registry_test.go +++ b/internal/runnerd/registry_test.go @@ -1,7 +1,14 @@ // internal/runnerd/registry_test.go package runnerd -import "testing" +import ( + "context" + "net" + "testing" + "time" + + "github.com/tokencanopy/rainier/internal/relay" +) func TestRegistryTracksAndWaits(t *testing.T) { r := newRegistry() @@ -18,3 +25,87 @@ func TestRegistryTracksAndWaits(t *testing.T) { t.Fatal("removed entry still present") } } + +// TestSetHubReportsTheHubItDisplacesSoItCanBeRetired is review round 2, +// finding 2. +// +// setHub used to overwrite the field and drop the previous hub on the floor, +// naming nobody. The ordinary re-register is a hub whose conn already died, +// so nothing was visibly wrong — but a SECOND live connection claiming one +// session left the first hub running with nothing able to reach it: its +// readLoop never errored, and its conn, its fd and its attachments were held +// for the life of the process, because the peer that decides when that conn +// dies is the sandbox. +// +// It is reported rather than closed here, because a displaced hub still has +// frames to deliver — see retireDisplacedHub, which is what ends it. +func TestSetHubReportsTheHubItDisplacesSoItCanBeRetired(t *testing.T) { + r := newRegistry() + r.put("s1", &sessionEntry{id: "s1", state: "running"}) + + firstGuest, firstHost := net.Pipe() + defer firstGuest.Close() + defer firstHost.Close() + first := relay.NewHub(context.Background(), relay.NetConn(firstHost)) + if displaced, ok := r.setHub("s1", first); !ok || displaced != nil { + t.Fatalf("the first setHub displaced %p (ok=%v); there was nothing to displace", displaced, ok) + } + + secondGuest, secondHost := net.Pipe() + defer secondGuest.Close() + defer secondHost.Close() + second := relay.NewHub(context.Background(), relay.NetConn(secondHost)) + displaced, ok := r.setHub("s1", second) + if !ok { + t.Fatal("the entry did not take the second hub") + } + if displaced != first { + t.Fatalf("setHub reported %p as displaced, want the first hub %p — an unreported hub is a leaked one", displaced, first) + } + got, ok := r.hub("s1") + if !ok || got != second { + t.Fatalf("the registry holds %p, want the second hub %p", got, second) + } + + // Setting the SAME hub again displaces nothing: a caller told to retire + // it would be closing the session's own live connection. + if displaced, ok := r.setHub("s1", second); !ok || displaced != nil { + t.Fatalf("re-setting a session's own hub reported it as displaced (%p, ok=%v)", displaced, ok) + } + + // And an entry that vanished reports neither. + r.remove("s1") + if displaced, ok := r.setHub("s1", second); ok || displaced != nil { + t.Fatalf("setHub on a removed entry = (%p, %v), want (nil, false)", displaced, ok) + } +} + +// TestRetireDisplacedHubWaitsAndThenCloses pins both halves of the grace. +// +// A hub that is going to drain what it already read gets to: the child_exited +// on a conn a redial replaced can be the only copy of it +// (TestAChildExitAcrossARedialIsRecordedEndToEnd). A hub whose peer simply +// never hangs up does not get to hold a conn, an fd and its attachments +// forever. +func TestRetireDisplacedHubWaitsAndThenCloses(t *testing.T) { + guest, host := net.Pipe() + defer guest.Close() + defer host.Close() + h := relay.NewHub(context.Background(), relay.NetConn(host)) + + // Within the grace it is left alone, live conn and all. + go retireDisplacedHub("s1", h, time.Hour) + select { + case <-h.Done(): + t.Fatal("a displaced hub was closed inside its grace, dropping what it had already read") + case <-time.After(100 * time.Millisecond): + } + + // And when the grace is up it goes, peer or no peer. + go retireDisplacedHub("s1", h, time.Millisecond) + select { + case <-h.Done(): + case <-time.After(3 * time.Second): + t.Fatal("a displaced hub outlived its grace; its conn, its fd and its attachments are leaked") + } +} diff --git a/internal/runnerd/runnerd.go b/internal/runnerd/runnerd.go index e07439db..c2eb6621 100644 --- a/internal/runnerd/runnerd.go +++ b/internal/runnerd/runnerd.go @@ -105,6 +105,14 @@ type Server struct { // written immediately after New. suspendAckWait time.Duration suspendReadyWait time.Duration + // coldSuspendReadyWait is the "are they gone" budget for a COLD suspend, + // which asks the sandbox for strictly more work than a freeze does. Same + // reason it is a field: a test must not spend the production one. + coldSuspendReadyWait time.Duration + // displacedHubGrace is how long a hub a re-register replaced keeps + // delivering before it is closed. A field for the same reason, and + // written only immediately after New. + displacedHubGrace time.Duration // now is the clock every idle-stop decision reads: time.Now in // production, a test's own function in tests, so the thirty minutes a // session has to sit idle can be a table row rather than a sleep. A field @@ -121,6 +129,10 @@ type Server struct { // value New leaves — means "derive it from the timeout" (see // idleSweepInterval); a test sets it directly to keep its loop short. idleSweep time.Duration + // runnerRPC is the pending table for the session-RPC requests this + // RUNNER originates, which today is exactly one: the bootstrap token a + // microVM cold resume needs. See runnerRPCTable and microvm.go. + runnerRPC *runnerRPCTable } // suspendWaiter is one in-flight suspend's two answers, matched by nonce. @@ -211,10 +223,21 @@ func (e *egressError) Error() string { return "egress setup: " + e.err.Error() } func (e *egressError) Unwrap() error { return e.err } func New(drv driver.Driver, dialBase, egressAdmin, proxyURL string) *Server { - return &Server{drv: drv, reg: newRegistry(), dialBase: dialBase, egressAdmin: egressAdmin, + s := &Server{drv: drv, reg: newRegistry(), dialBase: dialBase, egressAdmin: egressAdmin, proxyURL: proxyURL, hubWait: defaultHubWait, now: time.Now, suspendAckWait: defaultSuspendAckWait, suspendReadyWait: defaultSuspendReadyWait, - suspends: map[string]*suspendWaiter{}} + coldSuspendReadyWait: defaultColdSuspendReadyWait, + displacedHubGrace: defaultDisplacedHubGrace, + suspends: map[string]*suspendWaiter{}, runnerRPC: newRunnerRPCTable()} + // The microVM driver is composed BELOW this server and needs two things + // from above it — where a guest's control conn goes, and where a cold + // resume's bootstrap token comes from — so the runner installs itself + // here rather than being passed into the driver's constructor, which + // runs first. See driver.MicrovmHost. + if hd, ok := drv.(driver.HostedDriver); ok { + hd.SetHost(s) + } + return s } // Recover rebuilds the in-memory registry from the driver's labeled @@ -384,6 +407,16 @@ func (s *Server) createWithID(ctx context.Context, id string, spec driver.Spec, spec.SessionID = id spec.DialURL = s.dialBase + "/register" spec.ProxyURL = s.proxyURL + if s.withholdsSecrets() { + // There is no URL to dial. A microVM guest's control channel is the + // vsock conn IT opens to the host, which the driver is already + // listening for by the time the VM starts, so a dial URL here would + // be a promise of an endpoint the guest has no route to and no + // business reaching. Empty is also what makes NO_PROXY inside the + // guest the base list and nothing else: there is no dial host to + // exempt (design note §4, step 2). + spec.DialURL = "" + } h, err := s.drv.Create(ctx, spec) if err != nil { s.reg.remove(id) @@ -650,9 +683,22 @@ func (s *Server) Op(ctx context.Context, id, op string, warm bool) error { // design's lifetime rule. Sent only for the warm case, so the cold // path keeps exactly the shape it has today. if warm { - s.quiesceExecs(ctx, id) + s.quiesceExecs(ctx, id, false) } if !warm { + // A cold microVM suspend TERMINATES the VM, and no memory image + // is written anywhere (ADR-0003 §2.2), so the sandbox is told + // first — and told that it is COLD: flush what it has, unmount + // the agent home, and forget every secret the bootstrap exchange + // delivered, before the VM goes. + // + // The Docker path is deliberately untouched here. `docker stop` + // delivers the SIGTERM sessiond's own handler already answers, + // and a notice sent there would be a behaviour change on the one + // path this design promises to leave exactly as it is. + if s.withholdsSecrets() { + s.quiesceExecs(ctx, id, true) + } // Cold suspend (docker stop) kills the container's sessiond, // which closes its /register conn — the exact same socket-level // event as a crash. Mark "suspending" BEFORE calling Suspend so @@ -930,6 +976,26 @@ func (s *Server) register(w http.ResponseWriter, r *http.Request) { return } c.SetReadLimit(16 << 20) + s.serveSessionConn(r.Context(), id, relay.WSConn(c)) +} + +// serveSessionConn is what a registered session's conn MEANS to this runner, +// with the transport that carried it already resolved: build the hub, publish +// it, report the session running, and then hold the goroutine until the conn +// dies so the entry can be settled. +// +// It is shared by the two doors a sandbox can arrive at. The WebSocket one +// above is the Docker path and every session that exists today, where the +// container dials in over the network and asserts its session id in a query +// parameter. The vsock one is a microVM guest, whose connection Firecracker +// forwards to a socket inside that VM's own directory — so the session id is +// the PATH's and not the guest's claim, which is strictly stronger than the +// hop it replaces (tenancy §18 item 53). +// +// Everything below this line was in the /register handler and is unchanged; +// the extraction is what keeps the two doors from drifting into meaning two +// different things. +func (s *Server) serveSessionConn(ctx context.Context, id string, conn relay.Conn) { // Read BEFORE the hub, because NewHubWithControl starts the read loop. // Every control frame from this conn carries it, and the registry drops // the ones that name a boot the session has moved past. Note READ, not @@ -944,7 +1010,7 @@ func (s *Server) register(w http.ResponseWriter, r *http.Request) { // process on the other end may not be the one that numbered the last // report. See sessionEntry.execReg. reg := s.reg.registration() - hub := relay.NewHubWithControl(r.Context(), relay.WSConn(c), func(payload []byte) { + hub := relay.NewHubWithControl(ctx, conn, func(payload []byte) { // On its own goroutine, deliberately: this runs on the hub's read // loop, the single goroutine demultiplexing every attachment // multiplexed over this session's conn, and routeControl ends in the @@ -965,7 +1031,8 @@ func (s *Server) register(w http.ResponseWriter, r *http.Request) { // its own and this hop stayed as it is. go s.routeControl(id, boot, reg, payload) }) - if !s.reg.setHub(id, hub) { + displaced, ok := s.reg.setHub(id, hub) + if !ok { // The entry vanished between our existence check above and now — a // concurrent DELETE raced this dial-in (session torn down while its // container was still booting). No registry entry will ever exist to @@ -974,6 +1041,9 @@ func (s *Server) register(w http.ResponseWriter, r *http.Request) { hub.Close() return } + if displaced != nil { + go retireDisplacedHub(id, displaced, s.displacedHubGrace) + } log.Printf("session %s registered", id) s.fireEvent(id, "running") // Block on the hub's own liveness signal, not r.Context(). websocket.Accept @@ -1171,6 +1241,16 @@ func (s *Server) routeControl(id string, boot, reg uint64, payload []byte) { log.Printf("session %s: control response with no id; dropping", id) return } + // The id space is fenced for responses too. A response is not a + // request, so there is no method here to refuse — but the number on + // it is the same number, and a sandbox answering with one out of the + // runner's own space has put a runner-space id on the wire for the + // control plane to echo. Dropped rather than refused: answering an + // answer is meaningless, and there was never a caller for this one. + if refusal := refuseSandboxOrigin("resp", ev.ID); refusal != "" { + log.Printf("session %s: dropping a control response from a sandbox: %s", id, refusal) + return + } if !s.fireSessionRPC(id, runner.RPCEnvelope{ID: ev.ID, Method: "resp", OK: ev.OK, Payload: ev.Payload}) { // Nothing to report to and nothing to answer: answering an answer // is meaningless, and whoever asked has already given up (its @@ -1184,6 +1264,21 @@ func (s *Server) routeControl(id string, boot, reg uint64, payload []byte) { log.Printf("session %s: unknown control kind %q", id, ev.Kind) return } + // The forwarder is generic, and that is its whole value — but it is + // also an untrusted peer's door into the control plane's method + // table, so the two things that are NOT a sandbox's to send are + // refused here rather than upstream. controld cannot make this check + // itself: a "session_req" proves only that some runner sent it, and + // which end of the runner originated it is a fact only the runner + // has. + if refusal := refuseSandboxOrigin(method, ev.ID); refusal != "" { + log.Printf("session %s: refusing %q from a sandbox: %s", id, method, refusal) + if err := s.sendSessionRPC(id, runner.RPCEnvelope{ID: ev.ID, Method: "resp", + Payload: rpcErrorPayload(refusal)}); err != nil { + log.Printf("session %s: refusing %q locally: %v", id, method, err) + } + return + } if s.fireSessionRPC(id, runner.RPCEnvelope{ID: ev.ID, Method: method, Payload: ev.Payload}) { return } @@ -1213,9 +1308,35 @@ func (s *Server) routeControl(id string, boot, reg uint64, payload []byte) { // arrived — so only a sandbox that actually speaks the vocabulary can spend // it. It is comfortably longer than the sandbox's own quiesce budget, so the // ordinary answer is the acknowledgement rather than this timeout. +// defaultColdSuspendReadyWait is the third budget, and it exists because a +// cold suspend is not a freeze and cannot be paid for at a freeze's price. +// +// A warm suspend asks the sandbox for one thing: end your execs. Twelve +// seconds is comfortably over the ten the sandbox gives that +// (execQuiesceBudget in cmd/sessiond), because the ten is itself the +// SIGTERM-then-SIGKILL escalation and nothing follows it. +// +// A cold suspend asks for three things, in series, and this VM is about to +// cease to exist with no memory image (ADR-0003 §2.2), so anything not +// finished is lost rather than deferred: +// +// - a synchronous agent-home flush, which is a write to a block device; +// - the same ten-second exec kill; +// - an unmount of /rainier/agents, which syncs that device before it goes. +// +// Ten seconds of that is the kill alone, so twelve leaves two for a flush and +// an unmount — and the first of those two to run slowly is the one whose work +// is thrown away. Thirty is the kill with a factor of three over it, which is +// headroom rather than a second stopwatch: a sandbox that answers takes +// milliseconds, and only one that is genuinely stuck spends any of this. +// +// The ACK budget is shared with the warm path deliberately. The cold handler +// answers the ack before it does any of the work above, so "did you hear me" +// is the same question at the same price on both paths. const ( - defaultSuspendAckWait = 2 * time.Second - defaultSuspendReadyWait = 12 * time.Second + defaultSuspendAckWait = 2 * time.Second + defaultSuspendReadyWait = 12 * time.Second + defaultColdSuspendReadyWait = 30 * time.Second ) // quiesceExecs tells the sandbox it is about to be frozen and waits for it to @@ -1226,7 +1347,12 @@ const ( // sessiond that predates the notice drops it, a conn that has died takes the // whole question with it, and a cancelled dispatch is one whose suspend is // about to fail on its own. None of those is worse than not sending it at all. -func (s *Server) quiesceExecs(ctx context.Context, id string) { +// cold says which kind of suspend is coming. False is a freeze and is the +// notice this function has always sent, byte for byte: `cold` is omitempty, +// so a warm notice is unchanged for every sandbox already running. True is +// the end of this VM, and asks the sandbox for the two things a freeze does +// not need — a flush and an unmount — before it answers. +func (s *Server) quiesceExecs(ctx context.Context, id string, cold bool) { hub, ok := s.reg.hub(id) if !ok { return @@ -1235,7 +1361,7 @@ func (s *Server) quiesceExecs(ctx context.Context, id string) { w := s.armSuspendWaiter(id, nonce) defer s.disarmSuspendWaiter(id, w) - b, err := json.Marshal(relay.ControlEvent{Kind: relay.KindSuspending, ID: nonce}) + b, err := json.Marshal(relay.ControlEvent{Kind: relay.KindSuspending, ID: nonce, Cold: cold}) if err != nil { log.Printf("session %s: encoding the suspend notice: %v", id, err) return @@ -1263,17 +1389,73 @@ func (s *Server) quiesceExecs(ctx context.Context, id string) { return } - // "Are they gone." Only a sandbox that answered above can spend this. - readyTimer := time.NewTimer(s.suspendReadyWait) + // "Are they gone." Only a sandbox that answered above can spend this, + // and a COLD suspend is given its own budget: it asked for a flush and + // an unmount around the same kill, and work this one does not finish is + // lost rather than deferred, because there is no memory image and no VM + // on the other side of it. + readyWait := s.suspendReadyWait + if cold { + readyWait = s.coldSuspendReadyWait + } + readyTimer := time.NewTimer(readyWait) defer readyTimer.Stop() select { case <-w.ready: case <-hub.Done(): case <-ctx.Done(): case <-readyTimer.C: + what := "ending its commands" + if cold { + what = "flushing, ending its commands and unmounting the agent home" + } log.Printf("session %s: the sandbox heard the suspend but had not finished "+ - "ending its commands within %s; suspending anyway", id, s.suspendReadyWait) + "%s within %s; suspending anyway", id, what, readyWait) + } +} + +// defaultDisplacedHubGrace is how long a hub a re-register replaced has to +// finish delivering what it had already read, before it is closed. +// +// A displaced hub is not immediately useless, which is why this is a grace +// and not a close. sessiond survives losing its conn and redials, and a hub +// read loop that was stalled writing to a wedged viewer drains its buffered +// frames whenever it comes back — and the child_exited on that older conn can +// be the ONLY copy, because sessiond re-sends only what it never delivered. +// Dropping it leaves a finished session holding its slot for the life of the +// runner (TestAChildExitAcrossARedialIsRecordedEndToEnd). +// +// But it cannot be kept forever either, and "until its conn dies" is forever +// when the peer is a sandbox that declines to hang up: nothing in the +// registry points at a displaced hub, so its readLoop, its conn, its fd and +// its attachments are held by a structure nothing can reach (review round 2, +// finding 2). +// +// Ninety seconds is over clientWriteBase (70s, internal/relay), which is the +// longest one stalled write can legitimately hold that read loop up — so a +// hub that is going to drain has drained, and one that is not is closed. The +// ordinary case costs nothing: a redial happens because the old conn broke, +// so the old hub is usually already done before this timer is armed. +const defaultDisplacedHubGrace = 90 * time.Second + +// retireDisplacedHub ends a hub a newer connection replaced, once its grace +// is up. It returns without closing anything if the hub ends on its own +// first, which is what nearly every redial looks like. +// +// The goroutine that was serving the displaced hub is parked on hub.Done(), +// so closing here is what lets it go; its hubDied finds a newer hub in the +// registry and settles nothing, which is already how a stale hub is handled. +func retireDisplacedHub(id string, h *relay.Hub, grace time.Duration) { + timer := time.NewTimer(grace) + defer timer.Stop() + select { + case <-h.Done(): + return + case <-timer.C: } + log.Printf("session %s: closing the connection a re-register replaced; it had %s to deliver what it had already read", + id, grace) + h.Close() } // armSuspendWaiter registers this suspend's waiter, replacing any stale entry: diff --git a/protocol/runner/bootstrap_test.go b/protocol/runner/bootstrap_test.go new file mode 100644 index 00000000..3d1418f4 --- /dev/null +++ b/protocol/runner/bootstrap_test.go @@ -0,0 +1,157 @@ +package runner_test + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/tokencanopy/rainier/protocol/runner" +) + +// TestBootstrapFieldsAreAdditive is the compatibility promise of the microVM +// bootstrap design, at the hop where it is made: controld rolls before +// runners, so every create a control plane that KNOWS about the token +// dispatches to a runner that does not must be byte-identical to the one that +// runner already reads. +// +// The golden create is TestPublicRunnerWireShapes'. It is repeated here +// literally rather than referenced, because the point of a golden is that a +// change has to edit the bytes and say so. +func TestBootstrapFieldsAreAdditive(t *testing.T) { + create := runner.ToRunner{ + Type: "create", ReqID: 7, Session: "sess_example", + Spec: &runner.Spec{Image: "example.invalid/agent@sha256:0000", Cmd: []string{"bash"}}, + } + got, err := json.Marshal(create) + if err != nil { + t.Fatal(err) + } + const want = `{"type":"create","req_id":7,"session":"sess_example",` + + `"spec":{"image":"example.invalid/agent@sha256:0000","cmd":["bash"]}}` + if string(got) != want { + t.Fatalf("a create with no bootstrap = %s\nwant %s", got, want) + } + for _, tag := range []string{"bootstrap_token", "secret_names"} { + if strings.Contains(string(got), `"`+tag+`"`) { + t.Fatalf("a create with no bootstrap leaked %q: %s", tag, got) + } + } + + // And a Docker create carrying secret values keeps carrying them: the + // withholding is keyed on the capability, not on the field existing. + docker, err := json.Marshal(runner.Spec{Image: "img", Env: map[string]string{"TOKEN": "value_example"}}) + if err != nil { + t.Fatal(err) + } + if string(docker) != `{"image":"img","env":{"TOKEN":"value_example"}}` { + t.Fatalf("a docker create = %s", docker) + } +} + +// TestBootstrapSpecRoundTrip pins the two tags a microVM create adds and +// proves they survive the hop — an omitempty field that never carried its +// value would fence nobody. +func TestBootstrapSpecRoundTrip(t *testing.T) { + in := runner.ToRunner{Type: "create", ReqID: 21, Session: "sess_example", Spec: &runner.Spec{ + Image: "img", + BootstrapToken: "dG9rZW5fZXhhbXBsZQ", + SecretNames: []string{"DEPLOY_KEY", "NPM_TOKEN"}, + }} + b, err := json.Marshal(in) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `"bootstrap_token":"dG9rZW5fZXhhbXBsZQ"`, + `"secret_names":["DEPLOY_KEY","NPM_TOKEN"]`, + } { + if !strings.Contains(string(b), want) { + t.Fatalf("bootstrap tags wrong on the wire: %s", b) + } + } + // A microVM create carries the NAMES and no values at all. + if strings.Contains(string(b), `"env"`) { + t.Fatalf("a withheld create still carried an env block: %s", b) + } + var out runner.ToRunner + if err := json.Unmarshal(b, &out); err != nil { + t.Fatal(err) + } + if out.Spec == nil || out.Spec.BootstrapToken != in.Spec.BootstrapToken || + len(out.Spec.SecretNames) != 2 || out.Spec.SecretNames[1] != "NPM_TOKEN" { + t.Fatalf("round trip mangled the bootstrap: %+v", out.Spec) + } +} + +// TestBootstrapVocabulary pins the three wire words. They are read by a +// control plane, a runner and a sandbox that are three separately released +// binaries, so a typo in one of them would be a silently unanswered method +// rather than a build failure. +func TestBootstrapVocabulary(t *testing.T) { + if runner.MethodFetchSessionSecrets != "fetch_session_secrets" { + t.Fatalf("fetch method = %q", runner.MethodFetchSessionSecrets) + } + if runner.MethodMintSessionBootstrap != "mint_session_bootstrap" { + t.Fatalf("mint method = %q", runner.MethodMintSessionBootstrap) + } + if runner.CapabilityMicrovmV1 != "microvm.v1" { + t.Fatalf("microvm capability = %q", runner.CapabilityMicrovmV1) + } + if runner.SessionBootstrapProtocolVersion != 1 { + t.Fatalf("bootstrap protocol = %d", runner.SessionBootstrapProtocolVersion) + } +} + +// TestBootConfigWireShape pins the bytes a host sends a guest as its first +// control frame. sessiond ships inside the session image and a session keeps +// the one it booted with for life, so these tags are a contract between +// builds that may be months apart. +func TestBootConfigWireShape(t *testing.T) { + minimal, err := json.Marshal(runner.BootConfig{ + Protocol: runner.SessionBootstrapProtocolVersion, + SessionID: "sess_example", + Cmd: []string{"bash"}, + }) + if err != nil { + t.Fatal(err) + } + const wantMinimal = `{"protocol":1,"session_id":"sess_example","cmd":["bash"]}` + if string(minimal) != wantMinimal { + t.Fatalf("a minimal boot config = %s\nwant %s", minimal, wantMinimal) + } + + full := runner.BootConfig{ + Protocol: runner.SessionBootstrapProtocolVersion, + SessionID: "sess_example", + Cmd: []string{"bash", "-l"}, + ProxyURL: "http://sess_example@proxy.invalid:3128", + NoProxy: "127.0.0.1,localhost", + EgressAllow: []string{"registry.npmjs.org"}, + Setup: "apt-get install -y jq\n", + SetupTimeoutSec: 900, + Init: "make dev\n", + InitTimeoutSec: 300, + Repos: []runner.RepoSpec{{Owner: "acme", Name: "app", BaseBranch: "main", + SessionBranch: "rainier/work", Dir: "app"}}, + GitAuthorName: "example", + GitAuthorEmail: "42+example@users.noreply.github.com", + Env: map[string]string{"CLAUDE_CONFIG_DIR": "/rainier/agents/claude"}, + SecretNames: []string{"DEPLOY_KEY"}, + BootstrapToken: "dG9rZW5fZXhhbXBsZQ", + } + b, err := json.Marshal(full) + if err != nil { + t.Fatal(err) + } + var back runner.BootConfig + if err := json.Unmarshal(b, &back); err != nil { + t.Fatal(err) + } + if back.SessionID != full.SessionID || back.NoProxy != full.NoProxy || + back.SetupTimeoutSec != 900 || back.InitTimeoutSec != 300 || + len(back.Repos) != 1 || back.Repos[0].Dir != "app" || + back.Env["CLAUDE_CONFIG_DIR"] != "/rainier/agents/claude" || + len(back.SecretNames) != 1 || back.BootstrapToken != full.BootstrapToken { + t.Fatalf("boot config round trip mangled: %+v", back) + } +} diff --git a/protocol/runner/messages.go b/protocol/runner/messages.go index 7fabe245..888def23 100644 --- a/protocol/runner/messages.go +++ b/protocol/runner/messages.go @@ -55,6 +55,54 @@ const ( MethodRevokeAgentCredentials = "revoke_agent_credentials" ) +// The two methods a microVM session's bootstrap token rides, added by +// docs/design/2026-09-20-microvm-bootstrap-token-and-vsock.md §3. +// +// They ride the shapes that already exist — up as a FromRunner "session_req", +// down as a ToRunner "session_rpc" — so neither RPCEnvelope nor either +// direction's envelope changes. Only the switch that answers them grows. +// +// Neither puts a value anywhere a forwarder reads: runnerd is documented as +// passing RPCEnvelope.Payload through without parsing it, and the control +// plane's answer names a count in its log and never a name's value. +const ( + // MethodFetchSessionSecrets is sandbox → control plane, once per boot: + // {"protocol": 1, "token": ""} → {"env": {"NAME": "value"}}. + // Refused as the usual {"error": sentence} on ok:false when the token is + // spent, expired, or fenced by a placement generation that has moved. + // + // It buys exactly the environment's decrypted secret_refs and nothing + // else: a github credential and a coding agent's login set already have + // their own methods, unchanged by this one. + MethodFetchSessionSecrets = "fetch_session_secrets" + // MethodMintSessionBootstrap is runner → control plane, on a cold resume: + // {"protocol": 1} → {"token": "", "expires_in_sec": 120}. It + // carries no session id: the id is FromRunner.Session, which the + // placement guard has already checked, which is what keeps a runner + // holding session A from minting for session B. + MethodMintSessionBootstrap = "mint_session_bootstrap" +) + +// SessionBootstrapProtocolVersion is the independently negotiated version of +// the two methods above, carried as "protocol" in both request bodies for the +// same reason AgentCredentialProtocolVersion is: the sandbox ships inside the +// session image and keeps the sessiond it booted with for life, so the two +// ends of this exchange are routinely different builds and a mismatch must be +// an answer rather than a misparse. +const SessionBootstrapProtocolVersion = 1 + +// CapabilityMicrovmV1 is the capability token a runnerd built with the +// microVM driver announces. It is a fact about the BUILD and the configured +// driver, not a claim an operator makes, so runnerd appends it rather than +// waiting to be told — the same rule CapabilityExecV1 follows. +// +// It is also the fence the control plane keys withholding on: createSpec +// leaves an environment's decrypted secret values out of Spec.Env, and mints +// a bootstrap token in their place, exactly for a placement whose runner +// announced this. A runner that does not announce it is dispatched today's +// Spec, byte for byte. +const CapabilityMicrovmV1 = "microvm.v1" + // HomeMount is the agent home a create mounts into a sandbox: one writable // volume per (creator, workspace), landing at Path, inside which each coding // agent gets its own subdirectory. It is what makes "log in once" true across @@ -314,6 +362,13 @@ type Spec struct { GitAuthorEmail string `json:"git_author_email,omitempty"` // Env is injected into the container's environment. Values are secrets // as often as not, so this field is never logged verbatim. + // + // With one exception, which is the whole of the microVM bootstrap design: + // for a placement whose runner announced CapabilityMicrovmV1 the control + // plane leaves an environment's decrypted secret_refs OUT of this map and + // sets BootstrapToken and SecretNames instead, so what remains is + // configuration — agent-home paths and the agent manifest. Either the + // values or the token, never both. Env map[string]string `json:"env,omitempty"` // Home is the agent home this session mounts: the (creator, workspace) // volume every coding agent keeps its own configuration and credential @@ -323,6 +378,84 @@ type Spec struct { // does not know the field mounts nothing and the session's agents simply // ask for a login, which is the truthful state, not a failure. Home *HomeMount `json:"home,omitempty"` + // BootstrapToken is the single-use capability a microVM session exchanges + // for its environment's decrypted secrets, which for such a session are + // deliberately absent from Env. Minted per create and per cold resume, + // fenced by the placement generation, never written to host disk. Absent + // on every Docker create and from every older control plane, which is why + // Env keeps its meaning. + BootstrapToken string `json:"bootstrap_token,omitempty"` + // SecretNames are the NAMES the token will deliver — a name is not a + // value, as the launch-material resolver already says — so the guest can + // tell "no secrets declared" from "declared and never arrived". + SecretNames []string `json:"secret_names,omitempty"` +} + +// BootConfig is the first control frame a microVM session's host sends down +// the vsock channel, and the whole of what the guest is configured with. +// +// It lives here, beside Spec, rather than in internal/relay, because it is +// the same vocabulary a create resolved: every field below is a field of Spec +// or a value derived from one, and the two drifting apart would be a session +// configured with something the control plane never dispatched. relay carries +// it as an opaque payload on a FrameControl of kind relay.KindBootConfig and +// interprets none of it. +// +// It is NOT a runner-plane message: it never travels between runnerd and +// controld. It travels between a runner and the sandbox it booted, over the +// one channel that exists before the guest's network does. +// +// Nothing in it is a secret. That is the point of the whole design: Env below +// is the non-secret configuration block (agent-home paths and the agent +// manifest — paths and names), SecretNames are names, and BootstrapToken is a +// capability the control plane can refuse, not a credential. A microVM host +// writes no part of this to disk, and the values SecretNames names arrive in +// the guest over the token exchange, from the control plane, never from here. +type BootConfig struct { + // Protocol is SessionBootstrapProtocolVersion. A guest that reads a + // version it does not speak fails its boot chain rather than booting on a + // configuration it has half understood. + Protocol int `json:"protocol"` + // SessionID is the session this guest IS. On the WebSocket path the guest + // asserts its id in a query parameter and runnerd believes it; over vsock + // the listening socket is inside one VM's own directory, so the id is the + // host's statement and not the guest's claim. + SessionID string `json:"session_id"` + // Cmd is the agent argv, ProxyURL the egress proxy every outbound request + // must route through (already carrying the session's identity as URL + // userinfo), and EgressAllow the allowlist the host enforces and the + // guest is told about. + Cmd []string `json:"cmd,omitempty"` + ProxyURL string `json:"proxy_url,omitempty"` + NoProxy string `json:"no_proxy,omitempty"` + EgressAllow []string `json:"egress_allow,omitempty"` + // The boot chain, in the shape Spec carries it rather than base64 in an + // environment block: a JSON string holds a multi-line script directly, and + // the encoding only ever existed because `docker run -e K=V` carries one + // line per variable. + Setup string `json:"setup,omitempty"` + SetupTimeoutSec int `json:"setup_timeout_sec,omitempty"` + Init string `json:"init,omitempty"` + InitTimeoutSec int `json:"init_timeout_sec,omitempty"` + Repos []RepoSpec `json:"repos,omitempty"` + GitAuthorName string `json:"git_author_name,omitempty"` + GitAuthorEmail string `json:"git_author_email,omitempty"` + // Env is the session's NON-SECRET configuration environment: for a + // microVM session that is exactly the agent-home path variables and the + // agent manifest (RAINIER_AGENTS_B64), which are paths and names. An + // environment's decrypted secret_refs are never in it — the control plane + // withholds them from Spec.Env for a runner announcing + // CapabilityMicrovmV1, and a host that finds values here with no token + // refuses the create rather than delivering them. + Env map[string]string `json:"env,omitempty"` + // SecretNames and BootstrapToken are the exchange: which names are + // expected, and the one-shot capability that fetches their values. + // SecretNames non-empty with an empty token is a boot that must FAIL — + // the guest was promised credentials it cannot get — and that refusal is + // the guest's, not the host's, because only the guest knows whether it + // has already been given them. + SecretNames []string `json:"secret_names,omitempty"` + BootstrapToken string `json:"bootstrap_token,omitempty"` } // Attach is the dial_attach block of a ToRunner: controld tells the runner