diff --git a/go/cmd/compass-runner/main.go b/go/cmd/compass-runner/main.go index dc1db2fa..1cec32c3 100644 --- a/go/cmd/compass-runner/main.go +++ b/go/cmd/compass-runner/main.go @@ -194,6 +194,14 @@ type podmanPreflighter interface { VerifyUsernsRemapSupport(ctx context.Context) error } +// canaryBooter is the microVM backend's dynamic host-capability probe: it really +// boots a throwaway VM through the backend's own verbs, proving the whole boot +// chain. Kept a DISTINCT single-method interface from microVMPreflighter (not a +// widened two-method probe) so the single-method-probe discipline holds. +type canaryBooter interface { + BootCanary(ctx context.Context) (runtime.CanaryReport, error) +} + // verifyBackendPreflight runs the selected engine's static host-capability // preflight. It dispatches on the engine's concrete type, first match wins, // probing the microVM backend before podman; no engine satisfies both today, so @@ -204,7 +212,7 @@ type podmanPreflighter interface { func verifyBackendPreflight(ctx context.Context, engine runtime.ContainerRuntime) error { switch e := engine.(type) { case microVMPreflighter: - return e.VerifyMicroVMSupport(ctx) + return runMicroVMPreflight(ctx, e, engine) case podmanPreflighter: return e.VerifyUsernsRemapSupport(ctx) default: @@ -212,6 +220,30 @@ func verifyBackendPreflight(ctx context.Context, engine runtime.ContainerRuntime } } +// runMicroVMPreflight runs the microVM backend's two-stage startup gate: the +// static VerifyMicroVMSupport check, then — only once it passes — the dynamic +// BootCanary, logging the returned CanaryReport at info. A microVM engine that +// satisfies microVMPreflighter but not canaryBooter is a fail-closed startup +// error naming the type, never a silent skip (same posture as the neither-probe +// default). Split out so verifyBackendPreflight stays within funlen. +func runMicroVMPreflight(ctx context.Context, pre microVMPreflighter, engine runtime.ContainerRuntime) error { + if err := pre.VerifyMicroVMSupport(ctx); err != nil { + return err + } + canary, ok := engine.(canaryBooter) + if !ok { + return fmt.Errorf("microVM backend %T exposes no boot canary probe", engine) + } + report, err := canary.BootCanary(ctx) + if err != nil { + return err + } + slog.Info("microvm boot canary passed", + "boot_latency", report.BootLatency, + "guest_rss_bytes", report.GuestRSSBytes) + return nil +} + // setupOtel installs the tracer and meter providers off the env-only OTLP // endpoint, returning one shutdown that flushes both. When // OTEL_EXPORTER_OTLP_ENDPOINT is empty the providers are no-ops and the shutdown diff --git a/go/cmd/compass-runner/main_test.go b/go/cmd/compass-runner/main_test.go index 0a9327ce..67d0b4e4 100644 --- a/go/cmd/compass-runner/main_test.go +++ b/go/cmd/compass-runner/main_test.go @@ -20,6 +20,7 @@ import ( var ( _ microVMPreflighter = (*runtime.MicroVMRuntime)(nil) _ podmanPreflighter = (*runtime.PodmanCLI)(nil) + _ canaryBooter = (*runtime.MicroVMRuntime)(nil) ) // parseMount is the operator surface for --mount: a malformed value must be @@ -86,14 +87,41 @@ func (e podmanOnlyEngine) VerifyUsernsRemapSupport(context.Context) error { return e.err } -// microVMOnlyEngine exposes only the microVM probe. +// microVMOnlyEngine exposes the microVM static probe AND the canary probe — a +// real microVM engine satisfies both, and the gate now runs the canary after the +// static check passes, so a fake missing BootCanary would trip the fail-closed +// canary assertion rather than exercise the static-probe dispatch. type microVMOnlyEngine struct { + runtime.ContainerRuntime + called *bool + canaryCalled *bool + err error + report runtime.CanaryReport + canaryErr error +} + +func (e microVMOnlyEngine) VerifyMicroVMSupport(context.Context) error { + *e.called = true + return e.err +} + +func (e microVMOnlyEngine) BootCanary(context.Context) (runtime.CanaryReport, error) { + if e.canaryCalled != nil { + *e.canaryCalled = true + } + return e.report, e.canaryErr +} + +// microVMNoCanaryEngine exposes ONLY the static microVM probe, not the canary — +// a microVM backend that cannot boot-canary. The gate must fail closed on it, +// naming the type, never silently skipping the canary. +type microVMNoCanaryEngine struct { runtime.ContainerRuntime called *bool err error } -func (e microVMOnlyEngine) VerifyMicroVMSupport(context.Context) error { +func (e microVMNoCanaryEngine) VerifyMicroVMSupport(context.Context) error { *e.called = true return e.err } @@ -123,6 +151,10 @@ func (e bothProbesEngine) VerifyUsernsRemapSupport(context.Context) error { return e.err } +func (e bothProbesEngine) BootCanary(context.Context) (runtime.CanaryReport, error) { + return runtime.CanaryReport{}, nil +} + // verifyBackendPreflight dispatches on the selected engine's concrete type // (RIG-2496): microVM first, then podman, first match wins; the matched probe // runs and its error is returned verbatim; an engine exposing neither probe is a @@ -141,14 +173,53 @@ func TestVerifyBackendPreflight(t *testing.T) { } }) - t.Run("microvm probe dispatched", func(t *testing.T) { - called := false - err := verifyBackendPreflight(context.Background(), microVMOnlyEngine{called: &called}) + t.Run("microvm static probe then canary dispatched", func(t *testing.T) { + called, canaryCalled := false, false + err := verifyBackendPreflight(context.Background(), + microVMOnlyEngine{called: &called, canaryCalled: &canaryCalled}) if err != nil { t.Fatalf("verifyBackendPreflight = %v, want nil", err) } if !called { - t.Error("microVM probe was not called") + t.Error("microVM static probe was not called") + } + if !canaryCalled { + t.Error("boot canary was not called after the static probe passed") + } + }) + + t.Run("static probe error skips the canary", func(t *testing.T) { + called, canaryCalled := false, false + err := verifyBackendPreflight(context.Background(), + microVMOnlyEngine{called: &called, canaryCalled: &canaryCalled, err: sentinel}) + if !errors.Is(err, sentinel) { + t.Fatalf("verifyBackendPreflight = %v, want the sentinel error", err) + } + if canaryCalled { + t.Error("boot canary ran after the static probe failed") + } + }) + + t.Run("canary error returned verbatim", func(t *testing.T) { + called := false + err := verifyBackendPreflight(context.Background(), + microVMOnlyEngine{called: &called, canaryErr: sentinel}) + if !errors.Is(err, sentinel) { + t.Errorf("verifyBackendPreflight = %v, want the canary sentinel error", err) + } + }) + + t.Run("microVM without canary is fail-closed naming the type", func(t *testing.T) { + called := false + err := verifyBackendPreflight(context.Background(), microVMNoCanaryEngine{called: &called}) + if err == nil { + t.Fatal("verifyBackendPreflight = nil, want a fail-closed canary refusal") + } + if !called { + t.Error("microVM static probe was not called") + } + if !strings.Contains(err.Error(), "microVMNoCanaryEngine") { + t.Errorf("error %q does not name the engine type", err) } }) diff --git a/go/internal/microvmtest/canary_microvm_test.go b/go/internal/microvmtest/canary_microvm_test.go index 07cc5c41..4800c524 100644 --- a/go/internal/microvmtest/canary_microvm_test.go +++ b/go/internal/microvmtest/canary_microvm_test.go @@ -22,7 +22,10 @@ // microVM is V2a's job and needs the runtime this record does not own. Asserting // the resolved Env is fully populated and the two image paths exist on disk is // the strongest claim this slice can make WITHOUT a boot — and it is a real -// assertion, never a skip-always stub. +// assertion, never a skip-always stub. This is distinct from the V5 boot canary, +// runtime.(*MicroVMRuntime).BootCanary (microvm_preflight.go), which DOES do a +// real Create→Start→Exec→Remove boot as the microVM startup preflight; despite +// the shared "canary" word the two are unrelated artifacts (record §(g)). // // It lives in the EXTERNAL test package `microvmtest_test` (not in-package) and // calls the EXPORTED microvmtest.Require, for two reasons that both matter: diff --git a/go/internal/runtime/boot_canary_microvm_test.go b/go/internal/runtime/boot_canary_microvm_test.go new file mode 100644 index 00000000..1bc87850 --- /dev/null +++ b/go/internal/runtime/boot_canary_microvm_test.go @@ -0,0 +1,67 @@ +//go:build microvm && unix + +package runtime + +// The KVM-gated BootCanary e2e (record §(e)/W3 test cycle): it drives the real +// (*MicroVMRuntime).BootCanary against live hardware, proving the whole boot +// chain — KVM, vsock, image, guest supervisor, exec gate — end to end, and that +// the canary owns its own teardown (no orphan session, no leftover runtime dir). +// It calls microvmtest.Require(t) first (skip-on-absent-KVM, hard-fail under +// COMPASS_REQUIRE_MICROVM=1) and passes t.Context() as the caller ctx, mirroring +// TestMicroVMQBudget + e2eConfig. It rides the existing CI microVM leg +// (go test -tags microvm -race -timeout 15m ./...), budgeted well inside 15m. + +import ( + "os" + "path/filepath" + "testing" + + "github.com/RigelBuild/compass/go/internal/microvmtest" +) + +// TestBootCanary boots a real canary VM through BootCanary and asserts the report +// is populated (BootLatency in (0, canaryDeadline], GuestRSSBytes > 0) and that +// BootCanary tore down its own VM and runtime dir — nothing left in the session +// table and no leftover /microvm/* dir. Unlike TestMicroVMQBudget (which +// Removes in cleanup), the canary owns its teardown, so this asserts it happened. +func TestBootCanary(t *testing.T) { + env := microvmtest.Require(t) + cfg := e2eConfig(t, env) + m := NewMicroVMRuntime(cfg) + + report, err := m.BootCanary(t.Context()) + if err != nil { + t.Fatalf("BootCanary = %v, want nil", err) + } + if report.BootLatency <= 0 { + t.Errorf("BootLatency = %v, want > 0", report.BootLatency) + } + if report.BootLatency > canaryDeadline { + t.Errorf("BootLatency = %v, want <= the %v canary deadline", report.BootLatency, canaryDeadline) + } + if report.GuestRSSBytes <= 0 { + t.Errorf("GuestRSSBytes = %d, want > 0", report.GuestRSSBytes) + } + t.Logf("BootCanary: boot latency = %s, guest PSS = %d bytes", report.BootLatency, report.GuestRSSBytes) + + // BootCanary owns its teardown: no session leaked in the table. + m.mu.Lock() + n := len(m.sessions) + m.mu.Unlock() + if n != 0 { + t.Errorf("session table has %d entries after BootCanary, want 0 (canary must tear down its own session)", n) + } + + // And no per-session runtime dir left under /microvm. + microvmDir := filepath.Join(cfg.RunRoot, "microvm") + entries, statErr := os.ReadDir(microvmDir) + if statErr != nil { + if os.IsNotExist(statErr) { + return // never created, or fully removed — both fine + } + t.Fatalf("reading %s: %v", microvmDir, statErr) + } + if len(entries) != 0 { + t.Errorf("%s has %d leftover session dirs after BootCanary, want 0", microvmDir, len(entries)) + } +} diff --git a/go/internal/runtime/boot_canary_test.go b/go/internal/runtime/boot_canary_test.go new file mode 100644 index 00000000..8234f704 --- /dev/null +++ b/go/internal/runtime/boot_canary_test.go @@ -0,0 +1,504 @@ +//go:build unix + +package runtime + +// The hermetic BootCanary suite: it drives BootCanary behind the launchFunc + +// newGuestClient seams so no real cloud-hypervisor boots and no real vsock is +// dialed, proving the canary's Create→Start→Exec→Remove sequencing, report +// assembly, deadline derivation, teardown-on-failure, and reserved naming with +// no KVM (record §(e)/(f)/(g)). It is //go:build unix because the seams and the +// guestVM interface it fakes are unix-only. +// +// KEY SEAM SUBTLETY: BootCanary calls Create INTERNALLY, minting a fresh boot +// nonce the test cannot pre-read. So the fake launchFunc parses the hex nonce out +// of cfg.Cmdline ("compass.boot_nonce="+hex) and hands it to a fake guestVM whose +// Health echoes THAT nonce — otherwise awaitHealthy's identity binding fails. The +// fake guestClient's Exec echoes the command's argument back on stdout with exit +// 0, so BootCanary's echo-nonce round-trip check passes without a real guest. + +import ( + "context" + "encoding/hex" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "connectrpc.com/connect" + + compassv1 "github.com/RigelBuild/compass/go/internal/gen/compass/v1" + "github.com/RigelBuild/compass/go/internal/gen/compass/v1/compassv1internalconnect" + "github.com/RigelBuild/compass/go/internal/runtime/microvm" +) + +// canaryFakeVM is a guestVM handle for the canary path: Health answers ready and +// echoes the boot nonce the fake launchFunc decoded from cfg.Cmdline (so +// awaitHealthy's identity binding passes), PSS returns a configurable non-empty +// map (so GuestRSSBytes is assertable), and Shutdown is recorded (the teardown +// assertion). +type canaryFakeVM struct { + nonce []byte + pss map[string]int64 + pssErr error + mu sync.Mutex + shutdown bool +} + +func (f *canaryFakeVM) Health(context.Context) (*compassv1.HealthResponse, error) { + return &compassv1.HealthResponse{ + NetProvisioned: true, + WorkspaceMounted: true, + BootNonce: f.nonce, + }, nil +} + +func (f *canaryFakeVM) Shutdown(context.Context) error { + f.mu.Lock() + f.shutdown = true + f.mu.Unlock() + return nil +} + +func (f *canaryFakeVM) WaitVMMExit(_ time.Duration) bool { return true } + +func (f *canaryFakeVM) PSS() (map[string]int64, error) { return f.pss, f.pssErr } + +func (f *canaryFakeVM) wasShutdown() bool { + f.mu.Lock() + defer f.mu.Unlock() + return f.shutdown +} + +var _ guestVM = (*canaryFakeVM)(nil) + +// canaryLaunchRecorder is the fake launchFunc: it decodes the boot nonce from +// cfg.Cmdline, builds a canaryFakeVM echoing it, and records the launched VMs + +// the deadline the launch ctx carried (for the ctx-derivation assertions). A +// non-nil launchErr makes launch fail (the Start-failure case). +type canaryLaunchRecorder struct { + mu sync.Mutex + pss map[string]int64 + pssErr error + launchErr error + vms []*canaryFakeVM + calls int + lastDeadline time.Time + lastHasDeadln bool +} + +func (r *canaryLaunchRecorder) launch(ctx context.Context, cfg microvm.BootConfig) (guestVM, error) { + r.mu.Lock() + defer r.mu.Unlock() + r.calls++ + deadline, ok := ctx.Deadline() + r.lastDeadline = deadline + r.lastHasDeadln = ok + if r.launchErr != nil { + return nil, r.launchErr + } + nonce, err := parseBootNonce(cfg.Cmdline) + if err != nil { + return nil, err + } + vm := &canaryFakeVM{nonce: nonce, pss: r.pss, pssErr: r.pssErr} + r.vms = append(r.vms, vm) + return vm, nil +} + +func (r *canaryLaunchRecorder) snapshot() (calls int, deadline time.Time, hasDeadline bool) { + r.mu.Lock() + defer r.mu.Unlock() + return r.calls, r.lastDeadline, r.lastHasDeadln +} + +// parseBootNonce decodes the raw boot nonce out of the cmdline bootConfig set +// ("compass.boot_nonce="+hex), the seam-level value the fake launchFunc sees +// before Launch would append its own console/vsock params. +func parseBootNonce(cmdline string) ([]byte, error) { + const prefix = "compass.boot_nonce=" + if !strings.HasPrefix(cmdline, prefix) { + return nil, fmt.Errorf("canary fake: cmdline %q missing %q prefix", cmdline, prefix) + } + return hex.DecodeString(strings.TrimPrefix(cmdline, prefix)) +} + +// canaryFakeClient is a GuestControlClient for the canary path: Provision +// succeeds, and Exec echoes the command's argument back on stdout with a +// configurable exit code (default 0) unless execErr forces a transport failure. +// stdout, when non-nil, overrides the echoed output verbatim so a test can drive +// the exit-0-but-wrong-stdout case (the nonce-mismatch branch). +type canaryFakeClient struct { + execErr error + stdout *string + exitCode int32 +} + +func (c *canaryFakeClient) Provision(context.Context, *connect.Request[compassv1.ProvisionRequest]) (*connect.Response[compassv1.ProvisionResponse], error) { + return connect.NewResponse(&compassv1.ProvisionResponse{}), nil +} + +func (c *canaryFakeClient) Exec(_ context.Context, req *connect.Request[compassv1.ExecRequest]) (*connect.Response[compassv1.ExecResponse], error) { + if c.execErr != nil { + return nil, c.execErr + } + // Echo the argument(s) after the command name, mirroring `echo `, so + // BootCanary's nonce-round-trip check passes — unless stdout overrides it. + cmd := req.Msg.GetCommand() + var out string + if len(cmd) > 1 { + out = strings.Join(cmd[1:], " ") + } + if c.stdout != nil { + out = *c.stdout + } + return connect.NewResponse(&compassv1.ExecResponse{ + Stdout: []byte(out + "\n"), + ExitCode: c.exitCode, + }), nil +} + +func (c *canaryFakeClient) Health(context.Context, *connect.Request[compassv1.HealthRequest]) (*connect.Response[compassv1.HealthResponse], error) { + return nil, errors.New("canaryFakeClient: Health not used on the canary path") +} + +func (c *canaryFakeClient) ExecStream(context.Context) *connect.BidiStreamForClient[compassv1.ExecStreamRequest, compassv1.ExecStreamResponse] { + return nil +} + +func (c *canaryFakeClient) Signal(context.Context, *connect.Request[compassv1.SignalRequest]) (*connect.Response[compassv1.SignalResponse], error) { + return nil, errors.New("canaryFakeClient: Signal not used on the canary path") +} + +var _ compassv1internalconnect.GuestControlClient = (*canaryFakeClient)(nil) + +// seamCanary wires a MicroVMRuntime's launch + client seams to the canary fakes +// over a short runroot, returning the runtime, the launch recorder, and the +// client so a test can tune failures. +func seamCanary(t *testing.T, pss map[string]int64) (*MicroVMRuntime, *canaryLaunchRecorder, *canaryFakeClient) { + t.Helper() + m := NewMicroVMRuntime(MicroVMConfig{RunRoot: shortRunRoot(t)}) + rec := &canaryLaunchRecorder{pss: pss} + client := &canaryFakeClient{} + m.launchFunc = rec.launch + m.newGuestClient = func(string, uint32) compassv1internalconnect.GuestControlClient { return client } + return m, rec, client +} + +// sessionCount reads the live session-table size under the lock. +func sessionCount(m *MicroVMRuntime) int { + m.mu.Lock() + defer m.mu.Unlock() + return len(m.sessions) +} + +// canaryTempDirs is the set of leftover throwaway-workspace dirs BootCanary +// creates under os.TempDir(), so a test can assert it removed its own. +func canaryTempDirs(t *testing.T) map[string]bool { + t.Helper() + matches, err := filepath.Glob(filepath.Join(os.TempDir(), canaryNamePrefix+"*")) + if err != nil { + t.Fatalf("globbing canary temp dirs: %v", err) + } + set := make(map[string]bool, len(matches)) + for _, p := range matches { + set[p] = true + } + return set +} + +// assertNoTempLeak fails if any canary workspace dir exists now that did not +// before the call — proving BootCanary deleted the throwaway dir it minted. +func assertNoTempLeak(t *testing.T, before map[string]bool) { + t.Helper() + for p := range canaryTempDirs(t) { + if !before[p] { + t.Errorf("canary leaked a throwaway workspace dir: %s", p) + } + } +} + +// TestBootCanarySequencing pins the happy path: Create→Start→Exec→Remove run in +// order, the report is populated (BootLatency > 0 from the Start wall time, +// GuestRSSBytes summed from PSS kB→bytes), the launched VM is torn down, the +// session table is empty after, and no throwaway workspace leaks (record §(e)). +func TestBootCanarySequencing(t *testing.T) { + before := canaryTempDirs(t) + m, rec, _ := seamCanary(t, map[string]int64{"cloud-hypervisor": 100, "virtiofsd": 50}) + + report, err := m.BootCanary(t.Context()) + if err != nil { + t.Fatalf("BootCanary = %v, want nil", err) + } + if report.BootLatency <= 0 { + t.Errorf("BootLatency = %v, want > 0", report.BootLatency) + } + // (100 + 50) kB * 1024 = 153600 bytes. + if want := int64((100 + 50) * 1024); report.GuestRSSBytes != want { + t.Errorf("GuestRSSBytes = %d, want %d (PSS kB summed and converted to bytes)", report.GuestRSSBytes, want) + } + if calls, _, _ := rec.snapshot(); calls != 1 { + t.Errorf("launch called %d times, want 1", calls) + } + if len(rec.vms) != 1 || !rec.vms[0].wasShutdown() { + t.Error("canary VM was not shut down on teardown") + } + if n := sessionCount(m); n != 0 { + t.Errorf("session table has %d entries after BootCanary, want 0", n) + } + assertNoTempLeak(t, before) +} + +// TestBootCanaryStartFailureLeaksNothing: a launchFunc error fails BootCanary and +// leaves no session in the table and no throwaway workspace on disk — Create's +// entry is torn down by the always-run Remove (record §(e)/(f)). +func TestBootCanaryStartFailureLeaksNothing(t *testing.T) { + before := canaryTempDirs(t) + m, rec, _ := seamCanary(t, nil) + rec.launchErr = errors.New("boom: launch refused") + + _, err := m.BootCanary(t.Context()) + if err == nil { + t.Fatal("BootCanary = nil, want the launch error") + } + if !strings.Contains(err.Error(), "boom") { + t.Errorf("error %q does not carry the launch failure", err) + } + if n := sessionCount(m); n != 0 { + t.Errorf("session table has %d entries after a failed BootCanary, want 0", n) + } + assertNoTempLeak(t, before) +} + +// TestBootCanaryEchoFailureStillTearsDown: an echo-exec transport failure fails +// BootCanary, but the always-run Remove still shuts the VM down, empties the +// session table, and deletes the throwaway workspace (record §(e)). +func TestBootCanaryEchoFailureStillTearsDown(t *testing.T) { + before := canaryTempDirs(t) + m, rec, client := seamCanary(t, nil) + client.execErr = errors.New("boom: exec refused") + + _, err := m.BootCanary(t.Context()) + if err == nil { + t.Fatal("BootCanary = nil, want the echo-exec error") + } + if len(rec.vms) != 1 || !rec.vms[0].wasShutdown() { + t.Error("canary VM was not shut down after the echo failure") + } + if n := sessionCount(m); n != 0 { + t.Errorf("session table has %d entries after a failed BootCanary, want 0", n) + } + assertNoTempLeak(t, before) +} + +// TestBootCanaryNonZeroExitFails: an echo exec that returns a non-zero exit code +// (a successful call, not a transport error) is still a canary failure — the +// canary is a health gate, so a bad exit fails it. +func TestBootCanaryNonZeroExitFails(t *testing.T) { + before := canaryTempDirs(t) + m, rec, client := seamCanary(t, nil) + client.exitCode = 3 + + _, err := m.BootCanary(t.Context()) + if err == nil { + t.Fatal("BootCanary = nil, want a non-zero-exit failure") + } + if !strings.Contains(err.Error(), "exited 3") { + t.Errorf("error %q does not name the non-zero exit", err) + } + if len(rec.vms) != 1 || !rec.vms[0].wasShutdown() { + t.Error("canary VM was not shut down after the non-zero-exit failure") + } + if n := sessionCount(m); n != 0 { + t.Errorf("session table has %d entries after a failed BootCanary, want 0", n) + } + assertNoTempLeak(t, before) +} + +// TestBootCanaryNonceMismatchFails: an echo exec that returns exit 0 but stdout +// NOT containing the boot nonce is a canary failure. Exit 0 alone only proves a +// call returned; the nonce round-trip is the sole assertion that the guest really +// ran OUR command and returned OUR data — the "exec gate" leg of the whole-chain +// claim (record §(e)). A regression dropping this check (or a guestd returning +// exit 0 with stubbed/empty stdout) would let a broken exec path pass the startup +// canary as a healthy boot, the exact fail-open the gate prevents; this test +// breaks on that. The always-run teardown must still fire. +func TestBootCanaryNonceMismatchFails(t *testing.T) { + before := canaryTempDirs(t) + m, rec, client := seamCanary(t, nil) + wrong := "not-the-nonce" + client.stdout = &wrong // exit 0, but stdout never carries the minted nonce + + _, err := m.BootCanary(t.Context()) + if err == nil { + t.Fatal("BootCanary = nil, want a nonce-mismatch failure") + } + if !strings.Contains(err.Error(), "does not contain the nonce") { + t.Errorf("error %q does not name the nonce mismatch", err) + } + if len(rec.vms) != 1 || !rec.vms[0].wasShutdown() { + t.Error("canary VM was not shut down after the nonce-mismatch failure") + } + if n := sessionCount(m); n != 0 { + t.Errorf("session table has %d entries after a failed BootCanary, want 0", n) + } + assertNoTempLeak(t, before) +} + +// TestBootCanaryPSSErrorNonFatal pins the one fail-open seam on an otherwise +// fail-closed startup gate (record §(e)/OQ-10): a PSS read error is telemetry, +// never fatal — the canary still succeeds with GuestRSSBytes == 0 and the +// session still tears down. A regression flipping this branch to fatal would +// make the canary spuriously refuse Runner startup on a host with an unreadable +// smaps_rollup, and this test breaks on that flip. +func TestBootCanaryPSSErrorNonFatal(t *testing.T) { + before := canaryTempDirs(t) + m, rec, _ := seamCanary(t, nil) + rec.pssErr = errors.New("boom: smaps_rollup unreadable") + + report, err := m.BootCanary(t.Context()) + if err != nil { + t.Fatalf("BootCanary = %v, want nil (a PSS read error is best-effort telemetry, never fatal)", err) + } + if report.GuestRSSBytes != 0 { + t.Errorf("GuestRSSBytes = %d, want 0 on a PSS read error", report.GuestRSSBytes) + } + if len(rec.vms) != 1 || !rec.vms[0].wasShutdown() { + t.Error("canary VM was not shut down after a PSS read error") + } + if n := sessionCount(m); n != 0 { + t.Errorf("session table has %d entries after BootCanary, want 0", n) + } + assertNoTempLeak(t, before) +} + +// TestBootCanaryPartialPSSStillReported pins the OTHER half of the PSS contract: +// the real VM.PSS() (microvm/launch.go) returns a PARTIAL map ALONGSIDE a non-nil +// joined error when some children read and some do not — that is its normal shape, +// not an edge case. BootCanary must sum what it could read rather than discarding +// the partial map on any error. A regression zeroing the map inside the error +// branch (a plausible "tidy the error path" refactor) silently drops real +// telemetry, and this test breaks on that (record §(e)/OQ-10). +func TestBootCanaryPartialPSSStillReported(t *testing.T) { + before := canaryTempDirs(t) + m, rec, _ := seamCanary(t, map[string]int64{"cloud-hypervisor": 100, "virtiofsd": 50}) + rec.pssErr = errors.New("boom: one child's smaps_rollup unreadable") + + report, err := m.BootCanary(t.Context()) + if err != nil { + t.Fatalf("BootCanary = %v, want nil (a partial PSS read is best-effort telemetry, never fatal)", err) + } + // (100 + 50) kB * 1024 = 153600 bytes — the partial map is still summed + // despite the accompanying error. + if want := int64((100 + 50) * 1024); report.GuestRSSBytes != want { + t.Errorf("GuestRSSBytes = %d, want %d (a partial PSS map must still be reported)", report.GuestRSSBytes, want) + } + if len(rec.vms) != 1 || !rec.vms[0].wasShutdown() { + t.Error("canary VM was not shut down after a partial PSS read") + } + if n := sessionCount(m); n != 0 { + t.Errorf("session table has %d entries after BootCanary, want 0", n) + } + assertNoTempLeak(t, before) +} + +// TestBootCanaryDerivesDeadlineWhenCallerHasNone: with a deadline-less caller +// ctx, BootCanary derives the internal canaryDeadline bound and threads it into +// Start (so a wedged boot cannot hang Runner startup) — the launch ctx carries a +// deadline ~canaryDeadline out (record §(f)). +func TestBootCanaryDerivesDeadlineWhenCallerHasNone(t *testing.T) { + m, rec, _ := seamCanary(t, nil) + + if _, err := m.BootCanary(t.Context()); err != nil { + t.Fatalf("BootCanary = %v, want nil", err) + } + _, deadline, ok := rec.snapshot() + if !ok { + t.Fatal("launch ctx carried no deadline; BootCanary did not derive the canary bound") + } + remaining := time.Until(deadline) + if remaining <= 0 || remaining > canaryDeadline { + t.Errorf("derived deadline %v out, want within (0, %v]", remaining, canaryDeadline) + } +} + +// TestBootCanaryHonorsCallerDeadline: a caller ctx WITH a deadline is used as-is +// — BootCanary does NOT re-derive a fresh canaryDeadline over it, so the launch +// ctx carries the caller's shorter deadline (record §(f)). +func TestBootCanaryHonorsCallerDeadline(t *testing.T) { + m, rec, _ := seamCanary(t, nil) + callerBound := 5 * time.Second + ctx, cancel := context.WithTimeout(t.Context(), callerBound) + defer cancel() + + if _, err := m.BootCanary(ctx); err != nil { + t.Fatalf("BootCanary = %v, want nil", err) + } + _, deadline, ok := rec.snapshot() + if !ok { + t.Fatal("launch ctx carried no deadline") + } + // The caller's 5s deadline must be honored as-is, not widened to the 90s + // canaryDeadline: remaining must be well under canaryDeadline. + if remaining := time.Until(deadline); remaining > callerBound { + t.Errorf("launch deadline %v out exceeds the caller's %v bound; BootCanary re-derived instead of honoring the caller", remaining, callerBound) + } +} + +// TestBootCanaryHonorsLongerCallerDeadline: a caller ctx whose deadline EXCEEDS +// canaryDeadline is used as-is — BootCanary does NOT clamp it down to the 90s +// canary bound. This is the case the `if _, ok := ctx.Deadline(); !ok` guard +// actually protects: a shorter caller deadline is enforced by context.WithTimeout +// regardless of the guard, so only a longer one distinguishes honoring the caller +// from re-deriving. A regression dropping the guard (always re-deriving +// canaryDeadline) would clamp the caller's longer deadline, and this test breaks +// on that (record §(f)). +func TestBootCanaryHonorsLongerCallerDeadline(t *testing.T) { + m, rec, _ := seamCanary(t, nil) + callerBound := 10 * time.Minute + ctx, cancel := context.WithTimeout(t.Context(), callerBound) + defer cancel() + + if _, err := m.BootCanary(ctx); err != nil { + t.Fatalf("BootCanary = %v, want nil", err) + } + _, deadline, ok := rec.snapshot() + if !ok { + t.Fatal("launch ctx carried no deadline") + } + // The caller's 10m deadline must pass through, not be clamped to the 90s + // canaryDeadline: remaining must be materially greater than canaryDeadline. + if remaining := time.Until(deadline); remaining <= canaryDeadline { + t.Errorf("launch deadline %v out was clamped to the canary bound; BootCanary re-derived instead of honoring the caller's longer %v deadline", remaining, callerBound) + } +} + +// TestCanaryNameReserved pins the naming contract (record §(e)/(g)): a minted +// canary name carries the reserved compass-canary- prefix, never the agent +// session prefix, so a canary can never collide with a real agent session; two +// mints differ. +func TestCanaryNameReserved(t *testing.T) { + name, err := canaryName() + if err != nil { + t.Fatalf("canaryName = %v, want nil", err) + } + if !strings.HasPrefix(name, canaryNamePrefix) { + t.Errorf("canary name %q lacks the reserved %q prefix", name, canaryNamePrefix) + } + // The agent session prefix is "compass-agent-" (runner.AgentContainerNamePrefix, + // not imported here to avoid the runner→runtime import cycle). + const agentSessionPrefix = "compass-agent-" + if strings.HasPrefix(name, agentSessionPrefix) { + t.Errorf("canary name %q collides with the agent session prefix %q", name, agentSessionPrefix) + } + other, err := canaryName() + if err != nil { + t.Fatalf("canaryName (second) = %v, want nil", err) + } + if name == other { + t.Errorf("two canary names collided: %q", name) + } +} diff --git a/go/internal/runtime/microvm_preflight.go b/go/internal/runtime/microvm_preflight.go index 15db0899..fd6409ee 100644 --- a/go/internal/runtime/microvm_preflight.go +++ b/go/internal/runtime/microvm_preflight.go @@ -5,6 +5,7 @@ package runtime import ( "bufio" "context" + "crypto/rand" "crypto/sha256" "encoding/hex" "errors" @@ -15,7 +16,9 @@ import ( "os/exec" //nolint:depguard // microVM preflight: LookPath + fixed-arg --version probes of the VMM userspace trio "path/filepath" "strings" + "time" + "github.com/RigelBuild/compass/go/internal/agentuid" "github.com/RigelBuild/compass/go/internal/hostcheck" "github.com/RigelBuild/compass/go/internal/runtime/microvm" ) @@ -244,3 +247,167 @@ func parseManifest(path string) (map[string]string, error) { } return out, nil } + +// canaryDeadline bounds a whole BootCanary call end to end when the caller's ctx +// carries no deadline of its own: boot (≤60s worst case, bootDeadline) plus +// headroom for provision, one echo exec, and the severed teardown (record §(f)). +const canaryDeadline = 90 * time.Second + +// canaryTeardownGrace is the fresh short grace the severed teardown ctx carries: +// the canary's own bounded ctx may already have expired (a mid-boot timeout), so +// teardown runs under a WithoutCancel copy of it so the VM is torn down cleanly +// instead of against an already-dead ctx (record §(f)). NB: this bounds Remove +// only if/when Remove honors its ctx deadline — today Remove is deadline-agnostic +// (vm.Shutdown re-strips cancellation and bounds itself with its own reapGrace +// timer, os.RemoveAll ignores ctx), so the grace is not yet an enforced ceiling. +const canaryTeardownGrace = 30 * time.Second + +// canaryNamePrefix is the reserved name prefix every canary session carries. It +// sits outside runner.AgentContainerNamePrefix ("compass-agent-") so a canary +// can never collide with a real agent session in the table (record §(e)). +const canaryNamePrefix = "compass-canary-" + +// CanaryReport is the measurement a successful BootCanary produces: the boot +// latency the boot chain took to reach ready, and the guest's memory footprint. +type CanaryReport struct { + // BootLatency is the wall time of the canary's Start call — the full + // Launch→Health-OK→Provision "time to ready" window a real session waits + // (record §(e); mirrors TestMicroVMQBudget's Start timing). + BootLatency time.Duration + // GuestRSSBytes is the summed proportional-set-size (PSS) of the VM's + // host-side processes (VMM/virtiofsd/passt), converted from the kB + // smaps_rollup reports to bytes. PSS, not RSS: guest RAM is one shared + // mapping and PSS divides shared pages among mappers, so it is the honest + // per-VM share (record §(e)/OQ-10, launch.go PSS). The value undercounts by + // the passt share on a healthy host (passt sets PR_SET_DUMPABLE=0, so its + // smaps_rollup Pss is unreadable and drops out of the sum), so treat it as a + // vmm+virtiofsd-dominated lower bound, not an exact footprint. Zero when PSS + // is wholly unreadable — the canary's gate is the boot chain, RSS is + // telemetry, so a PSS read error is reported, never fatal. + GuestRSSBytes int64 +} + +// BootCanary is the dynamic startup preflight for the microVM backend: it really +// boots a throwaway canary VM through the backend's OWN lifecycle verbs +// (Create→Start→Exec→Remove), proving the whole chain — KVM, vsock, image, +// guest supervisor, exec gate — not just binary presence, and returns the boot +// latency + guest PSS the observability surface consumes (record §(e)). It owns +// the VM's entire lifetime inside the call, so it derives a bounded ctx spanning +// boot→teardown from the caller's ctx (the caller's deadline when present, else +// canaryDeadline) — the ctx-lifetime footgun guards VMs that OUTLIVE the bound, +// which this VM structurally cannot (record §(f)). Teardown is severed from that +// deadline so a mid-boot timeout still tears the VM down; the teardown and the +// throwaway-workspace cleanup errors are joined into the return, never discarded. +func (m *MicroVMRuntime) BootCanary(ctx context.Context) (report CanaryReport, err error) { + // Derive the bound only when the caller carries none: under a caller + // deadline the whole canary honors it as-is (mirrors bootPollContext). + if _, ok := ctx.Deadline(); !ok { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, canaryDeadline) + defer cancel() + } + + name, err := canaryName() + if err != nil { + return CanaryReport{}, err + } + + // A NON-empty mount set is required: an empty set leaves FSSharedDir empty, + // yet Launch always runs virtiofsd, and the guest health gate blocks forever + // on workspace_mounted (record §(e)). A freshly-minted throwaway dir is the + // real virtio-fs share a session boots. + workspace, err := os.MkdirTemp("", canaryNamePrefix) + if err != nil { + return CanaryReport{}, fmt.Errorf("microvm: canary: creating throwaway workspace: %w", err) + } + // Registered before the Remove defer so it runs AFTER teardown (LIFO): the + // VM is gone before its backing share is deleted. Joins into the return. + defer func() { + if rmErr := os.RemoveAll(workspace); rmErr != nil { + err = errors.Join(err, fmt.Errorf("microvm: canary: removing throwaway workspace %s: %w", workspace, rmErr)) + } + }() + + id, err := m.Create(ctx, ContainerSpec{ + Name: name, + UID: agentuid.AgentUID, + Mounts: []Mount{{HostPath: workspace, ContainerPath: workspaceMountPath}}, + }) + if err != nil { + return CanaryReport{}, fmt.Errorf("microvm: canary: creating session: %w", err) + } + // Remove ALWAYS runs — even when a later step errors — under a ctx severed + // from the canary deadline (record §(f)), so a timed-out boot still tears + // down. Its error is joined, never discarded (mirrors Remove's own + // errors.Join, microvm_lifecycle.go). + defer func() { + teardownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), canaryTeardownGrace) + defer cancel() + if rmErr := m.Remove(teardownCtx, id); rmErr != nil { + err = errors.Join(err, fmt.Errorf("microvm: canary: removing session %s: %w", id, rmErr)) + } + }() + + start := time.Now() + if startErr := m.Start(ctx, id); startErr != nil { + return CanaryReport{}, fmt.Errorf("microvm: canary: starting session: %w", startErr) + } + report.BootLatency = time.Since(start) + + nonce, err := canaryNonce() + if err != nil { + return CanaryReport{}, err + } + out, err := m.Exec(ctx, id, NewExecSpec("echo", nonce)) + if err != nil { + return CanaryReport{}, fmt.Errorf("microvm: canary: echo exec: %w", err) + } + if out.ExitCode != 0 { + return CanaryReport{}, fmt.Errorf("microvm: canary: echo exec exited %d, want 0 (stderr: %q)", out.ExitCode, out.Stderr) + } + if !strings.Contains(strings.TrimSpace(out.Stdout), nonce) { + return CanaryReport{}, fmt.Errorf("microvm: canary: echo output %q does not contain the nonce", out.Stdout) + } + + session, err := m.session(id) + if err != nil { + return CanaryReport{}, fmt.Errorf("microvm: canary: resolving session for PSS: %w", err) + } + // PSS is best-effort telemetry, not the boot gate (record §(e)): a read + // error (or a sandboxed helper with no readable smaps_rollup) leaves + // GuestRSSBytes at 0, logged, never failing the canary. + m.mu.Lock() + vm := session.vm + m.mu.Unlock() + if vm != nil { + pss, pssErr := vm.PSS() + if pssErr != nil { + slog.Warn("microvm: canary: reading guest PSS (best-effort)", "error", pssErr) + } + for _, kb := range pss { + report.GuestRSSBytes += kb * 1024 + } + } + return report, nil +} + +// canaryName mints a reserved canary session name: canaryNamePrefix plus 8 hex +// (4 random bytes), outside the agent-session prefix so it cannot collide with a +// real session (record §(e)). +func canaryName() (string, error) { + var b [4]byte + if _, err := rand.Read(b[:]); err != nil { + return "", fmt.Errorf("microvm: canary: minting name suffix: %w", err) + } + return canaryNamePrefix + hex.EncodeToString(b[:]), nil +} + +// canaryNonce mints a fresh random hex token the echo exec must round-trip on +// stdout, proving the guest exec path really ran (record §(e)). +func canaryNonce() (string, error) { + var b [16]byte + if _, err := rand.Read(b[:]); err != nil { + return "", fmt.Errorf("microvm: canary: minting echo nonce: %w", err) + } + return hex.EncodeToString(b[:]), nil +}