feat: qualify B1 guest and runner recovery

cmd/rainier/lifecycle.go

@@ -115,7 +115,7 @@ func eligibilityText(verdict, note string) string {
 // yet" from "never".
 func attachNote(s session) string {
 	switch s.State {
-	case "queued", "creating":
+	case "queued", "creating", "resuming":
 		return "waits for it to start"
 	case "suspended_warm", "suspended_cold":
 		return "resumes it first"
@@ -141,7 +141,7 @@ func stopNote(s session) string {
 		return "this build does not know this state; the server decides"
 	}
 	switch s.State {
-	case "queued", "creating":
+	case "queued", "creating", "resuming":
 		return "only a running session can be stopped"
 	case "suspended_warm", "suspended_cold":
 		return "already stopped"

cmd/rainier/sessionstate.go

@@ -77,7 +77,7 @@ const (
 // exited would be the collapse this file exists to prevent.
 func lifecycleOf(s session) string {
 	switch s.State {
-	case "queued", "creating":
+	case "queued", "creating", "resuming":
 		return lifecycleStarting
 	case "running":
 		// Regardless of child_exit_code. The sandbox is up.
@@ -188,7 +188,7 @@ const (
 // Everything else is refused by the endpoint.
 func canAttach(s session) string {
 	switch s.State {
-	case "running", "queued", "creating", "suspended_warm", "suspended_cold":
+	case "running", "queued", "creating", "resuming", "suspended_warm", "suspended_cold":
 		return eligibleYes
 	case "failed":
 		if s.Reachable {
@@ -216,7 +216,7 @@ func canStop(s session) string {
 	switch s.State {
 	case "running":
 		return eligibleYes
-	case "queued", "creating", "suspended_warm", "suspended_cold",
+	case "queued", "creating", "resuming", "suspended_warm", "suspended_cold",
 		"failed", "dead", "canceled", "destroyed":
 		return eligibleNo
 	default:
@@ -231,7 +231,7 @@ func canStop(s session) string {
 // a live container and deleting it is the cleanup.
 func canDelete(s session) string {
 	switch s.State {
-	case "running", "queued", "suspended_warm", "suspended_cold",
+	case "running", "queued", "resuming", "suspended_warm", "suspended_cold",
 		"failed", "dead", "canceled":
 		return eligibleYes
 	case "creating":

cmd/rainier/sessionstate_test.go

@@ -28,6 +28,7 @@ func TestThreeDimensions(t *testing.T) {
 			"creating", session{State: "creating", Reachable: true},
 			lifecycleStarting, processNone, connectionAvailable,
 		},
+		{"resuming", session{State: "resuming", Reachable: true}, lifecycleStarting, processNone, connectionAvailable},
 		{
 			"running with a live child", session{State: "running", Reachable: true},
 			lifecycleRunning, processRunning, connectionAvailable,
@@ -159,6 +160,7 @@ func TestActionEligibilityFollowsRawStates(t *testing.T) {
 			// it with a conflict because a dispatch may be in flight.
 			"creating", session{State: "creating"}, eligibleYes, eligibleNo, eligibleNo,
 		},
+		{"resuming", session{State: "resuming"}, eligibleYes, eligibleNo, eligibleYes},
 		{"suspended warm", session{State: "suspended_warm"}, eligibleYes, eligibleNo, eligibleYes},
 		{"suspended cold", session{State: "suspended_cold"}, eligibleYes, eligibleNo, eligibleYes},
 		{

cmd/rainier/wait.go

@@ -45,7 +45,7 @@ func initialAttachGuidance(ctx context.Context, cfg cli.Config, id string, since
 	} else {
 		row := resp.Session
 		switch row.State {
-		case "queued", "creating":
+		case "queued", "creating", "resuming":
 			explanation = "session is " + row.State + " and not ready to attach"
 			if row.QueueReason != "" {
 				// Queue text is server-owned prose. Redact known credentials before

cmd/runnerd/main.go

@@ -34,6 +34,8 @@ func main() {
 		"bearer token for the controld dial (required when --controld is set; or set RAINIER_RUNNER_TOKEN, which keeps it out of the process list)")
 	driverFlag := flag.String("driver", envDefault("RAINIER_RUNNER_DRIVER", "docker"),
 		"execution driver for session sandboxes: docker | microvm")
+	microvmGuestReconnect := flag.Bool("microvm-guest-reconnect", os.Getenv("RAINIER_MICROVM_GUEST_RECONNECT") == "1",
+		"opt in to authenticated recovery of surviving microVM guests after control connection or runner restart; requires a matching control plane and guest image")
 	kernelPath := flag.String("kernel", envDefault("RAINIER_KERNEL_PATH", ""),
 		"guest vmlinux kernel path for the microvm driver (required when --driver=microvm; the runner refuses to start without a readable one)")
 	rootfsPath := flag.String("rootfs", envDefault("RAINIER_ROOTFS_PATH", ""),
@@ -149,14 +151,15 @@ func main() {
 			log.Fatalf("--driver=microvm: %v", err)
 		}
 		mvm, err := driver.NewMicrovm(driver.MicrovmOpts{
-			Checkpoint:  ckpt,
-			KernelPath:  *kernelPath,
-			BaseRootfs:  *rootfsPath,
-			StateDir:    *microvmStateDir,
-			TotalSlots:  *slots,
-			VCPU:        *microvmVCPUs,
-			MemoryMiB:   *microvmMemoryMiB,
-			ImageSource: imageSource,
+			GuestReconnect: *microvmGuestReconnect,
+			Checkpoint:     ckpt,
+			KernelPath:     *kernelPath,
+			BaseRootfs:     *rootfsPath,
+			StateDir:       *microvmStateDir,
+			TotalSlots:     *slots,
+			VCPU:           *microvmVCPUs,
+			MemoryMiB:      *microvmMemoryMiB,
+			ImageSource:    imageSource,
 
 			SlotGuestCIDR:  *microvmGuestCIDR,
 			SlotUplinkCIDR: *microvmUplinkCIDR,

cmd/sessiond/bootconfig.go

@@ -65,6 +65,10 @@ const (
 // each connection and, when that configuration brings a token it has not
 // already spent, exchanges it for the session's secrets.
 //
+// Opted-in fresh boots instead retain a signing identity and require the
+// fail-closed reconnect preamble in reconnect.go. The following describes the
+// unchanged legacy mode.
+//
 // It is stateful for one reason, and it is the reason a redial is not a
 // resume: a bootstrap token is SINGLE-USE. A sessiond that re-exchanged on
 // every connection would be refused on the first redial inside a live VM and
@@ -74,7 +78,11 @@ const (
 // exactly "at boot and again after every resume", with no second signal
 // needed.
 type bootstrapper struct {
-	mu sync.Mutex
+	mu                   sync.Mutex
+	guest                *guestReconnectIdentity
+	reconnecting         bool
+	configured           []string
+	configurationApplied func(context.Context) error
 	// attempted is the last token this session sent a request for, whatever
 	// the answer was.
 	//
@@ -161,6 +169,9 @@ func readBootConfig(ctx context.Context, conn relay.Conn) (runner.BootConfig, er
 // dispatcher because the dispatcher is not serving yet: the relay it rides on
 // is started later, by dialLoop, once there is a session to serve.
 func (b *bootstrapper) exchange(ctx context.Context, conn relay.Conn, cfg runner.BootConfig) (map[string]string, error) {
+	if cfg.GuestReconnect != 0 {
+		return b.enrollGuest(ctx, conn, cfg)
+	}
 	if len(cfg.SecretNames) == 0 {
 		// A clean boot. "No secrets declared" and "declared and never
 		// arrived" are different facts, and this is the first.
@@ -291,11 +302,16 @@ func (b *bootstrapper) forget() int {
 //
 // RAINIER_DIAL is deliberately NOT set. There is nothing to dial.
 func applyBootConfig(cfg runner.BootConfig, secrets map[string]string) error {
+	return visitBootConfig(cfg, secrets, os.Setenv)
+}
+
+// visitBootConfig shares the legacy precedence and encoding with reconnect.
+func visitBootConfig(cfg runner.BootConfig, secrets map[string]string, put func(string, string) error) error {
 	set := func(k, v string) error {
 		if v == "" {
 			return nil
 		}
-		return os.Setenv(k, v)
+		return put(k, v)
 	}
 	b64 := func(s string) string {
 		if s == "" {
@@ -312,7 +328,7 @@ func applyBootConfig(cfg runner.BootConfig, secrets map[string]string) error {
 	// either. (The plane also drops such a name from SecretNames, so this is
 	// the second of two fences rather than the only one.)
 	for k, v := range secrets {
-		if err := os.Setenv(k, v); err != nil {
+		if err := put(k, v); err != nil {
 			return fmt.Errorf("applying this session's environment: %w", err)
 		}
 	}
@@ -364,7 +380,7 @@ func applyBootConfig(cfg runner.BootConfig, secrets map[string]string) error {
 		return err
 	}
 	for k, v := range cfg.Env {
-		if err := os.Setenv(k, v); err != nil {
+		if err := put(k, v); err != nil {
 			return fmt.Errorf("applying this session's configuration: %w", err)
 		}
 	}
@@ -422,7 +438,13 @@ func bootOverVsock(ctx context.Context, dial dialSession, b *bootstrapper) (rela
 		_ = conn.Close()
 		conn = nil
 	}
-	if err := applyBootConfig(cfg, secrets); err != nil {
+	apply := applyBootConfig
+	if cfg.GuestReconnect != 0 {
+		apply = func(cfg runner.BootConfig, secrets map[string]string) error {
+			return b.refreshConfiguration(ctx, cfg, secrets)
+		}
+	}
+	if err := apply(cfg, secrets); err != nil {
 		if conn != nil {
 			_ = conn.Close()
 		}
@@ -434,7 +456,8 @@ func bootOverVsock(ctx context.Context, dial dialSession, b *bootstrapper) (rela
 	return conn, cfg, failure, nil
 }
 
-// reBootstrap is the preamble on every LATER connection: read the
+// reBootstrap selects authenticated reconnect for an opted-in guest. It never
+// downgrades that guest to legacy bootstrap. In legacy mode it reads the
 // configuration again, and exchange again when it brings a token this
 // session has not spent — which is what a cold resume brings and a redial
 // does not.
@@ -453,10 +476,20 @@ func bootOverVsock(ctx context.Context, dial dialSession, b *bootstrapper) (rela
 // which for a spent token is forever. So it is logged and the connection is
 // served.
 func reBootstrap(ctx context.Context, conn relay.Conn, b *bootstrapper) error {
+	b.mu.Lock()
+	guestMode := b.guest != nil
+	b.mu.Unlock()
+	if guestMode {
+		return b.reconnectGuest(ctx, conn)
+	}
 	cfg, err := readBootConfig(ctx, conn)
 	if err != nil {
 		return err
 	}
+	// Capability selection belongs to the first boot, never a legacy redial.
+	if cfg.GuestReconnect != 0 {
+		return errGuestReconnect
+	}
 	secrets, xerr := b.exchange(ctx, conn, cfg)
 	if xerr != nil {
 		log.Printf("this session's secrets were not re-delivered on a new connection (%v); "+

cmd/sessiond/bootlaunch.go

@@ -0,0 +1,31 @@
+package main
+
+import (
+	"context"
+	"slices"
+
+	"github.com/tokencanopy/rainier/internal/relay"
+	"github.com/tokencanopy/rainier/protocol/runner"
+)
+
+// guestBoot is the launch state consumed by main before preparing the boot chain.
+type guestBoot struct {
+	conn    relay.Conn
+	config  runner.BootConfig
+	argv    []string
+	failure *bootFailure
+}
+
+func bootGuest(ctx context.Context, dial dialSession, boots *bootstrapper, imageCommand []string) (guestBoot, error) {
+	conn, cfg, failure, err := bootOverVsock(ctx, dial, boots)
+	if err != nil {
+		return guestBoot{}, err
+	}
+	// The image command only supplies a fallback. A microVM's per-session
+	// command arrives on the authenticated boot channel, not in PID 1's argv.
+	command := imageCommand
+	if len(cfg.Cmd) > 0 {
+		command = cfg.Cmd
+	}
+	return guestBoot{conn: conn, config: cfg, argv: slices.Clone(command), failure: failure}, nil
+}

cmd/sessiond/bootlaunch_test.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"context"
+	"os/exec"
+	"testing"
+	"time"
+
+	"github.com/tokencanopy/rainier/protocol/runner"
+)
+
+func TestGuestBootLaunchesRequestedCommand(t *testing.T) {
+	for _, requested := range []bool{true, false} {
+		name := "image_fallback"
+		if requested {
+			name = "requested_command"
+		}
+		t.Run(name, func(t *testing.T) {
+			cleanEnv(t)
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			defer cancel()
+			dial, hosts := fakeTransport(t)
+			type result struct {
+				boot guestBoot
+				err  error
+			}
+			done := make(chan result, 1)
+			go func() {
+				boot, err := bootGuest(ctx, dial, &bootstrapper{}, []string{"/bin/sh", "-c", "printf image_default"})
+				done <- result{boot, err}
+			}()
+			host := <-hosts
+			cfg := runner.BootConfig{Protocol: runner.SessionBootstrapProtocolVersion, SessionID: "session_command_test"}
+			want := "image_default"
+			if requested {
+				// Shell-looking text must remain one literal argument, not be re-parsed.
+				want = "requested ; $(false) ' quoted"
+				cfg.Cmd = []string{"/bin/sh", "-c", `printf '%s' "$1"`, "command-test", want}
+			}
+			host.sendBootConfig(cfg)
+			got := <-done
+			if got.err != nil || got.boot.failure != nil {
+				t.Fatalf("guest boot failed: %v %v", got.err, got.boot.failure)
+			}
+			defer got.boot.conn.Close()
+			output, err := exec.CommandContext(ctx, got.boot.argv[0], got.boot.argv[1:]...).Output()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if string(output) != want {
+				t.Fatalf("launched output %q, want %q", output, want)
+			}
+		})
+	}
+}

cmd/sessiond/main.go

@@ -112,20 +112,21 @@ func main() {
 	if *transport == transportVsock {
 		dialer = vsockTransport()
 		preamble = func(ctx context.Context, c relay.Conn) error { return reBootstrap(ctx, c, boots) }
-		conn, cfg, failure, err := bootOverVsock(context.Background(), dialer, boots)
+		boot, err := bootGuest(context.Background(), dialer, boots, argv)
 		if err != nil {
 			// Deliberately fatal, and the same judgement prepareBoot's own
 			// failure gets: a guest that could not read its configuration
 			// does not know what session it is, what to run, or where its
 			// egress goes. Dying is what makes the runner notice.
 			log.Fatalf("microvm boot: %v", err)
 		}
-		firstConn, overVsock = conn, true
+		firstConn, overVsock = boot.conn, true
+		argv = boot.argv
 		if *sessionID == "" {
-			*sessionID = cfg.SessionID
+			*sessionID = boot.config.SessionID
 		}
-		if failure != nil {
-			secretsFailure = failure.Error()
+		if boot.failure != nil {
+			secretsFailure = boot.failure.Error()
 		}
 	} else if *transport != transportWebSocket {
 		log.Fatalf("unknown --transport %q (valid: %s, %s)", *transport, transportWebSocket, transportVsock)
@@ -251,6 +252,9 @@ func main() {
 	execs := sandboxexec.NewRunner(workspaceRoot,
 		sandboxexec.SessionEnv(os.Environ(), envAssignments(chainVars)),
 		sandboxexec.NewSpawner().Start)
+	if overVsock {
+		boots.bindExecEnvironment(execs, envAssignments(chainVars))
+	}
 
 	// What the RUNNER needs to know about those commands, and the only thing
 	// it needs: how many are running. runnerd's idle auto-stop stops a session