diff --git a/apps/daemon/internal/agent/clirunner/handle.go b/apps/daemon/internal/agent/clirunner/handle.go index c676efef1..e331ec3ca 100644 --- a/apps/daemon/internal/agent/clirunner/handle.go +++ b/apps/daemon/internal/agent/clirunner/handle.go @@ -12,11 +12,11 @@ import ( "time" ) -// Handle is a running process that clirunner did not start, such as a Harness in an agent-host Session view. Its end also ends every descendant. +// Handle is a running process that clirunner did not start, such as a Harness in an agent-host Session view. The implementation says which of the process's descendants Signal reaches and Wait waits for. type Handle interface { - // Signal delivers sig to the process and every descendant it still has. Once the process itself has exited it delivers nothing and returns an error that matches os.ErrProcessDone, even while its descendants are still ending. + // Signal delivers sig to the process and the descendants the implementation reaches. Once the process itself has exited it delivers nothing and returns an error that matches os.ErrProcessDone, even while its descendants are still ending. Signal(syscall.Signal) error - // Wait returns once the process and its descendants have ended. The code is -1 when a signal ended the process. An error means the exit is unknown. + // Wait returns once the process, and the descendants the implementation waits for, have ended. The code is -1 when a signal ended the process. An error means the exit is unknown. Wait() (int, error) // Close kills whatever still runs and releases the handle. It never closes the stdio ends in HandleOptions, which the Process owns. It is safe to call more than once and after Wait. Close() error diff --git a/apps/daemon/internal/agent/harness.go b/apps/daemon/internal/agent/harness.go index 050fa4b46..2f46dae28 100644 --- a/apps/daemon/internal/agent/harness.go +++ b/apps/daemon/internal/agent/harness.go @@ -150,6 +150,15 @@ var ErrInvalidView = errors.New("agent: invalid view declaration") // a connection option. var ErrViewHandoff = errors.New("agent: view request carries a connection outside the Session's gateway") +// ViewSession.Launch and ViewSession.Spawn outcomes. +var ( + // ErrNotLocalExec is a Launch or Spawn whose Binary is not a LocalExec path. + ErrNotLocalExec = errors.New("agent: binary is not a LocalExec path") + // ErrNoLiveView is a Spawn while no view runs its Harness: none was + // launched yet, or its Harness has exited or its view has ended. + ErrNoLiveView = errors.New("agent: the Session has no live view") +) + // View declares how the Harness runs in an agent-host Session view. View // paths are absolute and clean. The closure, Exec overlays and the shim are // the only executable mounts, and all are read-only; the world, the home and @@ -250,6 +259,22 @@ type ViewSession struct { // TERM unless Cancel already did, and ends once they exit or KillTimeout // passes from the first TERM. Launch func(clirunner.StartOptions) (*clirunner.Process, error) + // Spawn runs Binary, a LocalExec path, as another process in the live + // view while its Harness runs, with StartOptions as Launch takes them. The + // process runs as the Harness does: as the same user, in the same + // namespaces, view cgroup, world and network, with no capabilities, + // no_new_privs and the same seccomp filter, in a process group of its own. + // The view starts one Spawn at a time, and Parent bounds the wait for its + // turn and for the start; once Parent ends, Spawn returns its error and + // kills a process that starts after all. Cancel sends TERM to the group + // and kills it after KillTimeout; once the process has exited, Cancel + // delivers nothing and what it left runs on as other processes in the view + // do. The view's end ends them all: a Cancel of the Harness reaches them, + // and when the Harness exits they are among the processes that remain. A + // view that ends after Spawn returned shows in the process's Wait. Spawn + // returns ErrNotLocalExec, ErrNoLiveView, or the error that kept the + // process from starting. + Spawn func(clirunner.StartOptions) (*clirunner.Process, error) } // checkViewHandoff enforces, before the factory runs, that the view request diff --git a/apps/daemon/internal/agenthost/agenthost.go b/apps/daemon/internal/agenthost/agenthost.go index a36551661..90131ce69 100644 --- a/apps/daemon/internal/agenthost/agenthost.go +++ b/apps/daemon/internal/agenthost/agenthost.go @@ -125,7 +125,7 @@ var ( // ErrWorld is a world that no longer shows the sandbox faithfully, or // that cannot show that its attachment holds nothing. ErrWorld = errors.New("agenthost: world lost") - // ErrLaunch is a view that could not be launched. + // ErrLaunch is a view, or a process in a view, that could not be started. ErrLaunch = errors.New("agenthost: launch failed") // ErrProcessBroker is a view's process broker that could not start, or // whose process relay was lost while the view ran diff --git a/apps/daemon/internal/agenthost/doc.go b/apps/daemon/internal/agenthost/doc.go index 4e3baa4b8..f374136c2 100644 --- a/apps/daemon/internal/agenthost/doc.go +++ b/apps/daemon/internal/agenthost/doc.go @@ -40,12 +40,13 @@ // and calls the view's Executor factory. Each ViewSession.Launch builds one // sessionview view, of which one at a time is live, over the world that // worldfs serves from the attachment's File service, with the gateway -// listening in the view's network namespace. A view with a shim gets its own -// process broker, started once the view runs and closed once it has ended. The -// broker runs the shims' commands over the attachment's Process service in the -// strongest scope the service declares, with the view's ForwardEnv and the -// Session's Environment, and cancels a forwarded process whose shim is lost -// with the launch's kill timeout as its grace. +// listening in the view's network namespace. ViewSession.Spawn runs another +// process in the live view (sessionview.View.Spawn). A view with a shim gets +// its own process broker, started once the view runs and closed once it has +// ended. The broker runs the shims' commands over the attachment's Process +// service in the strongest scope the service declares, with the view's +// ForwardEnv and the Session's Environment, and cancels a forwarded process +// whose shim is lost with the launch's kill timeout as its grace. // // Each view presents the closure directories read-only and executable, the // Session home read-write and noexec, the agent host's /etc/passwd, group, diff --git a/apps/daemon/internal/agenthost/launch_linux.go b/apps/daemon/internal/agenthost/launch_linux.go index 3b53be029..6df910e06 100644 --- a/apps/daemon/internal/agenthost/launch_linux.go +++ b/apps/daemon/internal/agenthost/launch_linux.go @@ -38,6 +38,7 @@ type runningView interface { Wait() (sessionview.Exit, error) Close() error Relay() *os.File + Spawn(ctx context.Context, path string, args, env []string, dir string, stdin bool) (*sessionview.Spawned, error) } // viewWorld is the part of *worldfs.World the Session watches. @@ -61,15 +62,23 @@ func (s *session) closeLive() { } } -// launch is ViewSession.Launch: it builds one view and runs opts.Binary in it. -func (s *session) launch(opts clirunner.StartOptions) (*clirunner.Process, error) { +// checkStart checks the options of Launch and Spawn. +func (s *session) checkStart(opts clirunner.StartOptions) error { switch { case !slices.Contains(s.plan.view.LocalExec, opts.Binary): - return nil, &Error{Kind: ErrLaunch, Err: fmt.Errorf("%q is not a LocalExec path", opts.Binary)} + return &Error{Kind: ErrLaunch, Err: fmt.Errorf("%w: %q", agent.ErrNotLocalExec, opts.Binary)} case !isViewPath(opts.Dir): - return nil, &Error{Kind: ErrLaunch, Err: fmt.Errorf("directory %q is not absolute and clean", opts.Dir)} + return &Error{Kind: ErrLaunch, Err: fmt.Errorf("directory %q is not absolute and clean", opts.Dir)} case !opts.OwnProcessGroup: - return nil, &Error{Kind: ErrLaunch, Err: errors.New("a view process runs in its own process group")} + return &Error{Kind: ErrLaunch, Err: errors.New("a view process runs in its own process group")} + } + return nil +} + +// launch is ViewSession.Launch: it builds one view and runs opts.Binary in it. +func (s *session) launch(opts clirunner.StartOptions) (*clirunner.Process, error) { + if err := s.checkStart(opts); err != nil { + return nil, err } if opts.Parent == nil { opts.Parent = context.Background() @@ -93,6 +102,40 @@ func (s *session) launch(opts clirunner.StartOptions) (*clirunner.Process, error return s.start(lv, opts) } +// spawn is ViewSession.Spawn: it runs opts.Binary in the live view. +func (s *session) spawn(opts clirunner.StartOptions) (*clirunner.Process, error) { + if err := s.checkStart(opts); err != nil { + return nil, err + } + var v runningView + s.mu.Lock() + if s.live != nil { + v = s.live.view + } + s.mu.Unlock() + if v == nil { + return nil, &Error{Kind: ErrLaunch, Op: "spawn", Err: agent.ErrNoLiveView} + } + if opts.Parent == nil { + opts.Parent = context.Background() + } + p, err := v.Spawn(opts.Parent, opts.Binary, append([]string{opts.Binary}, opts.Args...), opts.Env, opts.Dir, opts.NeedStdin) + if err != nil { + // Only a view that has ended has no live view; any other failure keeps its own error. + if errors.Is(err, sessionview.ErrExited) || errors.Is(err, sessionview.ErrClosed) { + err = fmt.Errorf("%w: %w", agent.ErrNoLiveView, err) + } + return nil, &Error{Kind: ErrLaunch, Op: "spawn", Err: err} + } + var stdin io.WriteCloser + if p.Stdin != nil { + stdin = p.Stdin + } + // FromHandle fails only without stdout and stderr, which a spawned process always has. + process, _ := clirunner.FromHandle(p, clirunner.HandleOptions{Parent: opts.Parent, Stdin: stdin, Stdout: p.Stdout, Stderr: p.Stderr, KillTimeout: opts.KillTimeout}) + return process, nil +} + // release frees the view slot and ends the launch's count. func (s *session) release(lv *liveView) { s.mu.Lock() diff --git a/apps/daemon/internal/agenthost/run_linux.go b/apps/daemon/internal/agenthost/run_linux.go index d51fde195..7f8bd4041 100644 --- a/apps/daemon/internal/agenthost/run_linux.go +++ b/apps/daemon/internal/agenthost/run_linux.go @@ -104,6 +104,7 @@ func run(ctx context.Context, cfg Config, in Session, d deps) error { Proxy: s.plan.proxy, MCP: s.plan.mcp, Launch: s.launch, + Spawn: s.spawn, }) if err != nil { err = executorError(err) diff --git a/apps/daemon/internal/agenthost/session_linux_test.go b/apps/daemon/internal/agenthost/session_linux_test.go index 9ba453393..b77848bb0 100644 --- a/apps/daemon/internal/agenthost/session_linux_test.go +++ b/apps/daemon/internal/agenthost/session_linux_test.go @@ -77,6 +77,23 @@ func TestViewEndReleasesTheSlotBeforeTheProcessEnds(t *testing.T) { } } +// TestSpawnKeepsItsErrors checks that only a view that has ended makes a Spawn fail with ErrNoLiveView, and that every other failure keeps its own error. +func TestSpawnKeepsItsErrors(t *testing.T) { + emfile := &sessionview.Error{Kind: sessionview.ErrLauncher, Op: "pipe", Err: syscall.EMFILE} + for _, c := range []struct { + err error + ended bool + }{{sessionview.ErrExited, true}, {sessionview.ErrClosed, true}, {emfile, false}, {sessionview.ErrExec, false}, {context.Canceled, false}} { + s := newOwnerSession(t) + s.plan = &plan{view: agent.View{LocalExec: []string{"/bin/true"}}} + s.live = &liveView{view: &fakeView{exit: make(chan struct{}), spawnErr: c.err}} + _, err := s.spawn(clirunner.StartOptions{Binary: "/bin/true", Dir: "/", OwnProcessGroup: true}) + if !errors.Is(err, c.err) || errors.Is(err, agent.ErrNoLiveView) != c.ended { + t.Errorf("Spawn failing with %v = %v; want that error, and ErrNoLiveView only for an ended view", c.err, err) + } + } +} + func TestFailureDuringTeardownCounts(t *testing.T) { s := newOwnerSession(t) // The relay revokes the attachment while Executor.Close waits. @@ -334,16 +351,21 @@ func (t *fakeTurn) AwaitSettlement(context.Context) (agent.TurnSettlement, error return agent.TurnSettlement{Reusable: t.settleErr == nil}, t.settleErr } -// fakeView is a view that ends when closed. +// fakeView is a view that ends when closed and whose spawns fail with spawnErr. type fakeView struct { - exit chan struct{} - once sync.Once + exit chan struct{} + once sync.Once + spawnErr error } func (v *fakeView) Signal(syscall.Signal) error { return nil } func (v *fakeView) Relay() *os.File { return nil } +func (v *fakeView) Spawn(context.Context, string, []string, []string, string, bool) (*sessionview.Spawned, error) { + return nil, v.spawnErr +} + func (v *fakeView) Wait() (sessionview.Exit, error) { <-v.exit return sessionview.Exit{}, nil diff --git a/apps/daemon/internal/agenthost/view_linux_test.go b/apps/daemon/internal/agenthost/view_linux_test.go index f2e0a4df0..04a88b1b0 100644 --- a/apps/daemon/internal/agenthost/view_linux_test.go +++ b/apps/daemon/internal/agenthost/view_linux_test.go @@ -90,6 +90,7 @@ func TestSessionRunsInAViewOverItsAttachment(t *testing.T) { t.Fatal(err) } copyExecutable(t, filepath.Join(closure, "harness")) + executors := make(chan *testExecutor, 1) register(reg, "test", &agent.View{ Closure: []agent.ViewMount{{Name: "harness", HostDir: closure}}, Masks: []agent.ViewMask{{Path: "/etc/ld.so.preload"}, {Path: "/etc/hostname"}, {Path: "/etc/apt", Dir: true}}, @@ -101,8 +102,13 @@ func TestSessionRunsInAViewOverItsAttachment(t *testing.T) { if err != nil { return nil, err } - return &testExecutor{session: s, dir: req.LocalEnvironment.WorkspaceRoot, - env: []string{harnessEnv + "=1", modelEnv + "=" + provider.BaseURL, caEnv + "=" + cfg.CADir}}, nil + e := &testExecutor{session: s, dir: req.LocalEnvironment.WorkspaceRoot, + env: []string{harnessEnv + "=1", modelEnv + "=" + provider.BaseURL, caEnv + "=" + cfg.CADir}} + select { + case executors <- e: + default: + } + return e, nil }, }) sb.auth.AddRuntime(cfg.Credential, cfg.RuntimeID) @@ -132,6 +138,14 @@ func TestSessionRunsInAViewOverItsAttachment(t *testing.T) { if r.Exit != "" { t.Errorf("Harness: %s; stderr %s", r.Exit, r.Stderr) } + // The Turn waited for its Harness, so no view runs. + e := <-executors + if _, err := e.session.Spawn(clirunner.StartOptions{Binary: harnessPath, Dir: e.dir, OwnProcessGroup: true}); !errors.Is(err, agent.ErrNoLiveView) { + t.Errorf("Spawn after the Harness exited = %v, want ErrNoLiveView", err) + } + if _, err := e.session.Spawn(clirunner.StartOptions{Binary: "/bin/sh", Dir: e.dir, OwnProcessGroup: true}); !errors.Is(err, agent.ErrNotLocalExec) { + t.Errorf("Spawn of a binary outside LocalExec = %v, want ErrNotLocalExec", err) + } select { case ok := <-keyed: if !ok { diff --git a/apps/daemon/internal/sessionview/control_linux.go b/apps/daemon/internal/sessionview/control_linux.go index 8a7411dd3..2e28bf14f 100644 --- a/apps/daemon/internal/sessionview/control_linux.go +++ b/apps/daemon/internal/sessionview/control_linux.go @@ -4,13 +4,13 @@ package sessionview import ( "bytes" + "context" "encoding/gob" "errors" "fmt" "io" "net" "os" - "sync" "syscall" "time" @@ -33,16 +33,21 @@ type launchSpec struct { Private []PrivateDir Overlays []Overlay Shim Shim - Path string - Args []string - Env []string - Dir string + Command command UID uint32 GID uint32 Groups []uint32 Grace time.Duration } +// command is what a process of the view runs. +type command struct { + Path string + Args []string + Env []string + Dir string +} + type msgKind uint8 const ( @@ -50,13 +55,17 @@ const ( msgProceed // daemon: the world serves and the network is set up; carries the mount targets msgStarted // launcher: the process runs msgFailed // launcher: construction failed - msgExited // launcher: the process ended - msgSignal // daemon: signal the process - msgSignaled // launcher: whether msgSignal reached the running process + msgExited // launcher: the process, or the spawned process ID, ended + msgSignal // daemon: signal the process, or the spawned process Spawn + msgSignaled // launcher: whether msgSignal reached it + msgSpawn // daemon: start another process; carries a pipe with its command, then its stdin, stdout and stderr + msgSpawned // launcher: the spawned process's Pid, or why none started ) type message struct { Kind msgKind + ID uint64 // pairs a reply with its request; a spawn's ID also names the process it started, and 0 names the process + Spawn uint64 // the ID of the spawned process msgSignal signals, or 0 Pid int Signal syscall.Signal Delivered bool @@ -65,7 +74,7 @@ type message struct { Targets map[string]string // each mountpoint's view path to the path the world presents it at } -// failure carries a launcher *Error across the control socket. +// failure carries a launcher *Error across the control socket. A failure to start a command leaves the command out, so that no message grows with what a caller passed: the daemon has the command, and startErr puts it back. type failure struct { Kind int Op string @@ -108,10 +117,23 @@ func (f failure) err() error { return e } +// startErr returns the error f reports while the launcher starts c, with the directory or path of c that a failed chdir or exec concerns. +func (f failure) startErr(c command) error { + if f.Path == "" { + switch f.Op { + case "chdir": + f.Path = c.Dir + case "exec": + f.Path = c.Path + } + } + return f.err() +} + // control is one end of the launcher's SOCK_SEQPACKET control socket. Each packet holds one gob-encoded message. type control struct { - conn *net.UnixConn - mu sync.Mutex + conn *net.UnixConn + sending chan struct{} // held by the send in progress } func newControl(f *os.File) (*control, error) { @@ -125,10 +147,11 @@ func newControl(f *os.File) (*control, error) { c.Close() return nil, fmt.Errorf("control socket is a %T", c) } - return &control{conn: uc}, nil + return &control{conn: uc, sending: make(chan struct{}, 1)}, nil } -func (c *control) send(m message, fds ...int) error { +// send sends m with fds. It returns ctx's error when ctx ends before m is sent, whether it waits for another send or for room on the socket; a packet goes whole or not at all. Once the socket is shut down, a waiting send fails. +func (c *control) send(ctx context.Context, m message, fds ...int) error { var buf bytes.Buffer if err := gob.NewEncoder(&buf).Encode(m); err != nil { return err @@ -137,16 +160,32 @@ func (c *control) send(m message, fds ...int) error { if len(fds) > 0 { oob = unix.UnixRights(fds...) } - c.mu.Lock() - defer c.mu.Unlock() + select { + case c.sending <- struct{}{}: + case <-ctx.Done(): + return ctx.Err() + } + defer func() { <-c.sending }() + expired := make(chan struct{}) + stop := context.AfterFunc(ctx, func() { + c.conn.SetWriteDeadline(time.Unix(1, 0)) + close(expired) + }) _, _, err := c.conn.WriteMsgUnix(buf.Bytes(), oob, nil) + if !stop() { + <-expired + c.conn.SetWriteDeadline(time.Time{}) + if err != nil { + err = ctx.Err() + } + } return err } // recv returns the next message and the files it carries. It returns io.EOF once the peer has closed its end. func (c *control) recv() (message, []*os.File, error) { buf := make([]byte, 64<<10) - oob := make([]byte, unix.CmsgSpace(2*4)) + oob := make([]byte, unix.CmsgSpace(4*4)) n, oobn, flags, _, err := c.conn.ReadMsgUnix(buf, oob) if err != nil { return message{}, nil, err @@ -169,7 +208,7 @@ func (c *control) recv() (message, []*os.File, error) { return m, files, nil } -// interrupt shuts the daemon's end down in both directions, so that a pending recv returns io.EOF and every later send fails. +// interrupt shuts the socket down in both directions, whoever else holds it, so that a pending recv at either end returns io.EOF and every later send fails. func (c *control) interrupt() { c.conn.CloseRead() c.conn.CloseWrite() diff --git a/apps/daemon/internal/sessionview/control_linux_test.go b/apps/daemon/internal/sessionview/control_linux_test.go new file mode 100644 index 000000000..01c2ec337 --- /dev/null +++ b/apps/daemon/internal/sessionview/control_linux_test.go @@ -0,0 +1,72 @@ +//go:build linux + +package sessionview + +import ( + "context" + "errors" + "os" + "strings" + "testing" + "time" + + "golang.org/x/sys/unix" +) + +// TestSendIsBounded checks that a send waiting for room on the socket or for another send returns once its context ends, without disturbing the send in progress, and that a shutdown ends the send in progress. +func TestSendIsBounded(t *testing.T) { + pair, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_SEQPACKET|unix.SOCK_CLOEXEC, 0) + if err != nil { + t.Fatal(err) + } + peer := os.NewFile(uintptr(pair[1]), "peer") + defer peer.Close() + c, err := newControl(os.NewFile(uintptr(pair[0]), "control")) + if err != nil { + t.Fatal(err) + } + defer c.close() + big := message{Targets: map[string]string{"/": strings.Repeat("x", 32<<10)}} + bounded := func(d time.Duration) error { + ctx, cancel := context.WithTimeout(context.Background(), d) + defer cancel() + return c.send(ctx, big) + } + // The peer reads nothing, so the socket fills. + for { + start := time.Now() + err := bounded(20 * time.Millisecond) + if time.Since(start) > 2*time.Second { + t.Fatalf("send took %v with a 20ms bound", time.Since(start)) + } + if errors.Is(err, context.DeadlineExceeded) { + break + } + if err != nil { + t.Fatal(err) + } + } + blocked := make(chan error, 1) + go func() { blocked <- c.send(context.Background(), big) }() + for len(c.sending) == 0 { + time.Sleep(time.Millisecond) + } + start := time.Now() + if err := bounded(20 * time.Millisecond); !errors.Is(err, context.DeadlineExceeded) || time.Since(start) > 2*time.Second { + t.Errorf("send behind a blocked send = %v after %v, want context.DeadlineExceeded within 20ms", err, time.Since(start)) + } + select { + case err := <-blocked: + t.Fatalf("unbounded send returned %v with the socket full", err) + case <-time.After(50 * time.Millisecond): + } + c.interrupt() + select { + case err := <-blocked: + if err == nil { + t.Error("send after the shutdown succeeded") + } + case <-time.After(2 * time.Second): + t.Fatal("send still blocked 2s after the shutdown") + } +} diff --git a/apps/daemon/internal/sessionview/doc.go b/apps/daemon/internal/sessionview/doc.go index cc13e2585..8e1c63099 100644 --- a/apps/daemon/internal/sessionview/doc.go +++ b/apps/daemon/internal/sessionview/doc.go @@ -1,4 +1,4 @@ -// Package sessionview runs one process inside a per-Session view on the agent host. +// Package sessionview runs a process, and others spawned beside it, inside a per-Session view on the agent host. // // A view is a private mount, PID and network namespace whose root is the Session's world: a FUSE file system that the daemon serves over a /dev/fuse connection. The launcher adds the local pieces on top of the world: private directories under /.oac, trusted overlays, the command shim, a fresh /proc and a minimal /dev. The world presents a mountpoint for each piece and reports where, following the sandbox's symlinks, and the launcher mounts at those paths without following any symlink itself. The process starts with no capabilities, no_new_privs, a seccomp filter and only stdin, stdout and stderr open. Its network namespace has only loopback up. // @@ -6,6 +6,8 @@ // // The daemon calls [Init] first thing in main. [Start] re-executes the daemon binary as the launcher, which becomes PID 1 of the view: it builds the view, starts the process, delivers signals to every process in the view, reaps orphans and exits with the process status. Once the process has exited, Signal reports [ErrExited] and delivers nothing. When the process exits while others remain, the launcher sends them TERM unless one already went to the view, and waits for them up to [Process].Grace from the first TERM. Its exit kills what remains and tears the view down. // +// While the process runs, [View.Spawn] has the launcher start another process in the view, passing the command on a pipe and the stdio pipes that Spawn makes over the control socket. A view starts one spawn at a time, and a spawn waiting for its turn holds no descriptors. The launcher starts each from the thread that carries the restrictions, as it started the process, so it runs as the same user, in the view's namespaces and cgroup, under the same restrictions and in a session of its own. That thread does nothing else once the process runs. It enters each command's directory itself, with the process's file system identity, before it forks, so the child inherits the directory and a directory stuck on the world holds up no other thread: reaping, signals, the drain and the exit go on. A command's path does not belong in the world: a child that waits on the world before its exec holds the fork, and with it the launcher, until the world answers. The launcher reaps as SIGCHLD reports exits. While a spawn forks, it reaps only the processes it has registered, so the pid of a child that exits before its registration stays its own until the registration ties it to its spawn. A spawned process counts like any other process in the view: a signal to the view reaches it, and when the process exits it is among those the launcher sends TERM and waits for. Each spawn has an ID, which [Spawned] uses rather than the pid: [Spawned].Signal reaches the process group until the launcher reaps the process, and reports [ErrExited] from then on. What the process leaves runs on as other processes in the view do, and the view's end ends it. +// // A view that declares a [Shim] also runs the Session's process relay, the shim binary in relay mode (package processshim). The launcher starts it before the process, under the same restrictions and as the same user, with a listening socket at processshim.SocketPath on a read-only mount and its end of a socket pair whose other end is [View.Relay]. The relay is the only process that receives the descriptors a shim hands over; the broker outside the view holds only its end of the pair. The launcher's drain ignores the relay, which ends with the view. // // Each view runs in a cgroup v2 of its own, which Start creates in [Spec].CgroupParent. The launcher is cloned into it with CLONE_INTO_CGROUP, so no process of the view ever runs outside it. The cgroup hierarchy is the only record of what the views own: [Recover] ends every cgroup an earlier owner of the parent left, with all its processes. diff --git a/apps/daemon/internal/sessionview/errors.go b/apps/daemon/internal/sessionview/errors.go index 1bc15eddc..8c0466de9 100644 --- a/apps/daemon/internal/sessionview/errors.go +++ b/apps/daemon/internal/sessionview/errors.go @@ -25,12 +25,12 @@ var ( ErrCgroup = errors.New("sessionview: view cgroup unavailable") // ErrCleanup reports a teardown or recovery that did not finish within its bound, or a cgroup that could not be ended and removed. The cgroup stays for Recover, and what remains finishes in the background if it can. ErrCleanup = errors.New("sessionview: cleanup incomplete") - // ErrExited is Signal's result once the process has exited. It also matches os.ErrProcessDone. + // ErrExited is Signal's and Spawn's result once the process has exited. Signal's also matches os.ErrProcessDone. ErrExited = errors.New("sessionview: process exited") ) // errorKinds fixes the wire code of each kind the launcher reports. -var errorKinds = []error{ErrLauncher, ErrNoFUSE, ErrMountDenied, ErrMountTarget, ErrNetwork, ErrRestrict, ErrExec} +var errorKinds = []error{ErrLauncher, ErrNoFUSE, ErrMountDenied, ErrMountTarget, ErrNetwork, ErrRestrict, ErrExec, ErrExited} // Error is a typed sessionview failure. It matches Kind and, when present, Err. type Error struct { diff --git a/apps/daemon/internal/sessionview/launcher_linux.go b/apps/daemon/internal/sessionview/launcher_linux.go index f1ed252e8..440be1d62 100644 --- a/apps/daemon/internal/sessionview/launcher_linux.go +++ b/apps/daemon/internal/sessionview/launcher_linux.go @@ -3,6 +3,7 @@ package sessionview import ( + "context" "encoding/gob" "fmt" "os" @@ -39,10 +40,18 @@ type launcher struct { proc int // the view's /proc relay int // the relay's pid, or 0 - // mu orders signals against the process's exit. running holds from the process's start until it is reaped; termAt is when TERM first went to the view. + spawns chan func() // to the restricted thread; the view sends one spawn at a time + chld chan os.Signal // SIGCHLD, and a wake once a spawn has registered its child + ready chan struct{} // wakes the writer + + // mu orders signals, spawns and messages against the reaping. running holds from the process's start until it is reaped, and code is how it exited; termAt is when TERM first went to the view. spawned maps the pid of the process and of each spawned process to its ID until the pid is reaped; forking holds while a spawn starts a child it has not registered yet. out holds the messages the writer sends next. mu sync.Mutex running bool + code int termAt time.Time + spawned map[int]uint64 + forking bool + out []message } func runLauncher() int { @@ -51,42 +60,42 @@ func runLauncher() int { fmt.Fprintf(os.Stderr, "sessionview launcher: %v\n", err) return 1 } - l := &launcher{ctl: ctl, proceed: make(chan struct{})} - code, err := l.run() - if err != nil { - _ = ctl.send(message{Kind: msgFailed, Fail: failureOf(err)}) - return 1 - } - return code + l := &launcher{ctl: ctl, proceed: make(chan struct{}), spawns: make(chan func(), 1), chld: make(chan os.Signal, 1), ready: make(chan struct{}, 1), spawned: map[int]uint64{}} + // run returns only when the build fails; once the process runs, the writer exits. + _ = l.ctl.send(context.Background(), message{Kind: msgFailed, Fail: failureOf(l.run())}) + return 1 } -func (l *launcher) run() (int, error) { +func (l *launcher) run() error { spec, err := readSpec() if err != nil { - return 0, err + return err } - go l.serveControl() + go l.serveControl(spec) if err := unix.Mount("", "/", "", unix.MS_REC|unix.MS_PRIVATE, ""); err != nil { - return 0, mountError("make-rprivate", "/", err) + return mountError("make-rprivate", "/", err) } if err := loopbackUp(); err != nil { - return 0, &Error{Kind: ErrNetwork, Op: "loopback", Err: err} + return &Error{Kind: ErrNetwork, Op: "loopback", Err: err} } if err := l.mountWorld(spec.Staging); err != nil { - return 0, err + return err } <-l.proceed b := &builder{root: -1, proc: -1, listener: -1, targets: l.targets} defer b.close() if err := b.build(spec); err != nil { - return 0, err + return err } l.proc = b.proc if err := switchRoot(b.root); err != nil { - return 0, err + return err } if err := restrict(); err != nil { - return 0, err + return err + } + if err := takeIdentity(spec); err != nil { + return err } b.closeBuild() if b.listener >= 0 { @@ -95,28 +104,32 @@ func (l *launcher) run() (int, error) { b.closeListener() unix.Close(relayFD) if err != nil { - return 0, err + return err } } - pid, err := startProcess(spec) + if err := chdir(spec.Command.Dir); err != nil { + return err + } + pid, err := startProcess(spec, spec.Command, []uintptr{stdinFD, stdoutFD, stderrFD}) if err != nil { - return 0, err + return err } l.mu.Lock() - l.running = true + l.running, l.spawned[pid] = true, 0 l.mu.Unlock() for _, fd := range []int{stdinFD, stdoutFD, stderrFD} { unix.Close(fd) } - if err := l.ctl.send(message{Kind: msgStarted, Pid: pid}); err != nil { - return 0, &Error{Kind: ErrLauncher, Op: "report start", Err: err} + if err := l.ctl.send(context.Background(), message{Kind: msgStarted, Pid: pid}); err != nil { + return &Error{Kind: ErrLauncher, Op: "report start", Err: err} } l.forwardSignals() - code, err := l.reap(pid) - if err == nil { - l.drain(spec.Grace) + go l.write() + go l.reap(spec.Grace) + // Only this thread carries the restrictions a process inherits, so every spawn starts here. + for { + (<-l.spawns)() } - return code, err } func readSpec() (*launchSpec, error) { @@ -130,10 +143,9 @@ func readSpec() (*launchSpec, error) { } // serveControl handles daemon messages. When the daemon goes away the view goes with it. -func (l *launcher) serveControl() { +func (l *launcher) serveControl(spec *launchSpec) { for { m, files, err := l.ctl.recv() - closeFiles(files) if err != nil { os.Exit(1) } @@ -144,15 +156,95 @@ func (l *launcher) serveControl() { close(l.proceed) }) case msgSignal: - _ = l.ctl.send(message{Kind: msgSignaled, Delivered: l.signal(m.Signal)}) + l.mu.Lock() + l.post(message{Kind: msgSignaled, ID: m.ID, Delivered: l.signal(m.Spawn, m.Signal)}) + l.mu.Unlock() + case msgSpawn: + // The restricted thread takes each spawn before it forks, and the view sends the next only once the last has been answered, so this never waits. + l.spawns <- func() { l.spawn(spec, m.ID, files) } + continue } + closeFiles(files) } } -// signal delivers sig to every process in the view while the process runs and reports whether it did. As PID 1 of the view, the launcher reaches them all with kill(-1) and is itself spared. Once the process has been reaped, only the drain signals what remains. -func (l *launcher) signal(sig syscall.Signal) bool { +// spawn starts the command read from the first of files as the spawned process id, with the other three as its stdin, stdout and stderr, as the process's user and in a session of its own, and answers with its pid. It enters the command's directory before it forks, so a directory the world is slow to answer blocks this thread in an ordinary syscall. It forks without holding mu; while it does, the reaper leaves unregistered children, so the child's pid stays its own until it is registered. +func (l *launcher) spawn(spec *launchSpec, id uint64, files []*os.File) { + defer closeFiles(files) + var c command + err := gob.NewDecoder(files[0]).Decode(&c) + if err != nil { + err = &Error{Kind: ErrLauncher, Op: "read spawn", Err: err} + } else { + err = chdir(c.Dir) + } l.mu.Lock() - defer l.mu.Unlock() + if err == nil && !l.running { + err = &Error{Kind: ErrExited, Op: "spawn"} + } + l.forking = err == nil + l.mu.Unlock() + m := message{Kind: msgSpawned, ID: id} + if err == nil { + m.Pid, err = startProcess(spec, c, []uintptr{files[1].Fd(), files[2].Fd(), files[3].Fd()}) + } + l.mu.Lock() + if err != nil { + m.Fail = failureOf(err) + } else { + l.spawned[m.Pid] = id + } + l.forking = false + l.post(m) + l.mu.Unlock() + // SIGCHLD does not queue: the child may have exited while the reaper left it. + select { + case l.chld <- unix.SIGCHLD: + default: + } +} + +// post queues m for the writer, which sends the queued messages in order; a message without a kind exits the launcher once those before it are sent. mu is held. +func (l *launcher) post(m message) { + l.out = append(l.out, m) + select { + case l.ready <- struct{}{}: + default: + } +} + +// write sends what post queues, so that neither the reaper nor a signal waits on the socket. +func (l *launcher) write() { + for range l.ready { + l.mu.Lock() + out := l.out + l.out = nil + l.mu.Unlock() + for _, m := range out { + if m.Kind == 0 { + // A spawn blocked on the world keeps the launcher, and this end, from closing until the teardown stops the world, so the daemon learns of the exit from the shutdown. + l.ctl.interrupt() + os.Exit(l.code) + } + // A send that fails ends the control channel, as a receive that fails does, so that no request waits for a reply that will not come. + if err := l.ctl.send(context.Background(), m); err != nil { + l.ctl.interrupt() + os.Exit(1) + } + } + } +} + +// signal delivers sig and reports whether it did. mu is held. With id 0 it signals every process in the view while the process runs: as PID 1 of the view, the launcher reaches them all with kill(-1) and is itself spared. Once the process has been reaped, only the drain signals what remains. Otherwise it signals the process group of the spawned process id until that is reaped; until then its pid, and so its group, cannot be reused. +func (l *launcher) signal(id uint64, sig syscall.Signal) bool { + if id != 0 { + for pid, sid := range l.spawned { + if sid == id { + return unix.Kill(-pid, sig) == nil + } + } + return false + } if !l.running { return false } @@ -168,7 +260,9 @@ func (l *launcher) forwardSignals() { signal.Notify(sigs, unix.SIGHUP, unix.SIGINT, unix.SIGQUIT, unix.SIGTERM, unix.SIGUSR1, unix.SIGUSR2, unix.SIGWINCH) go func() { for s := range sigs { - l.signal(s.(syscall.Signal)) + l.mu.Lock() + l.signal(0, s.(syscall.Signal)) + l.mu.Unlock() } }() } @@ -184,21 +278,16 @@ func (l *launcher) drain(grace time.Duration) { if grace <= 0 || (termAt.IsZero() && unix.Kill(-1, unix.SIGTERM) != nil) { return } - reaped := make(chan struct{}) - go func() { - defer close(reaped) - // The last process other than the relay to exit is a child of the launcher by then, so its exit ends the Wait4. - for l.othersRemain() { - if _, err := unix.Wait4(-1, nil, 0, nil); err != nil && err != unix.EINTR { - return - } - } - }() timer := time.NewTimer(grace) defer timer.Stop() - select { - case <-reaped: - case <-timer.C: + // The last process other than the relay to exit is a child of the launcher by then, so its SIGCHLD ends the wait. + for l.othersRemain() { + select { + case <-l.chld: + l.reapChildren() + case <-timer.C: + return + } } } @@ -222,34 +311,60 @@ func (l *launcher) othersRemain() bool { return false } -// reap collects every child, since orphans in the view reparent to PID 1, until the process exits. -func (l *launcher) reap(pid int) (int, error) { - for { - var ws unix.WaitStatus - wpid, err := unix.Wait4(-1, &ws, 0, nil) - if err == unix.EINTR { - continue - } - if err != nil { - return 0, &Error{Kind: ErrLauncher, Op: "wait", Err: err} - } - if wpid != pid { - continue +// reap reaps children as SIGCHLD reports them, starting with those that exited before it began, until the process has exited and the drain has ended, and then has the writer exit the launcher. +func (l *launcher) reap(grace time.Duration) { + signal.Notify(l.chld, unix.SIGCHLD) + for l.reapChildren() { + <-l.chld + } + l.drain(grace) + l.mu.Lock() + l.post(message{}) + l.mu.Unlock() +} + +// reapChildren reaps the children that have exited and reports whether the process still runs. It reaps registered children at any time and other children, the view's orphans and the relay, only while no spawn forks: until its registration, a spawned child that has exited stays a zombie and keeps its pid. +func (l *launcher) reapChildren() bool { + l.mu.Lock() + defer l.mu.Unlock() + var ws unix.WaitStatus + if l.forking { + for pid := range l.spawned { + if wpid, _ := unix.Wait4(pid, &ws, unix.WNOHANG, nil); wpid == pid { + l.reaped(pid, ws) + } } - l.mu.Lock() - l.running = false - l.mu.Unlock() - var exit Exit - code := ws.ExitStatus() - if ws.Signaled() { - exit.Signal, exit.CoreDumped = ws.Signal(), ws.CoreDump() - code = 128 + int(exit.Signal) - } else { - exit.Code = code + return l.running + } + for { + pid, _ := unix.Wait4(-1, &ws, unix.WNOHANG, nil) + if pid <= 0 { + return l.running } - _ = l.ctl.send(message{Kind: msgExited, Exit: exit}) - return code, nil + l.reaped(pid, ws) + } +} + +// reaped retires the ID of pid, if it has one, and reports its exit. mu is held. +func (l *launcher) reaped(pid int, ws unix.WaitStatus) { + id, ok := l.spawned[pid] + if !ok { + return } + delete(l.spawned, pid) + exit, code := exitOf(ws) + if id == 0 { + l.running, l.code = false, code + } + l.post(message{Kind: msgExited, ID: id, Exit: exit}) +} + +// exitOf describes how a process ended, with the code the launcher exits with for it. +func exitOf(ws unix.WaitStatus) (Exit, int) { + if ws.Signaled() { + return Exit{Signal: ws.Signal(), CoreDumped: ws.CoreDump()}, 128 + int(ws.Signal()) + } + return Exit{Code: ws.ExitStatus()}, ws.ExitStatus() } func (l *launcher) mountWorld(staging string) error { @@ -267,7 +382,7 @@ func (l *launcher) mountWorld(staging string) error { return &Error{Kind: ErrNetwork, Op: "open", Path: "/proc/self/ns/net", Err: err} } defer unix.Close(netns) - if err := l.ctl.send(message{Kind: msgMounted}, dev, netns); err != nil { + if err := l.ctl.send(context.Background(), message{Kind: msgMounted}, dev, netns); err != nil { return &Error{Kind: ErrLauncher, Op: "report mount", Err: err} } return nil @@ -298,7 +413,6 @@ func startRelay(spec *launchSpec, listener int) (int, error) { } files[processshim.RelayBrokerFD], files[processshim.RelayListenerFD] = relayFD, uintptr(listener) pid, err := syscall.ForkExec(processshim.RelayPath, processshim.RelayArgs, &syscall.ProcAttr{ - Dir: "/", Env: []string{}, Files: files, Sys: &syscall.SysProcAttr{ @@ -312,18 +426,60 @@ func startRelay(spec *launchSpec, listener int) (int, error) { return pid, nil } -func startProcess(spec *launchSpec) (int, error) { - pid, err := syscall.ForkExec(spec.Path, spec.Args, &syscall.ProcAttr{ - Dir: spec.Dir, - Env: spec.Env, - Files: []uintptr{stdinFD, stdoutFD, stderrFD}, +// takeIdentity gives this thread a working directory of its own and the process's file system identity, with no capability but the two a fork needs to set the child's user, so that it enters a directory as the process would. The other threads keep theirs. +func takeIdentity(spec *launchSpec) error { + groups := make([]int, len(spec.Groups)) + for i, g := range spec.Groups { + groups[i] = int(g) + } + const setID = 1<= spec.Process.Grace { t.Errorf("view ended after %v, want once the helper exited", elapsed) } - if got, err := os.ReadFile(filepath.Join(f.world, "data", "cleaned")); err != nil || string(got) != "done" { - t.Errorf("helper cleanup = %q, %v; want it finished", got, err) + for _, name := range []string{"cleaned", "cleaned-spawned"} { + if got, err := os.ReadFile(filepath.Join(f.world, "data", name)); err != nil || string(got) != "done" { + t.Errorf("helper cleanup %s = %q, %v; want it finished", name, got, err) + } + } + if code, err := spawned.Wait(); err != nil || code != 7 { + t.Errorf("spawned Wait = %d, %v; want exit code 7", code, err) + } + // Once the spawned process has exited, its handle no longer reaches the group it led. + if err := spawned.Signal(syscall.SIGTERM); !errors.Is(err, ErrExited) { + t.Errorf("Signal after the spawned process exited = %v, want ErrExited", err) + } +} + +// TestSpawnRunsInTheView checks that a spawned process runs as the process does, with its stdio and exit coming back, that its handle reaches nothing once it has exited, that a spawn whose pipes, command or directory fail fails alone, that one whose context ended before it began returns that error and leaves no process, and that the view's end ends it. +func TestSpawnRunsInTheView(t *testing.T) { + requireView(t) + f := newFixture(t) + w := &loopbackWorld{dir: f.world} + v, err := Start(context.Background(), f.spec(w, "identity")) + if err != nil { + t.Fatalf("Start: %v", err) + } + defer v.Close() + token := fmt.Sprintf("oac-spawned-%d", time.Now().UnixNano()) + sleeper, err := spawnHelper(context.Background(), v, "identity", "/data", token) + if err != nil { + t.Fatalf("Spawn: %v", err) + } + defer closeStdio(sleeper) + var ids [2]map[string]string + for i, stdout := range []io.Reader{v.Stdout(), sleeper.Stdout} { + if err := json.NewDecoder(stdout).Decode(&ids[i]); err != nil { + t.Fatal(err) + } + } + if len(ids[0]) != 20 || !maps.Equal(ids[0], ids[1]) { + t.Errorf("spawned process runs as and in %v, the process as and in %v", ids[1], ids[0]) + } + report, err := v.Spawn(context.Background(), "/.oac/harness/harness", []string{"harness"}, []string{helperEnv + "=report"}, "/data", true) + if err != nil { + t.Fatalf("Spawn: %v", err) + } + report.Stdin.Write([]byte("ping")) + report.Stdin.Close() + out, _ := io.ReadAll(report.Stdout) + errOut, _ := io.ReadAll(report.Stderr) + closeStdio(report) + if string(out) != "ping /data hello from the world " || string(errOut) != "to stderr" { + t.Errorf("spawned process wrote %q and %q", out, errOut) + } + if code, err := report.Wait(); err != nil || code != 3 { + t.Fatalf("spawned Wait = %d, %v; want exit code 3", code, err) + } + if err := report.Signal(syscall.SIGKILL); !errors.Is(err, ErrExited) { + t.Errorf("Signal after the spawned process exited = %v, want ErrExited", err) + } + // The directory is entered as the process's user. + if err := os.Mkdir(filepath.Join(f.harness, "root-only"), 0o700); err != nil { + t.Fatal(err) + } + if _, err := spawnHelper(context.Background(), v, "noop", "/.oac/harness/root-only"); !errors.Is(err, ErrExec) || !errors.Is(err, syscall.EACCES) { + t.Errorf("Spawn in a directory only root may enter = %v, want ErrExec with EACCES", err) + } + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + late := fmt.Sprintf("oac-cancelled-%d", time.Now().UnixNano()) + for range 20 { + if _, err := spawnHelper(cancelled, v, "sleep", "/data", late); !errors.Is(err, context.Canceled) { + t.Fatalf("Spawn with a cancelled context = %v, want context.Canceled", err) + } + } + // A directory longer than a control packet fails alone. This spawn begins once what the cancelled ones started is reaped. + bounded, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if _, err := spawnHelper(bounded, v, "noop", "/"+strings.Repeat("x", 1<<20)); !errors.Is(err, ErrExec) || !errors.Is(err, syscall.ENAMETOOLONG) { + t.Errorf("Spawn in a 1 MiB directory = %.200v, want ErrExec with ENAMETOOLONG", err) + } + if n := len(pidsWith(t, late)); n != 0 { + t.Errorf("the cancelled spawns left %d processes", n) + } + if err := sleeper.Signal(0); err != nil { + t.Errorf("Signal to the running spawned process = %v", err) + } + // A spawn that cannot make its pipes fails with the reason. + var limit unix.Rlimit + if err := unix.Prlimit(0, unix.RLIMIT_NOFILE, nil, &limit); err != nil { + t.Fatal(err) + } + if err := unix.Prlimit(0, unix.RLIMIT_NOFILE, &unix.Rlimit{Max: limit.Max}, nil); err != nil { + t.Fatal(err) + } + _, err = spawnHelper(context.Background(), v, "noop", "/data") + if err := unix.Prlimit(0, unix.RLIMIT_NOFILE, &limit, nil); err != nil { + t.Fatal(err) + } + if !errors.Is(err, ErrLauncher) || !errors.Is(err, syscall.EMFILE) { + t.Errorf("Spawn without descriptors = %v, want ErrLauncher with EMFILE", err) + } + // One argument above the control socket's packet size starts; one above exec's limit fails alone. + for size, want := range map[int]error{100 << 10: nil, 200 << 10: ErrExec} { + s, err := spawnHelper(context.Background(), v, "noop", "/data", strings.Repeat("x", size)) + if err == nil { + closeStdio(s) + _, err = s.Wait() + } + if !errors.Is(err, want) { + t.Errorf("Spawn with a %d byte argument = %v, want %v", size, err, want) + } + } + if err := v.Signal(0); err != nil || len(pidsWith(t, token)) != 1 { + t.Fatalf("after the spawns, the view's Signal = %v and the sleeper runs %d times; want both running", err, len(pidsWith(t, token))) + } + if err := v.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + if _, err := sleeper.Wait(); !errors.Is(err, ErrClosed) { + t.Errorf("spawned Wait after the view ended = %v, want ErrClosed", err) + } + if n := len(pidsWith(t, token)); n != 0 { + t.Errorf("%d spawned processes survived the view", n) + } +} + +// TestStalledSpawnBlocksNothingElse checks that while a spawn's directory is stuck on the world, the spawns behind it wait holding no descriptors and return once their contexts end, and that the view's signals, the exits of its other processes, the stuck caller's context, the process's exit and the teardown all go on. +func TestStalledSpawnBlocksNothingElse(t *testing.T) { + v, w := startStalled(t) + sleeper, err := spawnHelper(context.Background(), v, "sleep", "/data") + if err != nil { + t.Fatalf("Spawn: %v", err) + } + defer closeStdio(sleeper) + launcherFDs := fdCount(t, v.cmd.Process.Pid) + ctx, cancel := context.WithCancel(context.Background()) + stuck := spawnAsync(ctx, v, "sleep", "/data/stall") + await(t, w.stalled, "the spawn's lookup in the world") + fds := fdCount(t, os.Getpid()) + waitCtx, stopWaiting := context.WithCancel(context.Background()) + var waiting []<-chan spawnResult + for range 50 { + waiting = append(waiting, spawnAsync(waitCtx, v, "noop", "/data")) + } + eventually(t, "50 spawns waiting", func() bool { return inSpawn() == 51 }) + if n := fdCount(t, os.Getpid()); n > fds { + t.Errorf("%d descriptors with 50 spawns waiting, %d before", n, fds) + } + // The launcher holds the stuck spawn's descriptors, nothing for the spawns waiting. + if n := fdCount(t, v.cmd.Process.Pid); n > launcherFDs+4 { + t.Errorf("launcher holds %d descriptors with a spawn stuck and 50 waiting, %d before", n, launcherFDs) + } + stopWaiting() + for _, r := range waiting { + if r := await(t, r, "a waiting spawn's return"); !errors.Is(r.err, context.Canceled) { + t.Errorf("waiting Spawn = %v, want context.Canceled", r.err) + } + } + // Enough requests that the launcher's heap would call for a collection. + signaled := make(chan error, 1) + go func() { + for range 1000 { + if err := v.Signal(0); err != nil { + signaled <- err + return + } + } + signaled <- nil + }() + if err := await(t, signaled, "1000 signals"); err != nil { + t.Errorf("Signal while a spawn is stuck = %v", err) + } + if err := sleeper.Signal(syscall.SIGKILL); err != nil { + t.Fatalf("Signal to the sleeper = %v", err) + } + if code := await(t, waitFor(sleeper), "the sleeper's exit"); code != -1 { + t.Errorf("sleeper Wait = %d, want -1", code) + } + cancel() + if r := await(t, stuck, "the stuck spawn's return"); !errors.Is(r.err, context.Canceled) { + t.Errorf("stuck Spawn = %v, want context.Canceled", r.err) + } + if err := v.Signal(syscall.SIGTERM); err != nil { + t.Fatalf("Signal: %v", err) + } + exited := make(chan error, 1) + go func() { + exit, err := v.Wait() + if exit != (Exit{Code: 7}) { + err = errors.Join(err, fmt.Errorf("exit %+v", exit)) + } + exited <- err + }() + if err := await(t, exited, "the view's end"); err != nil { + t.Errorf("Wait with a spawn still starting: %v, want exit code 7", err) + } +} + +// TestLateSpawnIsEnded checks that a process whose spawn was cancelled before it started is killed once it starts, before the next spawn begins. +func TestLateSpawnIsEnded(t *testing.T) { + v, w := startStalled(t) + token := fmt.Sprintf("oac-late-%d", time.Now().UnixNano()) + ctx, cancel := context.WithCancel(context.Background()) + late := spawnAsync(ctx, v, "sleep", "/data/stall", token) + await(t, w.stalled, "the spawn's lookup in the world") + cancel() + if r := await(t, late, "the cancelled spawn's return"); !errors.Is(r.err, context.Canceled) { + t.Fatalf("Spawn = %v, want context.Canceled", r.err) + } + next := spawnAsync(context.Background(), v, "noop", "/data") + eventually(t, "the next spawn waiting", func() bool { return inSpawn() == 1 }) + w.unstall() + r := await(t, next, "the next spawn") + if r.err != nil { + t.Fatalf("next Spawn: %v", r.err) + } + closeStdio(r.s) + if n := len(pidsWith(t, token)); n != 0 { + t.Errorf("the late process runs %d times once the next spawn started", n) + } + if code, err := r.s.Wait(); err != nil || code != 0 { + t.Errorf("next Wait = %d, %v", code, err) + } +} + +// TestSpawnExitsBeforeItsRegistration checks that a spawned child that dies before its exec, while orphans exit around it, reports its own exit to its own handle and to no other, that a handle whose process has exited reaches nothing, and that Close ends what remains. While the child is stuck, its fork may hold up the launcher, so the test acts on the processes directly. +func TestSpawnExitsBeforeItsRegistration(t *testing.T) { + v, w := startStalled(t) + launcher := v.cmd.Process.Pid + token := fmt.Sprintf("oac-bystander-%d", time.Now().UnixNano()) + bystander, err := spawnHelper(context.Background(), v, "sleep", "/data", token) + if err != nil { + t.Fatalf("Spawn: %v", err) + } + defer closeStdio(bystander) + orphanToken := fmt.Sprintf("oac-orphan-%d", time.Now().UnixNano()) + parent, err := v.Spawn(context.Background(), "/.oac/harness/harness", []string{"harness"}, []string{helperEnv + "=wait", "OAC_VIEW_TOKEN=" + orphanToken}, "/data", false) + if err != nil { + t.Fatalf("Spawn: %v", err) + } + defer closeStdio(parent) + if line, err := bufio.NewReader(parent.Stdout).ReadString('\n'); err != nil || line != "ready\n" { + t.Fatalf("parent said %q, %v", line, err) + } + orphan := pidsWith(t, orphanToken) + if len(orphan) != 1 { + t.Fatalf("orphans: %v, want 1", orphan) + } + group, _ := strconv.Atoi(stat(orphan[0])[2]) + // The child's exec stays on the world. + pending := make(chan spawnResult, 1) + go func() { + s, err := v.Spawn(context.Background(), "/data/stall", []string{"stall"}, nil, "/data", false) + pending <- spawnResult{s, err} + }() + await(t, w.stalled, "the exec's lookup in the world") + // The parent's group ends while the child forks; its orphan stays unreaped until the child is registered. + if err := unix.Kill(-group, unix.SIGKILL); err != nil { + t.Fatal(err) + } + eventually(t, "the orphan's exit", func() bool { return slices.Contains(zombies(t, launcher), orphan[0]) }) + child := pidsWith(t, launcherArg0) + child = slices.DeleteFunc(child, func(pid int) bool { return pid == launcher }) + if len(child) != 1 { + t.Fatalf("children before their exec: %v, want 1", child) + } + // The child dies before its exec, once the world answers. + if err := unix.Kill(child[0], unix.SIGKILL); err != nil { + t.Fatal(err) + } + w.unstall() + r := await(t, pending, "the spawn") + if r.err != nil { + t.Fatalf("Spawn = %v, want the child that died before its exec", r.err) + } + defer closeStdio(r.s) + if code := await(t, waitFor(r.s), "the child's exit"); code != -1 { + t.Errorf("child Wait = %d, want -1", code) + } + if code := await(t, waitFor(parent), "the parent's exit"); code != -1 { + t.Errorf("parent Wait = %d, want -1", code) + } + for _, s := range []*Spawned{r.s, parent} { + if err := s.Signal(0); !errors.Is(err, ErrExited) { + t.Errorf("Signal after the process exited = %v, want ErrExited", err) + } + } + if err := bystander.Signal(0); err != nil { + t.Errorf("Signal to the bystander = %v", err) + } + eventually(t, "the orphans reaped", func() bool { return len(zombies(t, launcher)) == 0 }) + if err := v.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + if _, err := bystander.Wait(); !errors.Is(err, ErrClosed) { + t.Errorf("bystander Wait after Close = %v, want ErrClosed", err) + } + if n := len(pidsWith(t, token)); n != 0 { + t.Errorf("%d bystanders survived the view", n) } } @@ -327,7 +630,7 @@ func TestRemoveCgroupKeepsItPastTheDeadline(t *testing.T) { func TestCancelledStartStopsTheWorld(t *testing.T) { requireView(t) f := newFixture(t) - w := &stallWorld{loopbackWorld: loopbackWorld{dir: f.world}, stalled: make(chan struct{}), release: make(chan struct{})} + w := &stallWorld{loopbackWorld: loopbackWorld{dir: f.world}, name: "proc", stalled: make(chan struct{}), release: make(chan struct{})} spec := f.spec(&w.loopbackWorld, "noop") spec.World = w.serve ctx, cancel := context.WithCancel(context.Background()) @@ -609,20 +912,23 @@ func (n *hangNode) Write(context.Context, gofs.FileHandle, []byte, int64) (uint3 return 0, syscall.EIO } -// stallWorld is a loopbackWorld that answers no lookup of proc until Stop. +// stallWorld is a loopbackWorld that answers no lookup of name until unstall or Stop. type stallWorld struct { loopbackWorld - stalled, release chan struct{} // stalled closes at the first lookup of proc - stallOnce sync.Once + name string + stalled, release chan struct{} // stalled closes at the first lookup of name + stallOnce, releaseOnce sync.Once } +func (w *stallWorld) unstall() { w.releaseOnce.Do(func() { close(w.release) }) } + func (w *stallWorld) serve(_ context.Context, dev *os.File, mount WorldMount) (WorldServer, Presentation, error) { root, err := gofs.NewLoopbackRoot(w.dir) if err != nil { return nil, Presentation{}, err } root.(*gofs.LoopbackNode).RootData.NewNode = func(r *gofs.LoopbackRoot, _ *gofs.Inode, name string, _ *syscall.Stat_t) gofs.InodeEmbedder { - if name == "proc" { + if name == w.name { w.stallOnce.Do(func() { close(w.stalled) }) <-w.release } @@ -636,7 +942,7 @@ func (w *stallWorld) serve(_ context.Context, dev *os.File, mount WorldMount) (W } func (w *stallWorld) Stop() error { - close(w.release) + w.unstall() return w.loopbackWorld.Stop() } @@ -677,20 +983,171 @@ func serveBroker(t *testing.T) func(*os.File) error { } } -// processesWith counts processes whose command line contains token. -func processesWith(t *testing.T, token string) int { +// pidsWith lists the processes whose command line contains token. +func pidsWith(t *testing.T, token string) []int { t.Helper() cmdlines, err := filepath.Glob("/proc/[0-9]*/cmdline") if err != nil { t.Fatal(err) } - n := 0 + var pids []int for _, p := range cmdlines { if b, err := os.ReadFile(p); err == nil && bytes.Contains(b, []byte(token)) { - n++ + pid, _ := strconv.Atoi(filepath.Base(filepath.Dir(p))) + pids = append(pids, pid) + } + } + return pids +} + +// stat returns the fields of pid's stat after its command name, which may hold anything: the state, the parent's pid and the process group come first. It returns nil once pid is gone. +func stat(pid int) []string { + b, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid)) + if err != nil { + return nil + } + return strings.Fields(string(b[bytes.LastIndexByte(b, ')')+1:])) +} + +// zombies returns the children of pid that have exited and are not reaped yet. +func zombies(t *testing.T, pid int) []int { + t.Helper() + stats, err := filepath.Glob("/proc/[0-9]*/stat") + if err != nil { + t.Fatal(err) + } + var z []int + for _, p := range stats { + child, _ := strconv.Atoi(filepath.Base(filepath.Dir(p))) + if f := stat(child); len(f) > 1 && f[0] == "Z" && f[1] == strconv.Itoa(pid) { + z = append(z, child) + } + } + return z +} + +func fdCount(t *testing.T, pid int) int { + t.Helper() + fds, err := os.ReadDir(fmt.Sprintf("/proc/%d/fd", pid)) + if err != nil { + t.Fatal(err) + } + return len(fds) +} + +// inSpawn counts the goroutines in View.Spawn. +func inSpawn() int { + buf := make([]byte, 1<<20) + return strings.Count(string(buf[:runtime.Stack(buf, true)]), "sessionview.(*View).Spawn(") +} + +// startStalled starts a view whose process waits for TERM and whose world answers no lookup of /data/stall until w.unstall. +func startStalled(t *testing.T) (*View, *stallWorld) { + t.Helper() + requireView(t) + f := newFixture(t) + mkdir(t, filepath.Join(f.world, "data", "stall")) + w := &stallWorld{loopbackWorld: loopbackWorld{dir: f.world}, name: "stall", stalled: make(chan struct{}), release: make(chan struct{})} + spec := f.spec(&w.loopbackWorld, "wait", "OAC_VIEW_TOKEN=oac-unused") + spec.World = w.serve + v, err := Start(context.Background(), spec) + if err != nil { + t.Fatalf("Start: %v", err) + } + t.Cleanup(func() { v.Close() }) + if line, err := bufio.NewReader(v.Stdout()).ReadString('\n'); err != nil || line != "ready\n" { + t.Fatalf("harness said %q, %v", line, err) + } + return v, w +} + +// spawnHelper spawns this binary as helper mode in dir, with args after its name and no stdin. +func spawnHelper(ctx context.Context, v *View, mode, dir string, args ...string) (*Spawned, error) { + return v.Spawn(ctx, "/.oac/harness/harness", append([]string{"harness"}, args...), []string{helperEnv + "=" + mode}, dir, false) +} + +type spawnResult struct { + s *Spawned + err error +} + +func spawnAsync(ctx context.Context, v *View, mode, dir string, args ...string) <-chan spawnResult { + ch := make(chan spawnResult, 1) + go func() { + s, err := spawnHelper(ctx, v, mode, dir, args...) + ch <- spawnResult{s, err} + }() + return ch +} + +// waitFor delivers s's exit code, or -2 when Wait fails. +func waitFor(s *Spawned) <-chan int { + ch := make(chan int, 1) + go func() { + code, err := s.Wait() + if err != nil { + code = -2 + } + ch <- code + }() + return ch +} + +func closeStdio(s *Spawned) { closeFiles([]*os.File{s.Stdin, s.Stdout, s.Stderr}) } + +// await returns what ch delivers, failing the test when nothing comes within 10 seconds. +func await[T any](t *testing.T, ch <-chan T, what string) T { + t.Helper() + select { + case v := <-ch: + return v + case <-time.After(10 * time.Second): + t.Fatalf("%s: nothing within 10s", what) + } + var zero T + return zero +} + +// eventually polls cond, failing the test when it does not hold within 10 seconds. +func eventually(t *testing.T, what string, cond func() bool) { + t.Helper() + for deadline := time.Now().Add(10 * time.Second); !cond(); time.Sleep(5 * time.Millisecond) { + if time.Now().After(deadline) { + t.Fatalf("%s not within 10s", what) + } + } +} + +// identity describes what this process runs as and in: its credentials and restrictions, its namespaces, its cgroup and its root. +func identity() (map[string]string, error) { + status, err := os.ReadFile("/proc/self/status") + if err != nil { + return nil, err + } + id := map[string]string{} + for _, line := range strings.Split(string(status), "\n") { + k, v, _ := strings.Cut(line, ":") + switch k { + case "Uid", "Gid", "Groups", "CapInh", "CapPrm", "CapEff", "CapBnd", "CapAmb", "NoNewPrivs", "Seccomp", "Seccomp_filters": + id[k] = strings.TrimSpace(v) + } + } + for _, ns := range []string{"mnt", "net", "pid", "ipc", "uts", "user", "cgroup"} { + if id["ns "+ns], err = os.Readlink("/proc/self/ns/" + ns); err != nil { + return nil, err } } - return n + cgroup, err := os.ReadFile("/proc/self/cgroup") + if err != nil { + return nil, err + } + id["cgroup"] = string(cgroup) + var st unix.Stat_t + if err := unix.Stat("/", &st); err != nil { + return nil, err + } + id["root"] = fmt.Sprintf("%d:%d", st.Dev, st.Ino) + return id, nil } func runHelper(mode string) int { @@ -725,7 +1182,7 @@ func runHelper(mode string) int { sigs := make(chan os.Signal, 1) signal.Notify(sigs, syscall.SIGTERM) child := exec.Command("/.oac/harness/harness") - child.Env = []string{helperEnv + "=slow-term"} + child.Env = []string{helperEnv + "=slow-term", "OAC_VIEW_TOKEN=" + os.Getenv("OAC_VIEW_TOKEN")} child.Stdout = os.Stdout if err := child.Start(); err != nil { fmt.Fprintln(os.Stderr, err) @@ -741,13 +1198,31 @@ func runHelper(mode string) int { <-sigs signal.Reset(syscall.SIGTERM) time.Sleep(300 * time.Millisecond) - if err := os.WriteFile("/data/cleaned", []byte("done"), 0o644); err != nil { + if err := os.WriteFile("/data/cleaned"+os.Getenv("OAC_VIEW_TOKEN"), []byte("done"), 0o644); err != nil { fmt.Fprintln(os.Stderr, err) return 1 } return 0 case "noop": return 0 + case "identity": + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGTERM) + id, err := identity() + if err != nil { + fmt.Fprintln(os.Stderr, err) + return 1 + } + json.NewEncoder(os.Stdout).Encode(id) + <-sigs + return 7 + case "report": + in, err := io.ReadAll(os.Stdin) + wd, werr := os.Getwd() + data, rerr := os.ReadFile("in.txt") + fmt.Printf("%s %s %s %v", in, wd, data, errors.Join(err, werr, rerr)) + fmt.Fprint(os.Stderr, "to stderr") + return 3 case "hang": f, err := os.OpenFile("/data/hang", os.O_WRONLY, 0) if err != nil { diff --git a/contracts/agents-api/harness-onboarding.md b/contracts/agents-api/harness-onboarding.md index dd4ca62d4..8a510833e 100644 --- a/contracts/agents-api/harness-onboarding.md +++ b/contracts/agents-api/harness-onboarding.md @@ -289,7 +289,7 @@ An agent host runs the Harness outside the sandbox, in a per-Session view. The v ### Executables -Only mount flags grant execution. The closure, `Exec` overlays and the shim are read-only and are the only executable mounts; the sandbox's files and the home are noexec. `Launch` accepts only a `LocalExec` path as `Binary`. A dynamic binary, such as `node`, needs its ELF interpreter as an `Exec` overlay at its `PT_INTERP` path, and every library it loads in the closure, reached through `LD_LIBRARY_PATH`. Nothing loads from the sandbox's files. `viewloader.For` builds this from the binaries' ELF headers: the interpreter's host directory as the `lib` closure mount, the interpreter overlay, empty masks over `/etc/ld.so.preload` and `/etc/ld.so.cache`, and the `LD_LIBRARY_PATH` value. A layout it cannot present, such as a library outside the interpreter's directory, returns `ErrUnsupportedOperation`. +Only mount flags grant execution. The closure, `Exec` overlays and the shim are read-only and are the only executable mounts; the sandbox's files and the home are noexec. `Launch` and `Spawn` accept only a `LocalExec` path as `Binary` and otherwise return `ErrNotLocalExec`. A dynamic binary, such as `node`, needs its ELF interpreter as an `Exec` overlay at its `PT_INTERP` path, and every library it loads in the closure, reached through `LD_LIBRARY_PATH`. Nothing loads from the sandbox's files. `viewloader.For` builds this from the binaries' ELF headers: the interpreter's host directory as the `lib` closure mount, the interpreter overlay, empty masks over `/etc/ld.so.preload` and `/etc/ld.so.cache`, and the `LD_LIBRARY_PATH` value. A layout it cannot present, such as a library outside the interpreter's directory, returns `ErrUnsupportedOperation`. ### Shims @@ -323,6 +323,15 @@ With `ViewProxyNone`, `ViewSession.Proxy` is empty and the view has no generic p - When the Harness exits while other processes remain, the view sends them TERM unless Cancel already did, and ends once they exit or `KillTimeout` passes from the first TERM. - `Wait` closes the stdio ends, returns the context error when Cancel's TERM reached the running Harness and it then exited 0, and `ExitCode` reports the exit once `Done` closes. +### Spawn + +`ViewSession.Spawn` runs a `LocalExec` binary as another process in the live view while the Harness runs, such as a reader of the Harness's native history. Harness-side code that reads Harness-written data runs here, never on the agent host outside the view and never in a view of its own. `Spawn` takes `StartOptions` as `Launch` does and returns the same `clirunner.Process`. The process runs as the Harness does: as the same user, in the same namespaces, view cgroup, world and network, with no capabilities, `no_new_privs` and the same seccomp filter, in a process group of its own. + +- The view starts one `Spawn` at a time. `Parent` bounds the wait for its turn and for the start. Once it ends, `Spawn` returns its error and kills a process that starts after all. +- Cancel sends TERM to its process group and kills the group after `KillTimeout`. Once the process has exited, Cancel delivers nothing, and what it left runs on as the view's other processes do. +- The view's end ends them all. A Cancel of the Harness reaches them, and when the Harness exits they are among the processes that remain. A view that ends after `Spawn` returned shows in the process's `Wait`. +- `Spawn` returns `ErrNotLocalExec` when `Binary` is not a `LocalExec` path, and `ErrNoLiveView` when no view runs its Harness: none was launched yet, or its Harness has exited or its view has ended. Any other failure, such as a binary that does not start or no descriptors left, keeps its own error. + ### Qualify the view Run the adapter's Turns, cancellation and continuation in a view, then qualify each declared entry: diff --git a/contracts/agents-api/zh/harness-onboarding.md b/contracts/agents-api/zh/harness-onboarding.md index f9e9f1871..66fa44adc 100644 --- a/contracts/agents-api/zh/harness-onboarding.md +++ b/contracts/agents-api/zh/harness-onboarding.md @@ -1,7 +1,7 @@ --- title: "将原生 Harness 添加到 OpenAgentCore" source: contracts/agents-api/harness-onboarding.md -source_hash: 54cdb8254d7f00fbcd78f8ea4a836556db90f12f21dd606a933adb0ead24f652 +source_hash: aa5ee32e9ae72b923addad7b049d586a32ec18319e6454c52cc21a96e0f69614 --- **Harness** 是一种运行模型和工具循环的原生代理引擎(Codex、Claude Code、MiniMax Code)。**Harness 适配器**将 Runtime 的 Executor 和 Turn 契约转换到该引擎的 SDK 或协议。本文档定义 Runtime–Harness 协议:适配器接口及其生命周期义务、注册、Core 资格认定和验收。[Harness capabilities](harness-capabilities.md) 记录了当前每个 Harness 支持的功能。 @@ -291,7 +291,7 @@ agent host 在沙箱之外、在每个 Session 一个的视图中运行 Harness ### 可执行文件 {#executables} -只有挂载标志授予执行权限。closure、`Exec` overlay 和 shim 是只读的,也是仅有的可执行挂载;沙箱的文件和 home 都是 noexec。`Launch` 只接受 `LocalExec` 路径作为 `Binary`。动态二进制(例如 `node`)需要把它的 ELF 解释器作为 `Exec` overlay 放在其 `PT_INTERP` 路径上,并且它加载的每个库都要在 closure 中,通过 `LD_LIBRARY_PATH` 找到。任何内容都不从沙箱的文件加载。`viewloader.For` 根据二进制的 ELF header 构建这些内容:解释器所在的主机目录作为 `lib` closure 挂载、解释器 overlay、覆盖 `/etc/ld.so.preload` 和 `/etc/ld.so.cache` 的空 mask,以及 `LD_LIBRARY_PATH` 的值。它无法呈现的布局(例如位于解释器目录之外的库)返回 `ErrUnsupportedOperation`。 +只有挂载标志授予执行权限。closure、`Exec` overlay 和 shim 是只读的,也是仅有的可执行挂载;沙箱的文件和 home 都是 noexec。`Launch` 和 `Spawn` 只接受 `LocalExec` 路径作为 `Binary`,否则返回 `ErrNotLocalExec`。动态二进制(例如 `node`)需要把它的 ELF 解释器作为 `Exec` overlay 放在其 `PT_INTERP` 路径上,并且它加载的每个库都要在 closure 中,通过 `LD_LIBRARY_PATH` 找到。任何内容都不从沙箱的文件加载。`viewloader.For` 根据二进制的 ELF header 构建这些内容:解释器所在的主机目录作为 `lib` closure 挂载、解释器 overlay、覆盖 `/etc/ld.so.preload` 和 `/etc/ld.so.cache` 的空 mask,以及 `LD_LIBRARY_PATH` 的值。它无法呈现的布局(例如位于解释器目录之外的库)返回 `ErrUnsupportedOperation`。 ### Shim {#shims} @@ -325,6 +325,15 @@ agent host 根据声明推导进程 broker 的映射表:`/.oac/bin/` 在 - Harness 退出而仍有其他进程时,除非 Cancel 已发送过 TERM,视图会向它们发送 TERM,并在它们退出或自首次 TERM 起经过 `KillTimeout` 后结束。 - `Wait` 关闭 stdio 端;当 Cancel 的 TERM 到达运行中的 Harness 且 Harness 随后以 0 退出时,`Wait` 返回 context 错误;`Done` 关闭后,`ExitCode` 报告退出结果。 +### Spawn {#spawn} + +`ViewSession.Spawn` 在 Harness 运行期间,把一个 `LocalExec` 二进制作为另一个进程运行在活动视图中,例如读取 Harness 原生历史的程序。读取 Harness 所写数据的 Harness 侧代码在这里运行,从不在视图之外的 agent host 上运行,也从不获得自己的视图。`Spawn` 像 `Launch` 一样接收 `StartOptions`,并返回同样的 `clirunner.Process`。该进程的运行方式与 Harness 相同:同一用户,同一组命名空间、视图 cgroup、world 和网络,没有 capability,设置 `no_new_privs` 并使用同一 seccomp 过滤器,且位于自己的进程组中。 + +- 视图一次只启动一个 `Spawn`。`Parent` 限定等待轮次和等待启动的时间。它结束后,`Spawn` 返回它的错误,并杀死此后仍然启动的进程。 +- Cancel 向它的进程组发送 TERM,并在 `KillTimeout` 后杀死该进程组。进程退出后,Cancel 不再投递任何信号,它遗留的进程像视图中的其他进程一样继续运行。 +- 视图结束时它们全部随之结束。Harness 的 Cancel 会到达它们;Harness 退出时,它们属于仍然存在的进程。`Spawn` 返回之后视图才结束的情况,体现在该进程的 `Wait` 中。 +- `Binary` 不是 `LocalExec` 路径时,`Spawn` 返回 `ErrNotLocalExec`;没有视图在运行其 Harness 时返回 `ErrNoLiveView`:尚未启动视图,或其 Harness 已退出,或其视图已结束。其他失败(例如二进制无法启动或描述符耗尽)保留各自的错误。 + ### 认定视图资格 {#qualify-the-view} 在视图中运行适配器的 Turn、取消和续接,然后逐项认定每个声明条目: