From 3acd5abc7cecef65a4c3800bdce795d6c5c7ed3e Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 15:11:48 +0900 Subject: [PATCH 01/37] test(g01): preserve paired handoff red cases --- .../g01-scaleset/cmd/g01-live/main_test.go | 119 +++++++++++++++++- experiments/g02-auth/broker_test.go | 9 ++ 2 files changed, 127 insertions(+), 1 deletion(-) diff --git a/experiments/g01-scaleset/cmd/g01-live/main_test.go b/experiments/g01-scaleset/cmd/g01-live/main_test.go index a1658da..799c101 100644 --- a/experiments/g01-scaleset/cmd/g01-live/main_test.go +++ b/experiments/g01-scaleset/cmd/g01-live/main_test.go @@ -5,12 +5,15 @@ package main import ( "bytes" "encoding/json" - "github.com/1XP-AI/gh-runnerd/experiments/g01-scaleset/livecanary" + "io" "os" "path/filepath" "strings" "testing" "time" + + "github.com/1XP-AI/gh-runnerd/experiments/g01-scaleset/livecanary" + "github.com/1XP-AI/gh-runnerd/experiments/g01-scaleset/liveworker" ) type unreadable struct{ t *testing.T } @@ -20,6 +23,17 @@ func (r unreadable) Read([]byte) (int, error) { return 0, nil } +type countedInput struct { + io.Reader + reads int +} + +func (r *countedInput) Close() error { return nil } +func (r *countedInput) Read(p []byte) (int, error) { + r.reads++ + return r.Reader.Read(p) +} + func TestPlanAndRefusalsNeverReadCredentialsOrEchoInputs(t *testing.T) { for _, args := range [][]string{{"--plan"}, {}, {"--synthetic-secret=do-not-print"}, {"--execute-approved-canary", "--approval=synthetic-secret", "--state-dir=synthetic-secret", "--phase=create"}} { var out bytes.Buffer @@ -87,3 +101,106 @@ func TestPreparationCommandNeverReadsCredentialsOrRunsRemotePhase(t *testing.T) }) } } + +func TestPairedTerminalModeReadsControllerInputAfterAllGates(t *testing.T) { + a := livecanary.Approval{AppID: 11, InstallationID: 12, Organization: "fixture-org", Repository: "canary", RepositoryID: 42, RunnerGroupID: 3, OwnerNonce: strings.Repeat("a", 32), HarnessSHA: strings.Repeat("b", 40), WorkflowSHA: strings.Repeat("c", 40), WorkflowPath: ".github/workflows/canary.yml", WorkflowRunID: 5, Controller: "fixture-controller", ExpiresAt: time.Now().Add(time.Hour), ActionsHosts: []string{"fixture.actions.githubusercontent.com"}, Phases: []string{"create", "before-ack", "after-ack", "before-acquire", "acquire-loss", "inspect", "cleanup", "jit-loss"}} + worker := liveworker.Approval{RunnerUpdatesDisabled: true, HarnessSHA: a.HarnessSHA, WorkflowSHA: a.WorkflowSHA, OwnerNonce: a.OwnerNonce, Controller: a.Controller, Endpoint: "/tmp/g01-paired-docker.sock", DaemonID: "fixture-daemon", ImageID: "sha256:" + strings.Repeat("d", 64), Image: liveworker.ImageReference, ExpiresAt: a.ExpiresAt, Phases: []string{"create", "start", "inspect", "cleanup"}} + root := t.TempDir() + controllerState := filepath.Join(root, "controller-state") + workerState := filepath.Join(root, "worker-state") + if err := os.Mkdir(controllerState, 0700); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(workerState, 0700); err != nil { + t.Fatal(err) + } + controllerPath := filepath.Join(root, "controller.json") + workerPath := filepath.Join(root, "worker.json") + controllerData, err := json.Marshal(a) + if err != nil { + t.Fatal(err) + } + workerData, err := json.Marshal(worker) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(controllerPath, controllerData, 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(workerPath, workerData, 0600); err != nil { + t.Fatal(err) + } + input := &countedInput{Reader: strings.NewReader(`{}`)} + var out bytes.Buffer + code := runWithPreparation([]string{"--execute-approved-paired-terminal", "--approval", controllerPath, "--state-dir", controllerState, "--worker-approval", workerPath, "--worker-state-dir", workerState}, input, &out, func() (string, bool) { return a.HarnessSHA, true }, func(string, livecanary.Approval, string) (livecanary.PreparationReceipt, error) { + t.Fatal("paired mode entered controller-only preparation") + return livecanary.PreparationReceipt{}, nil + }) + if code == 0 || input.reads == 0 { + t.Fatalf("paired mode did not reach its bounded controller input gate: code=%d reads=%d output=%q", code, input.reads, out.String()) + } +} + +func TestPairedTerminalModeRejectsUnusedPhaseAndControllerFlagsBeforeInput(t *testing.T) { + a := livecanary.Approval{AppID: 11, InstallationID: 12, Organization: "fixture-org", Repository: "canary", RepositoryID: 42, RunnerGroupID: 3, OwnerNonce: strings.Repeat("a", 32), HarnessSHA: strings.Repeat("b", 40), WorkflowSHA: strings.Repeat("c", 40), WorkflowPath: ".github/workflows/canary.yml", WorkflowRunID: 5, Controller: "fixture-controller", ExpiresAt: time.Now().Add(time.Hour), ActionsHosts: []string{"fixture.actions.githubusercontent.com"}, Phases: []string{"create", "before-ack", "after-ack", "before-acquire", "acquire-loss", "inspect", "cleanup", "jit-loss"}} + worker := liveworker.Approval{RunnerUpdatesDisabled: true, HarnessSHA: a.HarnessSHA, WorkflowSHA: a.WorkflowSHA, OwnerNonce: a.OwnerNonce, Controller: a.Controller, Endpoint: "/tmp/g01-paired-docker.sock", DaemonID: "fixture-daemon", ImageID: "sha256:" + strings.Repeat("d", 64), Image: liveworker.ImageReference, ExpiresAt: a.ExpiresAt, Phases: []string{"create", "start", "inspect", "cleanup"}} + root := t.TempDir() + controllerState := filepath.Join(root, "controller-state") + workerState := filepath.Join(root, "worker-state") + if err := os.Mkdir(controllerState, 0700); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(workerState, 0700); err != nil { + t.Fatal(err) + } + controllerPath := filepath.Join(root, "controller.json") + workerPath := filepath.Join(root, "worker.json") + controllerData, _ := json.Marshal(a) + workerData, _ := json.Marshal(worker) + if err := os.WriteFile(controllerPath, controllerData, 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(workerPath, workerData, 0600); err != nil { + t.Fatal(err) + } + base := []string{"--execute-approved-paired-terminal", "--approval", controllerPath, "--state-dir", controllerState, "--worker-approval", workerPath, "--worker-state-dir", workerState} + for _, extra := range [][]string{{"--phase", "cleanup"}, {"--execute-approved-canary"}} { + input := &countedInput{Reader: strings.NewReader(`{}`)} + args := append(append([]string(nil), base...), extra...) + var out bytes.Buffer + if code := runWithPreparation(args, input, &out, func() (string, bool) { return a.HarnessSHA, true }, nil); code == 0 || input.reads != 0 { + t.Fatalf("incompatible paired flags reached input: extra=%v code=%d reads=%d", extra, code, input.reads) + } + } +} + +func TestPairedTerminalModeRequiresWorkflowVerificationAuthorityBeforeInput(t *testing.T) { + a := livecanary.Approval{AppID: 11, InstallationID: 12, Organization: "fixture-org", Repository: "canary", RepositoryID: 42, RunnerGroupID: 3, OwnerNonce: strings.Repeat("a", 32), HarnessSHA: strings.Repeat("b", 40), WorkflowSHA: strings.Repeat("c", 40), WorkflowPath: ".github/workflows/canary.yml", WorkflowRunID: 5, Controller: "fixture-controller", ExpiresAt: time.Now().Add(time.Hour), ActionsHosts: []string{"fixture.actions.githubusercontent.com"}, Phases: []string{"create", "inspect", "cleanup"}} + worker := liveworker.Approval{RunnerUpdatesDisabled: true, HarnessSHA: a.HarnessSHA, WorkflowSHA: a.WorkflowSHA, OwnerNonce: a.OwnerNonce, Controller: a.Controller, Endpoint: "/tmp/g01-paired-docker.sock", DaemonID: "fixture-daemon", ImageID: "sha256:" + strings.Repeat("d", 64), Image: liveworker.ImageReference, ExpiresAt: a.ExpiresAt, Phases: []string{"create", "start", "inspect", "cleanup"}} + root := t.TempDir() + controllerState := filepath.Join(root, "controller-state") + workerState := filepath.Join(root, "worker-state") + if err := os.Mkdir(controllerState, 0700); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(workerState, 0700); err != nil { + t.Fatal(err) + } + controllerPath := filepath.Join(root, "controller.json") + workerPath := filepath.Join(root, "worker.json") + controllerData, _ := json.Marshal(a) + workerData, _ := json.Marshal(worker) + if err := os.WriteFile(controllerPath, controllerData, 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(workerPath, workerData, 0600); err != nil { + t.Fatal(err) + } + input := &countedInput{Reader: strings.NewReader(`{}`)} + var out bytes.Buffer + args := []string{"--execute-approved-paired-terminal", "--approval", controllerPath, "--state-dir", controllerState, "--worker-approval", workerPath, "--worker-state-dir", workerState} + code := runWithPreparation(args, input, &out, func() (string, bool) { return a.HarnessSHA, true }, nil) + if code == 0 || input.reads != 0 { + t.Fatalf("paired mode accepted missing verification authority or read input: code=%d reads=%d output=%q", code, input.reads, out.String()) + } +} diff --git a/experiments/g02-auth/broker_test.go b/experiments/g02-auth/broker_test.go index f3df413..ad3de66 100644 --- a/experiments/g02-auth/broker_test.go +++ b/experiments/g02-auth/broker_test.go @@ -17,6 +17,15 @@ func brokerApprovalFixture() BrokerApproval { return BrokerApproval{OwnerNonce: strings.Repeat("a", 32), Mode: "discover-actions-host", AppID: 71, AppName: "synthetic-app", AppOwner: "org-a", AppOwnerID: 101, InstallationID: 201, Organization: "org-a", OrganizationID: 101, Repository: "canary", RepositoryID: 501, RunnerGroupID: 3, RunnerGroupName: "synthetic-group", ExpiresAt: time.Now().Add(time.Hour)} } +func TestPairedTerminalBrokerApprovalUsesDedicatedMode(t *testing.T) { + a := brokerApprovalFixture() + a.Mode = "paired-terminal" + a.Phase = "paired-terminal" + if err := a.validate(time.Now()); err != nil { + t.Fatalf("paired terminal approval was refused: %v", err) + } +} + type brokerHTTPFixture struct { t *testing.T calls []string From 507850c80875e5928ea39d245ca13087a9908330 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 15:25:44 +0900 Subject: [PATCH 02/37] feat(g01): wire paired terminal through bounded broker --- docs/evidence/g01-paired-broker.md | 91 ++++++++++ experiments/g01-scaleset/cmd/g01-live/main.go | 60 ++++++- .../livecanary/paired_terminal.go | 151 ++++++++++++++++ experiments/g02-auth/broker.go | 17 +- experiments/g02-auth/broker_admission.go | 44 ++++- experiments/g02-auth/broker_entry.go | 44 ++++- experiments/g02-auth/broker_files.go | 10 +- experiments/g02-auth/broker_paired_test.go | 166 ++++++++++++++++++ experiments/g02-auth/broker_plan.go | 138 ++++++++++++++- experiments/g02-auth/broker_process.go | 44 +++++ experiments/g02-auth/broker_process_test.go | 29 +++ experiments/g02-auth/broker_snapshot_test.go | 2 +- experiments/g02-auth/cmd/g01-broker/main.go | 4 +- 13 files changed, 775 insertions(+), 25 deletions(-) create mode 100644 docs/evidence/g01-paired-broker.md create mode 100644 experiments/g01-scaleset/livecanary/paired_terminal.go create mode 100644 experiments/g02-auth/broker_paired_test.go diff --git a/docs/evidence/g01-paired-broker.md b/docs/evidence/g01-paired-broker.md new file mode 100644 index 0000000..8399eae --- /dev/null +++ b/docs/evidence/g01-paired-broker.md @@ -0,0 +1,91 @@ +# G01g: paired terminal executable and bounded broker handoff + +Issue [60](https://github.com/1XP-AI/gh-runnerd/issues/60) connects the reviewed +paired terminal sequence to one tagged `g01-live` executable and a dedicated +`g01-broker` mode. This is an offline experiment continuation, not a live +authorization, production daemon, or closure of G01/G02. + +## Implementation boundary + +`g01-live` now has a mutually exclusive +`--execute-approved-paired-terminal` mode. It accepts only the controller +approval/state and explicit worker approval/state inputs; phase, controller-only +execution and worker flags are rejected before controller credential stdin is +read. The executable validates both approvals, shared nonce/harness/workflow/ +controller identity, workflow-run authority, expiries, required terminal phases, +immutable build revision and distinct private state roots. It then reads one +bounded controller credential payload and invokes the existing terminal sequence +in the same process. Controller and worker journals are acquired in order and +released in reverse order; worker JIT remains an in-memory handoff and is never +sent to a worker stdin or subprocess. + +The broker accepts a dedicated `paired-terminal` approval and requires worker +approval/state flags only for that mode. It holds exact worker approval bytes, +approval-file identity and state-root identity through preparation, binds them +into the native-account admission event, requires workflow identity verification +before launch, mints once, and invokes one fixed `g01-live` argv with a minimal +environment. PEM remains in the broker; only bounded controller credentials are +sent to the child. Existing controller-only, discovery and separate worker +paths retain their prior mode/refusal behavior. + +Failure boundaries remain fail-stop: changed worker approval/state, mismatched +shared identity, consumed paired admission, invalid phases/expiry/build, +cancellation, storage uncertainty, bounded child output overflow or child +failure returns a fixed refusal/quarantine result without retry or resume. + +## TDD evidence and checks + +Red tests were preserved in commit `3acd5ab` (`test(g01): preserve paired +handoff red cases`). With the implementation temporarily absent, the meaningful +CLI test failed before input consumption: + +```text +TestPairedTerminalModeReadsControllerInputAfterAllGates: code=1 reads=0 +output="canary refused; approval, authority or private state requires review" +``` + +The broker mode test independently failed with the fixed broker refusal. The +green implementation adds the same-process adapter, broker worker binding and +fixed child handoff, plus negative tests for unused flags, missing workflow +verification authority, changed worker approval and mismatched shared identity. + +The paired broker behavioral fixture uses only generated temporary files, +synthetic HTTP responses and bounded child test processes. It verifies worker +binding before authenticated work, workflow verification before launch, one +installation-token mint, one handoff, no PEM in the payload or admission +ledger, and the dedicated `paired_terminal_completed` result. Existing tagged +private TLS/Unix fixtures continue to exercise one acquire/JIT/create/start, +original-session close, non-force worker delete with separate absence, owned-set +delete with absence, complete rosters and secret-free journals. + +Final offline checks on Go 1.26.8/Darwin ARM64: + +```text +GOTOOLCHAIN=go1.26.8 go test -race -count=1 -timeout=90s ./... # G01 passed +GOTOOLCHAIN=go1.26.8 go vet ./... # G01 passed +GOTOOLCHAIN=go1.26.8 make experiments # passed: 2 module(s) +``` + +The offline gate also passed its tagged `g01-live,g01-worker` CLI tests, +tagged vet, collection/terminal/storage fixture partitions, and both G02 +module suites. `git diff --check` passed. No live GitHub endpoint, App, +credential, runner/group/workflow, Docker/Lima configuration, Keychain, +launchd service or existing runner was touched. + +The repository-wide `GOTOOLCHAIN=go1.26.8 make check` also passed: formatting, +build, root unit/race tests, the configured fuzz smoke, module verification, +license inventory, both offline experiment modules and pinned `govulncheck` +(`v1.7.0`, no vulnerabilities found). + +## Remaining gates and rollback + +Live execution remains unperformed and requires the exact reviewed immutable +artifact, explicit maintainer dispatch and separately approved resources. The +same-UID private-file model is not hostile-code isolation; crash recovery, +uncertain acquisition/JIT/session reconciliation and any successor live run +remain separate evidence gates. Independent Luna/max review, hosted CI and +exact-head GitHub Codex review are still required before merge. + +Offline rollback is source-only: revert the focused issue-60 commits. The +experiment creates no persistent live resource and requires no runner, Docker, +Keychain, launchd or GitHub cleanup. diff --git a/experiments/g01-scaleset/cmd/g01-live/main.go b/experiments/g01-scaleset/cmd/g01-live/main.go index 7acfd31..8631160 100644 --- a/experiments/g01-scaleset/cmd/g01-live/main.go +++ b/experiments/g01-scaleset/cmd/g01-live/main.go @@ -16,6 +16,7 @@ import ( "time" "github.com/1XP-AI/gh-runnerd/experiments/g01-scaleset/livecanary" + "github.com/1XP-AI/gh-runnerd/experiments/g01-scaleset/liveworker" ) func buildRevision() (string, bool) { @@ -56,10 +57,13 @@ func runWithPreparation(args []string, in io.Reader, out io.Writer, revisionForB flags.SetOutput(io.Discard) plan := flags.Bool("plan", false, "") execute := flags.Bool("execute-approved-canary", false, "") + pairedExecute := flags.Bool("execute-approved-paired-terminal", false, "") prepare := flags.Bool("prepare-approved-journal", false, "") approvalPath := flags.String("approval", "", "") statePath := flags.String("state-dir", "", "") phase := flags.String("phase", "", "") + workerApprovalPath := flags.String("worker-approval", "", "") + workerStatePath := flags.String("worker-state-dir", "", "") reject := func() int { fmt.Fprintln(out, "canary refused; approval, authority or private state requires review") return 1 @@ -67,21 +71,50 @@ func runWithPreparation(args []string, in io.Reader, out io.Writer, revisionForB if flags.Parse(args) != nil || flags.NArg() != 0 { return reject() } - if *plan && !*execute && !*prepare { - fmt.Fprintln(out, "Controller-only phases: create, before-ack, after-ack, before-acquire, acquire-loss, jit-loss, inspect, cleanup. No worker launch or workflow dispatch. Live execution requires an immutable reviewed build, exact private approval and controller-side broker input.") + workerInputs := *workerApprovalPath != "" || *workerStatePath != "" + if *plan && !*execute && !*pairedExecute && !*prepare { + if *approvalPath != "" || *statePath != "" || *phase != "" || workerInputs { + return reject() + } + fmt.Fprintln(out, "Controller-only phases: create, before-ack, after-ack, before-acquire, acquire-loss, jit-loss, inspect, cleanup. Paired terminal mode uses one same-process executable with explicit worker approval/state inputs and fixed terminal sequencing; no worker launch or workflow dispatch. Live execution requires an immutable reviewed build, exact private approval and controller-side broker input.") return 0 } - if (*execute == *prepare) || *plan || *approvalPath == "" || *statePath == "" || *phase == "" { + modeCount := 0 + if *execute { + modeCount++ + } + if *pairedExecute { + modeCount++ + } + if *prepare { + modeCount++ + } + if modeCount != 1 || *plan || *approvalPath == "" || *statePath == "" { + return reject() + } + if *pairedExecute { + if *phase != "" || *workerApprovalPath == "" || *workerStatePath == "" { + return reject() + } + } else if *phase == "" || workerInputs { return reject() } a, err := livecanary.ReadApproval(*approvalPath) - if err != nil || a.Validate(time.Now()) != nil || !slices.Contains(a.Phases, *phase) { + if err != nil || a.Validate(time.Now()) != nil || (!*pairedExecute && !slices.Contains(a.Phases, *phase)) { return reject() } revision, ok := revisionForBuild() if !ok || revision != a.HarnessSHA { return reject() } + var worker liveworker.Approval + if *pairedExecute { + var workerErr error + worker, workerErr = liveworker.ReadApproval(*workerApprovalPath) + if workerErr != nil || livecanary.ValidatePairedApprovals(a, worker) != nil || livecanary.ValidatePairedStatePaths(*statePath, *workerStatePath) != nil { + return reject() + } + } if *prepare { if prepareJournal == nil { return reject() @@ -95,11 +128,14 @@ func runWithPreparation(args []string, in io.Reader, out io.Writer, revisionForB } return 0 } - j, err := livecanary.OpenJournal(*statePath, a) - if err != nil { - return reject() + var j *livecanary.FileJournal + if !*pairedExecute { + j, err = livecanary.OpenJournal(*statePath, a) + if err != nil { + return reject() + } + defer j.Close() } - defer j.Close() closable, ok := in.(io.ReadCloser) if !ok { return reject() @@ -119,6 +155,14 @@ func runWithPreparation(args []string, in io.Reader, out io.Writer, revisionForB if livecanary.DecodeStrict(data, &credentials) != nil { return reject() } + if *pairedExecute { + if livecanary.RunPairedTerminal(context.Background(), livecanary.PairedTerminalFiles{ControllerApprovalPath: *approvalPath, ControllerStateDirectory: *statePath, WorkerApprovalPath: *workerApprovalPath, WorkerStateDirectory: *workerStatePath}, credentials) != nil { + fmt.Fprintln(out, "paired terminal stopped; retain private state and all uncertain resources; no automatic retry") + return 1 + } + fmt.Fprintln(out, "paired terminal completed; inspect private evidence") + return 0 + } api, err := livecanary.NewSDKAPI(a, credentials) if err != nil { return reject() diff --git a/experiments/g01-scaleset/livecanary/paired_terminal.go b/experiments/g01-scaleset/livecanary/paired_terminal.go new file mode 100644 index 0000000..0261238 --- /dev/null +++ b/experiments/g01-scaleset/livecanary/paired_terminal.go @@ -0,0 +1,151 @@ +package livecanary + +import ( + "context" + "os" + "path/filepath" + "strings" + "syscall" + "time" + + "github.com/1XP-AI/gh-runnerd/experiments/g01-scaleset/liveworker" +) + +// PairedTerminalFiles identify the two already-prepared private state domains. +// The paired terminal mode has one process and one controller input stream; the +// worker is never launched as a child process. +type PairedTerminalFiles struct { + ControllerApprovalPath string + ControllerStateDirectory string + WorkerApprovalPath string + WorkerStateDirectory string +} + +// ValidatePairedApprovals is the credential-free front door shared by the CLI +// and the same-process runner. PairInput is still derived from the controller +// journal and approval; worker input only proves the intended trusted worker +// profile and shared immutable identity. +func ValidatePairedApprovals(controller Approval, worker liveworker.Approval) error { + now := time.Now() + if controller.Validate(now) != nil || worker.Validate(now) != nil { + return ErrApproval + } + if controller.OwnerNonce != worker.OwnerNonce || controller.HarnessSHA != worker.HarnessSHA || controller.WorkflowSHA != worker.WorkflowSHA || controller.Controller != worker.Controller || !controller.ExpiresAt.Equal(worker.ExpiresAt) { + return ErrApproval + } + if controller.WorkflowRunID <= 0 { + return ErrApproval + } + verificationPhase := false + for _, phase := range controller.Phases { + if phase == "before-ack" || phase == "after-ack" || phase == "before-acquire" || phase == "acquire-loss" { + verificationPhase = true + break + } + } + if !verificationPhase { + return ErrApproval + } + for _, phase := range []string{"create", "inspect", "cleanup"} { + found := false + for _, candidate := range controller.Phases { + if candidate == phase { + found = true + break + } + } + if !found { + return ErrApproval + } + } + for _, phase := range []string{"create", "start", "inspect", "cleanup"} { + found := false + for _, candidate := range worker.Phases { + if candidate == phase { + found = true + break + } + } + if !found { + return ErrApproval + } + } + return nil +} + +// ValidatePairedStatePaths checks the private roots before the controller +// credential stream is read. OpenJournal repeats the inode/claim checks while +// holding each domain lease; this early check only prevents an invalid or +// aliased root from reaching credential handling. +func ValidatePairedStatePaths(controller, worker string) error { + if !privateStateDirectory(controller) || !privateStateDirectory(worker) { + return ErrJournal + } + controllerReal, err := filepath.EvalSymlinks(controller) + if err != nil { + return ErrJournal + } + workerReal, err := filepath.EvalSymlinks(worker) + if err != nil || workerReal == controllerReal { + return ErrJournal + } + return nil +} + +func privateStateDirectory(path string) bool { + if !filepath.IsAbs(path) || filepath.Clean(path) != path || len(path) > 4096 { + return false + } + info, err := os.Lstat(path) + if err != nil || !info.IsDir() || info.Mode().Perm() != 0700 { + return false + } + stat, ok := info.Sys().(*syscall.Stat_t) + return ok && int(stat.Uid) == os.Geteuid() +} + +// RunPairedTerminal opens both existing journals, constructs the pinned SDK and +// direct Unix-socket runtime, and runs the reviewed terminal sequence in this +// process. It returns only fixed error categories; no private SDK/Docker error +// or path is part of the result boundary. +func RunPairedTerminal(ctx context.Context, files PairedTerminalFiles, credentials Credentials) error { + if ctx == nil { + return ErrApproval + } + controller, err := ReadApproval(files.ControllerApprovalPath) + if err != nil { + return ErrApproval + } + worker, err := liveworker.ReadApproval(files.WorkerApprovalPath) + if err != nil || ValidatePairedApprovals(controller, worker) != nil || ValidatePairedStatePaths(files.ControllerStateDirectory, files.WorkerStateDirectory) != nil { + return ErrApproval + } + if credentials.validate(controller, time.Now()) != nil || len(credentials.VerificationToken) < 20 || len(credentials.VerificationToken) > 1024 || credentials.VerificationToken == credentials.InstallationToken || strings.ContainsAny(credentials.VerificationToken, "\r\n\x00") { + return ErrApproval + } + controllerJournal, err := OpenJournal(files.ControllerStateDirectory, controller) + if err != nil { + return ErrJournal + } + defer controllerJournal.Close() + workerJournal, err := liveworker.OpenJournal(files.WorkerStateDirectory, worker) + if err != nil { + return ErrJournal + } + // LIFO closes the worker claim before the controller claim, matching the + // paired cleanup order even when the terminal exits through an error path. + defer workerJournal.Close() + api, err := NewSDKAPI(controller, credentials) + if err != nil { + return ErrApproval + } + docker, err := liveworker.NewDocker(worker) + if err != nil { + return ErrJournal + } + result, err := runPairedTerminal(ctx, &Driver{Approval: controller, Journal: controllerJournal, API: api}, &liveworker.Driver{Approval: worker, Journal: workerJournal, Runtime: docker}) + if err != nil || result.Terminal != terminalComplete { + return ErrQuarantine + } + return nil +} diff --git a/experiments/g02-auth/broker.go b/experiments/g02-auth/broker.go index ce31098..6acfda1 100644 --- a/experiments/g02-auth/broker.go +++ b/experiments/g02-auth/broker.go @@ -58,7 +58,15 @@ func (a BrokerApproval) validate(now time.Time) error { if a.Phase != "" || a.AllowVerificationAuthority { return errBroker } - } else if a.Mode != "controller" || !brokerPhases[a.Phase] { + } else if a.Mode == "controller" { + if !brokerPhases[a.Phase] { + return errBroker + } + } else if a.Mode == "paired-terminal" { + if a.Phase != "" && a.Phase != "paired-terminal" { + return errBroker + } + } else { return errBroker } return nil @@ -75,7 +83,7 @@ func validBrokerToken(token string) bool { return true } func brokerExecute(parent context.Context, a BrokerApproval, input brokerInput, path string, api *brokerAPI, plan *brokerControllerPlan) (BrokerResult, error) { - if parent == nil || api == nil || a.validate(api.now()) != nil || (a.AllowVerificationAuthority && input.VerificationToken == "") || (input.VerificationToken != "" && (!a.AllowVerificationAuthority || !validBrokerToken(input.VerificationToken))) || (a.Mode == "controller" && plan == nil) || (a.Mode != "controller" && plan != nil) { + if parent == nil || api == nil || a.validate(api.now()) != nil || (a.AllowVerificationAuthority && input.VerificationToken == "") || (input.VerificationToken != "" && (!a.AllowVerificationAuthority || !validBrokerToken(input.VerificationToken))) || ((a.Mode == "controller" || a.Mode == "paired-terminal") && plan == nil) || (a.Mode != "controller" && a.Mode != "paired-terminal" && plan != nil) { return BrokerResult{}, errBroker } ctx, cancel := context.WithDeadline(parent, minTime(a.ExpiresAt, api.now().Add(10*time.Minute))) @@ -194,9 +202,12 @@ func brokerExecute(parent context.Context, a BrokerApproval, input brokerInput, if len(data) > 16384 || j.append("controller_handoff_started", nil) != nil { return BrokerResult{}, errBroker } - if (plan.controller.needsVerification() && api.verifyWorkflow(ctx, a, plan.controller, input.VerificationToken) != nil) || guard() != nil || plan.launch(ctx, data, filepath.Join(path, "controller-approval.json")) != nil || j.append("controller_completed", nil) != nil || claim.complete() != nil { + if ((a.Mode == "paired-terminal" || plan.controller.needsVerification()) && api.verifyWorkflow(ctx, a, plan.controller, input.VerificationToken) != nil) || guard() != nil || plan.launch(ctx, data, filepath.Join(path, "controller-approval.json")) != nil || j.append("controller_completed", nil) != nil || claim.complete() != nil { return BrokerResult{}, errBroker } + if a.Mode == "paired-terminal" { + return BrokerResult{Status: "paired_terminal_completed"}, nil + } return BrokerResult{Status: "controller_completed"}, nil } func minTime(a, b time.Time) time.Time { diff --git a/experiments/g02-auth/broker_admission.go b/experiments/g02-auth/broker_admission.go index f608077..e9094bf 100644 --- a/experiments/g02-auth/broker_admission.go +++ b/experiments/g02-auth/broker_admission.go @@ -80,6 +80,11 @@ type brokerControllerBinding struct { Harness string `json:"harness"` State brokerInode `json:"state"` } +type brokerWorkerBinding struct { + Approval string `json:"approval"` + ApprovalFile brokerInode `json:"approval_file"` + State brokerInode `json:"state"` +} type brokerControllerAuthority struct { Digest string `json:"digest"` Approval controllerApproval `json:"approval"` @@ -91,6 +96,7 @@ type brokerClaimEvent struct { Attempt brokerInode `json:"attempt"` Journal brokerInode `json:"journal"` Controller *brokerControllerBinding `json:"controller,omitempty"` + Worker *brokerWorkerBinding `json:"worker,omitempty"` Authority *brokerControllerAuthority `json:"authority,omitempty"` Snapshot brokerInode `json:"snapshot"` SnapshotDigest string `json:"snapshot_digest,omitempty"` @@ -258,6 +264,7 @@ func openBrokerAdmission(directory string, a BrokerApproval, j *brokerJournal, p slots := map[string]brokerClaimEvent{} done := map[string]bool{} var binding *brokerControllerBinding + var workerBinding *brokerWorkerBinding var authority *brokerControllerAuthority latestSlot := "" for _, line := range lines[1 : len(lines)-1] { @@ -268,7 +275,7 @@ func openBrokerAdmission(directory string, a BrokerApproval, j *brokerJournal, p switch event.Kind { case "claim": for previous, receipt := range slots { - if (!done[previous] && event.Slot != "inspect" && event.Slot != "cleanup") || receipt.Attempt == event.Attempt || receipt.Journal == event.Journal || (event.Controller != nil && receipt.Controller != nil && receipt.Snapshot == event.Snapshot) { + if (!done[previous] && event.Slot != "inspect" && event.Slot != "cleanup") || receipt.Attempt == event.Attempt || receipt.Journal == event.Journal || (event.Controller != nil && receipt.Controller != nil && receipt.Snapshot == event.Snapshot) || (event.Worker != nil && receipt.Worker != nil && receipt.Worker.State == event.Worker.State) { return nil, errBroker } } @@ -287,6 +294,12 @@ func openBrokerAdmission(directory string, a BrokerApproval, j *brokerJournal, p return nil, errBroker } binding = event.Controller + if event.Worker != nil { + if workerBinding != nil && *workerBinding != *event.Worker { + return nil, errBroker + } + workerBinding = event.Worker + } authority = event.Authority } slots[event.Slot] = event @@ -304,6 +317,8 @@ func openBrokerAdmission(directory string, a BrokerApproval, j *brokerJournal, p slot := a.Phase if a.Mode == "discover-actions-host" { slot = a.Mode + } else if a.Mode == "paired-terminal" { + slot = "paired-terminal" } if _, ok := slots[slot]; ok { return nil, errBroker @@ -332,6 +347,15 @@ func openBrokerAdmission(directory string, a BrokerApproval, j *brokerJournal, p c.event.Authority = &next c.event.Snapshot = brokerFileIdentity(p.snapshotInfo) c.event.SnapshotDigest = a.ControllerApprovalSHA256 + if p.worker != nil { + worker, e := p.worker.binding() + if e != nil || (workerBinding != nil && *workerBinding != worker) { + return nil, errBroker + } + c.event.Worker = &worker + } else if a.Mode == "paired-terminal" { + return nil, errBroker + } } else if a.Mode == "controller" { return nil, errBroker } @@ -366,16 +390,28 @@ func validBrokerClaimEvent(a BrokerApproval, e brokerClaimEvent) bool { return false } if e.Slot == "discover-actions-host" { - return e.Controller == nil && e.Authority == nil && e.Snapshot == (brokerInode{}) && e.SnapshotDigest == "" + return e.Controller == nil && e.Worker == nil && e.Authority == nil && e.Snapshot == (brokerInode{}) && e.SnapshotDigest == "" } - if !brokerPhases[e.Slot] || e.Controller == nil || e.Authority == nil || e.Controller.State.Inode == 0 || e.Snapshot.Inode == 0 || !brokerSHA256.MatchString(e.SnapshotDigest) || !brokerSHA256.MatchString(e.Controller.Ownership) || !brokerSHA256.MatchString(e.Controller.Binary) || !brokerSHA40.MatchString(e.Controller.Harness) { + paired := e.Slot == "paired-terminal" + if (!brokerPhases[e.Slot] && !paired) || e.Controller == nil || e.Authority == nil || e.Controller.State.Inode == 0 || e.Snapshot.Inode == 0 || !brokerSHA256.MatchString(e.SnapshotDigest) || !brokerSHA256.MatchString(e.Controller.Ownership) || !brokerSHA256.MatchString(e.Controller.Binary) || !brokerSHA40.MatchString(e.Controller.Harness) { + return false + } + if paired { + if a.Mode != "paired-terminal" || e.Worker == nil || !brokerSHA256.MatchString(e.Worker.Approval) || e.Worker.ApprovalFile.Device == 0 || e.Worker.ApprovalFile.Inode == 0 || e.Worker.State.Device == 0 || e.Worker.State.Inode == 0 { + return false + } + } else if e.Worker != nil || a.Mode == "paired-terminal" { return false } c := e.Authority.Approval if c.ExpiresAt.IsZero() || e.Authority.Digest != brokerDigest(c) { return false } - a.Mode = "controller" + if paired { + a.Mode = "paired-terminal" + } else { + a.Mode = "controller" + } a.Phase = e.Slot a.ExpiresAt = c.ExpiresAt a.ControllerHarnessSHA = e.Controller.Harness diff --git a/experiments/g02-auth/broker_entry.go b/experiments/g02-auth/broker_entry.go index 6cb20ce..efb37c0 100644 --- a/experiments/g02-auth/broker_entry.go +++ b/experiments/g02-auth/broker_entry.go @@ -43,7 +43,19 @@ func (c controllerApproval) needsVerification() bool { }) } func (c controllerApproval) validate(a BrokerApproval, now time.Time) error { - if c.OwnerNonce != a.OwnerNonce || c.AppID != a.AppID || c.InstallationID != a.InstallationID || c.Organization != a.Organization || c.Repository != a.Repository || c.RepositoryID != a.RepositoryID || c.RunnerGroupID != a.RunnerGroupID || c.HarnessSHA != a.ControllerHarnessSHA || !brokerSHA40.MatchString(c.HarnessSHA) || !brokerSHA40.MatchString(c.WorkflowSHA) || !brokerNonce.MatchString(c.OwnerNonce) || !brokerWorkflow.MatchString(c.WorkflowPath) || !brokerComponent.MatchString(c.Controller) || !c.ExpiresAt.After(now.Add(time.Minute)) || c.ExpiresAt.After(now.Add(24*time.Hour)) || c.ExpiresAt.Before(a.ExpiresAt) || len(c.ActionsHosts) < 1 || len(c.ActionsHosts) > 8 || len(c.Phases) < 1 || len(c.Phases) > 8 || !slices.Contains(c.Phases, a.Phase) || c.needsVerification() != a.AllowVerificationAuthority || (c.needsVerification() && c.WorkflowRunID < 1) { + if c.OwnerNonce != a.OwnerNonce || c.AppID != a.AppID || c.InstallationID != a.InstallationID || c.Organization != a.Organization || c.Repository != a.Repository || c.RepositoryID != a.RepositoryID || c.RunnerGroupID != a.RunnerGroupID || c.HarnessSHA != a.ControllerHarnessSHA || !brokerSHA40.MatchString(c.HarnessSHA) || !brokerSHA40.MatchString(c.WorkflowSHA) || !brokerNonce.MatchString(c.OwnerNonce) || !brokerWorkflow.MatchString(c.WorkflowPath) || !brokerComponent.MatchString(c.Controller) || !c.ExpiresAt.After(now.Add(time.Minute)) || c.ExpiresAt.After(now.Add(24*time.Hour)) || c.ExpiresAt.Before(a.ExpiresAt) || len(c.ActionsHosts) < 1 || len(c.ActionsHosts) > 8 || len(c.Phases) < 1 || len(c.Phases) > 8 || c.needsVerification() != a.AllowVerificationAuthority || (c.needsVerification() && c.WorkflowRunID < 1) { + return errBroker + } + if a.Mode == "paired-terminal" { + if !a.AllowVerificationAuthority || !c.needsVerification() || c.WorkflowRunID < 1 { + return errBroker + } + for _, phase := range []string{"create", "inspect", "cleanup"} { + if !slices.Contains(c.Phases, phase) { + return errBroker + } + } + } else if !slices.Contains(c.Phases, a.Phase) { return errBroker } seen := map[string]bool{} @@ -112,11 +124,18 @@ func runBrokerWithAPI(ctx context.Context, files BrokerFiles, input *os.File, ap if _, err := readBrokerPrivateJSON(files.ApprovalPath, &approval); err != nil || approval.validate(api.now()) != nil { return BrokerResult{}, errBroker } + if approval.Mode != "paired-terminal" && (files.WorkerApproval != "" || files.WorkerStateDirectory != "") { + return BrokerResult{}, errBroker + } + if approval.Mode == "paired-terminal" && (files.WorkerApproval == "" || files.WorkerStateDirectory == "") { + return BrokerResult{}, errBroker + } var binary *verifiedBrokerBinary var controller controllerApproval var controllerData []byte var controllerRoot *os.Root - if approval.Mode == "controller" { + var workerPlan *brokerWorkerPlan + if approval.Mode == "controller" || approval.Mode == "paired-terminal" { if !brokerSHA256.MatchString(approval.ControllerBinarySHA256) || !brokerSHA256.MatchString(approval.ControllerApprovalSHA256) || !brokerSHA40.MatchString(approval.ControllerHarnessSHA) { return BrokerResult{}, errBroker } @@ -136,7 +155,14 @@ func runBrokerWithAPI(ctx context.Context, files BrokerFiles, input *os.File, ap return BrokerResult{}, errBroker } defer controllerRoot.Close() - } else if files.ControllerBinary != "" || files.ControllerApproval != "" || files.ControllerStateDirectory != "" || approval.ControllerBinarySHA256 != "" || approval.ControllerApprovalSHA256 != "" || approval.ControllerHarnessSHA != "" { + if approval.Mode == "paired-terminal" { + workerPlan, err = openBrokerWorkerPlan(files.WorkerApproval, files.WorkerStateDirectory, files.ControllerStateDirectory, approval, controller) + if err != nil { + return BrokerResult{}, errBroker + } + defer workerPlan.close() + } + } else if files.ControllerBinary != "" || files.ControllerApproval != "" || files.ControllerStateDirectory != "" || files.WorkerApproval != "" || files.WorkerStateDirectory != "" || approval.ControllerBinarySHA256 != "" || approval.ControllerApprovalSHA256 != "" || approval.ControllerHarnessSHA != "" { return BrokerResult{}, errBroker } credentialInput, err := readBrokerInput(ctx, input) @@ -145,19 +171,27 @@ func runBrokerWithAPI(ctx context.Context, files BrokerFiles, input *os.File, ap } var plan *brokerControllerPlan - if approval.Mode == "controller" { + if approval.Mode == "controller" || approval.Mode == "paired-terminal" { plan, err = newBrokerControllerPlan(approval, controller, controllerData, controllerRoot, files.ControllerStateDirectory, binary.check, func(ctx context.Context, data []byte, snapshotPath string) error { if plan.check() != nil { return errBroker } + if approval.Mode == "paired-terminal" { + return invokeBrokerPairedTerminal(ctx, binary, files.StateDirectory, snapshotPath, files.ControllerStateDirectory, files.WorkerApproval, files.WorkerStateDirectory, data) + } return invokeBrokerController(ctx, binary, files.StateDirectory, snapshotPath, files.ControllerStateDirectory, approval.Phase, data) }) if err != nil { return BrokerResult{}, errBroker } plan.localPrepare = func(ctx context.Context, snapshotPath string) (brokerPreparationReceipt, error) { - return invokeBrokerPreparation(ctx, binary, files.StateDirectory, snapshotPath, files.ControllerStateDirectory, approval.Phase) + phase := approval.Phase + if approval.Mode == "paired-terminal" { + phase = "cleanup" + } + return invokeBrokerPreparation(ctx, binary, files.StateDirectory, snapshotPath, files.ControllerStateDirectory, phase) } + plan.worker = workerPlan } return brokerExecute(ctx, approval, credentialInput, files.StateDirectory, api, plan) } diff --git a/experiments/g02-auth/broker_files.go b/experiments/g02-auth/broker_files.go index d1d6fb1..1c1f5f7 100644 --- a/experiments/g02-auth/broker_files.go +++ b/experiments/g02-auth/broker_files.go @@ -59,4 +59,12 @@ func openBrokerPrivateDirectory(path string) (*os.Root, error) { return root, nil } -type BrokerFiles struct{ ApprovalPath, StateDirectory, ControllerBinary, ControllerApproval, ControllerStateDirectory string } +type BrokerFiles struct { + ApprovalPath string + StateDirectory string + ControllerBinary string + ControllerApproval string + ControllerStateDirectory string + WorkerApproval string + WorkerStateDirectory string +} diff --git a/experiments/g02-auth/broker_paired_test.go b/experiments/g02-auth/broker_paired_test.go new file mode 100644 index 0000000..0795019 --- /dev/null +++ b/experiments/g02-auth/broker_paired_test.go @@ -0,0 +1,166 @@ +package enrollment + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func pairedPlanInputs(t *testing.T) (BrokerApproval, controllerApproval, string, string, string) { + t.Helper() + root := t.TempDir() + if err := os.Chmod(root, 0700); err != nil { + t.Fatal(err) + } + controllerState := filepath.Join(root, "controller-state") + workerState := filepath.Join(root, "worker-state") + if err := os.Mkdir(controllerState, 0700); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(workerState, 0700); err != nil { + t.Fatal(err) + } + now := time.Now().Add(time.Hour) + nonce := strings.Repeat("a", 32) + harness := strings.Repeat("b", 40) + workflow := strings.Repeat("c", 40) + a := brokerApprovalFixture() + a.Mode, a.Phase = "paired-terminal", "paired-terminal" + a.ExpiresAt = now + a.ControllerHarnessSHA = harness + c := controllerApproval{AppID: a.AppID, InstallationID: a.InstallationID, Organization: a.Organization, Repository: a.Repository, RepositoryID: a.RepositoryID, RunnerGroupID: a.RunnerGroupID, OwnerNonce: nonce, HarnessSHA: harness, WorkflowSHA: workflow, WorkflowPath: ".github/workflows/canary.yml", Controller: "trusted-controller", ExpiresAt: now, ActionsHosts: []string{"fixture.actions.githubusercontent.com"}, Phases: []string{"create", "before-ack", "after-ack", "before-acquire", "acquire-loss", "inspect", "cleanup", "jit-loss"}} + worker := pairedWorkerApproval{RunnerUpdatesDisabled: true, HarnessSHA: harness, WorkflowSHA: workflow, OwnerNonce: nonce, Controller: c.Controller, Endpoint: "/tmp/g01-paired-docker.sock", DaemonID: "fixture-daemon", ImageID: "sha256:" + strings.Repeat("d", 64), Image: pairedWorkerImage, ExpiresAt: now, Phases: []string{"create", "start", "inspect", "cleanup"}} + data, err := json.Marshal(worker) + if err != nil { + t.Fatal(err) + } + workerPath := filepath.Join(root, "worker.json") + if err := os.WriteFile(workerPath, data, 0600); err != nil { + t.Fatal(err) + } + return a, c, controllerState, workerState, workerPath +} + +func TestPairedWorkerBindingRetainsApprovalAndStateIdentity(t *testing.T) { + a, c, controllerState, workerState, workerPath := pairedPlanInputs(t) + plan, err := openBrokerWorkerPlan(workerPath, workerState, controllerState, a, c) + if err != nil { + t.Fatal("valid paired worker plan refused") + } + defer plan.close() + if _, err := plan.binding(); err != nil { + t.Fatal("valid paired worker binding unavailable") + } + if err := os.Rename(workerPath, workerPath+".retained"); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(workerPath, plan.raw, 0600); err != nil { + t.Fatal(err) + } + if _, err := plan.binding(); err == nil { + t.Fatal("replaced worker approval retained authority") + } +} + +func TestPairedWorkerApprovalMismatchRefusesBeforeBinding(t *testing.T) { + a, c, controllerState, workerState, workerPath := pairedPlanInputs(t) + raw, err := os.ReadFile(workerPath) + if err != nil { + t.Fatal(err) + } + var worker pairedWorkerApproval + if err := decodeBrokerJSON(raw, &worker, true); err != nil { + t.Fatal(err) + } + worker.WorkflowSHA = strings.Repeat("e", 40) + raw, err = json.Marshal(worker) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(workerPath, raw, 0600); err != nil { + t.Fatal(err) + } + if _, err := openBrokerWorkerPlan(workerPath, workerState, controllerState, a, c); err == nil { + t.Fatal("mismatched worker identity accepted") + } +} + +func TestPairedBrokerBindsWorkerBeforeWorkflowVerifiedHandoff(t *testing.T) { + a, candidate, api, fixture, attempt := newBrokerFixture(t) + a.Mode, a.Phase, a.AllowVerificationAuthority = "paired-terminal", "paired-terminal", true + parent := filepath.Dir(attempt) + launches := 0 + plan := brokerTestPlan(t, &a, parent, func(_ context.Context, data []byte, _ string) error { + launches++ + verified := false + for _, call := range fixture.calls { + if strings.HasPrefix(call, "GET /repos/org-a/canary/actions/runs/") { + verified = true + break + } + } + if !verified { + t.Fatal("paired handoff launched before workflow verification") + } + var payload map[string]any + if json.Unmarshal(data, &payload) != nil || payload["installation_token"] != fixture.token || payload["verification_token"] != "synthetic-private-verification-token" { + t.Fatal("paired handoff lost broker-issued credentials") + } + if _, found := payload["pem"]; found { + t.Fatal("PEM crossed the broker boundary") + } + return nil + }) + + // The paired mode requires the same reviewed workflow verification authority + // as the original terminal fixture before the child is launched. + plan.controller.Phases = []string{"create", "before-ack", "inspect", "cleanup"} + plan.controller.WorkflowRunID = 7 + plan.raw, _ = json.Marshal(plan.controller) + a.ControllerApprovalSHA256 = brokerBytesDigest(plan.raw) + plan.approval = a + + workerState := filepath.Join(parent, "worker-state") + if err := os.Mkdir(workerState, 0700); err != nil { + t.Fatal(err) + } + worker := pairedWorkerApproval{RunnerUpdatesDisabled: true, HarnessSHA: plan.controller.HarnessSHA, WorkflowSHA: plan.controller.WorkflowSHA, OwnerNonce: plan.controller.OwnerNonce, Controller: plan.controller.Controller, Endpoint: "/tmp/g01-paired-broker.sock", DaemonID: "fixture-daemon", ImageID: "sha256:" + strings.Repeat("d", 64), Image: pairedWorkerImage, ExpiresAt: plan.controller.ExpiresAt, Phases: []string{"create", "start", "inspect", "cleanup"}} + workerData, err := json.Marshal(worker) + if err != nil { + t.Fatal(err) + } + workerPath := filepath.Join(parent, "worker-approval.json") + if err := os.WriteFile(workerPath, workerData, 0600); err != nil { + t.Fatal(err) + } + workerPlan, err := openBrokerWorkerPlan(workerPath, workerState, plan.statePath, a, plan.controller) + if err != nil { + t.Fatal("valid paired worker plan refused") + } + plan.worker = workerPlan + + result, err := brokerExecute(context.Background(), a, brokerInput{PEM: string(candidate.PEM), VerificationToken: "synthetic-private-verification-token"}, attempt, api, plan) + if err != nil || result.Status != "paired_terminal_completed" || fixture.tokenCalls != 1 || launches != 1 { + t.Fatalf("paired broker handoff incomplete: result=%+v err=%v mints=%d launches=%d", result, err, fixture.tokenCalls, launches) + } + verified := false + for _, call := range fixture.calls { + if strings.HasPrefix(call, "GET /repos/org-a/canary/actions/runs/") { + verified = true + } + } + if !verified { + t.Fatal("workflow identity was not verified") + } + if fixture.tokenCalls != 1 { + t.Fatal("paired handoff retried issuance") + } + ledger, err := os.ReadFile(filepath.Join(fixture.admissionRoot, "broker-admission.jsonl")) + if err != nil || !strings.Contains(string(ledger), `"slot":"paired-terminal"`) || !strings.Contains(string(ledger), `"worker"`) || strings.Contains(string(ledger), fixture.token) || strings.Contains(string(ledger), string(candidate.PEM)) { + t.Fatal("paired admission did not retain bounded worker binding") + } +} diff --git a/experiments/g02-auth/broker_plan.go b/experiments/g02-auth/broker_plan.go index 1ad5279..efd7687 100644 --- a/experiments/g02-auth/broker_plan.go +++ b/experiments/g02-auth/broker_plan.go @@ -5,10 +5,137 @@ import ( "io" "os" "path/filepath" + "strings" "syscall" "time" ) +const pairedWorkerImage = "ghcr.io/actions/actions-runner@sha256:f5a0d9a3d857315f2aed7075a02a29f46927ad198221c3b1c66585ae9fe36c0d" + +type pairedWorkerApproval struct { + RunnerUpdatesDisabled bool `json:"runner_updates_disabled"` + HarnessSHA string `json:"harness_sha"` + WorkflowSHA string `json:"workflow_sha"` + OwnerNonce string `json:"owner_nonce"` + Controller string `json:"controller"` + Endpoint string `json:"endpoint"` + DaemonID string `json:"daemon_id"` + ImageID string `json:"image_id"` + Image string `json:"image"` + ExpiresAt time.Time `json:"expires_at"` + Phases []string `json:"phases"` +} + +func (a pairedWorkerApproval) validate(now time.Time) error { + if !a.RunnerUpdatesDisabled || !brokerSHA40.MatchString(a.HarnessSHA) || !brokerSHA40.MatchString(a.WorkflowSHA) || !brokerNonce.MatchString(a.OwnerNonce) || !brokerComponent.MatchString(a.Controller) || !brokerComponent.MatchString(a.DaemonID) || !strings.HasPrefix(a.ImageID, "sha256:") || !brokerSHA256.MatchString(strings.TrimPrefix(a.ImageID, "sha256:")) || a.Image != pairedWorkerImage || !filepath.IsAbs(a.Endpoint) || filepath.Clean(a.Endpoint) != a.Endpoint || len(a.Endpoint) > 103 || strings.ContainsAny(a.Endpoint, "\x00\r\n") || !a.ExpiresAt.After(now) || a.ExpiresAt.After(now.Add(24*time.Hour)) || len(a.Phases) != 4 { + return errBroker + } + seen := map[string]bool{} + for _, phase := range a.Phases { + if phase != "create" && phase != "start" && phase != "inspect" && phase != "cleanup" || seen[phase] { + return errBroker + } + seen[phase] = true + } + return nil +} + +type brokerWorkerPlan struct { + approval pairedWorkerApproval + raw []byte + approvalPath string + approvalFile *os.File + approvalInfo os.FileInfo + statePath string + state *os.Root + stateInfo os.FileInfo +} + +func (p *brokerWorkerPlan) close() { + if p == nil { + return + } + if p.approvalFile != nil { + _ = p.approvalFile.Close() + } + if p.state != nil { + _ = p.state.Close() + } +} + +func (p *brokerWorkerPlan) check() error { + if p == nil || p.approvalFile == nil || p.state == nil || !filepath.IsAbs(p.approvalPath) || filepath.Clean(p.approvalPath) != p.approvalPath || !filepath.IsAbs(p.statePath) || filepath.Clean(p.statePath) != p.statePath { + return errBroker + } + approvalInfo, err := p.approvalFile.Stat() + namedApproval, namedErr := os.Lstat(p.approvalPath) + stateInfo, stateErr := p.state.Stat(".") + namedState, namedStateErr := os.Lstat(p.statePath) + if err != nil || namedErr != nil || stateErr != nil || namedStateErr != nil || !privateFile(p.approvalFile) || !brokerOwnedDirectory(namedState, true) || !os.SameFile(approvalInfo, p.approvalInfo) || !os.SameFile(namedApproval, p.approvalInfo) || !os.SameFile(stateInfo, p.stateInfo) || !os.SameFile(namedState, p.stateInfo) { + return errBroker + } + data, err := io.ReadAll(io.NewSectionReader(p.approvalFile, 0, 16385)) + if err != nil || len(data) != len(p.raw) || brokerBytesDigest(data) != brokerBytesDigest(p.raw) { + return errBroker + } + return nil +} + +func (p *brokerWorkerPlan) binding() (brokerWorkerBinding, error) { + if p.check() != nil { + return brokerWorkerBinding{}, errBroker + } + return brokerWorkerBinding{Approval: brokerBytesDigest(p.raw), ApprovalFile: brokerFileIdentity(p.approvalInfo), State: brokerFileIdentity(p.stateInfo)}, nil +} + +func openBrokerWorkerPlan(path, statePath, controllerStatePath string, a BrokerApproval, c controllerApproval) (*brokerWorkerPlan, error) { + if a.Mode != "paired-terminal" || !filepath.IsAbs(path) || filepath.Clean(path) != path || !filepath.IsAbs(statePath) || filepath.Clean(statePath) != statePath || filepath.Clean(statePath) == filepath.Clean(controllerStatePath) { + return nil, errBroker + } + controllerReal, err := filepath.EvalSymlinks(controllerStatePath) + workerReal, workerErr := filepath.EvalSymlinks(statePath) + if err != nil || workerErr != nil || controllerReal == workerReal { + return nil, errBroker + } + file, err := openBrokerPrivateFile(path, 0600, 16384) + if err != nil { + return nil, errBroker + } + data, err := io.ReadAll(io.NewSectionReader(file, 0, 16385)) + if err != nil || len(data) > 16384 { + file.Close() + return nil, errBroker + } + var worker pairedWorkerApproval + if decodeBrokerJSON(data, &worker, true) != nil || worker.validate(time.Now()) != nil || worker.OwnerNonce != c.OwnerNonce || worker.HarnessSHA != c.HarnessSHA || worker.WorkflowSHA != c.WorkflowSHA || worker.Controller != c.Controller || !worker.ExpiresAt.Equal(c.ExpiresAt) { + file.Close() + return nil, errBroker + } + root, err := openBrokerPrivateDirectory(statePath) + if err != nil { + file.Close() + return nil, errBroker + } + stateInfo, err := root.Stat(".") + if err != nil { + file.Close() + root.Close() + return nil, errBroker + } + approvalInfo, err := file.Stat() + if err != nil { + file.Close() + root.Close() + return nil, errBroker + } + plan := &brokerWorkerPlan{approval: worker, raw: data, approvalPath: path, approvalFile: file, approvalInfo: approvalInfo, statePath: statePath, state: root, stateInfo: stateInfo} + if plan.check() != nil { + plan.close() + return nil, errBroker + } + return plan, nil +} + // Private prepared inputs are constructed only after the front door verifies // source, binary, authority and private paths. Test launchers are synthetic. type brokerControllerPlan struct { @@ -25,6 +152,7 @@ type brokerControllerPlan struct { localPrepare func(context.Context, string) (brokerPreparationReceipt, error) preparationReceipt brokerPreparationReceipt prepared *brokerPreparedState + worker *brokerWorkerPlan launch func(context.Context, []byte, string) error } @@ -35,6 +163,9 @@ func (p *brokerControllerPlan) close() { if p != nil && p.snapshot != nil { p.snapshot.Close() } + if p != nil { + p.worker.close() + } } func (p *brokerControllerPlan) authority() brokerControllerAuthority { return brokerControllerAuthority{brokerDigest(p.controller), p.controller} @@ -54,7 +185,10 @@ func (p *brokerControllerPlan) binding() (brokerControllerBinding, error) { return brokerControllerBinding{brokerDigest(c), p.approval.ControllerBinarySHA256, p.approval.ControllerHarnessSHA, brokerFileIdentity(i)}, nil } func (p *brokerControllerPlan) prepare(a BrokerApproval, j *brokerJournal, now time.Time) error { - if p == nil || p.binaryCheck == nil || p.launch == nil || p.localPrepare == nil || brokerDigest(p.approval) != brokerDigest(a) || p.controller.validate(a, now) != nil || brokerBytesDigest(p.raw) != a.ControllerApprovalSHA256 || p.binaryCheck() != nil { + if p == nil || p.binaryCheck == nil || p.launch == nil || p.localPrepare == nil || brokerDigest(p.approval) != brokerDigest(a) || p.controller.validate(a, now) != nil || brokerBytesDigest(p.raw) != a.ControllerApprovalSHA256 || p.binaryCheck() != nil || (a.Mode == "paired-terminal" && p.worker == nil) { + return errBroker + } + if p.worker != nil && p.worker.check() != nil { return errBroker } if _, e := p.binding(); e != nil { @@ -79,7 +213,7 @@ func (p *brokerControllerPlan) check() error { if _, e := p.binding(); e != nil { return errBroker } - if p.binaryCheck() != nil || p.snapshot == nil || p.journal.check() != nil || !privateFile(p.snapshot) { + if p.binaryCheck() != nil || p.snapshot == nil || p.journal.check() != nil || !privateFile(p.snapshot) || (p.worker != nil && p.worker.check() != nil) { return errBroker } named, e := p.journal.root.Lstat("controller-approval.json") diff --git a/experiments/g02-auth/broker_process.go b/experiments/g02-auth/broker_process.go index 86644df..75b0134 100644 --- a/experiments/g02-auth/broker_process.go +++ b/experiments/g02-auth/broker_process.go @@ -145,3 +145,47 @@ func invokeBrokerController(parent context.Context, binary *verifiedBrokerBinary } return nil } + +// invokeBrokerPairedTerminal has one fixed argv shape. The worker is supplied +// as approval/state input to the same g01-live process; it is never started as +// a separate child and no arbitrary phase/command reaches exec. +func invokeBrokerPairedTerminal(parent context.Context, binary *verifiedBrokerBinary, workingDirectory, approvalPath, stateDirectory, workerApprovalPath, workerStateDirectory string, data []byte) error { + if len(data) > 16384 || binary == nil || binary.check() != nil || !filepath.IsAbs(approvalPath) || !filepath.IsAbs(stateDirectory) || !filepath.IsAbs(workerApprovalPath) || !filepath.IsAbs(workerStateDirectory) || filepath.Clean(approvalPath) != approvalPath || filepath.Clean(stateDirectory) != stateDirectory || filepath.Clean(workerApprovalPath) != workerApprovalPath || filepath.Clean(workerStateDirectory) != workerStateDirectory { + return errBroker + } + ctx, cancel := context.WithCancel(parent) + defer cancel() + command := exec.CommandContext(ctx, binary.path, "--execute-approved-paired-terminal", "--approval", approvalPath, "--state-dir", stateDirectory, "--worker-approval", workerApprovalPath, "--worker-state-dir", workerStateDirectory) + command.Dir = workingDirectory + command.Env = []string{"LANG=C", "LC_ALL=C"} + command.WaitDelay = time.Second + output := &brokerOutputBudget{cancel: cancel} + command.Stdout = output + command.Stderr = output + pipe, err := command.StdinPipe() + if err != nil { + return errBroker + } + if command.Start() != nil { + pipe.Close() + return errBroker + } + wrote := make(chan error, 1) + go func() { + _, e := pipe.Write(data) + closeErr := pipe.Close() + if e == nil { + e = closeErr + } + wrote <- e + }() + waited := command.Wait() + writeErr := <-wrote + output.mu.Lock() + overflow := output.overflow + output.mu.Unlock() + if waited != nil || writeErr != nil || overflow || ctx.Err() != nil { + return errBroker + } + return nil +} diff --git a/experiments/g02-auth/broker_process_test.go b/experiments/g02-auth/broker_process_test.go index f871995..a7c062d 100644 --- a/experiments/g02-auth/broker_process_test.go +++ b/experiments/g02-auth/broker_process_test.go @@ -71,6 +71,22 @@ func TestMain(m *testing.M) { } os.Exit(0) } + if len(os.Args) > 1 && os.Args[1] == "--execute-approved-paired-terminal" { + if len(os.Args) != 10 || os.Args[2] != "--approval" || os.Args[4] != "--state-dir" || os.Args[6] != "--worker-approval" || os.Args[8] != "--worker-state-dir" { + os.Exit(3) + } + for _, entry := range os.Environ() { + if entry != "LANG=C" && entry != "LC_ALL=C" { + os.Exit(4) + } + } + data, err := io.ReadAll(io.LimitReader(os.Stdin, 16385)) + if err != nil || !bytes.Contains(data, []byte("synthetic-private-installation-token")) || bytes.Contains(data, []byte("PRIVATE KEY")) { + os.Exit(5) + } + fmt.Fprintln(os.Stdout, "synthetic-private-paired-output") + os.Exit(0) + } os.Exit(m.Run()) } func testBrokerBinary(t *testing.T) *verifiedBrokerBinary { @@ -99,6 +115,19 @@ func TestBrokerPipeUsesFixedArgsMinimalEnvAndDiscardsChildSecrets(t *testing.T) t.Fatal("private fixed child handoff failed") } } + +func TestBrokerPairedTerminalPipeUsesFixedArgsAndOneControllerInput(t *testing.T) { + binary := testBrokerBinary(t) + root := t.TempDir() + workerState := filepath.Join(root, "worker-state") + if err := os.Mkdir(workerState, 0700); err != nil { + t.Fatal(err) + } + data := []byte(`{"installation_token":"synthetic-private-installation-token"}`) + if err := invokeBrokerPairedTerminal(context.Background(), binary, root, filepath.Join(root, "approval.json"), root, filepath.Join(root, "worker.json"), workerState, data); err != nil { + t.Fatal("private fixed paired terminal handoff failed") + } +} func TestBrokerChildBoundsAndReplacedBinaryRefuse(t *testing.T) { binary := testBrokerBinary(t) root := t.TempDir() diff --git a/experiments/g02-auth/broker_snapshot_test.go b/experiments/g02-auth/broker_snapshot_test.go index d0c4744..ac56140 100644 --- a/experiments/g02-auth/broker_snapshot_test.go +++ b/experiments/g02-auth/broker_snapshot_test.go @@ -101,7 +101,7 @@ func brokerSnapshotFrontDoorFixture(t *testing.T, path string, supported bool) { input, _ := os.Open(inputPath) defer input.Close() - _, e = runBrokerWithAPI(context.Background(), BrokerFiles{approvalPath, root, binary, ctrlPath, state}, input, api) + _, e = runBrokerWithAPI(context.Background(), BrokerFiles{ApprovalPath: approvalPath, StateDirectory: root, ControllerBinary: binary, ControllerApproval: ctrlPath, ControllerStateDirectory: state}, input, api) if e == nil || f.tokenCalls != 0 || (!supported && len(f.calls) != 0) { t.Fatalf("snapshot collision accepted or minted: mint=%d", f.tokenCalls) } diff --git a/experiments/g02-auth/cmd/g01-broker/main.go b/experiments/g02-auth/cmd/g01-broker/main.go index 2060980..58f715e 100644 --- a/experiments/g02-auth/cmd/g01-broker/main.go +++ b/experiments/g02-auth/cmd/g01-broker/main.go @@ -35,11 +35,13 @@ func run(ctx context.Context, args []string, input *os.File, out io.Writer) (cod flags.StringVar(&files.ControllerBinary, "controller-binary", "", "") flags.StringVar(&files.ControllerApproval, "controller-approval", "", "") flags.StringVar(&files.ControllerStateDirectory, "controller-state-dir", "", "") + flags.StringVar(&files.WorkerApproval, "worker-approval", "", "") + flags.StringVar(&files.WorkerStateDirectory, "worker-state-dir", "", "") if flags.Parse(args) != nil || flags.NArg() != 0 { return reject() } if *plan && !*execute { - fmt.Fprintln(out, "Modes in the exact private approval: discover-actions-host or controller. Required fixed private native-account admission root and owner nonce; one permanently consumed issuance slot per approved phase, including one inspect and cleanup; temporary registration/admin authentication only in approved discovery; one hash-verified controller phase only in controller mode. No worker, App creation, workflow dispatch or persistent credentials. Live execution requires separate explicit authorization.") + fmt.Fprintln(out, "Modes in the exact private approval: discover-actions-host, controller, or paired-terminal. Paired-terminal uses one fixed g01-live executable invocation with explicit worker approval/state inputs and one bounded controller credential stdin; no worker subprocess, App creation, workflow dispatch or persistent credentials. Required fixed private native-account admission root and owner nonce; one permanently consumed issuance slot per approved mode/phase. Live execution requires separate explicit authorization.") return 0 } if !*execute || *plan || files.ApprovalPath == "" || files.StateDirectory == "" { From ad7c2cf44ce1af8621aff9f42cd4335c837ee257 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 16:43:32 +0900 Subject: [PATCH 03/37] fix(g01): complete paired broker handoff --- docs/evidence/g01-paired-broker.md | 154 ++++++----- experiments/g01-scaleset/cmd/g01-live/main.go | 33 ++- .../g01-scaleset/cmd/g01-live/main_test.go | 85 +++++- .../livecanary/baseline_integration.go | 11 +- .../livecanary/baseline_terminal.go | 5 +- .../baseline_terminal_fixture_test.go | 242 ++++++++++++++++++ .../livecanary/paired_terminal.go | 199 +++++++++++++- .../g01-scaleset/livecanary/preparation.go | 71 +++++ .../livecanary/preparation_fixture_test.go | 32 +++ .../livecanary/preparation_test.go | 29 +++ experiments/g01-scaleset/livecanary/sdk.go | 17 +- .../liveworker/paired_fixture_admission.go | 13 + experiments/g02-auth/broker.go | 21 +- experiments/g02-auth/broker_entry.go | 13 +- experiments/g02-auth/broker_paired_test.go | 82 ++++++ experiments/g02-auth/broker_plan.go | 37 +++ experiments/g02-auth/broker_preparation.go | 41 ++- experiments/g02-auth/broker_process.go | 27 +- experiments/g02-auth/broker_process_test.go | 132 +++++++++- 19 files changed, 1127 insertions(+), 117 deletions(-) create mode 100644 experiments/g01-scaleset/liveworker/paired_fixture_admission.go diff --git a/docs/evidence/g01-paired-broker.md b/docs/evidence/g01-paired-broker.md index 8399eae..178e5ea 100644 --- a/docs/evidence/g01-paired-broker.md +++ b/docs/evidence/g01-paired-broker.md @@ -7,85 +7,111 @@ authorization, production daemon, or closure of G01/G02. ## Implementation boundary -`g01-live` now has a mutually exclusive -`--execute-approved-paired-terminal` mode. It accepts only the controller -approval/state and explicit worker approval/state inputs; phase, controller-only -execution and worker flags are rejected before controller credential stdin is -read. The executable validates both approvals, shared nonce/harness/workflow/ -controller identity, workflow-run authority, expiries, required terminal phases, -immutable build revision and distinct private state roots. It then reads one -bounded controller credential payload and invokes the existing terminal sequence -in the same process. Controller and worker journals are acquired in order and -released in reverse order; worker JIT remains an in-memory handoff and is never -sent to a worker stdin or subprocess. - -The broker accepts a dedicated `paired-terminal` approval and requires worker -approval/state flags only for that mode. It holds exact worker approval bytes, -approval-file identity and state-root identity through preparation, binds them -into the native-account admission event, requires workflow identity verification -before launch, mints once, and invokes one fixed `g01-live` argv with a minimal -environment. PEM remains in the broker; only bounded controller credentials are -sent to the child. Existing controller-only, discovery and separate worker -paths retain their prior mode/refusal behavior. - -Failure boundaries remain fail-stop: changed worker approval/state, mismatched -shared identity, consumed paired admission, invalid phases/expiry/build, -cancellation, storage uncertainty, bounded child output overflow or child -failure returns a fixed refusal/quarantine result without retry or resume. +The paired broker approval now requires the explicit `paired-terminal` phase. +The canonical `runBrokerWithAPI` path invokes a dedicated fixed +`--prepare-approved-paired-journal` child contract and accepts only its +paired-terminal preparation receipt. The preparation contract proves a fresh +controller journal and admission claim under controller authority; it does +not borrow cleanup authority, read credentials, contact a remote service, or +authorize worker effects. + +After preparation, the broker captures the exact controller snapshot and +worker approval byte hashes plus controller/worker approval and state-root +device/inode identities. The fixed child argv carries only those paths and a +bounded credential-free binding; the controller-only credential payload is +bounded and contains no PEM. The child validates the binding before reading +controller credentials and before constructing SDK/Docker adapters, compares +the argv binding with the broker-supplied payload binding, and revalidates the +same identities throughout the terminal sequence and before completion. The +canonical controller approval and journal-derived `PairInput` remain the only +pairing authority; the binding is identity evidence, not a second pairing +manifest. + +The exported `livecanary.RunPairedTerminal` path now owns paired journal/API/ +Unix adapter construction. A private `g01_pair_fixture` seam redirects only +generated temporary journal/admission roots, the synthetic private TLS API, +and the Unix fixture; it does not expose runtime authority or alter account, +Keychain, runner, Docker, or service state. The tagged fixture calls the +exported entrypoint and verifies two session acknowledgements, one acquire, +JIT, create/start, original-session close, non-force worker deletion plus +absence, owned-set deletion plus absence, complete rosters, real journal/lease +behavior, and secret-free journals. Cancellation, lost response, reopened +history, changed approval bytes/inodes, changed state roots, and symlinked +roots fail before new effects. + +Paired child execution has a fixed argv and `LANG=C`/`LC_ALL=C` environment, +bounded output, a 30-second child timeout, cancellation handling, no retry, +and one logical controller stdin consumption. Existing controller-only, +discovery, and separate-worker paths retain their prior refusal and authority +boundaries. ## TDD evidence and checks -Red tests were preserved in commit `3acd5ab` (`test(g01): preserve paired -handoff red cases`). With the implementation temporarily absent, the meaningful -CLI test failed before input consumption: +The immutable review baseline retained meaningful red behavior in commit +`3acd5ab`: ```text -TestPairedTerminalModeReadsControllerInputAfterAllGates: code=1 reads=0 -output="canary refused; approval, authority or private state requires review" +GOTOOLCHAIN=go1.26.8 go test -race -count=1 -timeout=60s -tags=g01_live -run '^TestPairedTerminalMode' ./cmd/g01-live +exit 1: paired mode refused before its credential input gate (reads=0) + +GOTOOLCHAIN=go1.26.8 go test -race -count=1 -timeout=60s -run '^TestPairedTerminalBrokerApprovalUsesDedicatedMode$' ./... +exit 1: paired terminal broker approval was refused +``` + +Before implementing the canonical entrypoint fix, the new behavioral +regression test was run against the frozen implementation: + +```text +GOTOOLCHAIN=go1.26.8 go test -count=1 -run '^TestPairedBrokerRealEntrypointUsesPairedPreparationClosure$' . +exit 1: real paired entrypoint did not complete one handoff ... mints=0 ``` -The broker mode test independently failed with the fixed broker refusal. The -green implementation adds the same-process adapter, broker worker binding and -fixed child handoff, plus negative tests for unused flags, missing workflow -verification authority, changed worker approval and mismatched shared identity. +The focused green checks then passed: + +```text +GOTOOLCHAIN=go1.26.8 go test -count=1 -run '^TestPairedBrokerRealEntrypointUsesPairedPreparationClosure$|^TestBrokerPaired' . +ok github.com/1XP-AI/gh-runnerd/experiments/g02-auth 1.665s + +GOTOOLCHAIN=go1.26.8 go test -tags g01_live -count=1 ./cmd/g01-live +ok github.com/1XP-AI/gh-runnerd/experiments/g01-scaleset/cmd/g01-live 0.846s -The paired broker behavioral fixture uses only generated temporary files, -synthetic HTTP responses and bounded child test processes. It verifies worker -binding before authenticated work, workflow verification before launch, one -installation-token mint, one handoff, no PEM in the payload or admission -ledger, and the dedicated `paired_terminal_completed` result. Existing tagged -private TLS/Unix fixtures continue to exercise one acquire/JIT/create/start, -original-session close, non-force worker delete with separate absence, owned-set -delete with absence, complete rosters and secret-free journals. +GOTOOLCHAIN=go1.26.8 go test -tags g01_pair_fixture -count=1 -timeout=120s -run '^TestPairedTerminalExported|^TestPairedTerminalBinding' ./livecanary +ok github.com/1XP-AI/gh-runnerd/experiments/g01-scaleset/livecanary +``` -Final offline checks on Go 1.26.8/Darwin ARM64: +Pinned verification completed without live resources: ```text -GOTOOLCHAIN=go1.26.8 go test -race -count=1 -timeout=90s ./... # G01 passed -GOTOOLCHAIN=go1.26.8 go vet ./... # G01 passed -GOTOOLCHAIN=go1.26.8 make experiments # passed: 2 module(s) +GOTOOLCHAIN=go1.26.8 bash scripts/check-offline-experiments.sh +offline experiment checks passed: 2 module(s) + +GOTOOLCHAIN=go1.26.8 go test -race -count=1 ./... +ok: root module + +GOTOOLCHAIN=go1.26.8 go vet ./... +ok: root module + +git diff --check +ok ``` -The offline gate also passed its tagged `g01-live,g01-worker` CLI tests, -tagged vet, collection/terminal/storage fixture partitions, and both G02 -module suites. `git diff --check` passed. No live GitHub endpoint, App, -credential, runner/group/workflow, Docker/Lima configuration, Keychain, -launchd service or existing runner was touched. +The offline gate covers both module race/vet suites, tagged `g01-live`/ +`g01-worker` CLI tests, tagged paired fixture partitions, and the G02 module +suites on Go 1.26.8. The command and tooling files owned by issue #61 were not +edited; their canonical command updates still need integration by that task. -The repository-wide `GOTOOLCHAIN=go1.26.8 make check` also passed: formatting, -build, root unit/race tests, the configured fuzz smoke, module verification, -license inventory, both offline experiment modules and pinned `govulncheck` -(`v1.7.0`, no vulnerabilities found). +No live GitHub endpoint, App, credential, runner/group/workflow, Docker/Lima +configuration, Keychain, launchd service, reboot, or manually installed +runner was touched. The same-UID private-file model is not hostile-code +isolation, and no production daemon or G01 recovery/live completion is +claimed. ## Remaining gates and rollback -Live execution remains unperformed and requires the exact reviewed immutable -artifact, explicit maintainer dispatch and separately approved resources. The -same-UID private-file model is not hostile-code isolation; crash recovery, -uncertain acquisition/JIT/session reconciliation and any successor live run -remain separate evidence gates. Independent Luna/max review, hosted CI and -exact-head GitHub Codex review are still required before merge. +Independent Luna/max review, hosted CI, and exact-head GitHub Codex review are +still required. The coordinator owns those review, stale-finding, CI, and +merge gates; a pending or unreviewed exact head blocks merge. -Offline rollback is source-only: revert the focused issue-60 commits. The -experiment creates no persistent live resource and requires no runner, Docker, -Keychain, launchd or GitHub cleanup. +Offline rollback is source-only: revert the focused issue-60 commit(s). The +tests create only temporary local TLS/Unix fixtures and require no runner, +Docker, Keychain, launchd, or GitHub cleanup. diff --git a/experiments/g01-scaleset/cmd/g01-live/main.go b/experiments/g01-scaleset/cmd/g01-live/main.go index 8631160..bbd3ced 100644 --- a/experiments/g01-scaleset/cmd/g01-live/main.go +++ b/experiments/g01-scaleset/cmd/g01-live/main.go @@ -59,11 +59,13 @@ func runWithPreparation(args []string, in io.Reader, out io.Writer, revisionForB execute := flags.Bool("execute-approved-canary", false, "") pairedExecute := flags.Bool("execute-approved-paired-terminal", false, "") prepare := flags.Bool("prepare-approved-journal", false, "") + pairedPrepare := flags.Bool("prepare-approved-paired-journal", false, "") approvalPath := flags.String("approval", "", "") statePath := flags.String("state-dir", "", "") phase := flags.String("phase", "", "") workerApprovalPath := flags.String("worker-approval", "", "") workerStatePath := flags.String("worker-state-dir", "", "") + pairedBinding := flags.String("paired-binding", "", "") reject := func() int { fmt.Fprintln(out, "canary refused; approval, authority or private state requires review") return 1 @@ -72,8 +74,8 @@ func runWithPreparation(args []string, in io.Reader, out io.Writer, revisionForB return reject() } workerInputs := *workerApprovalPath != "" || *workerStatePath != "" - if *plan && !*execute && !*pairedExecute && !*prepare { - if *approvalPath != "" || *statePath != "" || *phase != "" || workerInputs { + if *plan && !*execute && !*pairedExecute && !*prepare && !*pairedPrepare { + if *approvalPath != "" || *statePath != "" || *phase != "" || workerInputs || *pairedBinding != "" { return reject() } fmt.Fprintln(out, "Controller-only phases: create, before-ack, after-ack, before-acquire, acquire-loss, jit-loss, inspect, cleanup. Paired terminal mode uses one same-process executable with explicit worker approval/state inputs and fixed terminal sequencing; no worker launch or workflow dispatch. Live execution requires an immutable reviewed build, exact private approval and controller-side broker input.") @@ -89,18 +91,25 @@ func runWithPreparation(args []string, in io.Reader, out io.Writer, revisionForB if *prepare { modeCount++ } + if *pairedPrepare { + modeCount++ + } if modeCount != 1 || *plan || *approvalPath == "" || *statePath == "" { return reject() } if *pairedExecute { - if *phase != "" || *workerApprovalPath == "" || *workerStatePath == "" { + if *phase != "" || *workerApprovalPath == "" || *workerStatePath == "" || *pairedBinding == "" { + return reject() + } + } else if *pairedPrepare { + if *phase != "" || workerInputs || *pairedBinding != "" { return reject() } - } else if *phase == "" || workerInputs { + } else if *phase == "" || workerInputs || *pairedBinding != "" { return reject() } a, err := livecanary.ReadApproval(*approvalPath) - if err != nil || a.Validate(time.Now()) != nil || (!*pairedExecute && !slices.Contains(a.Phases, *phase)) { + if err != nil || a.Validate(time.Now()) != nil || (!*pairedExecute && !*pairedPrepare && !slices.Contains(a.Phases, *phase)) { return reject() } revision, ok := revisionForBuild() @@ -108,12 +117,23 @@ func runWithPreparation(args []string, in io.Reader, out io.Writer, revisionForB return reject() } var worker liveworker.Approval + var binding livecanary.PairedTerminalBinding if *pairedExecute { var workerErr error worker, workerErr = liveworker.ReadApproval(*workerApprovalPath) if workerErr != nil || livecanary.ValidatePairedApprovals(a, worker) != nil || livecanary.ValidatePairedStatePaths(*statePath, *workerStatePath) != nil { return reject() } + if livecanary.DecodeStrict([]byte(*pairedBinding), &binding) != nil || livecanary.ValidatePairedTerminalBinding(livecanary.PairedTerminalFiles{ControllerApprovalPath: *approvalPath, ControllerStateDirectory: *statePath, WorkerApprovalPath: *workerApprovalPath, WorkerStateDirectory: *workerStatePath}, binding) != nil { + return reject() + } + } + if *pairedPrepare { + receipt, e := livecanary.PreparePairedJournal(*statePath, a) + if e != nil || json.NewEncoder(out).Encode(receipt) != nil { + return reject() + } + return 0 } if *prepare { if prepareJournal == nil { @@ -156,6 +176,9 @@ func runWithPreparation(args []string, in io.Reader, out io.Writer, revisionForB return reject() } if *pairedExecute { + if credentials.PairedBinding == nil || *credentials.PairedBinding != binding { + return reject() + } if livecanary.RunPairedTerminal(context.Background(), livecanary.PairedTerminalFiles{ControllerApprovalPath: *approvalPath, ControllerStateDirectory: *statePath, WorkerApprovalPath: *workerApprovalPath, WorkerStateDirectory: *workerStatePath}, credentials) != nil { fmt.Fprintln(out, "paired terminal stopped; retain private state and all uncertain resources; no automatic retry") return 1 diff --git a/experiments/g01-scaleset/cmd/g01-live/main_test.go b/experiments/g01-scaleset/cmd/g01-live/main_test.go index 799c101..97804a0 100644 --- a/experiments/g01-scaleset/cmd/g01-live/main_test.go +++ b/experiments/g01-scaleset/cmd/g01-live/main_test.go @@ -4,11 +4,14 @@ package main import ( "bytes" + "crypto/sha256" + "encoding/hex" "encoding/json" "io" "os" "path/filepath" "strings" + "syscall" "testing" "time" @@ -25,15 +28,77 @@ func (r unreadable) Read([]byte) (int, error) { type countedInput struct { io.Reader - reads int + reads int + eofs int + readsAfterEOF int +} + +func pairedBindingFixture(t *testing.T, controllerPath, controllerState, workerPath, workerState string) livecanary.PairedTerminalBinding { + t.Helper() + identity := func(path string) (uint64, uint64) { + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + t.Fatal("fixture identity") + } + return uint64(stat.Dev), stat.Ino + } + digest := func(path string) string { + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) + } + controllerDevice, controllerInode := identity(controllerPath) + controllerStateDevice, controllerStateInode := identity(controllerState) + workerDevice, workerInode := identity(workerPath) + workerStateDevice, workerStateInode := identity(workerState) + return livecanary.PairedTerminalBinding{ControllerApprovalSHA256: digest(controllerPath), ControllerApprovalDevice: controllerDevice, ControllerApprovalInode: controllerInode, ControllerStateDevice: controllerStateDevice, ControllerStateInode: controllerStateInode, WorkerApprovalSHA256: digest(workerPath), WorkerApprovalDevice: workerDevice, WorkerApprovalInode: workerInode, WorkerStateDevice: workerStateDevice, WorkerStateInode: workerStateInode} } func (r *countedInput) Close() error { return nil } func (r *countedInput) Read(p []byte) (int, error) { r.reads++ - return r.Reader.Read(p) + if r.eofs != 0 { + r.readsAfterEOF++ + } + n, err := r.Reader.Read(p) + if err == io.EOF { + r.eofs++ + } + return n, err +} + +type chunkedControllerInput struct { + chunks [][]byte + index int + reads int + eofs int + readsAfterEOF int +} + +func (r *chunkedControllerInput) Read(p []byte) (int, error) { + r.reads++ + if r.eofs != 0 { + r.readsAfterEOF++ + return 0, io.EOF + } + if r.index == len(r.chunks) { + r.eofs++ + return 0, io.EOF + } + n := copy(p, r.chunks[r.index]) + r.index++ + return n, nil } +func (r *chunkedControllerInput) Close() error { return nil } + func TestPlanAndRefusalsNeverReadCredentialsOrEchoInputs(t *testing.T) { for _, args := range [][]string{{"--plan"}, {}, {"--synthetic-secret=do-not-print"}, {"--execute-approved-canary", "--approval=synthetic-secret", "--state-dir=synthetic-secret", "--phase=create"}} { var out bytes.Buffer @@ -130,14 +195,16 @@ func TestPairedTerminalModeReadsControllerInputAfterAllGates(t *testing.T) { if err := os.WriteFile(workerPath, workerData, 0600); err != nil { t.Fatal(err) } - input := &countedInput{Reader: strings.NewReader(`{}`)} + binding := pairedBindingFixture(t, controllerPath, controllerState, workerPath, workerState) + input := &chunkedControllerInput{chunks: [][]byte{[]byte(`{`), []byte(`}`)}} var out bytes.Buffer - code := runWithPreparation([]string{"--execute-approved-paired-terminal", "--approval", controllerPath, "--state-dir", controllerState, "--worker-approval", workerPath, "--worker-state-dir", workerState}, input, &out, func() (string, bool) { return a.HarnessSHA, true }, func(string, livecanary.Approval, string) (livecanary.PreparationReceipt, error) { + bindingData, _ := json.Marshal(binding) + code := runWithPreparation([]string{"--execute-approved-paired-terminal", "--approval", controllerPath, "--state-dir", controllerState, "--worker-approval", workerPath, "--worker-state-dir", workerState, "--paired-binding", string(bindingData)}, input, &out, func() (string, bool) { return a.HarnessSHA, true }, func(string, livecanary.Approval, string) (livecanary.PreparationReceipt, error) { t.Fatal("paired mode entered controller-only preparation") return livecanary.PreparationReceipt{}, nil }) - if code == 0 || input.reads == 0 { - t.Fatalf("paired mode did not reach its bounded controller input gate: code=%d reads=%d output=%q", code, input.reads, out.String()) + if code == 0 || input.reads != 3 || input.eofs != 1 || input.readsAfterEOF != 0 { + t.Fatalf("paired mode did not consume one logical controller input: code=%d reads=%d eofs=%d rereads=%d output=%q", code, input.reads, input.eofs, input.readsAfterEOF, out.String()) } } @@ -163,7 +230,8 @@ func TestPairedTerminalModeRejectsUnusedPhaseAndControllerFlagsBeforeInput(t *te if err := os.WriteFile(workerPath, workerData, 0600); err != nil { t.Fatal(err) } - base := []string{"--execute-approved-paired-terminal", "--approval", controllerPath, "--state-dir", controllerState, "--worker-approval", workerPath, "--worker-state-dir", workerState} + bindingData, _ := json.Marshal(pairedBindingFixture(t, controllerPath, controllerState, workerPath, workerState)) + base := []string{"--execute-approved-paired-terminal", "--approval", controllerPath, "--state-dir", controllerState, "--worker-approval", workerPath, "--worker-state-dir", workerState, "--paired-binding", string(bindingData)} for _, extra := range [][]string{{"--phase", "cleanup"}, {"--execute-approved-canary"}} { input := &countedInput{Reader: strings.NewReader(`{}`)} args := append(append([]string(nil), base...), extra...) @@ -196,9 +264,10 @@ func TestPairedTerminalModeRequiresWorkflowVerificationAuthorityBeforeInput(t *t if err := os.WriteFile(workerPath, workerData, 0600); err != nil { t.Fatal(err) } + bindingData, _ := json.Marshal(pairedBindingFixture(t, controllerPath, controllerState, workerPath, workerState)) input := &countedInput{Reader: strings.NewReader(`{}`)} var out bytes.Buffer - args := []string{"--execute-approved-paired-terminal", "--approval", controllerPath, "--state-dir", controllerState, "--worker-approval", workerPath, "--worker-state-dir", workerState} + args := []string{"--execute-approved-paired-terminal", "--approval", controllerPath, "--state-dir", controllerState, "--worker-approval", workerPath, "--worker-state-dir", workerState, "--paired-binding", string(bindingData)} code := runWithPreparation(args, input, &out, func() (string, bool) { return a.HarnessSHA, true }, nil) if code == 0 || input.reads != 0 { t.Fatalf("paired mode accepted missing verification authority or read input: code=%d reads=%d output=%q", code, input.reads, out.String()) diff --git a/experiments/g01-scaleset/livecanary/baseline_integration.go b/experiments/g01-scaleset/livecanary/baseline_integration.go index c443767..c828bdc 100644 --- a/experiments/g01-scaleset/livecanary/baseline_integration.go +++ b/experiments/g01-scaleset/livecanary/baseline_integration.go @@ -42,6 +42,7 @@ type pairedBaselineScope struct { observedWorkerDeletion *liveworker.DeletionReceipt terminalEnabled bool cadence pairedBaselineCadence + bindingCheck func() error ctx context.Context driver *Driver @@ -71,9 +72,12 @@ func runPairedBaseline(ctx context.Context, d *Driver, w *liveworker.Driver) (pa // The production entry fixes the real clock. Tests can advance only cadence; // approval, network and scope deadlines always use their original real context. func runPairedBaselineWithCadence(ctx context.Context, d *Driver, w *liveworker.Driver, cadence pairedBaselineCadence) (pairedBaselineCollection, error) { - return runPairedBaselineMode(ctx, d, w, cadence, false) + return runPairedBaselineModeWithBinding(ctx, d, w, cadence, false, nil) } func runPairedBaselineMode(ctx context.Context, d *Driver, w *liveworker.Driver, cadence pairedBaselineCadence, terminal bool) (out pairedBaselineCollection, err error) { + return runPairedBaselineModeWithBinding(ctx, d, w, cadence, terminal, nil) +} +func runPairedBaselineModeWithBinding(ctx context.Context, d *Driver, w *liveworker.Driver, cadence pairedBaselineCadence, terminal bool, bindingCheck func() error) (out pairedBaselineCollection, err error) { out = pairedBaselineCollection{Outcome: collectionUnresolved, OutstandingSession: sessionNone} if ctx == nil || ctx.Err() != nil || d == nil || w == nil || cadence.now == nil || cadence.wait == nil { return out, ErrApproval @@ -122,7 +126,7 @@ func runPairedBaselineMode(ctx context.Context, d *Driver, w *liveworker.Driver, } outer, cancel := context.WithDeadline(ctx, deadline) defer cancel() - s := &pairedBaselineScope{terminalEnabled: terminal, cadence: cadence, ctx: outer, driver: d, workerDriver: w, approval: a, workerApproval: wa, api: api, captured: initial.api, docker: docker, journal: j, workerJournal: wj, identity: initial.identity, creation: initial.creation, setID: history.setID, listener: initial} + s := &pairedBaselineScope{terminalEnabled: terminal, cadence: cadence, bindingCheck: bindingCheck, ctx: outer, driver: d, workerDriver: w, approval: a, workerApproval: wa, api: api, captured: initial.api, docker: docker, journal: j, workerJournal: wj, identity: initial.identity, creation: initial.creation, setID: history.setID, listener: initial} for _, e := range j.Events() { if e.Kind == "inventory" { if s.inventory.Sequence != 0 { @@ -154,6 +158,9 @@ func runPairedBaselineMode(ctx context.Context, d *Driver, w *liveworker.Driver, return out, nil } func (s *pairedBaselineScope) current() error { + if s.bindingCheck != nil && s.bindingCheck() != nil { + return ErrQuarantine + } if s.ctx.Err() != nil || s.driver.Journal != s.journal || s.driver.API != s.api || s.workerDriver.Journal != s.workerJournal || s.workerDriver.Runtime != s.docker || approvalDigest(s.driver.Approval) != approvalDigest(s.approval) || workerApprovalDigest(s.workerDriver.Approval) != workerApprovalDigest(s.workerApproval) || s.api.client != s.captured.client || s.api.rest != s.captured.rest || s.api.baseURL != s.captured.baseURL || s.api.credentials != s.captured.credentials || approvalDigest(s.api.approval) != approvalDigest(s.approval) { return ErrQuarantine } diff --git a/experiments/g01-scaleset/livecanary/baseline_terminal.go b/experiments/g01-scaleset/livecanary/baseline_terminal.go index 957e1a9..76f8b40 100644 --- a/experiments/g01-scaleset/livecanary/baseline_terminal.go +++ b/experiments/g01-scaleset/livecanary/baseline_terminal.go @@ -24,7 +24,10 @@ func runPairedTerminal(ctx context.Context, d *Driver, w *liveworker.Driver) (pa return runPairedTerminalWithCadence(ctx, d, w, realBaselineCadence()) } func runPairedTerminalWithCadence(ctx context.Context, d *Driver, w *liveworker.Driver, cadence pairedBaselineCadence) (pairedBaselineTerminalResult, error) { - out, err := runPairedBaselineMode(ctx, d, w, cadence, true) + return runPairedTerminalWithBinding(ctx, d, w, cadence, nil) +} +func runPairedTerminalWithBinding(ctx context.Context, d *Driver, w *liveworker.Driver, cadence pairedBaselineCadence, bindingCheck func() error) (pairedBaselineTerminalResult, error) { + out, err := runPairedBaselineModeWithBinding(ctx, d, w, cadence, true, bindingCheck) result := pairedBaselineTerminalResult{Collection: out, Terminal: terminalUnresolved} if out.Terminal != nil && err == nil { result.Terminal = out.Terminal.Outcome diff --git a/experiments/g01-scaleset/livecanary/baseline_terminal_fixture_test.go b/experiments/g01-scaleset/livecanary/baseline_terminal_fixture_test.go index 86b8e15..b471992 100644 --- a/experiments/g01-scaleset/livecanary/baseline_terminal_fixture_test.go +++ b/experiments/g01-scaleset/livecanary/baseline_terminal_fixture_test.go @@ -4,10 +4,18 @@ package livecanary import ( "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" "net/http" + "os" + "path/filepath" "strings" "sync/atomic" + "syscall" "testing" + + "github.com/1XP-AI/gh-runnerd/experiments/g01-scaleset/liveworker" ) type terminalFixture struct { @@ -101,6 +109,240 @@ func newTerminalFixtureWithControllerPhases(t *testing.T, phases, controllerPhas func (f *terminalFixture) run() (pairedBaselineTerminalResult, error) { return runPairedTerminalWithCadence(context.Background(), &Driver{Approval: f.c.a, Journal: f.c.j, API: f.c.api}, f.w, fastPairCadence()) } + +type exportedPairedFixture struct { + files PairedTerminalFiles + credentials Credentials +} + +func pairedFixtureIdentity(t *testing.T, path string) (uint64, uint64) { + t.Helper() + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + t.Fatal("fixture identity unavailable") + } + return uint64(stat.Dev), stat.Ino +} + +func pairedFixtureDigest(t *testing.T, path string) string { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + digest := sha256.Sum256(data) + return hex.EncodeToString(digest[:]) +} + +func newExportedPairedFixture(t *testing.T, f *terminalFixture) exportedPairedFixture { + t.Helper() + controllerState := f.c.j.directory + controllerAdmission := f.c.j.claim.directory + workerState := f.wf.StateDirectory() + workerAdmission := f.wf.AdmissionDirectory() + if f.c.j.Close() != nil || f.wf.Journal.Close() != nil { + t.Fatal("close initial fixture journals") + } + root := filepath.Dir(controllerState) + controllerPath := filepath.Join(root, "controller-approval.json") + workerPath := filepath.Join(root, "worker-approval.json") + controllerData, err := json.Marshal(f.c.a) + if err != nil || os.WriteFile(controllerPath, controllerData, 0600) != nil { + t.Fatal("controller approval fixture") + } + workerData, err := json.Marshal(f.w.Approval) + if err != nil || os.WriteFile(workerPath, workerData, 0600) != nil { + t.Fatal("worker approval fixture") + } + controllerDevice, controllerInode := pairedFixtureIdentity(t, controllerPath) + controllerStateDevice, controllerStateInode := pairedFixtureIdentity(t, controllerState) + workerDevice, workerInode := pairedFixtureIdentity(t, workerPath) + workerStateDevice, workerStateInode := pairedFixtureIdentity(t, workerState) + binding := &PairedTerminalBinding{ControllerApprovalSHA256: pairedFixtureDigest(t, controllerPath), ControllerApprovalDevice: controllerDevice, ControllerApprovalInode: controllerInode, ControllerStateDevice: controllerStateDevice, ControllerStateInode: controllerStateInode, WorkerApprovalSHA256: pairedFixtureDigest(t, workerPath), WorkerApprovalDevice: workerDevice, WorkerApprovalInode: workerInode, WorkerStateDevice: workerStateDevice, WorkerStateInode: workerStateInode} + files := PairedTerminalFiles{ControllerApprovalPath: controllerPath, ControllerStateDirectory: controllerState, WorkerApprovalPath: workerPath, WorkerStateDirectory: workerState} + oldAdapters, oldCadence := pairedTerminalFixtureAdapters, pairedTerminalFixtureCadence + pairedTerminalFixtureCadence = fastPairCadence + pairedTerminalFixtureAdapters = &pairedTerminalAdapters{ + openController: func(path string, a Approval) (*FileJournal, error) { + j, err := openJournalAtAdmission(path, a, controllerAdmission, func(file *os.File) error { return file.Sync() }) + if err == nil { + f.c.j = j + } + return j, err + }, + openWorker: func(path string, a liveworker.Approval) (*liveworker.FileJournal, error) { + j, err := liveworker.OpenJournalForPairedFixture(path, a, workerAdmission) + if err == nil { + f.wf.Journal = j + } + return j, err + }, + newAPI: func(a Approval, c Credentials) (*SDKAPI, error) { + f.c.api.approval = a + f.c.api.credentials = c + return f.c.api, nil + }, + newDocker: liveworker.NewDocker, + } + t.Cleanup(func() { + pairedTerminalFixtureAdapters, pairedTerminalFixtureCadence = oldAdapters, oldCadence + }) + return exportedPairedFixture{files: files, credentials: Credentials{InstallationToken: f.c.api.credentials.InstallationToken, VerificationToken: f.c.api.credentials.VerificationToken, AppID: f.c.a.AppID, InstallationID: f.c.a.InstallationID, Organization: f.c.a.Organization, ExpiresAt: f.c.api.credentials.ExpiresAt, SelfHostedRunners: "write", Metadata: "read", PairedBinding: binding}} +} + +func TestPairedTerminalExportedAdapterActualJournalsFinalize(t *testing.T) { + f := newTerminalFixture(t, true) + fixture := newExportedPairedFixture(t, f) + if err := RunPairedTerminal(context.Background(), fixture.files, fixture.credentials); err != nil { + t.Fatalf("exported paired adapter refused private TLS/Unix fixture: %v", err) + } + if f.c.acks.Load() != 2 || f.c.acquires.Load() != 1 || f.jit.Load() != 1 || f.creates.Load() != 1 || f.starts.Load() != 1 || f.sessionDeletes.Load() != 1 || f.workerDeletes.Load() != 1 || f.setDeletes.Load() != 1 || f.workerAbsences.Load() != 1 || f.setAbsences.Load() != 1 || f.rosters.Load() != 4 || f.cleanup.Load() != 0 || f.c.forbidden.Load() != 0 { + t.Fatalf("exported terminal sequence: ack=%d acquire=%d JIT=%d create=%d start=%d session=%d worker=%d set=%d worker404=%d set404=%d roster=%d forbidden=%d", f.c.acks.Load(), f.c.acquires.Load(), f.jit.Load(), f.creates.Load(), f.starts.Load(), f.sessionDeletes.Load(), f.workerDeletes.Load(), f.setDeletes.Load(), f.workerAbsences.Load(), f.setAbsences.Load(), f.rosters.Load(), f.c.forbidden.Load()) + } + raw, _ := json.Marshal(f.c.j.Events()) + workerRaw, _ := json.Marshal(f.wf.Journal.Events()) + for _, secret := range []string{fixture.credentials.InstallationToken, fixture.credentials.VerificationToken, pairFixtureJIT} { + if strings.Contains(string(raw), secret) || strings.Contains(string(workerRaw), secret) { + t.Fatal("exported fixture journal leaked credential or JIT") + } + } +} + +func TestPairedTerminalExportedAdapterCancellationAndReopenNoReplay(t *testing.T) { + f := newTerminalFixture(t, true) + fixture := newExportedPairedFixture(t, f) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + f.c.afterResponse = func(r *http.Request, response *http.Response) { + if r.Method == http.MethodDelete && strings.Contains(r.URL.Path, "/sessions/") && response.StatusCode == http.StatusNoContent { + cancel() + } + } + if err := RunPairedTerminal(ctx, fixture.files, fixture.credentials); err == nil || f.sessionDeletes.Load() != 1 || f.workerDeletes.Load() != 0 || f.setDeletes.Load() != 0 { + t.Fatalf("canceled exported terminal crossed cleanup boundary: err=%v session=%d worker=%d set=%d", err, f.sessionDeletes.Load(), f.workerDeletes.Load(), f.setDeletes.Load()) + } + before := [3]int32{f.sessionDeletes.Load(), f.workerDeletes.Load(), f.setDeletes.Load()} + reopened := newExportedPairedFixture(t, f) + if err := RunPairedTerminal(context.Background(), reopened.files, reopened.credentials); err == nil { + t.Fatal("reopened canceled exported terminal was replayed") + } + after := [3]int32{f.sessionDeletes.Load(), f.workerDeletes.Load(), f.setDeletes.Load()} + if before != after { + t.Fatalf("reopened canceled terminal issued another effect: before=%v after=%v", before, after) + } +} + +func TestPairedTerminalExportedAdapterLostResponseStopsWithoutReplay(t *testing.T) { + f := newTerminalFixture(t, true) + original := f.githubBefore + f.githubBefore = func(w http.ResponseWriter, r *http.Request) bool { + if r.Method == http.MethodDelete && strings.Contains(r.URL.Path, "/sessions/") { + terminalLose(w) + return true + } + return original(w, r) + } + fixture := newExportedPairedFixture(t, f) + if err := RunPairedTerminal(context.Background(), fixture.files, fixture.credentials); err == nil || f.sessionDeletes.Load() != 0 || f.workerDeletes.Load() != 0 || f.setDeletes.Load() != 0 { + t.Fatalf("lost exported session response crossed effect boundary: err=%v session=%d worker=%d set=%d", err, f.sessionDeletes.Load(), f.workerDeletes.Load(), f.setDeletes.Load()) + } + before := [3]int32{f.sessionDeletes.Load(), f.workerDeletes.Load(), f.setDeletes.Load()} + reopened := newExportedPairedFixture(t, f) + if err := RunPairedTerminal(context.Background(), reopened.files, reopened.credentials); err == nil { + t.Fatal("reopened lost-response terminal was replayed") + } + after := [3]int32{f.sessionDeletes.Load(), f.workerDeletes.Load(), f.setDeletes.Load()} + if before != after { + t.Fatalf("reopened lost-response terminal issued another effect: before=%v after=%v", before, after) + } +} + +func TestPairedTerminalBindingRechecksAfterJournalOpenBeforeEffects(t *testing.T) { + f := newTerminalFixture(t, true) + fixture := newExportedPairedFixture(t, f) + adapters := *pairedTerminalFixtureAdapters + originalOpen := adapters.openController + adapters.openController = func(path string, a Approval) (*FileJournal, error) { + j, err := originalOpen(path, a) + if err == nil { + data, readErr := os.ReadFile(fixture.files.WorkerApprovalPath) + if readErr != nil || os.WriteFile(fixture.files.WorkerApprovalPath, append(data, '\n'), 0600) != nil { + t.Fatal("worker approval replacement") + } + } + return j, err + } + pairedTerminalFixtureAdapters = &adapters + if err := RunPairedTerminal(context.Background(), fixture.files, fixture.credentials); err == nil || f.c.requests.Load() != 0 || f.c.acquires.Load() != 0 || f.jit.Load() != 0 || f.creates.Load() != 0 || f.starts.Load() != 0 || f.sessionDeletes.Load() != 0 || f.workerDeletes.Load() != 0 || f.setDeletes.Load() != 0 { + t.Fatalf("approval replacement after initial binding crossed pre-effect fence: err=%v requests=%d acquire=%d JIT=%d create=%d start=%d session=%d worker=%d set=%d", err, f.c.requests.Load(), f.c.acquires.Load(), f.jit.Load(), f.creates.Load(), f.starts.Load(), f.sessionDeletes.Load(), f.workerDeletes.Load(), f.setDeletes.Load()) + } +} + +func TestPairedTerminalBindingRejectsChangedBytesAndStateBeforeEffects(t *testing.T) { + for _, kind := range []string{"controller-bytes", "controller-inode", "controller-hash", "controller-state", "controller-state-symlink", "worker-bytes", "worker-inode", "worker-hash", "worker-state", "worker-state-symlink"} { + t.Run(kind, func(t *testing.T) { + f := newTerminalFixture(t, true) + fixture := newExportedPairedFixture(t, f) + mutateFile := func(path string, data []byte) { + if err := os.WriteFile(path, data, 0600); err != nil { + t.Fatal(err) + } + } + switch kind { + case "controller-bytes": + data, _ := os.ReadFile(fixture.files.ControllerApprovalPath) + mutateFile(fixture.files.ControllerApprovalPath, append(data, '\n')) + case "controller-inode": + data, _ := os.ReadFile(fixture.files.ControllerApprovalPath) + if err := os.Rename(fixture.files.ControllerApprovalPath, fixture.files.ControllerApprovalPath+".retained"); err != nil { + t.Fatal(err) + } + mutateFile(fixture.files.ControllerApprovalPath, data) + case "controller-hash": + binding := *fixture.credentials.PairedBinding + binding.ControllerApprovalSHA256 = strings.Repeat("0", 64) + fixture.credentials.PairedBinding = &binding + case "controller-state": + if err := os.Rename(fixture.files.ControllerStateDirectory, fixture.files.ControllerStateDirectory+".retained"); err != nil || os.Mkdir(fixture.files.ControllerStateDirectory, 0700) != nil { + t.Fatal("controller state replacement") + } + case "controller-state-symlink": + if err := os.Rename(fixture.files.ControllerStateDirectory, fixture.files.ControllerStateDirectory+".retained"); err != nil || os.Symlink(fixture.files.ControllerStateDirectory+".retained", fixture.files.ControllerStateDirectory) != nil { + t.Fatal("controller state symlink") + } + case "worker-bytes": + data, _ := os.ReadFile(fixture.files.WorkerApprovalPath) + mutateFile(fixture.files.WorkerApprovalPath, append(data, '\n')) + case "worker-inode": + data, _ := os.ReadFile(fixture.files.WorkerApprovalPath) + if err := os.Rename(fixture.files.WorkerApprovalPath, fixture.files.WorkerApprovalPath+".retained"); err != nil { + t.Fatal(err) + } + mutateFile(fixture.files.WorkerApprovalPath, data) + case "worker-hash": + binding := *fixture.credentials.PairedBinding + binding.WorkerApprovalSHA256 = strings.Repeat("0", 64) + fixture.credentials.PairedBinding = &binding + case "worker-state": + if err := os.Rename(fixture.files.WorkerStateDirectory, fixture.files.WorkerStateDirectory+".retained"); err != nil || os.Mkdir(fixture.files.WorkerStateDirectory, 0700) != nil { + t.Fatal("worker state replacement") + } + case "worker-state-symlink": + if err := os.Rename(fixture.files.WorkerStateDirectory, fixture.files.WorkerStateDirectory+".retained"); err != nil || os.Symlink(fixture.files.WorkerStateDirectory+".retained", fixture.files.WorkerStateDirectory) != nil { + t.Fatal("worker state symlink") + } + } + if err := RunPairedTerminal(context.Background(), fixture.files, fixture.credentials); err == nil || f.c.requests.Load() != 0 || f.c.acquires.Load() != 0 || f.jit.Load() != 0 || f.creates.Load() != 0 || f.starts.Load() != 0 || f.sessionDeletes.Load() != 0 || f.workerDeletes.Load() != 0 || f.setDeletes.Load() != 0 { + t.Fatalf("changed paired identity crossed pre-effect gate: err=%v requests=%d acquire=%d jit=%d create=%d start=%d session=%d worker=%d set=%d", err, f.c.requests.Load(), f.c.acquires.Load(), f.jit.Load(), f.creates.Load(), f.starts.Load(), f.sessionDeletes.Load(), f.workerDeletes.Load(), f.setDeletes.Load()) + } + }) + } +} + func TestPairedTerminalActualJournalsFinalize(t *testing.T) { f := newTerminalFixture(t, true) if len(f.c.j.Events()) != 3 || f.wf.Journal == nil { diff --git a/experiments/g01-scaleset/livecanary/paired_terminal.go b/experiments/g01-scaleset/livecanary/paired_terminal.go index 0261238..5aa2a16 100644 --- a/experiments/g01-scaleset/livecanary/paired_terminal.go +++ b/experiments/g01-scaleset/livecanary/paired_terminal.go @@ -2,6 +2,9 @@ package livecanary import ( "context" + "crypto/sha256" + "encoding/hex" + "io" "os" "path/filepath" "strings" @@ -21,6 +24,148 @@ type PairedTerminalFiles struct { WorkerStateDirectory string } +// PairedTerminalBinding is broker-captured identity evidence. It is not a +// pairing manifest or an authority source: controller approval plus the +// journal-derived PairInput remain authoritative for the terminal sequence. +// Every value is bounded and credential-free so it can cross the child argv +// boundary alongside the controller-only credential payload. +type PairedTerminalBinding struct { + ControllerApprovalSHA256 string `json:"controller_approval_sha256"` + ControllerApprovalDevice uint64 `json:"controller_approval_device"` + ControllerApprovalInode uint64 `json:"controller_approval_inode"` + ControllerStateDevice uint64 `json:"controller_state_device"` + ControllerStateInode uint64 `json:"controller_state_inode"` + WorkerApprovalSHA256 string `json:"worker_approval_sha256"` + WorkerApprovalDevice uint64 `json:"worker_approval_device"` + WorkerApprovalInode uint64 `json:"worker_approval_inode"` + WorkerStateDevice uint64 `json:"worker_state_device"` + WorkerStateInode uint64 `json:"worker_state_inode"` +} + +func (b PairedTerminalBinding) equal(other PairedTerminalBinding) bool { return b == other } + +func (b PairedTerminalBinding) valid() bool { + return len(b.ControllerApprovalSHA256) == 64 && isLowerHex(b.ControllerApprovalSHA256) && b.ControllerApprovalDevice != 0 && b.ControllerApprovalInode != 0 && b.ControllerStateDevice != 0 && b.ControllerStateInode != 0 && len(b.WorkerApprovalSHA256) == 64 && isLowerHex(b.WorkerApprovalSHA256) && b.WorkerApprovalDevice != 0 && b.WorkerApprovalInode != 0 && b.WorkerStateDevice != 0 && b.WorkerStateInode != 0 +} + +func isLowerHex(s string) bool { + for _, c := range s { + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return false + } + } + return true +} + +type pairedApprovalSnapshot struct { + raw []byte + info os.FileInfo +} + +func readPairedApproval(path string, target any, decode func([]byte, any) error) (pairedApprovalSnapshot, error) { + if !filepath.IsAbs(path) || filepath.Clean(path) != path { + return pairedApprovalSnapshot{}, ErrApproval + } + file, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NOFOLLOW, 0) + if err != nil { + return pairedApprovalSnapshot{}, ErrApproval + } + defer file.Close() + info, err := file.Stat() + if err != nil || !privatePairedApproval(info) || info.Size() > 16384 { + return pairedApprovalSnapshot{}, ErrApproval + } + data, err := io.ReadAll(io.LimitReader(file, 16385)) + if err != nil || len(data) > 16384 || decode(data, target) != nil { + return pairedApprovalSnapshot{}, ErrApproval + } + return pairedApprovalSnapshot{raw: data, info: info}, nil +} + +func privatePairedApproval(info os.FileInfo) bool { + stat, ok := info.Sys().(*syscall.Stat_t) + return ok && int(stat.Uid) == os.Geteuid() && info.Mode().Perm() == 0600 && info.Mode().IsRegular() && stat.Nlink == 1 +} + +func pairedIdentity(info os.FileInfo) (device, inode uint64, ok bool) { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return 0, 0, false + } + return uint64(stat.Dev), stat.Ino, true +} + +func pairedStateIdentity(path string) (device, inode uint64, err error) { + if !privateStateDirectory(path) { + return 0, 0, ErrJournal + } + info, err := os.Lstat(path) + if err != nil { + return 0, 0, ErrJournal + } + device, inode, ok := pairedIdentity(info) + if !ok || device == 0 || inode == 0 { + return 0, 0, ErrJournal + } + return device, inode, nil +} + +func readPairedSnapshots(files PairedTerminalFiles) (Approval, liveworker.Approval, pairedApprovalSnapshot, pairedApprovalSnapshot, error) { + var controller Approval + controllerSnapshot, err := readPairedApproval(files.ControllerApprovalPath, &controller, DecodeStrict) + if err != nil { + return Approval{}, liveworker.Approval{}, pairedApprovalSnapshot{}, pairedApprovalSnapshot{}, ErrApproval + } + var worker liveworker.Approval + workerSnapshot, err := readPairedApproval(files.WorkerApprovalPath, &worker, liveworker.DecodeStrict) + if err != nil { + return Approval{}, liveworker.Approval{}, pairedApprovalSnapshot{}, pairedApprovalSnapshot{}, ErrApproval + } + return controller, worker, controllerSnapshot, workerSnapshot, nil +} + +func validatePairedBinding(files PairedTerminalFiles, binding PairedTerminalBinding, controllerSnapshot, workerSnapshot pairedApprovalSnapshot) error { + if !binding.valid() { + return ErrApproval + } + controllerDevice, controllerInode, ok := pairedIdentity(controllerSnapshot.info) + if !ok || controllerDevice != binding.ControllerApprovalDevice || controllerInode != binding.ControllerApprovalInode || brokerDigestBytes(controllerSnapshot.raw) != binding.ControllerApprovalSHA256 { + return ErrApproval + } + workerDevice, workerInode, ok := pairedIdentity(workerSnapshot.info) + if !ok || workerDevice != binding.WorkerApprovalDevice || workerInode != binding.WorkerApprovalInode || brokerDigestBytes(workerSnapshot.raw) != binding.WorkerApprovalSHA256 { + return ErrApproval + } + controllerStateDevice, controllerStateInode, err := pairedStateIdentity(files.ControllerStateDirectory) + if err != nil || controllerStateDevice != binding.ControllerStateDevice || controllerStateInode != binding.ControllerStateInode { + return ErrJournal + } + workerStateDevice, workerStateInode, err := pairedStateIdentity(files.WorkerStateDirectory) + if err != nil || workerStateDevice != binding.WorkerStateDevice || workerStateInode != binding.WorkerStateInode { + return ErrJournal + } + return nil +} + +func brokerDigestBytes(data []byte) string { + digest := sha256.Sum256(data) + return hex.EncodeToString(digest[:]) +} + +// ValidatePairedTerminalBinding is the credential-free pre-input fence used +// by g01-live. It compares broker evidence with the exact approval bytes and +// state roots currently named by the fixed argv. +func ValidatePairedTerminalBinding(files PairedTerminalFiles, binding PairedTerminalBinding) error { + controller, worker, controllerSnapshot, workerSnapshot, err := readPairedSnapshots(files) + if err != nil { + return err + } + if controller.Validate(time.Now()) != nil || worker.Validate(time.Now()) != nil || ValidatePairedApprovals(controller, worker) != nil || ValidatePairedStatePaths(files.ControllerStateDirectory, files.WorkerStateDirectory) != nil { + return ErrApproval + } + return validatePairedBinding(files, binding, controllerSnapshot, workerSnapshot) +} + // ValidatePairedApprovals is the credential-free front door shared by the CLI // and the same-process runner. PairInput is still derived from the controller // journal and approval; worker input only proves the intended trusted worker @@ -104,6 +249,22 @@ func privateStateDirectory(path string) bool { return ok && int(stat.Uid) == os.Geteuid() } +type pairedTerminalAdapters struct { + openController func(string, Approval) (*FileJournal, error) + openWorker func(string, liveworker.Approval) (*liveworker.FileJournal, error) + newAPI func(Approval, Credentials) (*SDKAPI, error) + newDocker func(liveworker.Approval) (*liveworker.Docker, error) +} + +// This hook is nil in production. The g01_pair_fixture-only test support uses +// it solely to point real journal/SDK construction at generated private roots +// and a private TLS server; no caller-controlled runtime authority is exposed. +var pairedTerminalFixtureAdapters *pairedTerminalAdapters + +// Pair fixture tests may replace only the cadence; production keeps the real +// terminal timing and all API/journal/lease behavior. +var pairedTerminalFixtureCadence func() pairedBaselineCadence + // RunPairedTerminal opens both existing journals, constructs the pinned SDK and // direct Unix-socket runtime, and runs the reviewed terminal sequence in this // process. It returns only fixed error categories; no private SDK/Docker error @@ -112,40 +273,60 @@ func RunPairedTerminal(ctx context.Context, files PairedTerminalFiles, credentia if ctx == nil { return ErrApproval } - controller, err := ReadApproval(files.ControllerApprovalPath) - if err != nil { + if credentials.PairedBinding == nil { return ErrApproval } - worker, err := liveworker.ReadApproval(files.WorkerApprovalPath) - if err != nil || ValidatePairedApprovals(controller, worker) != nil || ValidatePairedStatePaths(files.ControllerStateDirectory, files.WorkerStateDirectory) != nil { + controller, worker, controllerSnapshot, workerSnapshot, err := readPairedSnapshots(files) + if err != nil || ValidatePairedApprovals(controller, worker) != nil || ValidatePairedStatePaths(files.ControllerStateDirectory, files.WorkerStateDirectory) != nil || validatePairedBinding(files, *credentials.PairedBinding, controllerSnapshot, workerSnapshot) != nil { return ErrApproval } if credentials.validate(controller, time.Now()) != nil || len(credentials.VerificationToken) < 20 || len(credentials.VerificationToken) > 1024 || credentials.VerificationToken == credentials.InstallationToken || strings.ContainsAny(credentials.VerificationToken, "\r\n\x00") { return ErrApproval } - controllerJournal, err := OpenJournal(files.ControllerStateDirectory, controller) + adapters := pairedTerminalAdapters{openController: OpenJournal, openWorker: liveworker.OpenJournal, newAPI: NewSDKAPI, newDocker: liveworker.NewDocker} + if pairedTerminalFixtureAdapters != nil { + adapters = *pairedTerminalFixtureAdapters + } + controllerJournal, err := adapters.openController(files.ControllerStateDirectory, controller) if err != nil { return ErrJournal } defer controllerJournal.Close() - workerJournal, err := liveworker.OpenJournal(files.WorkerStateDirectory, worker) + workerJournal, err := adapters.openWorker(files.WorkerStateDirectory, worker) if err != nil { return ErrJournal } // LIFO closes the worker claim before the controller claim, matching the // paired cleanup order even when the terminal exits through an error path. defer workerJournal.Close() - api, err := NewSDKAPI(controller, credentials) + if err := ValidatePairedTerminalBinding(files, *credentials.PairedBinding); err != nil { + return err + } + api, err := adapters.newAPI(controller, credentials) if err != nil { return ErrApproval } - docker, err := liveworker.NewDocker(worker) + docker, err := adapters.newDocker(worker) if err != nil { return ErrJournal } - result, err := runPairedTerminal(ctx, &Driver{Approval: controller, Journal: controllerJournal, API: api}, &liveworker.Driver{Approval: worker, Journal: workerJournal, Runtime: docker}) + bindingCheck := func() error { + _, _, controllerSnapshot, workerSnapshot, e := readPairedSnapshots(files) + if e != nil { + return ErrApproval + } + return validatePairedBinding(files, *credentials.PairedBinding, controllerSnapshot, workerSnapshot) + } + cadence := realBaselineCadence() + if pairedTerminalFixtureCadence != nil { + cadence = pairedTerminalFixtureCadence() + } + result, err := runPairedTerminalWithBinding(ctx, &Driver{Approval: controller, Journal: controllerJournal, API: api}, &liveworker.Driver{Approval: worker, Journal: workerJournal, Runtime: docker}, cadence, bindingCheck) if err != nil || result.Terminal != terminalComplete { return ErrQuarantine } + if bindingCheck() != nil { + return ErrQuarantine + } return nil } diff --git a/experiments/g01-scaleset/livecanary/preparation.go b/experiments/g01-scaleset/livecanary/preparation.go index a3012e5..1b4d4c3 100644 --- a/experiments/g01-scaleset/livecanary/preparation.go +++ b/experiments/g01-scaleset/livecanary/preparation.go @@ -53,6 +53,8 @@ type PreparationReceipt struct { ClaimDigest string `json:"claim_digest"` } +const pairedPreparationPhase = "paired-terminal" + func preparedIdentity(i os.FileInfo) PreparedIdentity { s := i.Sys().(*syscall.Stat_t) return PreparedIdentity{uint64(s.Dev), s.Ino} @@ -78,6 +80,75 @@ func preparedDigest(f *os.File, limit int64) (string, error) { func PrepareJournal(directory string, a Approval, phase string) (PreparationReceipt, error) { return prepareJournal(directory, a, phase, OpenJournal) } + +// PreparePairedJournal is the paired terminal's explicit local preparation +// contract. It proves a fresh controller journal and its admission claim under +// the controller authority; it does not borrow cleanup authority and never +// reads credentials, worker input or contacts a remote service. +func PreparePairedJournal(directory string, a Approval) (PreparationReceipt, error) { + return preparePairedJournal(directory, a, OpenJournal) +} + +func pairedPreparationReady(a Approval, now time.Time) bool { + if a.Validate(now) != nil || a.WorkflowRunID <= 0 { + return false + } + want := map[string]bool{"create": false, "inspect": false, "cleanup": false} + verification := false + for _, phase := range a.Phases { + if _, ok := want[phase]; ok { + want[phase] = true + } + if phase == "before-ack" || phase == "after-ack" || phase == "before-acquire" || phase == "acquire-loss" { + verification = true + } + } + return verification && want["create"] && want["inspect"] && want["cleanup"] +} + +func preparePairedJournal(directory string, a Approval, open func(string, Approval) (*FileJournal, error)) (receipt PreparationReceipt, err error) { + if !pairedPreparationReady(a, time.Now()) || open == nil { + return receipt, ErrApproval + } + j, e := open(directory, a) + if e != nil { + return receipt, ErrJournal + } + defer func() { + if j.Close() != nil { + receipt = PreparationReceipt{} + err = ErrJournal + } + }() + release, e := j.authorize(a) + if e != nil { + return receipt, ErrJournal + } + defer release() + s := replay(j.Events()) + if s.uncertain || s.deleted || s.setID != 0 || s.reserved || s.workObserved || len(j.Events()) != 0 { + return receipt, ErrQuarantine + } + jd, e := preparedDigest(j.file, 1<<20) + if e != nil { + return receipt, e + } + cd, e := preparedDigest(j.claim.file, 4096) + if e != nil { + return receipt, e + } + ji, e := j.file.Stat() + if e != nil { + return receipt, ErrJournal + } + ci, e := j.claim.file.Stat() + if e != nil || !j.ownsCurrentJournal() || !j.claim.matches(j) { + return receipt, ErrJournal + } + receipt = PreparationReceipt{1, "controller_journal_prepared", pairedPreparationPhase, approvalDigest(a), preparedIdentity(j.directoryInfo), preparedIdentity(ji), preparedIdentity(ci), jd, cd} + return receipt, nil +} + func prepareJournal(directory string, a Approval, phase string, open func(string, Approval) (*FileJournal, error)) (receipt PreparationReceipt, err error) { if a.Validate(time.Now()) != nil || !slices.Contains(a.Phases, phase) || open == nil { return receipt, ErrApproval diff --git a/experiments/g01-scaleset/livecanary/preparation_fixture_test.go b/experiments/g01-scaleset/livecanary/preparation_fixture_test.go index a56fe3d..003ce32 100644 --- a/experiments/g01-scaleset/livecanary/preparation_fixture_test.go +++ b/experiments/g01-scaleset/livecanary/preparation_fixture_test.go @@ -12,6 +12,38 @@ import ( // no injected root/entry point. It calls the canonical local preparation helper, // never Driver.Run or an API, and must receive an empty stdin and fixed argv. func TestMain(m *testing.M) { + if len(os.Args) > 1 && os.Args[1] == "--prepare-approved-paired-journal" { + if len(os.Args) != 6 || os.Args[2] != "--approval" || os.Args[4] != "--state-dir" { + os.Exit(2) + } + input, e := io.ReadAll(io.LimitReader(os.Stdin, 1)) + if e != nil || len(input) != 0 { + os.Exit(3) + } + for _, entry := range os.Environ() { + if entry != "LANG=C" && entry != "LC_ALL=C" { + os.Exit(4) + } + } + a, e := ReadApproval(os.Args[3]) + if e != nil { + os.Exit(5) + } + directory, e := filepath.EvalSymlinks(filepath.Join(filepath.Dir(os.Args[5]), "admission")) + if e != nil { + os.Exit(6) + } + receipt, e := preparePairedJournal(os.Args[5], a, func(path string, a Approval) (*FileJournal, error) { + return openJournalAtAdmission(path, a, directory, func(f *os.File) error { return f.Sync() }) + }) + if e != nil { + os.Exit(7) + } + if json.NewEncoder(os.Stdout).Encode(receipt) != nil { + os.Exit(8) + } + os.Exit(0) + } if len(os.Args) > 1 && os.Args[1] == "--prepare-approved-journal" { if len(os.Args) != 8 || os.Args[2] != "--approval" || os.Args[4] != "--state-dir" || os.Args[6] != "--phase" { os.Exit(2) diff --git a/experiments/g01-scaleset/livecanary/preparation_test.go b/experiments/g01-scaleset/livecanary/preparation_test.go index 9755e55..b044a93 100644 --- a/experiments/g01-scaleset/livecanary/preparation_test.go +++ b/experiments/g01-scaleset/livecanary/preparation_test.go @@ -44,6 +44,35 @@ func TestCanonicalPreparationRecordsNoPhaseOrRemoteIntent(t *testing.T) { t.Fatal("preparation recorded phase or remote intent") } } + +func TestPairedPreparationUsesDedicatedPhaseWithoutCleanupAuthority(t *testing.T) { + parent := privateDir(t) + directory := admissionState(t, parent, "paired-state") + capRoot := testAdmissionDirectory(t, directory) + a := approval() + open := func(path string, a Approval) (*FileJournal, error) { + return openJournalAtAdmission(path, a, capRoot, func(f *os.File) error { return f.Sync() }) + } + receipt, err := preparePairedJournal(directory, a, open) + if err != nil || receipt.Phase != pairedPreparationPhase || receipt.Status != "controller_journal_prepared" { + t.Fatalf("paired preparation did not produce its own receipt: receipt=%+v err=%v", receipt, err) + } + j, err := open(directory, a) + if err != nil { + t.Fatal("paired journal reopen") + } + defer j.Close() + if len(j.Events()) != 0 { + t.Fatal("paired preparation borrowed cleanup authority or recorded an effect") + } + withoutVerification := a + withoutVerification.Phases = []string{"create", "inspect", "cleanup"} + withoutDirectory := admissionState(t, parent, "without-verification") + if _, err := preparePairedJournal(withoutDirectory, withoutVerification, open); err == nil { + t.Fatal("paired preparation accepted missing verification authority") + } +} + func TestCanonicalPreparationRefusesInvalidJournalAndPhase(t *testing.T) { for _, kind := range []string{"locked", "malformed", "oversized", "permission", "authority", "seen phase", "unknown", "deleted", "unapproved"} { t.Run(kind, func(t *testing.T) { diff --git a/experiments/g01-scaleset/livecanary/sdk.go b/experiments/g01-scaleset/livecanary/sdk.go index 74837db..1ac0a65 100644 --- a/experiments/g01-scaleset/livecanary/sdk.go +++ b/experiments/g01-scaleset/livecanary/sdk.go @@ -26,14 +26,15 @@ import ( // Preflight independently proves installation-token use, repository scope and // current target policy. It cannot independently prove broker provenance. type Credentials struct { - InstallationToken string `json:"installation_token"` - VerificationToken string `json:"verification_token"` - AppID int64 `json:"app_id"` - InstallationID int64 `json:"installation_id"` - Organization string `json:"organization"` - ExpiresAt time.Time `json:"expires_at"` - SelfHostedRunners string `json:"organization_self_hosted_runners"` - Metadata string `json:"metadata"` + InstallationToken string `json:"installation_token"` + VerificationToken string `json:"verification_token"` + AppID int64 `json:"app_id"` + InstallationID int64 `json:"installation_id"` + Organization string `json:"organization"` + ExpiresAt time.Time `json:"expires_at"` + SelfHostedRunners string `json:"organization_self_hosted_runners"` + Metadata string `json:"metadata"` + PairedBinding *PairedTerminalBinding `json:"paired_binding,omitempty"` } func (c Credentials) validate(a Approval, now time.Time) error { diff --git a/experiments/g01-scaleset/liveworker/paired_fixture_admission.go b/experiments/g01-scaleset/liveworker/paired_fixture_admission.go new file mode 100644 index 0000000..debb152 --- /dev/null +++ b/experiments/g01-scaleset/liveworker/paired_fixture_admission.go @@ -0,0 +1,13 @@ +//go:build g01_pair_fixture && !g01_live && !g01_worker + +package liveworker + +import "os" + +// OpenJournalForPairedFixture is a private test-only seam. It accepts only a +// generated fixture admission root and is unavailable from ordinary builds; +// production OpenJournal continues to derive its permanent root from the +// native account. +func OpenJournalForPairedFixture(directory string, a Approval, admissionDirectory string) (*FileJournal, error) { + return openJournalAtAdmission(directory, a, admissionDirectory, func(file *os.File) error { return file.Sync() }) +} diff --git a/experiments/g02-auth/broker.go b/experiments/g02-auth/broker.go index 6acfda1..498f91d 100644 --- a/experiments/g02-auth/broker.go +++ b/experiments/g02-auth/broker.go @@ -63,7 +63,7 @@ func (a BrokerApproval) validate(now time.Time) error { return errBroker } } else if a.Mode == "paired-terminal" { - if a.Phase != "" && a.Phase != "paired-terminal" { + if a.Phase != "paired-terminal" { return errBroker } } else { @@ -118,16 +118,18 @@ func brokerExecute(parent context.Context, a BrokerApproval, input brokerInput, return BrokerResult{}, errBroker } defer claim.close() - guard := func() error { + guard := func(checkPrepared bool) error { if claim.check() != nil || j.check() != nil { return errBroker } - if plan != nil && (plan.check() != nil || plan.compatibleControllerClaim(claim.root) != nil || (plan.prepared != nil && plan.preparedState(claim.root, false) != nil)) { + if plan != nil && (plan.check() != nil || plan.compatibleControllerClaim(claim.root) != nil || (checkPrepared && plan.prepared != nil && plan.preparedState(claim.root, false) != nil)) { return errBroker } return nil } - if guard() != nil { + guardLive := func() error { return guard(true) } + guardPostChild := func() error { return guard(false) } + if guardLive() != nil { return BrokerResult{}, errBroker } @@ -140,16 +142,16 @@ func brokerExecute(parent context.Context, a BrokerApproval, input brokerInput, return BrokerResult{}, errBroker } plan.preparationReceipt = receipt - if guard() != nil || plan.preparedState(claim.root, true) != nil { + if guardLive() != nil || plan.preparedState(claim.root, true) != nil { return BrokerResult{}, errBroker } - if j.append("controller_state_prepared", map[string]any{"receipt": receipt}) != nil || guard() != nil { + if j.append("controller_state_prepared", map[string]any{"receipt": receipt}) != nil || guardLive() != nil { return BrokerResult{}, errBroker } } // Every authenticated call revalidates the still-held durable claim. - scoped := newBrokerAPI(api.now, brokerGuardTransport{api.client.Transport, guard}) + scoped := newBrokerAPI(api.now, brokerGuardTransport{api.client.Transport, guardLive}) api = scoped // JWT identity verification precedes the one token mint. Private repository // and runner-group APIs require that installation token, so scope preflight @@ -202,7 +204,10 @@ func brokerExecute(parent context.Context, a BrokerApproval, input brokerInput, if len(data) > 16384 || j.append("controller_handoff_started", nil) != nil { return BrokerResult{}, errBroker } - if ((a.Mode == "paired-terminal" || plan.controller.needsVerification()) && api.verifyWorkflow(ctx, a, plan.controller, input.VerificationToken) != nil) || guard() != nil || plan.launch(ctx, data, filepath.Join(path, "controller-approval.json")) != nil || j.append("controller_completed", nil) != nil || claim.complete() != nil { + if (a.Mode == "paired-terminal" || plan.controller.needsVerification()) && api.verifyWorkflow(ctx, a, plan.controller, input.VerificationToken) != nil { + return BrokerResult{}, errBroker + } + if guardLive() != nil || plan.launch(ctx, data, filepath.Join(path, "controller-approval.json")) != nil || guardPostChild() != nil || j.append("controller_completed", nil) != nil || claim.complete() != nil || guardPostChild() != nil { return BrokerResult{}, errBroker } if a.Mode == "paired-terminal" { diff --git a/experiments/g02-auth/broker_entry.go b/experiments/g02-auth/broker_entry.go index efb37c0..ea9070f 100644 --- a/experiments/g02-auth/broker_entry.go +++ b/experiments/g02-auth/broker_entry.go @@ -145,7 +145,7 @@ func runBrokerWithAPI(ctx context.Context, files BrokerFiles, input *os.File, ap if err != nil || hex.EncodeToString(digest[:]) != approval.ControllerApprovalSHA256 || controller.validate(approval, api.now()) != nil { return BrokerResult{}, errBroker } - binary, err = openBrokerBinary(files.ControllerBinary, approval) + binary, err = brokerBinaryOpener(files.ControllerBinary, approval) if err != nil { return BrokerResult{}, errBroker } @@ -177,7 +177,11 @@ func runBrokerWithAPI(ctx context.Context, files BrokerFiles, input *os.File, ap return errBroker } if approval.Mode == "paired-terminal" { - return invokeBrokerPairedTerminal(ctx, binary, files.StateDirectory, snapshotPath, files.ControllerStateDirectory, files.WorkerApproval, files.WorkerStateDirectory, data) + binding, err := plan.pairedBinding() + if err != nil { + return errBroker + } + return invokeBrokerPairedTerminal(ctx, binary, files.StateDirectory, snapshotPath, files.ControllerStateDirectory, files.WorkerApproval, files.WorkerStateDirectory, binding, data) } return invokeBrokerController(ctx, binary, files.StateDirectory, snapshotPath, files.ControllerStateDirectory, approval.Phase, data) }) @@ -185,11 +189,10 @@ func runBrokerWithAPI(ctx context.Context, files BrokerFiles, input *os.File, ap return BrokerResult{}, errBroker } plan.localPrepare = func(ctx context.Context, snapshotPath string) (brokerPreparationReceipt, error) { - phase := approval.Phase if approval.Mode == "paired-terminal" { - phase = "cleanup" + return invokeBrokerPairedPreparation(ctx, binary, files.StateDirectory, snapshotPath, files.ControllerStateDirectory) } - return invokeBrokerPreparation(ctx, binary, files.StateDirectory, snapshotPath, files.ControllerStateDirectory, phase) + return invokeBrokerPreparation(ctx, binary, files.StateDirectory, snapshotPath, files.ControllerStateDirectory, approval.Phase) } plan.worker = workerPlan } diff --git a/experiments/g02-auth/broker_paired_test.go b/experiments/g02-auth/broker_paired_test.go index 0795019..df4da1f 100644 --- a/experiments/g02-auth/broker_paired_test.go +++ b/experiments/g02-auth/broker_paired_test.go @@ -164,3 +164,85 @@ func TestPairedBrokerBindsWorkerBeforeWorkflowVerifiedHandoff(t *testing.T) { t.Fatal("paired admission did not retain bounded worker binding") } } + +// This is intentionally an entrypoint-level fixture. It uses the real +// BrokerFiles loader, paired preparation process and brokerExecute closure, so +// the canonical preparation receipt and fixed child handoff are both exercised. +func TestPairedBrokerRealEntrypointUsesPairedPreparationClosure(t *testing.T) { + a, candidate, api, fixture, attempt := newBrokerFixture(t) + parent := filepath.Dir(attempt) + controllerState := filepath.Join(parent, "paired-controller-state") + workerState := filepath.Join(parent, "paired-worker-state") + if err := os.Mkdir(controllerState, 0700); err != nil { + t.Fatal("controller state") + } + if err := os.Mkdir(workerState, 0700); err != nil { + t.Fatal("worker state") + } + now := time.Now().Add(time.Hour) + harness := strings.Repeat("c", 40) + workflow := strings.Repeat("b", 40) + a.Mode, a.Phase, a.AllowVerificationAuthority, a.ExpiresAt, a.ControllerHarnessSHA = "paired-terminal", "paired-terminal", true, now, harness + controller := controllerApproval{AppID: a.AppID, InstallationID: a.InstallationID, Organization: a.Organization, Repository: a.Repository, RepositoryID: a.RepositoryID, RunnerGroupID: a.RunnerGroupID, OwnerNonce: a.OwnerNonce, HarnessSHA: harness, WorkflowSHA: workflow, WorkflowPath: ".github/workflows/canary.yml", WorkflowRunID: 7, Controller: "trusted-controller", ExpiresAt: now, ActionsHosts: []string{"fixture.actions.githubusercontent.com"}, Phases: []string{"create", "before-ack", "after-ack", "before-acquire", "inspect", "cleanup"}} + controllerData, err := json.Marshal(controller) + if err != nil { + t.Fatal("controller approval") + } + controllerPath := filepath.Join(parent, "controller-approval.json") + if err := os.WriteFile(controllerPath, controllerData, 0600); err != nil { + t.Fatal("controller approval file") + } + worker := pairedWorkerApproval{RunnerUpdatesDisabled: true, HarnessSHA: harness, WorkflowSHA: workflow, OwnerNonce: a.OwnerNonce, Controller: controller.Controller, Endpoint: "/tmp/g01-paired-entry.sock", DaemonID: "fixture-daemon", ImageID: "sha256:" + strings.Repeat("d", 64), Image: pairedWorkerImage, ExpiresAt: now, Phases: []string{"create", "start", "inspect", "cleanup"}} + workerData, err := json.Marshal(worker) + if err != nil { + t.Fatal("worker approval") + } + workerPath := filepath.Join(parent, "worker-approval.json") + if err := os.WriteFile(workerPath, workerData, 0600); err != nil { + t.Fatal("worker approval file") + } + binary := testBrokerBinary(t) + a.ControllerBinarySHA256 = binary.digest + a.ControllerApprovalSHA256 = brokerBytesDigest(controllerData) + approvalPath := filepath.Join(parent, "broker-approval.json") + approvalData, err := json.Marshal(a) + if err != nil { + t.Fatal("broker approval") + } + if err := os.WriteFile(approvalPath, approvalData, 0600); err != nil { + t.Fatal("broker approval file") + } + inputPath := filepath.Join(parent, "broker-input.json") + inputData, err := json.Marshal(brokerInput{PEM: string(candidate.PEM), VerificationToken: "synthetic-private-verification-token"}) + if err != nil { + t.Fatal("broker input") + } + if err := os.WriteFile(inputPath, inputData, 0600); err != nil { + t.Fatal("broker input file") + } + input, err := os.Open(inputPath) + if err != nil { + t.Fatal("broker input open") + } + defer input.Close() + oldOpener := brokerBinaryOpener + brokerBinaryOpener = func(string, BrokerApproval) (*verifiedBrokerBinary, error) { return binary, nil } + defer func() { brokerBinaryOpener = oldOpener }() + result, err := runBrokerWithAPI(context.Background(), BrokerFiles{ApprovalPath: approvalPath, StateDirectory: attempt, ControllerBinary: binary.path, ControllerApproval: controllerPath, ControllerStateDirectory: controllerState, WorkerApproval: workerPath, WorkerStateDirectory: workerState}, input, api) + if err != nil || result.Status != "paired_terminal_completed" || fixture.tokenCalls != 1 { + t.Fatalf("real paired entrypoint did not complete one handoff: result=%+v err=%v mints=%d calls=%v", result, err, fixture.tokenCalls, fixture.calls) + } + retryPath := filepath.Join(parent, "broker-input-retry.json") + if err := os.WriteFile(retryPath, inputData, 0600); err != nil { + t.Fatal("retry input file") + } + retryInput, err := os.Open(retryPath) + if err != nil { + t.Fatal("retry input open") + } + defer retryInput.Close() + _, retryErr := runBrokerWithAPI(context.Background(), BrokerFiles{ApprovalPath: approvalPath, StateDirectory: attempt, ControllerBinary: binary.path, ControllerApproval: controllerPath, ControllerStateDirectory: controllerState, WorkerApproval: workerPath, WorkerStateDirectory: workerState}, retryInput, api) + if retryErr == nil || fixture.tokenCalls != 1 { + t.Fatalf("paired handoff replayed after a completed attempt: err=%v mints=%d", retryErr, fixture.tokenCalls) + } +} diff --git a/experiments/g02-auth/broker_plan.go b/experiments/g02-auth/broker_plan.go index efd7687..3e6ca6b 100644 --- a/experiments/g02-auth/broker_plan.go +++ b/experiments/g02-auth/broker_plan.go @@ -51,6 +51,26 @@ type brokerWorkerPlan struct { stateInfo os.FileInfo } +// brokerPairedBinding is immutable identity evidence carried to the child. +// It does not describe or authorize pairing; the controller approval and the +// journal-derived PairInput remain the canonical authority in G01. +type brokerPairedBinding struct { + ControllerApprovalSHA256 string `json:"controller_approval_sha256"` + ControllerApprovalDevice uint64 `json:"controller_approval_device"` + ControllerApprovalInode uint64 `json:"controller_approval_inode"` + ControllerStateDevice uint64 `json:"controller_state_device"` + ControllerStateInode uint64 `json:"controller_state_inode"` + WorkerApprovalSHA256 string `json:"worker_approval_sha256"` + WorkerApprovalDevice uint64 `json:"worker_approval_device"` + WorkerApprovalInode uint64 `json:"worker_approval_inode"` + WorkerStateDevice uint64 `json:"worker_state_device"` + WorkerStateInode uint64 `json:"worker_state_inode"` +} + +func (b brokerPairedBinding) valid() bool { + return brokerSHA256.MatchString(b.ControllerApprovalSHA256) && b.ControllerApprovalDevice != 0 && b.ControllerApprovalInode != 0 && b.ControllerStateDevice != 0 && b.ControllerStateInode != 0 && brokerSHA256.MatchString(b.WorkerApprovalSHA256) && b.WorkerApprovalDevice != 0 && b.WorkerApprovalInode != 0 && b.WorkerStateDevice != 0 && b.WorkerStateInode != 0 +} + func (p *brokerWorkerPlan) close() { if p == nil { return @@ -184,6 +204,23 @@ func (p *brokerControllerPlan) binding() (brokerControllerBinding, error) { c.Phases = nil return brokerControllerBinding{brokerDigest(c), p.approval.ControllerBinarySHA256, p.approval.ControllerHarnessSHA, brokerFileIdentity(i)}, nil } + +func (p *brokerControllerPlan) pairedBinding() (brokerPairedBinding, error) { + if p == nil || p.approval.Mode != "paired-terminal" || p.snapshotInfo == nil || p.worker == nil || p.check() != nil { + return brokerPairedBinding{}, errBroker + } + worker, err := p.worker.binding() + if err != nil { + return brokerPairedBinding{}, errBroker + } + controllerApproval := brokerFileIdentity(p.snapshotInfo) + controllerState := brokerFileIdentity(p.stateInfo) + binding := brokerPairedBinding{ControllerApprovalSHA256: p.approval.ControllerApprovalSHA256, ControllerApprovalDevice: controllerApproval.Device, ControllerApprovalInode: controllerApproval.Inode, ControllerStateDevice: controllerState.Device, ControllerStateInode: controllerState.Inode, WorkerApprovalSHA256: worker.Approval, WorkerApprovalDevice: worker.ApprovalFile.Device, WorkerApprovalInode: worker.ApprovalFile.Inode, WorkerStateDevice: worker.State.Device, WorkerStateInode: worker.State.Inode} + if !binding.valid() { + return brokerPairedBinding{}, errBroker + } + return binding, nil +} func (p *brokerControllerPlan) prepare(a BrokerApproval, j *brokerJournal, now time.Time) error { if p == nil || p.binaryCheck == nil || p.launch == nil || p.localPrepare == nil || brokerDigest(p.approval) != brokerDigest(a) || p.controller.validate(a, now) != nil || brokerBytesDigest(p.raw) != a.ControllerApprovalSHA256 || p.binaryCheck() != nil || (a.Mode == "paired-terminal" && p.worker == nil) { return errBroker diff --git a/experiments/g02-auth/broker_preparation.go b/experiments/g02-auth/broker_preparation.go index 6b48dec..716fa14 100644 --- a/experiments/g02-auth/broker_preparation.go +++ b/experiments/g02-auth/broker_preparation.go @@ -6,6 +6,7 @@ import ( "io" "os" "os/exec" + "path/filepath" "sync" "syscall" "time" @@ -24,7 +25,16 @@ type brokerPreparationReceipt struct { } func (r brokerPreparationReceipt) valid(p *brokerControllerPlan) bool { - return r.Version == 1 && r.Status == "controller_journal_prepared" && r.Phase == p.approval.Phase && r.ApprovalDigest == brokerDigest(p.controller) && r.State.Inode != 0 && r.Journal.Inode != 0 && r.Claim.Inode != 0 && brokerSHA256.MatchString(r.JournalDigest) && brokerSHA256.MatchString(r.ClaimDigest) + if p == nil { + return false + } + phase := p.approval.Phase + if p.approval.Mode == "paired-terminal" { + // Paired preparation is its own local authority contract. It is not a + // cleanup receipt lending cleanup authority to the full pair. + phase = "paired-terminal" + } + return r.Version == 1 && r.Status == "controller_journal_prepared" && r.Phase == phase && r.ApprovalDigest == brokerDigest(p.controller) && r.State.Device != 0 && r.State.Inode != 0 && r.Journal.Device != 0 && r.Journal.Inode != 0 && r.Claim.Device != 0 && r.Claim.Inode != 0 && brokerSHA256.MatchString(r.JournalDigest) && brokerSHA256.MatchString(r.ClaimDigest) } type brokerPreparationOutput struct { @@ -71,6 +81,35 @@ func invokeBrokerPreparation(parent context.Context, binary *verifiedBrokerBinar return receipt, nil } +// invokeBrokerPairedPreparation uses a distinct executable contract. The +// paired preparation receipt is not a cleanup receipt and carries no child +// credentials or worker authority. +func invokeBrokerPairedPreparation(parent context.Context, binary *verifiedBrokerBinary, workingDirectory, approvalPath, stateDirectory string) (receipt brokerPreparationReceipt, err error) { + if binary == nil || binary.check() != nil || !filepath.IsAbs(approvalPath) || !filepath.IsAbs(stateDirectory) || filepath.Clean(approvalPath) != approvalPath || filepath.Clean(stateDirectory) != stateDirectory { + return receipt, errBroker + } + ctx, cancel := context.WithTimeout(parent, 30*time.Second) + defer cancel() + command := exec.CommandContext(ctx, binary.path, "--prepare-approved-paired-journal", "--approval", approvalPath, "--state-dir", stateDirectory) + command.Dir = workingDirectory + command.Env = []string{"LANG=C", "LC_ALL=C"} + command.Stdin = bytes.NewReader(nil) + command.WaitDelay = time.Second + output := &brokerPreparationOutput{cancel: cancel} + errors := &brokerOutputBudget{cancel: cancel} + command.Stdout = output + command.Stderr = errors + e := command.Run() + output.mu.Lock() + defer output.mu.Unlock() + errors.mu.Lock() + defer errors.mu.Unlock() + if e != nil || ctx.Err() != nil || output.overflow || errors.overflow || decodeBrokerJSON(output.data, &receipt, true) != nil { + return receipt, errBroker + } + return receipt, nil +} + type brokerPreparedState struct { journal, claim *os.File journalInfo, claimInfo os.FileInfo diff --git a/experiments/g02-auth/broker_process.go b/experiments/g02-auth/broker_process.go index 75b0134..ab5f0d3 100644 --- a/experiments/g02-auth/broker_process.go +++ b/experiments/g02-auth/broker_process.go @@ -5,6 +5,7 @@ import ( "crypto/sha256" "debug/buildinfo" "encoding/hex" + "encoding/json" "io" "os" "os/exec" @@ -23,6 +24,11 @@ type verifiedBrokerBinary struct { digest string } +// brokerBinaryOpener is kept narrow so offline tests can exercise the real +// BrokerFiles entrypoint with the test executable without weakening the +// production build metadata gate. +var brokerBinaryOpener = openBrokerBinary + func validBrokerBuild(info *debug.BuildInfo, expected string) bool { if info == nil || info.GoVersion != "go1.26.8" || info.Path != "github.com/1XP-AI/gh-runnerd/experiments/g01-scaleset/cmd/g01-live" || !brokerSHA40.MatchString(expected) { return false @@ -149,13 +155,26 @@ func invokeBrokerController(parent context.Context, binary *verifiedBrokerBinary // invokeBrokerPairedTerminal has one fixed argv shape. The worker is supplied // as approval/state input to the same g01-live process; it is never started as // a separate child and no arbitrary phase/command reaches exec. -func invokeBrokerPairedTerminal(parent context.Context, binary *verifiedBrokerBinary, workingDirectory, approvalPath, stateDirectory, workerApprovalPath, workerStateDirectory string, data []byte) error { - if len(data) > 16384 || binary == nil || binary.check() != nil || !filepath.IsAbs(approvalPath) || !filepath.IsAbs(stateDirectory) || !filepath.IsAbs(workerApprovalPath) || !filepath.IsAbs(workerStateDirectory) || filepath.Clean(approvalPath) != approvalPath || filepath.Clean(stateDirectory) != stateDirectory || filepath.Clean(workerApprovalPath) != workerApprovalPath || filepath.Clean(workerStateDirectory) != workerStateDirectory { +func invokeBrokerPairedTerminal(parent context.Context, binary *verifiedBrokerBinary, workingDirectory, approvalPath, stateDirectory, workerApprovalPath, workerStateDirectory string, binding brokerPairedBinding, data []byte) error { + if len(data) > 16384 || binary == nil || binary.check() != nil || !binding.valid() || !filepath.IsAbs(approvalPath) || !filepath.IsAbs(stateDirectory) || !filepath.IsAbs(workerApprovalPath) || !filepath.IsAbs(workerStateDirectory) || filepath.Clean(approvalPath) != approvalPath || filepath.Clean(stateDirectory) != stateDirectory || filepath.Clean(workerApprovalPath) != workerApprovalPath || filepath.Clean(workerStateDirectory) != workerStateDirectory { return errBroker } - ctx, cancel := context.WithCancel(parent) + var payload map[string]json.RawMessage + if json.Unmarshal(data, &payload) != nil { + return errBroker + } + bindingData, err := json.Marshal(binding) + if err != nil { + return errBroker + } + payload["paired_binding"] = bindingData + data, err = json.Marshal(payload) + if err != nil || len(data) > 16384 { + return errBroker + } + ctx, cancel := context.WithTimeout(parent, 30*time.Second) defer cancel() - command := exec.CommandContext(ctx, binary.path, "--execute-approved-paired-terminal", "--approval", approvalPath, "--state-dir", stateDirectory, "--worker-approval", workerApprovalPath, "--worker-state-dir", workerStateDirectory) + command := exec.CommandContext(ctx, binary.path, "--execute-approved-paired-terminal", "--approval", approvalPath, "--state-dir", stateDirectory, "--worker-approval", workerApprovalPath, "--worker-state-dir", workerStateDirectory, "--paired-binding", string(bindingData)) command.Dir = workingDirectory command.Env = []string{"LANG=C", "LC_ALL=C"} command.WaitDelay = time.Second diff --git a/experiments/g02-auth/broker_process_test.go b/experiments/g02-auth/broker_process_test.go index a7c062d..2d78385 100644 --- a/experiments/g02-auth/broker_process_test.go +++ b/experiments/g02-auth/broker_process_test.go @@ -5,6 +5,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "encoding/json" "fmt" "io" "os" @@ -12,6 +13,7 @@ import ( "runtime" "runtime/debug" "strings" + "syscall" "testing" "time" ) @@ -20,6 +22,74 @@ import ( // production entry point must separately verify G01 build metadata before it // can construct verifiedBrokerBinary; these pipe tests isolate that handoff. func TestMain(m *testing.M) { + if len(os.Args) > 1 && os.Args[1] == "--prepare-approved-paired-journal" { + if len(os.Args) != 6 || os.Args[2] != "--approval" || os.Args[4] != "--state-dir" { + os.Exit(3) + } + for _, entry := range os.Environ() { + if entry != "LANG=C" && entry != "LC_ALL=C" { + os.Exit(4) + } + } + data, e := io.ReadAll(io.LimitReader(os.Stdin, 1)) + if e != nil || len(data) != 0 { + os.Exit(5) + } + var controller controllerApproval + _, e = readBrokerPrivateJSON(os.Args[3], &controller) + if e != nil { + os.Exit(6) + } + statePath := os.Args[5] + admissionPath := filepath.Join(filepath.Dir(statePath), "admission") + if e = os.Mkdir(admissionPath, 0700); e != nil && !os.IsExist(e) { + os.Exit(7) + } + journalPath := filepath.Join(statePath, "journal.jsonl") + if _, e = os.Stat(journalPath); os.IsNotExist(e) { + if e = os.WriteFile(journalPath, []byte("synthetic prepared journal\n"), 0600); e != nil { + os.Exit(8) + } + } + journalInfo, e := os.Stat(journalPath) + if e != nil { + os.Exit(9) + } + stateInfo, e := os.Stat(statePath) + if e != nil { + os.Exit(10) + } + claimPath := filepath.Join(admissionPath, "admission.json") + if _, e = os.Stat(claimPath); os.IsNotExist(e) { + stateID := brokerFileIdentity(stateInfo) + journalID := brokerFileIdentity(journalInfo) + ownership := controller + ownership.ExpiresAt = time.Time{} + ownership.Phases = nil + claim := map[string]any{"version": 1, "ownership": brokerDigest(ownership), "state_device": stateID.Device, "state_inode": stateID.Inode, "journal_device": journalID.Device, "journal_inode": journalID.Inode} + claimData, _ := json.Marshal(claim) + if e = os.WriteFile(claimPath, append(claimData, '\n'), 0600); e != nil { + os.Exit(11) + } + } + claimInfo, e := os.Stat(claimPath) + if e != nil { + os.Exit(12) + } + journalData, e := os.ReadFile(journalPath) + if e != nil { + os.Exit(13) + } + claimData, e := os.ReadFile(claimPath) + if e != nil { + os.Exit(14) + } + receipt := brokerPreparationReceipt{Version: 1, Status: "controller_journal_prepared", Phase: "paired-terminal", ApprovalDigest: brokerDigest(controller), State: brokerFileIdentity(stateInfo), Journal: brokerFileIdentity(journalInfo), Claim: brokerFileIdentity(claimInfo), JournalDigest: brokerBytesDigest(journalData), ClaimDigest: brokerBytesDigest(claimData)} + if json.NewEncoder(os.Stdout).Encode(receipt) != nil { + os.Exit(13) + } + os.Exit(0) + } if len(os.Args) > 1 && os.Args[1] == "--prepare-approved-journal" { if len(os.Args) != 8 || os.Args[2] != "--approval" || os.Args[4] != "--state-dir" || os.Args[6] != "--phase" { os.Exit(3) @@ -72,7 +142,7 @@ func TestMain(m *testing.M) { os.Exit(0) } if len(os.Args) > 1 && os.Args[1] == "--execute-approved-paired-terminal" { - if len(os.Args) != 10 || os.Args[2] != "--approval" || os.Args[4] != "--state-dir" || os.Args[6] != "--worker-approval" || os.Args[8] != "--worker-state-dir" { + if len(os.Args) != 12 || os.Args[2] != "--approval" || os.Args[4] != "--state-dir" || os.Args[6] != "--worker-approval" || os.Args[8] != "--worker-state-dir" || os.Args[10] != "--paired-binding" { os.Exit(3) } for _, entry := range os.Environ() { @@ -84,6 +154,28 @@ func TestMain(m *testing.M) { if err != nil || !bytes.Contains(data, []byte("synthetic-private-installation-token")) || bytes.Contains(data, []byte("PRIVATE KEY")) { os.Exit(5) } + var payload map[string]json.RawMessage + if json.Unmarshal(data, &payload) != nil { + os.Exit(6) + } + var argvBinding brokerPairedBinding + var payloadBinding brokerPairedBinding + bindingData, ok := payload["paired_binding"] + if !ok || decodeBrokerJSON(bindingData, &payloadBinding, true) != nil || !payloadBinding.valid() || decodeBrokerJSON([]byte(os.Args[11]), &argvBinding, true) != nil || payloadBinding != argvBinding { + os.Exit(7) + } + if journal, e := os.OpenFile(filepath.Join(os.Args[5], "journal.jsonl"), os.O_APPEND|os.O_WRONLY|syscall.O_NOFOLLOW, 0); e == nil { + _, _ = journal.WriteString("{\"paired_child\":true}\n") + _ = journal.Sync() + _ = journal.Close() + } + if strings.HasPrefix(argvBinding.ControllerApprovalSHA256, "e") { + fmt.Fprint(os.Stdout, strings.Repeat("synthetic-private-paired-overflow", 1000)) + os.Exit(0) + } + if strings.HasPrefix(argvBinding.ControllerApprovalSHA256, "f") { + time.Sleep(time.Minute) + } fmt.Fprintln(os.Stdout, "synthetic-private-paired-output") os.Exit(0) } @@ -124,10 +216,46 @@ func TestBrokerPairedTerminalPipeUsesFixedArgsAndOneControllerInput(t *testing.T t.Fatal(err) } data := []byte(`{"installation_token":"synthetic-private-installation-token"}`) - if err := invokeBrokerPairedTerminal(context.Background(), binary, root, filepath.Join(root, "approval.json"), root, filepath.Join(root, "worker.json"), workerState, data); err != nil { + binding := brokerPairedBinding{ControllerApprovalSHA256: strings.Repeat("a", 64), ControllerApprovalDevice: 1, ControllerApprovalInode: 2, ControllerStateDevice: 1, ControllerStateInode: 3, WorkerApprovalSHA256: strings.Repeat("b", 64), WorkerApprovalDevice: 1, WorkerApprovalInode: 4, WorkerStateDevice: 1, WorkerStateInode: 5} + if err := invokeBrokerPairedTerminal(context.Background(), binary, root, filepath.Join(root, "approval.json"), root, filepath.Join(root, "worker.json"), workerState, binding, data); err != nil { t.Fatal("private fixed paired terminal handoff failed") } } + +func TestBrokerPairedChildBoundsTimeoutOverflowAndCancel(t *testing.T) { + binary := testBrokerBinary(t) + root := t.TempDir() + data := []byte(`{"installation_token":"synthetic-private-installation-token"}`) + base := brokerPairedBinding{ControllerApprovalDevice: 1, ControllerApprovalInode: 2, ControllerStateDevice: 1, ControllerStateInode: 3, WorkerApprovalDevice: 1, WorkerApprovalInode: 4, WorkerStateDevice: 1, WorkerStateInode: 5} + for _, tc := range []struct { + name string + prefix string + ctx func() (context.Context, context.CancelFunc) + }{ + {name: "overflow", prefix: "e", ctx: func() (context.Context, context.CancelFunc) { + return context.WithTimeout(context.Background(), time.Second) + }}, + {name: "timeout", prefix: "f", ctx: func() (context.Context, context.CancelFunc) { + return context.WithTimeout(context.Background(), 100*time.Millisecond) + }}, + {name: "cancel", prefix: "a", ctx: func() (context.Context, context.CancelFunc) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + return ctx, func() {} + }}, + } { + t.Run(tc.name, func(t *testing.T) { + binding := base + binding.ControllerApprovalSHA256 = tc.prefix + strings.Repeat("a", 63) + binding.WorkerApprovalSHA256 = strings.Repeat("b", 64) + ctx, cancel := tc.ctx() + defer cancel() + if err := invokeBrokerPairedTerminal(ctx, binary, root, filepath.Join(root, "approval.json"), root, filepath.Join(root, "worker.json"), filepath.Join(root, "worker-state"), binding, data); err == nil { + t.Fatal("paired child bound failure accepted") + } + }) + } +} func TestBrokerChildBoundsAndReplacedBinaryRefuse(t *testing.T) { binary := testBrokerBinary(t) root := t.TempDir() From a0df276793ea5e7b956e15cc06c866cf869218df Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 17:16:50 +0900 Subject: [PATCH 04/37] test(g01): capture paired prerequisite history contract --- .../livecanary/preparation_test.go | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/experiments/g01-scaleset/livecanary/preparation_test.go b/experiments/g01-scaleset/livecanary/preparation_test.go index b044a93..2ca0ee0 100644 --- a/experiments/g01-scaleset/livecanary/preparation_test.go +++ b/experiments/g01-scaleset/livecanary/preparation_test.go @@ -73,6 +73,101 @@ func TestPairedPreparationUsesDedicatedPhaseWithoutCleanupAuthority(t *testing.T } } +func TestPairedPreparationPreservesCanonicalControllerHistory(t *testing.T) { + parent := privateDir(t) + directory := admissionState(t, parent, "paired-history") + capRoot := testAdmissionDirectory(t, directory) + a := approval() + open := func(path string, a Approval) (*FileJournal, error) { + return openJournalAtAdmission(path, a, capRoot, func(f *os.File) error { return f.Sync() }) + } + j, err := open(directory, a) + if err != nil { + t.Fatal("canonical controller journal") + } + for _, event := range []Event{ + {Kind: "phase", Operation: "create"}, + {Kind: "inventory", Digest: strings.Repeat("a", 64)}, + {Kind: "intent", Operation: "create"}, + {Kind: "result", Operation: "create", ID: 7}, + } { + if err := j.Append(event); err != nil { + t.Fatalf("canonical prerequisite event %q: %v", event.Operation, err) + } + } + if err := j.Close(); err != nil { + t.Fatal("close canonical controller journal") + } + before, err := os.ReadFile(filepath.Join(directory, "journal.jsonl")) + if err != nil { + t.Fatal("read prerequisite journal") + } + receipt, err := preparePairedJournal(directory, a, open) + if err != nil || receipt.Phase != pairedPreparationPhase { + t.Fatalf("valid canonical controller history refused: receipt=%+v err=%v", receipt, err) + } + after, err := os.ReadFile(filepath.Join(directory, "journal.jsonl")) + if err != nil { + t.Fatal("read prepared journal") + } + if string(before) != string(after) { + t.Fatal("paired preparation changed prerequisite controller history") + } + reopened, err := open(directory, a) + if err != nil { + t.Fatal("reopen prepared controller journal") + } + defer reopened.Close() + if events := reopened.Events(); len(events) != 4 || events[1].Kind != "inventory" || events[3].Operation != "create" || events[3].ID != 7 { + t.Fatalf("paired preparation did not preserve canonical events: %+v", events) + } +} + +func TestPairedPreparationRejectsFreshAndNonCanonicalHistory(t *testing.T) { + for _, kind := range []string{"fresh", "pending", "deleted", "previous-paired"} { + t.Run(kind, func(t *testing.T) { + parent := privateDir(t) + directory := admissionState(t, parent, "paired-"+kind) + capRoot := testAdmissionDirectory(t, directory) + a := approval() + open := func(path string, a Approval) (*FileJournal, error) { + return openJournalAtAdmission(path, a, capRoot, func(f *os.File) error { return f.Sync() }) + } + j, err := open(directory, a) + if err != nil { + t.Fatal("fixture journal") + } + switch kind { + case "pending": + _ = j.Append(Event{Kind: "phase", Operation: "create"}) + _ = j.Append(Event{Kind: "inventory", Digest: strings.Repeat("a", 64)}) + _ = j.Append(Event{Kind: "intent", Operation: "create"}) + case "deleted": + _ = j.Append(Event{Kind: "phase", Operation: "create"}) + _ = j.Append(Event{Kind: "inventory", Digest: strings.Repeat("a", 64)}) + _ = j.Append(Event{Kind: "intent", Operation: "create"}) + _ = j.Append(Event{Kind: "result", Operation: "create", ID: 7}) + _ = j.Append(Event{Kind: "intent", Operation: "delete"}) + _ = j.Append(Event{Kind: "result", Operation: "delete"}) + case "previous-paired": + // Baseline records are rejected by replayBaseline before preparation + // can issue a receipt, and must never be treated as controller setup. + _ = j.Append(Event{Kind: "phase", Operation: "create"}) + _ = j.Append(Event{Kind: "inventory", Digest: strings.Repeat("a", 64)}) + _ = j.Append(Event{Kind: "intent", Operation: "create"}) + _ = j.Append(Event{Kind: "result", Operation: "create", ID: 7}) + _ = j.Append(Event{Kind: "baseline", Baseline: &baselineRecord{Version: 1, Stage: "pair", Outcome: "intent", SetID: 7}}) + } + if err := j.Close(); err != nil { + t.Fatal("close fixture journal") + } + if _, err := preparePairedJournal(directory, a, open); err == nil { + t.Fatal("non-canonical paired preparation state accepted") + } + }) + } +} + func TestCanonicalPreparationRefusesInvalidJournalAndPhase(t *testing.T) { for _, kind := range []string{"locked", "malformed", "oversized", "permission", "authority", "seen phase", "unknown", "deleted", "unapproved"} { t.Run(kind, func(t *testing.T) { From 278d8e9b93354444f680b0be6c990e6acc0075ca Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 17:18:41 +0900 Subject: [PATCH 05/37] test(g01): cover paired admission history and bounds --- experiments/g02-auth/broker_admission_test.go | 50 +++++++++++++++++++ experiments/g02-auth/broker_paired_test.go | 43 ++++++++++++++++ 2 files changed, 93 insertions(+) diff --git a/experiments/g02-auth/broker_admission_test.go b/experiments/g02-auth/broker_admission_test.go index 86f42f4..85ac235 100644 --- a/experiments/g02-auth/broker_admission_test.go +++ b/experiments/g02-auth/broker_admission_test.go @@ -105,6 +105,56 @@ func TestBrokerFinitePhasesAndUnknownRetention(t *testing.T) { }) } } + +func TestBrokerPairedAdmissionAcceptsHistoricalControllerClaim(t *testing.T) { + a, c, api, f, root := newBrokerFixture(t) + parent := filepath.Dir(root) + a.Mode, a.Phase, a.AllowVerificationAuthority = "controller", "create", true + controllerPlan := brokerTestPlan(t, &a, parent, func(context.Context, []byte, string) error { return nil }) + controllerPlan.controller.Phases = []string{"create", "before-ack", "after-ack", "before-acquire", "acquire-loss", "inspect", "cleanup"} + controllerPlan.controller.WorkflowRunID = 7 + controllerPlan.raw, _ = json.Marshal(controllerPlan.controller) + a.ControllerApprovalSHA256 = brokerBytesDigest(controllerPlan.raw) + controllerPlan.approval = a + if _, err := brokerExecute(context.Background(), a, brokerInput{PEM: string(c.PEM), VerificationToken: "synthetic-private-workflow-token"}, root, api, controllerPlan); err != nil { + t.Fatalf("controller prerequisite claim: %v", err) + } + + // The paired attempt reuses the same controller identity and durable ledger, + // as the real controller-create -> paired-terminal sequence does. + a.Mode, a.Phase = "paired-terminal", "paired-terminal" + pairedRoot := filepath.Join(parent, "paired-attempt") + pairedPlan := brokerTestPlan(t, &a, parent, func(context.Context, []byte, string) error { return nil }) + pairedPlan.controller.Phases = append([]string(nil), controllerPlan.controller.Phases...) + pairedPlan.controller.WorkflowRunID = controllerPlan.controller.WorkflowRunID + pairedPlan.raw, _ = json.Marshal(pairedPlan.controller) + a.ControllerApprovalSHA256 = brokerBytesDigest(pairedPlan.raw) + pairedPlan.approval = a + workerState := filepath.Join(parent, "paired-worker-state") + if err := os.Mkdir(workerState, 0700); err != nil { + t.Fatal("worker state") + } + worker := pairedWorkerApproval{RunnerUpdatesDisabled: true, HarnessSHA: pairedPlan.controller.HarnessSHA, WorkflowSHA: pairedPlan.controller.WorkflowSHA, OwnerNonce: pairedPlan.controller.OwnerNonce, Controller: pairedPlan.controller.Controller, Endpoint: "/tmp/g01-paired-admission.sock", DaemonID: "fixture-daemon", ImageID: "sha256:" + strings.Repeat("d", 64), Image: pairedWorkerImage, ExpiresAt: pairedPlan.controller.ExpiresAt, Phases: []string{"create", "start", "inspect", "cleanup"}} + workerData, err := json.Marshal(worker) + if err != nil { + t.Fatal("worker approval") + } + workerPath := filepath.Join(parent, "paired-worker-approval.json") + if err := os.WriteFile(workerPath, workerData, 0600); err != nil { + t.Fatal("worker approval file") + } + pairedPlan.worker, err = openBrokerWorkerPlan(workerPath, workerState, pairedPlan.statePath, a, pairedPlan.controller) + if err != nil { + t.Fatalf("paired worker plan: %v", err) + } + defer pairedPlan.worker.close() + if _, err := brokerExecute(context.Background(), a, brokerInput{PEM: string(c.PEM), VerificationToken: "synthetic-private-workflow-token"}, pairedRoot, api, pairedPlan); err != nil { + t.Fatalf("paired attempt rejected historical controller claim: %v", err) + } + if f.tokenCalls != 2 { + t.Fatalf("historical controller claim blocked current paired issuance: mints=%d", f.tokenCalls) + } +} func TestBrokerAdmissionFailureBeforeAPIAndResync(t *testing.T) { for _, kind := range []string{"missing", "symlink", "sync", "existing-sync"} { t.Run(kind, func(t *testing.T) { diff --git a/experiments/g02-auth/broker_paired_test.go b/experiments/g02-auth/broker_paired_test.go index df4da1f..0ef7ed8 100644 --- a/experiments/g02-auth/broker_paired_test.go +++ b/experiments/g02-auth/broker_paired_test.go @@ -89,6 +89,49 @@ func TestPairedWorkerApprovalMismatchRefusesBeforeBinding(t *testing.T) { } } +func TestPairedWorkerDaemonIDMatchesCanonicalBoundaries(t *testing.T) { + for _, tc := range []struct { + name string + daemon string + valid bool + }{ + {name: "colon and 128 bytes", daemon: "a:" + strings.Repeat("d", 126), valid: true}, + {name: "129 bytes", daemon: "a" + strings.Repeat("d", 128), valid: false}, + {name: "invalid slash", daemon: "a/b", valid: false}, + {name: "invalid leading punctuation", daemon: ":daemon", valid: false}, + } { + t.Run(tc.name, func(t *testing.T) { + a, c, controllerState, workerState, workerPath := pairedPlanInputs(t) + data, err := os.ReadFile(workerPath) + if err != nil { + t.Fatal(err) + } + var worker pairedWorkerApproval + if err := decodeBrokerJSON(data, &worker, true); err != nil { + t.Fatal(err) + } + worker.DaemonID = tc.daemon + data, err = json.Marshal(worker) + if err != nil || os.WriteFile(workerPath, data, 0600) != nil { + t.Fatal("worker approval rewrite") + } + _, err = openBrokerWorkerPlan(workerPath, workerState, controllerState, a, c) + if (err == nil) != tc.valid { + t.Fatalf("daemon ID validity=%v want=%v: %v", err == nil, tc.valid, err) + } + }) + } +} + +func TestPairedApprovalRejectsInsufficientTerminalAuthority(t *testing.T) { + a := brokerApprovalFixture() + a.Mode, a.Phase = "paired-terminal", "paired-terminal" + a.ExpiresAt = time.Now().Add(90 * time.Second) + if err := a.validate(time.Now()); err == nil { + t.Fatal("paired approval accepted less than the bounded terminal completion budget") + } +} + func TestPairedBrokerBindsWorkerBeforeWorkflowVerifiedHandoff(t *testing.T) { a, candidate, api, fixture, attempt := newBrokerFixture(t) a.Mode, a.Phase, a.AllowVerificationAuthority = "paired-terminal", "paired-terminal", true From 419f9cd583c13d9e9e7657dc39af24fd6da14c49 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 18:01:25 +0900 Subject: [PATCH 06/37] fix(g01): preserve paired prerequisites and bound terminal authority --- .../cmd/g01-live/fixture_support.go | 70 ++ experiments/g01-scaleset/cmd/g01-live/main.go | 17 +- .../g01-scaleset/cmd/g01-live/main_test.go | 4 +- .../livecanary/paired_fixture_runtime.go | 150 ++++ .../g01-scaleset/livecanary/preparation.go | 58 +- .../livecanary/preparation_test.go | 27 +- .../liveworker/paired_fixture_admission.go | 2 +- experiments/g02-auth/broker.go | 62 +- experiments/g02-auth/broker_admission.go | 31 +- experiments/g02-auth/broker_admission_test.go | 18 + .../g02-auth/broker_paired_bridge_test.go | 801 ++++++++++++++++++ experiments/g02-auth/broker_plan.go | 2 +- experiments/g02-auth/broker_process.go | 20 +- experiments/g02-auth/broker_process_test.go | 21 + 14 files changed, 1255 insertions(+), 28 deletions(-) create mode 100644 experiments/g01-scaleset/cmd/g01-live/fixture_support.go create mode 100644 experiments/g01-scaleset/livecanary/paired_fixture_runtime.go create mode 100644 experiments/g02-auth/broker_paired_bridge_test.go diff --git a/experiments/g01-scaleset/cmd/g01-live/fixture_support.go b/experiments/g01-scaleset/cmd/g01-live/fixture_support.go new file mode 100644 index 0000000..b9a646d --- /dev/null +++ b/experiments/g01-scaleset/cmd/g01-live/fixture_support.go @@ -0,0 +1,70 @@ +//go:build g01_live && g01_pair_fixture + +package main + +import ( + "context" + "io" + "os" + "path/filepath" + "syscall" + + "github.com/1XP-AI/gh-runnerd/experiments/g01-scaleset/livecanary" +) + +type fixtureEndpointConfig struct { + BaseURL string `json:"base_url"` + CAPEM string `json:"ca_pem"` + AdmissionDirectory string `json:"admission_directory"` +} + +func readFixtureEndpointConfig(stateDirectory string) (fixtureEndpointConfig, error) { + var config fixtureEndpointConfig + path := filepath.Join(stateDirectory, "paired-fixture.json") + file, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NOFOLLOW, 0) + if err != nil { + return config, livecanary.ErrApproval + } + defer file.Close() + info, err := file.Stat() + stat, ok := info.Sys().(*syscall.Stat_t) + if err != nil || !ok || int(stat.Uid) != os.Geteuid() || info.Mode().Perm() != 0600 || !info.Mode().IsRegular() || stat.Nlink != 1 || info.Size() > 8192 { + return config, livecanary.ErrApproval + } + data, err := io.ReadAll(io.LimitReader(file, 8193)) + if err != nil || len(data) > 8192 || livecanary.DecodeStrict(data, &config) != nil || config.BaseURL == "" || config.CAPEM == "" || config.AdmissionDirectory == "" || !filepath.IsAbs(config.AdmissionDirectory) || filepath.Clean(config.AdmissionDirectory) != config.AdmissionDirectory { + return fixtureEndpointConfig{}, livecanary.ErrApproval + } + return config, nil +} + +func init() { + openJournalForCommand = func(stateDirectory string, a livecanary.Approval) (*livecanary.FileJournal, error) { + config, err := readFixtureEndpointConfig(stateDirectory) + if err != nil { + return nil, err + } + return livecanary.OpenJournalForPairedFixtureAt(stateDirectory, a, config.AdmissionDirectory) + } + pairedPrepareJournalForCommand = func(stateDirectory string, a livecanary.Approval) (livecanary.PreparationReceipt, error) { + config, err := readFixtureEndpointConfig(stateDirectory) + if err != nil { + return livecanary.PreparationReceipt{}, err + } + return livecanary.PreparePairedJournalForFixtureAt(stateDirectory, a, config.AdmissionDirectory) + } + newSDKAPIForCommand = func(a livecanary.Approval, c livecanary.Credentials, stateDirectory string) (*livecanary.SDKAPI, error) { + config, err := readFixtureEndpointConfig(stateDirectory) + if err != nil { + return nil, err + } + return livecanary.NewSDKAPIForPairedFixture(a, c, config.BaseURL, []byte(config.CAPEM)) + } + runPairedTerminalForCommand = func(ctx context.Context, files livecanary.PairedTerminalFiles, c livecanary.Credentials) error { + config, err := readFixtureEndpointConfig(files.ControllerStateDirectory) + if err != nil { + return err + } + return livecanary.RunPairedTerminalForFixture(ctx, files, c, config.BaseURL, []byte(config.CAPEM), config.AdmissionDirectory) + } +} diff --git a/experiments/g01-scaleset/cmd/g01-live/main.go b/experiments/g01-scaleset/cmd/g01-live/main.go index bbd3ced..76d41b6 100644 --- a/experiments/g01-scaleset/cmd/g01-live/main.go +++ b/experiments/g01-scaleset/cmd/g01-live/main.go @@ -41,6 +41,15 @@ func buildRevision() (string, bool) { return version, clean && sdk && len(version) == 40 } +var openJournalForCommand = livecanary.OpenJournal +var pairedPrepareJournalForCommand = livecanary.PreparePairedJournal +var newSDKAPIForCommand = func(a livecanary.Approval, c livecanary.Credentials, _ string) (*livecanary.SDKAPI, error) { + return livecanary.NewSDKAPI(a, c) +} +var runPairedTerminalForCommand = func(ctx context.Context, files livecanary.PairedTerminalFiles, c livecanary.Credentials) error { + return livecanary.RunPairedTerminal(ctx, files, c) +} + func run(args []string, in io.Reader, out io.Writer) int { return runWithPreparation(args, in, out, buildRevision, livecanary.PrepareJournal) } @@ -129,7 +138,7 @@ func runWithPreparation(args []string, in io.Reader, out io.Writer, revisionForB } } if *pairedPrepare { - receipt, e := livecanary.PreparePairedJournal(*statePath, a) + receipt, e := pairedPrepareJournalForCommand(*statePath, a) if e != nil || json.NewEncoder(out).Encode(receipt) != nil { return reject() } @@ -150,7 +159,7 @@ func runWithPreparation(args []string, in io.Reader, out io.Writer, revisionForB } var j *livecanary.FileJournal if !*pairedExecute { - j, err = livecanary.OpenJournal(*statePath, a) + j, err = openJournalForCommand(*statePath, a) if err != nil { return reject() } @@ -179,14 +188,14 @@ func runWithPreparation(args []string, in io.Reader, out io.Writer, revisionForB if credentials.PairedBinding == nil || *credentials.PairedBinding != binding { return reject() } - if livecanary.RunPairedTerminal(context.Background(), livecanary.PairedTerminalFiles{ControllerApprovalPath: *approvalPath, ControllerStateDirectory: *statePath, WorkerApprovalPath: *workerApprovalPath, WorkerStateDirectory: *workerStatePath}, credentials) != nil { + if runPairedTerminalForCommand(context.Background(), livecanary.PairedTerminalFiles{ControllerApprovalPath: *approvalPath, ControllerStateDirectory: *statePath, WorkerApprovalPath: *workerApprovalPath, WorkerStateDirectory: *workerStatePath}, credentials) != nil { fmt.Fprintln(out, "paired terminal stopped; retain private state and all uncertain resources; no automatic retry") return 1 } fmt.Fprintln(out, "paired terminal completed; inspect private evidence") return 0 } - api, err := livecanary.NewSDKAPI(a, credentials) + api, err := newSDKAPIForCommand(a, credentials, *statePath) if err != nil { return reject() } diff --git a/experiments/g01-scaleset/cmd/g01-live/main_test.go b/experiments/g01-scaleset/cmd/g01-live/main_test.go index 97804a0..2e34538 100644 --- a/experiments/g01-scaleset/cmd/g01-live/main_test.go +++ b/experiments/g01-scaleset/cmd/g01-live/main_test.go @@ -28,8 +28,8 @@ func (r unreadable) Read([]byte) (int, error) { type countedInput struct { io.Reader - reads int - eofs int + reads int + eofs int readsAfterEOF int } diff --git a/experiments/g01-scaleset/livecanary/paired_fixture_runtime.go b/experiments/g01-scaleset/livecanary/paired_fixture_runtime.go new file mode 100644 index 0000000..3592797 --- /dev/null +++ b/experiments/g01-scaleset/livecanary/paired_fixture_runtime.go @@ -0,0 +1,150 @@ +//go:build g01_pair_fixture + +package livecanary + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "log/slog" + "net" + "net/http" + "net/url" + "os" + "path/filepath" + "time" + + "github.com/actions/scaleset" + "github.com/hashicorp/go-retryablehttp" + + "github.com/1XP-AI/gh-runnerd/experiments/g01-scaleset/liveworker" +) + +// These constructors are only present in the explicitly tagged offline +// bridge fixture. The ordinary binary cannot select a caller-provided API +// endpoint or journal admission root. +func OpenJournalForPairedFixture(directory string, a Approval) (*FileJournal, error) { + return OpenJournalForPairedFixtureAt(directory, a, directory) +} + +func OpenJournalForPairedFixtureAt(directory string, a Approval, admissionDirectory string) (*FileJournal, error) { + return openJournalAtAdmission(directory, a, admissionDirectory, func(file *os.File) error { return file.Sync() }) +} + +func PreparePairedJournalForFixture(directory string, a Approval) (PreparationReceipt, error) { + return preparePairedJournal(directory, a, OpenJournalForPairedFixture) +} + +func PreparePairedJournalForFixtureAt(directory string, a Approval, admissionDirectory string) (PreparationReceipt, error) { + return preparePairedJournal(directory, a, func(path string, approval Approval) (*FileJournal, error) { + return OpenJournalForPairedFixtureAt(path, approval, admissionDirectory) + }) +} + +func fixtureEndpoint(baseURL string, caPEM []byte) (*url.URL, *x509.CertPool, *x509.Certificate, error) { + u, err := url.Parse(baseURL) + if err != nil || u.Scheme != "https" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || u.Path != "" { + return nil, nil, nil, ErrApproval + } + host := u.Hostname() + if net.ParseIP(host) == nil || !net.ParseIP(host).IsLoopback() || u.Port() == "" { + return nil, nil, nil, ErrApproval + } + block, _ := pem.Decode(caPEM) + if block == nil || block.Type != "CERTIFICATE" || len(block.Bytes) == 0 || len(caPEM) > 8192 { + return nil, nil, nil, ErrApproval + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, nil, nil, ErrApproval + } + pool := x509.NewCertPool() + pool.AddCert(cert) + return u, pool, cert, nil +} + +func NewSDKAPIForPairedFixture(a Approval, c Credentials, baseURL string, caPEM []byte) (*SDKAPI, error) { + u, roots, certificate, err := fixtureEndpoint(baseURL, caPEM) + if err != nil || a.Validate(time.Now()) != nil || c.validate(a, time.Now()) != nil { + return nil, ErrApproval + } + serverName := u.Hostname() + if err := certificate.VerifyHostname(serverName); err != nil { + if len(certificate.DNSNames) == 0 { + return nil, ErrApproval + } + serverName = certificate.DNSNames[0] + } + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.Protocols = new(http.Protocols) + transport.Protocols.SetHTTP1(true) + transport.Proxy = nil + transport.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: roots, ServerName: serverName, NextProtos: []string{"http/1.1"}} + address := u.Host + transport.DialContext = func(ctx context.Context, network, target string) (net.Conn, error) { + if network != "tcp" || target != address { + return nil, ErrApproval + } + return (&net.Dialer{Timeout: 10 * time.Second}).DialContext(ctx, network, target) + } + retry := retryablehttp.NewClient() + retry.RetryMax = 0 + retry.Logger = nil + httpClient := &http.Client{Transport: withResponseBudget(transport), Timeout: operationTimeout, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + retry.HTTPClient = httpClient + options := []scaleset.HTTPOption{scaleset.WithRetryableHTTPClint(retry), scaleset.WithLogger(slog.New(slog.DiscardHandler))} + client, err := scaleset.NewClientWithPersonalAccessToken(scaleset.NewClientWithPersonalAccessTokenConfig{GitHubConfigURL: baseURL + "/" + a.Organization, PersonalAccessToken: c.InstallationToken}, options...) + if err != nil { + return nil, ErrApproval + } + return &SDKAPI{client: client, rest: httpClient, baseURL: baseURL, approval: a, credentials: c, options: options}, nil +} + +func fixtureFastPairCadence() pairedBaselineCadence { + now := time.Now() + return pairedBaselineCadence{ + now: func() time.Time { return now }, + wait: func(ctx context.Context, delay time.Duration) error { + if ctx.Err() != nil { + return ErrQuarantine + } + now = now.Add(delay) + return nil + }, + } +} + +// RunPairedTerminalForFixture calls the exported production entrypoint with +// only generated private roots and a loopback TLS endpoint. The fast cadence +// is a test-only clock seam; it still retains every production HTTP, journal, +// binding, and effect ordering check. +func RunPairedTerminalForFixture(ctx context.Context, files PairedTerminalFiles, c Credentials, baseURL string, caPEM []byte, controllerAdmissionDirectory string) error { + if ctx == nil || filepath.Clean(files.ControllerStateDirectory) != files.ControllerStateDirectory || filepath.Clean(files.WorkerStateDirectory) != files.WorkerStateDirectory { + return ErrApproval + } + if _, _, _, err := fixtureEndpoint(baseURL, caPEM); err != nil { + return err + } + if !filepath.IsAbs(controllerAdmissionDirectory) || filepath.Clean(controllerAdmissionDirectory) != controllerAdmissionDirectory { + return ErrApproval + } + oldAdapters, oldCadence := pairedTerminalFixtureAdapters, pairedTerminalFixtureCadence + pairedTerminalFixtureAdapters = &pairedTerminalAdapters{ + openController: func(path string, a Approval) (*FileJournal, error) { + return OpenJournalForPairedFixtureAt(path, a, controllerAdmissionDirectory) + }, + openWorker: func(path string, a liveworker.Approval) (*liveworker.FileJournal, error) { + return liveworker.OpenJournalForPairedFixture(path, a, path) + }, + newAPI: func(a Approval, credentials Credentials) (*SDKAPI, error) { + return NewSDKAPIForPairedFixture(a, credentials, baseURL, caPEM) + }, + newDocker: liveworker.NewDocker, + } + pairedTerminalFixtureCadence = fixtureFastPairCadence + defer func() { + pairedTerminalFixtureAdapters, pairedTerminalFixtureCadence = oldAdapters, oldCadence + }() + return RunPairedTerminal(ctx, files, c) +} diff --git a/experiments/g01-scaleset/livecanary/preparation.go b/experiments/g01-scaleset/livecanary/preparation.go index 1b4d4c3..1d884c8 100644 --- a/experiments/g01-scaleset/livecanary/preparation.go +++ b/experiments/g01-scaleset/livecanary/preparation.go @@ -82,9 +82,10 @@ func PrepareJournal(directory string, a Approval, phase string) (PreparationRece } // PreparePairedJournal is the paired terminal's explicit local preparation -// contract. It proves a fresh controller journal and its admission claim under -// the controller authority; it does not borrow cleanup authority and never -// reads credentials, worker input or contacts a remote service. +// contract. It proves the already-completed controller create prerequisite and +// its admission claim under controller authority; it does not borrow cleanup +// authority and never reads credentials, worker input or contacts a remote +// service. func PreparePairedJournal(directory string, a Approval) (PreparationReceipt, error) { return preparePairedJournal(directory, a, OpenJournal) } @@ -106,6 +107,54 @@ func pairedPreparationReady(a Approval, now time.Time) bool { return verification && want["create"] && want["inspect"] && want["cleanup"] } +// pairedControllerPrerequisite accepts only the canonical controller-create +// prefix consumed by newBaselineListenerHeld. The paired preparation phase is +// deliberately not a second create/cleanup authority: it may inspect the +// completed create and inventory records, but it cannot reset, discard or +// append to them. Authority renewal records are structural and are ignored by +// this prefix parser; every other event must be one of the exact create +// protocol records below. +func pairedControllerPrerequisite(events []Event) (state, error) { + s := replay(events) + if s.uncertain || s.deleted || s.reserved || s.workObserved || s.setID <= 0 || s.inventory == "" { + return state{}, ErrQuarantine + } + phase, inventory, discoveryIntent, discoveryResult, createIntent, createResult := 0, 0, 0, 0, 0, 0 + phaseAt, inventoryAt, discoveryIntentAt, discoveryResultAt, createIntentAt, createResultAt := 0, 0, 0, 0, 0, 0 + for index, e := range events { + if e.Kind == "authority" { + continue + } + position := index + 1 + switch { + case e.Kind == "phase" && e.Operation == "create": + phase++ + phaseAt = position + case e.Kind == "inventory" && len(e.Digest) == 64 && isLowerHex(e.Digest): + inventory++ + inventoryAt = position + case e.Kind == "intent" && e.Operation == "observe-discovery": + discoveryIntent++ + discoveryIntentAt = position + case e.Kind == "result" && e.Operation == "observe-discovery" && e.Work == "": + discoveryResult++ + discoveryResultAt = position + case e.Kind == "intent" && e.Operation == "create": + createIntent++ + createIntentAt = position + case e.Kind == "result" && e.Operation == "create" && e.ID > 0 && e.Work == "": + createResult++ + createResultAt = position + default: + return state{}, ErrQuarantine + } + } + if phase != 1 || inventory != 1 || discoveryIntent != 1 || discoveryResult != 1 || createIntent != 1 || createResult != 1 || !(phaseAt < inventoryAt && inventoryAt < discoveryIntentAt && discoveryIntentAt < discoveryResultAt && discoveryResultAt < createIntentAt && createIntentAt < createResultAt) { + return state{}, ErrQuarantine + } + return s, nil +} + func preparePairedJournal(directory string, a Approval, open func(string, Approval) (*FileJournal, error)) (receipt PreparationReceipt, err error) { if !pairedPreparationReady(a, time.Now()) || open == nil { return receipt, ErrApproval @@ -125,8 +174,7 @@ func preparePairedJournal(directory string, a Approval, open func(string, Approv return receipt, ErrJournal } defer release() - s := replay(j.Events()) - if s.uncertain || s.deleted || s.setID != 0 || s.reserved || s.workObserved || len(j.Events()) != 0 { + if _, e = pairedControllerPrerequisite(j.Events()); e != nil { return receipt, ErrQuarantine } jd, e := preparedDigest(j.file, 1<<20) diff --git a/experiments/g01-scaleset/livecanary/preparation_test.go b/experiments/g01-scaleset/livecanary/preparation_test.go index 2ca0ee0..79a711d 100644 --- a/experiments/g01-scaleset/livecanary/preparation_test.go +++ b/experiments/g01-scaleset/livecanary/preparation_test.go @@ -53,16 +53,35 @@ func TestPairedPreparationUsesDedicatedPhaseWithoutCleanupAuthority(t *testing.T open := func(path string, a Approval) (*FileJournal, error) { return openJournalAtAdmission(path, a, capRoot, func(f *os.File) error { return f.Sync() }) } + j, err := open(directory, a) + if err != nil { + t.Fatal("canonical controller journal") + } + for _, event := range []Event{ + {Kind: "phase", Operation: "create"}, + {Kind: "inventory", Digest: strings.Repeat("a", 64)}, + {Kind: "intent", Operation: "observe-discovery"}, + {Kind: "result", Operation: "observe-discovery"}, + {Kind: "intent", Operation: "create"}, + {Kind: "result", Operation: "create", ID: 7}, + } { + if err := j.Append(event); err != nil { + t.Fatalf("canonical prerequisite event %q: %v", event.Operation, err) + } + } + if err := j.Close(); err != nil { + t.Fatal("close canonical controller journal") + } receipt, err := preparePairedJournal(directory, a, open) if err != nil || receipt.Phase != pairedPreparationPhase || receipt.Status != "controller_journal_prepared" { t.Fatalf("paired preparation did not produce its own receipt: receipt=%+v err=%v", receipt, err) } - j, err := open(directory, a) + j, err = open(directory, a) if err != nil { t.Fatal("paired journal reopen") } defer j.Close() - if len(j.Events()) != 0 { + if len(j.Events()) != 6 { t.Fatal("paired preparation borrowed cleanup authority or recorded an effect") } withoutVerification := a @@ -88,6 +107,8 @@ func TestPairedPreparationPreservesCanonicalControllerHistory(t *testing.T) { for _, event := range []Event{ {Kind: "phase", Operation: "create"}, {Kind: "inventory", Digest: strings.Repeat("a", 64)}, + {Kind: "intent", Operation: "observe-discovery"}, + {Kind: "result", Operation: "observe-discovery"}, {Kind: "intent", Operation: "create"}, {Kind: "result", Operation: "create", ID: 7}, } { @@ -118,7 +139,7 @@ func TestPairedPreparationPreservesCanonicalControllerHistory(t *testing.T) { t.Fatal("reopen prepared controller journal") } defer reopened.Close() - if events := reopened.Events(); len(events) != 4 || events[1].Kind != "inventory" || events[3].Operation != "create" || events[3].ID != 7 { + if events := reopened.Events(); len(events) != 6 || events[1].Kind != "inventory" || events[5].Operation != "create" || events[5].ID != 7 { t.Fatalf("paired preparation did not preserve canonical events: %+v", events) } } diff --git a/experiments/g01-scaleset/liveworker/paired_fixture_admission.go b/experiments/g01-scaleset/liveworker/paired_fixture_admission.go index debb152..2cb8ccc 100644 --- a/experiments/g01-scaleset/liveworker/paired_fixture_admission.go +++ b/experiments/g01-scaleset/liveworker/paired_fixture_admission.go @@ -1,4 +1,4 @@ -//go:build g01_pair_fixture && !g01_live && !g01_worker +//go:build g01_pair_fixture package liveworker diff --git a/experiments/g02-auth/broker.go b/experiments/g02-auth/broker.go index 498f91d..d509bdc 100644 --- a/experiments/g02-auth/broker.go +++ b/experiments/g02-auth/broker.go @@ -48,10 +48,28 @@ type BrokerResult struct { var errBroker = errors.New("broker stopped; retain private intent and review; no automatic retry") var brokerComponent = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]{0,99}$`) +var brokerWorkerComponent = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$`) var brokerPhases = map[string]bool{"create": true, "before-ack": true, "after-ack": true, "before-acquire": true, "acquire-loss": true, "jit-loss": true, "inspect": true, "cleanup": true} +const ( + // Terminal collection has seven five-second cadence gaps in production. + // Keep a complete cadence plus a separate execution margin available to the + // child, and reserve one more minute for the short-lived installation token + // context that precedes it. + pairedTerminalCadenceBudget = 35 * time.Second + pairedTerminalChildMargin = 25 * time.Second + pairedTerminalMinimumChildBudget = pairedTerminalCadenceBudget + pairedTerminalChildMargin + pairedTerminalCredentialMargin = time.Minute + pairedTerminalMinimumAuthority = pairedTerminalMinimumChildBudget + pairedTerminalCredentialMargin + pairedTerminalMaximumChildBudget = 10 * time.Minute +) + func (a BrokerApproval) validate(now time.Time) error { - if !brokerNonce.MatchString(a.OwnerNonce) || a.AppID < 1 || a.AppOwnerID < 1 || a.InstallationID < 1 || a.OrganizationID < 1 || a.RepositoryID < 1 || a.RunnerGroupID < 1 || !appSlug.MatchString(a.AppName) || !organizationLogin.MatchString(a.AppOwner) || !organizationLogin.MatchString(a.Organization) || !brokerComponent.MatchString(a.Repository) || !brokerComponent.MatchString(a.RunnerGroupName) || !a.ExpiresAt.After(now.Add(time.Minute)) || a.ExpiresAt.After(now.Add(24*time.Hour)) { + minimumLifetime := time.Minute + if a.Mode == "paired-terminal" { + minimumLifetime = pairedTerminalMinimumAuthority + } + if !brokerNonce.MatchString(a.OwnerNonce) || a.AppID < 1 || a.AppOwnerID < 1 || a.InstallationID < 1 || a.OrganizationID < 1 || a.RepositoryID < 1 || a.RunnerGroupID < 1 || !appSlug.MatchString(a.AppName) || !organizationLogin.MatchString(a.AppOwner) || !organizationLogin.MatchString(a.Organization) || !brokerComponent.MatchString(a.Repository) || !brokerComponent.MatchString(a.RunnerGroupName) || !a.ExpiresAt.After(now.Add(minimumLifetime)) || a.ExpiresAt.After(now.Add(24*time.Hour)) { return errBroker } if a.Mode == "discover-actions-host" { @@ -82,11 +100,48 @@ func validBrokerToken(token string) bool { } return true } + +func brokerPairedAuthorityDeadline(parent context.Context, a BrokerApproval, plan *brokerControllerPlan, now time.Time) (time.Time, error) { + if parent == nil || parent.Err() != nil { + return time.Time{}, errBroker + } + deadline := a.ExpiresAt + if plan != nil { + if plan.controller.ExpiresAt.IsZero() { + return time.Time{}, errBroker + } + deadline = minTime(deadline, plan.controller.ExpiresAt) + if plan.worker != nil { + if plan.worker.approval.ExpiresAt.IsZero() { + return time.Time{}, errBroker + } + deadline = minTime(deadline, plan.worker.approval.ExpiresAt) + } + } + if parentDeadline, ok := parent.Deadline(); ok { + deadline = minTime(deadline, parentDeadline) + } + if !deadline.After(now.Add(pairedTerminalMinimumAuthority)) { + return time.Time{}, errBroker + } + return deadline, nil +} + func brokerExecute(parent context.Context, a BrokerApproval, input brokerInput, path string, api *brokerAPI, plan *brokerControllerPlan) (BrokerResult, error) { if parent == nil || api == nil || a.validate(api.now()) != nil || (a.AllowVerificationAuthority && input.VerificationToken == "") || (input.VerificationToken != "" && (!a.AllowVerificationAuthority || !validBrokerToken(input.VerificationToken))) || ((a.Mode == "controller" || a.Mode == "paired-terminal") && plan == nil) || (a.Mode != "controller" && a.Mode != "paired-terminal" && plan != nil) { return BrokerResult{}, errBroker } - ctx, cancel := context.WithDeadline(parent, minTime(a.ExpiresAt, api.now().Add(10*time.Minute))) + now := api.now() + deadline := minTime(a.ExpiresAt, now.Add(10*time.Minute)) + if a.Mode == "paired-terminal" { + var err error + deadline, err = brokerPairedAuthorityDeadline(parent, a, plan, now) + if err != nil { + return BrokerResult{}, errBroker + } + deadline = minTime(deadline, now.Add(10*time.Minute)) + } + ctx, cancel := context.WithDeadline(parent, deadline) defer cancel() candidate := Candidate{AppID: a.AppID, PEM: []byte(input.PEM)} defer clear(candidate.PEM) @@ -171,6 +226,9 @@ func brokerExecute(parent context.Context, a BrokerApproval, input brokerInput, if err != nil { return BrokerResult{}, errBroker } + if a.Mode == "paired-terminal" && !issued.ExpiresAt.After(api.now().Add(pairedTerminalMinimumAuthority)) { + return BrokerResult{}, errBroker + } tokenCtx, stopToken := context.WithDeadline(ctx, issued.ExpiresAt.Add(-time.Minute)) defer stopToken() ctx = tokenCtx diff --git a/experiments/g02-auth/broker_admission.go b/experiments/g02-auth/broker_admission.go index e9094bf..4f773da 100644 --- a/experiments/g02-auth/broker_admission.go +++ b/experiments/g02-auth/broker_admission.go @@ -397,26 +397,39 @@ func validBrokerClaimEvent(a BrokerApproval, e brokerClaimEvent) bool { return false } if paired { - if a.Mode != "paired-terminal" || e.Worker == nil || !brokerSHA256.MatchString(e.Worker.Approval) || e.Worker.ApprovalFile.Device == 0 || e.Worker.ApprovalFile.Inode == 0 || e.Worker.State.Device == 0 || e.Worker.State.Inode == 0 { + if e.Worker == nil || !brokerSHA256.MatchString(e.Worker.Approval) || e.Worker.ApprovalFile.Device == 0 || e.Worker.ApprovalFile.Inode == 0 || e.Worker.State.Device == 0 || e.Worker.State.Inode == 0 { return false } - } else if e.Worker != nil || a.Mode == "paired-terminal" { + } else if e.Worker != nil { return false } c := e.Authority.Approval if c.ExpiresAt.IsZero() || e.Authority.Digest != brokerDigest(c) { return false } + // Validate a historical event against the mode and slot it records. The + // current request may be reciprocal (for example, a paired retry after a + // controller create, or an inspect/cleanup after a failed paired claim), so + // using its mode here would incorrectly turn current authority into a + // prerequisite for replaying old ledger records. + eventApproval := a if paired { - a.Mode = "paired-terminal" + eventApproval.Mode = "paired-terminal" } else { - a.Mode = "controller" + eventApproval.Mode = "controller" } - a.Phase = e.Slot - a.ExpiresAt = c.ExpiresAt - a.ControllerHarnessSHA = e.Controller.Harness - a.AllowVerificationAuthority = c.needsVerification() - if c.validate(a, c.ExpiresAt.Add(-2*time.Minute)) != nil { + eventApproval.Phase = e.Slot + eventApproval.ExpiresAt = c.ExpiresAt + eventApproval.ControllerHarnessSHA = e.Controller.Harness + eventApproval.AllowVerificationAuthority = c.needsVerification() + validationNow := c.ExpiresAt.Add(-2 * time.Minute) + if paired { + // Paired approvals reserve a two-minute completion budget. Validate the + // historical authority at a point before that budget, rather than at the + // exact expiry boundary where the current-mode minimum would fail. + validationNow = c.ExpiresAt.Add(-pairedTerminalMinimumAuthority - time.Second) + } + if c.validate(eventApproval, validationNow) != nil { return false } c.ExpiresAt = time.Time{} diff --git a/experiments/g02-auth/broker_admission_test.go b/experiments/g02-auth/broker_admission_test.go index 85ac235..f53c7b7 100644 --- a/experiments/g02-auth/broker_admission_test.go +++ b/experiments/g02-auth/broker_admission_test.go @@ -154,6 +154,24 @@ func TestBrokerPairedAdmissionAcceptsHistoricalControllerClaim(t *testing.T) { if f.tokenCalls != 2 { t.Fatalf("historical controller claim blocked current paired issuance: mints=%d", f.tokenCalls) } + ledger, err := os.ReadFile(filepath.Join(f.admissionRoot, "broker-admission.jsonl")) + if err != nil { + t.Fatal("paired ledger") + } + lines := strings.Split(strings.TrimSpace(string(ledger)), "\n") + var pairedEvent brokerClaimEvent + for _, line := range lines { + var candidate brokerClaimEvent + if json.Unmarshal([]byte(line), &candidate) == nil && candidate.Slot == "paired-terminal" && candidate.Kind == "claim" { + pairedEvent = candidate + break + } + } + controllerView := a + controllerView.Mode, controllerView.Phase = "controller", "inspect" + if pairedEvent.Slot == "" || !validBrokerClaimEvent(controllerView, pairedEvent) { + t.Fatal("historical paired claim was rejected under the reciprocal controller mode") + } } func TestBrokerAdmissionFailureBeforeAPIAndResync(t *testing.T) { for _, kind := range []string{"missing", "symlink", "sync", "existing-sync"} { diff --git a/experiments/g02-auth/broker_paired_bridge_test.go b/experiments/g02-auth/broker_paired_bridge_test.go new file mode 100644 index 0000000..caf1528 --- /dev/null +++ b/experiments/g02-auth/broker_paired_bridge_test.go @@ -0,0 +1,801 @@ +package enrollment + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "encoding/pem" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "runtime" + "slices" + "strconv" + "strings" + "sync" + "syscall" + "testing" + "time" +) + +const pairedBridgeJIT = "c3ludGhldGljLXByaXZhdGUtaml0LWNvbmZpZw==" + +// pairedBrokerBridge is an offline-only private transport. It serves the +// controller's real REST/Actions calls over a generated TLS root and the +// worker's real Docker calls over a generated Unix socket. It retains only +// bounded counters and protocol state; request bodies and credentials are +// never recorded. +type pairedBrokerBridge struct { + mu sync.Mutex + server *httptest.Server + dockerServer *http.Server + dockerListener net.Listener + dockerEndpoint string + installationToken string + verificationToken string + adminToken string + queueToken string + organization string + repository string + workflowSHA string + workflowPath string + workflowRunID int64 + runnerGroupID int64 + setName string + workerName string + workerImage string + workerImageID string + workerDaemonID string + setExists bool + setCreated bool + setDeleted bool + container map[string]any + containerStarted bool + containerDeleted bool + polls int + lastAck int + unexpected int + createCalls int + startCalls int + workerDeleteCalls int + workerAbsenceCalls int + setCreateCalls int + setDeleteCalls int + setAbsenceCalls int + sessionOpenCalls int + sessionCloseCalls int + jitCalls int + acquireCalls int + ackCalls int + rosterCalls int + controllerInventory int + registrationCalls int + exchangeCalls int + jobListCalls int + jobDetailCalls int + sdkRunnerCalls int + restRunnerCalls int +} + +func newPairedBrokerBridge(t *testing.T) *pairedBrokerBridge { + t.Helper() + // Unix socket paths are capped at 108 bytes on the supported hosts. Use a + // short private /tmp directory so the fixture remains robust under the + // workspace's long per-test temporary path. + root, err := os.MkdirTemp("/tmp", "g01p-") + if err != nil { + t.Fatal("private bridge root") + } + t.Cleanup(func() { _ = os.RemoveAll(root) }) + if err := os.Chmod(root, 0700); err != nil { + t.Fatal("private bridge root") + } + f := &pairedBrokerBridge{ + installationToken: "synthetic-private-installation-token", + verificationToken: "synthetic-private-workflow-token", + queueToken: "synthetic-private-queue-token", + organization: "org-a", + repository: "canary", + workflowSHA: strings.Repeat("b", 40), + workflowPath: ".github/workflows/canary.yml", + workflowRunID: 7, + runnerGroupID: 3, + setName: "g01-" + strings.Repeat("a", 32), + workerName: "g01-" + strings.Repeat("a", 32) + "-worker-1", + workerImage: pairedWorkerImage, + workerImageID: "sha256:" + strings.Repeat("d", 64), + workerDaemonID: "fixture-daemon", + } + listener, err := net.Listen("unix", filepath.Join(root, "docker.sock")) + if err != nil { + t.Fatal("private Docker socket") + } + if err := os.Chmod(listener.Addr().String(), 0600); err != nil { + _ = listener.Close() + t.Fatal("private Docker socket mode") + } + f.dockerListener = listener + f.dockerEndpoint = listener.Addr().String() + f.dockerServer = &http.Server{Handler: http.HandlerFunc(f.handleDocker)} + go func() { _ = f.dockerServer.Serve(listener) }() + + f.server = httptest.NewTLSServer(http.HandlerFunc(f.handleGitHub)) + t.Cleanup(func() { + f.server.Close() + _ = f.dockerServer.Close() + _ = f.dockerListener.Close() + }) + return f +} + +func (f *pairedBrokerBridge) markUnexpected() { + f.unexpected++ +} + +func writeBridgeJSON(w http.ResponseWriter, status int, value any) { + w.Header().Set("Content-Type", "application/json") + if status != http.StatusOK { + w.WriteHeader(status) + } + if status != http.StatusNoContent && value != nil { + _ = json.NewEncoder(w).Encode(value) + } +} + +func bridgeRepository() map[string]any { + return map[string]any{ + "id": 501, "full_name": "org-a/canary", "private": true, "fork": false, + "owner": map[string]any{"id": 101, "login": "org-a", "type": "Organization"}, + } +} + +func (f *pairedBrokerBridge) actionPath(path string) string { + if strings.HasPrefix(path, "/tenant/") { + return strings.TrimPrefix(path, "/tenant") + } + return path +} + +func (f *pairedBrokerBridge) auth(r *http.Request, want string) bool { + return r.Header.Get("Authorization") == want +} + +func (f *pairedBrokerBridge) scaleSet() map[string]any { + return map[string]any{ + "id": 7, "name": f.setName, "runnerGroupId": f.runnerGroupID, + "labels": []map[string]any{{"name": f.setName, "type": "System"}}, + "RunnerSetting": map[string]any{"disableUpdate": true}, + "statistics": map[string]int{"totalAvailableJobs": 0, "totalAcquiredJobs": 0, "totalAssignedJobs": 0, "totalRunningJobs": 0, "totalRegisteredRunners": 0, "totalBusyRunners": 0, "totalIdleRunners": 0}, + } +} + +func (f *pairedBrokerBridge) job() map[string]any { + status := "in_progress" + var conclusion any + if f.polls >= 2 { + status = "completed" + conclusion = "success" + } + return map[string]any{ + "id": 701, "run_id": f.workflowRunID, "run_attempt": 1, + "head_sha": f.workflowSHA, "status": status, "conclusion": conclusion, + "runner_id": 9001, "runner_name": f.workerName, "runner_group_id": f.runnerGroupID, + } +} + +func (f *pairedBrokerBridge) queueItems() []map[string]any { + base := map[string]any{ + "runnerRequestId": int64(42), "jobId": "fixture-job", "ownerName": f.organization, + "repositoryName": f.repository, "workflowRunId": f.workflowRunID, + "eventName": "workflow_dispatch", "jobWorkflowRef": "fixture-workflow-ref", + "acquireJobUrl": "https://invalid.example/acquire", "jobDisplayName": "fixture-job", + } + if f.polls < 2 { + item := cloneBridgeMap(base) + item["messageType"] = "JobAvailable" + return []map[string]any{item} + } + assigned := cloneBridgeMap(base) + assigned["messageType"] = "JobAssigned" + completed := cloneBridgeMap(base) + completed["messageType"] = "JobCompleted" + completed["runnerId"], completed["runnerName"], completed["result"] = 81, f.workerName, "succeeded" + started := cloneBridgeMap(base) + started["messageType"] = "JobStarted" + started["runnerId"], started["runnerName"] = 81, f.workerName + return []map[string]any{assigned, completed, started} +} + +func cloneBridgeMap(in map[string]any) map[string]any { + out := make(map[string]any, len(in)+2) + for key, value := range in { + out[key] = value + } + return out +} + +func (f *pairedBrokerBridge) handleGitHub(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + path := f.actionPath(r.URL.Path) + + // These are the only REST calls that carry the temporary installation or + // verification authorities. The bridge compares them but never records them. + if strings.HasSuffix(path, "/installation/repositories") && r.Method == http.MethodGet { + if !f.auth(r, "Bearer "+f.installationToken) { + f.markUnexpected() + writeBridgeJSON(w, http.StatusForbidden, nil) + return + } + writeBridgeJSON(w, http.StatusOK, map[string]any{"total_count": 1, "repositories": []any{bridgeRepository()}}) + return + } + if path == "/repos/"+f.organization+"/"+f.repository && r.Method == http.MethodGet { + if !f.auth(r, "Bearer "+f.installationToken) { + f.markUnexpected() + writeBridgeJSON(w, http.StatusForbidden, nil) + return + } + writeBridgeJSON(w, http.StatusOK, bridgeRepository()) + return + } + groupPath := "/orgs/" + f.organization + "/actions/runner-groups/3" + if path == groupPath && r.Method == http.MethodGet { + if !f.auth(r, "Bearer "+f.installationToken) { + f.markUnexpected() + writeBridgeJSON(w, http.StatusForbidden, nil) + return + } + writeBridgeJSON(w, http.StatusOK, map[string]any{"id": 3, "name": "fixture-group", "visibility": "selected", "default": false, "inherited": false, "allows_public_repositories": false}) + return + } + if path == groupPath+"/repositories" && r.Method == http.MethodGet { + if !f.auth(r, "Bearer "+f.installationToken) { + f.markUnexpected() + writeBridgeJSON(w, http.StatusForbidden, nil) + return + } + writeBridgeJSON(w, http.StatusOK, map[string]any{"total_count": 1, "repositories": []any{bridgeRepository()}}) + return + } + if strings.HasSuffix(path, "/actions/runners/registration-token") && r.Method == http.MethodPost { + if !f.auth(r, "Bearer "+f.installationToken) { + f.markUnexpected() + writeBridgeJSON(w, http.StatusForbidden, nil) + return + } + f.registrationCalls++ + writeBridgeJSON(w, http.StatusCreated, map[string]any{"token": "synthetic-private-registration-token", "expires_at": time.Now().Add(time.Hour)}) + return + } + if strings.HasSuffix(path, "/actions/runner-registration") && r.Method == http.MethodPost { + if !f.auth(r, "RemoteAuth synthetic-private-registration-token") { + f.markUnexpected() + writeBridgeJSON(w, http.StatusForbidden, nil) + return + } + f.exchangeCalls++ + claims, _ := json.Marshal(map[string]int64{"exp": time.Now().Add(time.Hour).Unix()}) + f.adminToken = "eyJhbGciOiJub25lIn0." + base64.RawURLEncoding.EncodeToString(claims) + "." + writeBridgeJSON(w, http.StatusOK, map[string]string{"url": f.server.URL + "/tenant/", "token": f.adminToken}) + return + } + + if strings.HasSuffix(path, "/actions/runners") && r.Method == http.MethodGet { + if !f.auth(r, "Bearer "+f.installationToken) { + f.markUnexpected() + writeBridgeJSON(w, http.StatusForbidden, nil) + return + } + if f.setCreated { + f.rosterCalls++ + } else { + f.controllerInventory++ + } + writeBridgeJSON(w, http.StatusOK, map[string]any{"total_count": 0, "runners": []any{}}) + return + } + if path == "/orgs/"+f.organization+"/actions/runners/9001" && r.Method == http.MethodGet { + if !f.auth(r, "Bearer "+f.installationToken) { + f.markUnexpected() + writeBridgeJSON(w, http.StatusForbidden, nil) + return + } + f.restRunnerCalls++ + if f.polls >= 2 { + writeBridgeJSON(w, http.StatusNotFound, map[string]string{"message": "runner absent"}) + return + } + writeBridgeJSON(w, http.StatusOK, map[string]any{"id": 9001, "name": f.workerName, "status": "online", "busy": true}) + return + } + workflowPath := "/repos/" + f.organization + "/" + f.repository + "/actions/runs/7" + if path == workflowPath && r.Method == http.MethodGet { + if !f.auth(r, "Bearer "+f.verificationToken) { + f.markUnexpected() + writeBridgeJSON(w, http.StatusForbidden, nil) + return + } + repo := bridgeRepository() + writeBridgeJSON(w, http.StatusOK, map[string]any{"id": f.workflowRunID, "head_sha": f.workflowSHA, "path": f.workflowPath, "event": "workflow_dispatch", "run_attempt": 1, "repository": repo, "head_repository": repo}) + return + } + jobsPath := "/repos/" + f.organization + "/" + f.repository + "/actions/runs/7/attempts/1/jobs" + if path == jobsPath && r.Method == http.MethodGet { + if !f.auth(r, "Bearer "+f.verificationToken) { + f.markUnexpected() + writeBridgeJSON(w, http.StatusForbidden, nil) + return + } + f.jobListCalls++ + writeBridgeJSON(w, http.StatusOK, map[string]any{"total_count": 1, "jobs": []any{f.job()}}) + return + } + if path == "/repos/"+f.organization+"/"+f.repository+"/actions/jobs/701" && r.Method == http.MethodGet { + if !f.auth(r, "Bearer "+f.verificationToken) { + f.markUnexpected() + writeBridgeJSON(w, http.StatusForbidden, nil) + return + } + f.jobDetailCalls++ + writeBridgeJSON(w, http.StatusOK, f.job()) + return + } + + // All remaining routes are Actions service calls. The SDK pins the admin + // token in the generated private client and never follows redirects. + if strings.HasPrefix(path, "/_apis/") || path == "/queue" || strings.HasPrefix(path, "/queue/") { + if f.adminToken != "" && !f.auth(r, "Bearer "+f.adminToken) && path != "/queue" && !strings.HasPrefix(path, "/queue/") { + f.markUnexpected() + writeBridgeJSON(w, http.StatusForbidden, nil) + return + } + f.handleActions(w, r, path) + return + } + f.markUnexpected() + writeBridgeJSON(w, http.StatusForbidden, nil) +} + +func (f *pairedBrokerBridge) handleActions(w http.ResponseWriter, r *http.Request, path string) { + if path == "/queue" && r.Method == http.MethodGet { + if !f.auth(r, "Bearer "+f.queueToken) { + f.markUnexpected() + writeBridgeJSON(w, http.StatusForbidden, nil) + return + } + f.polls++ + lastMessageID := r.URL.Query().Get("lastMessageId") + if f.lastAck == 0 { + if lastMessageID != "" { + f.markUnexpected() + } + } else if lastMessageID != strconv.Itoa(f.lastAck) { + f.markUnexpected() + } + items, _ := json.Marshal(f.queueItems()) + writeBridgeJSON(w, http.StatusOK, map[string]any{"messageId": 8 + f.polls, "messageType": "RunnerScaleSetJobMessages", "body": string(items), "statistics": map[string]int{"totalAvailableJobs": 0, "totalAcquiredJobs": 0, "totalAssignedJobs": 1, "totalRunningJobs": 0, "totalRegisteredRunners": 0, "totalBusyRunners": 0, "totalIdleRunners": 0}}) + return + } + if strings.HasPrefix(path, "/queue/") && r.Method == http.MethodDelete { + if !f.auth(r, "Bearer "+f.queueToken) { + f.markUnexpected() + writeBridgeJSON(w, http.StatusForbidden, nil) + return + } + id, err := strconv.Atoi(strings.TrimPrefix(path, "/queue/")) + if err != nil || id != 8+f.polls { + f.markUnexpected() + } + f.lastAck, f.ackCalls = id, f.ackCalls+1 + writeBridgeJSON(w, http.StatusNoContent, nil) + return + } + if strings.HasSuffix(path, "/runnerscalesets") { + if r.Method == http.MethodGet { + if f.setExists { + writeBridgeJSON(w, http.StatusOK, map[string]any{"count": 1, "value": []any{f.scaleSet()}}) + } else { + writeBridgeJSON(w, http.StatusOK, map[string]any{"count": 0, "value": []any{}}) + } + return + } + if r.Method == http.MethodPost { + f.setCreated, f.setExists, f.setCreateCalls = true, true, f.setCreateCalls+1 + writeBridgeJSON(w, http.StatusOK, f.scaleSet()) + return + } + } + if strings.HasSuffix(path, "/runnerscalesets/7") { + switch r.Method { + case http.MethodGet: + if !f.setExists { + f.setAbsenceCalls++ + writeBridgeJSON(w, http.StatusNotFound, map[string]string{"message": "scale set absent"}) + return + } + writeBridgeJSON(w, http.StatusOK, f.scaleSet()) + case http.MethodDelete: + if !f.setExists { + f.markUnexpected() + writeBridgeJSON(w, http.StatusNotFound, nil) + return + } + f.setExists, f.setDeleted, f.setDeleteCalls = false, true, f.setDeleteCalls+1 + writeBridgeJSON(w, http.StatusNoContent, nil) + default: + f.markUnexpected() + writeBridgeJSON(w, http.StatusForbidden, nil) + } + return + } + if strings.HasSuffix(path, "/runnerscalesets/7/sessions") && r.Method == http.MethodPost { + f.sessionOpenCalls++ + writeBridgeJSON(w, http.StatusOK, map[string]any{"sessionId": "00000000-0000-4000-8000-000000000001", "ownerName": f.setName, "messageQueueUrl": f.server.URL + "/queue", "messageQueueAccessToken": f.queueToken, "statistics": map[string]int{}}) + return + } + if strings.Contains(path, "/runnerscalesets/7/sessions/") && r.Method == http.MethodDelete { + f.sessionCloseCalls++ + writeBridgeJSON(w, http.StatusNoContent, nil) + return + } + if strings.HasSuffix(path, "/runnerscalesets/7/generatejitconfig") && r.Method == http.MethodPost { + f.jitCalls++ + writeBridgeJSON(w, http.StatusOK, map[string]any{"runner": map[string]any{"id": 81, "name": f.workerName, "runnerScaleSetId": 7}, "encodedJITConfig": pairedBridgeJIT}) + return + } + if strings.HasSuffix(path, "/acquirejobs") && r.Method == http.MethodPost { + f.acquireCalls++ + writeBridgeJSON(w, http.StatusOK, map[string]any{"count": 1, "value": []int64{42}}) + return + } + if strings.HasSuffix(path, "/agents/81") && r.Method == http.MethodGet { + f.sdkRunnerCalls++ + if f.polls >= 2 { + writeBridgeJSON(w, http.StatusNotFound, map[string]string{"message": "runner absent"}) + return + } + writeBridgeJSON(w, http.StatusOK, map[string]any{"id": 81, "name": f.workerName, "runnerScaleSetId": 7}) + return + } + if strings.HasSuffix(path, "/agents") && r.Method == http.MethodGet { + writeBridgeJSON(w, http.StatusOK, map[string]any{"count": 0, "value": []any{}}) + return + } + f.markUnexpected() + writeBridgeJSON(w, http.StatusForbidden, nil) +} + +func (f *pairedBrokerBridge) handleDocker(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + path := r.URL.Path + if path == "/version" && r.Method == http.MethodGet { + writeBridgeJSON(w, http.StatusOK, map[string]string{"ApiVersion": "1.51", "MinAPIVersion": "1.24"}) + return + } + if path == "/v1.45/info" && r.Method == http.MethodGet { + writeBridgeJSON(w, http.StatusOK, map[string]any{"ID": f.workerDaemonID, "OSType": "linux", "Architecture": "aarch64", "NCPU": 4, "MemTotal": int64(4 << 30), "MemoryLimit": true, "SwapLimit": true, "CpuCfsQuota": true, "PidsLimit": true, "Warnings": []string{}}) + return + } + if strings.HasPrefix(path, "/v1.45/images/") && strings.HasSuffix(path, "/json") && r.Method == http.MethodGet { + writeBridgeJSON(w, http.StatusOK, map[string]any{"Id": f.workerImageID, "Os": "linux", "Architecture": "arm64", "RepoDigests": []string{f.workerImage}, "Config": map[string]any{"Env": []string{"PATH=/usr/bin"}, "Labels": map[string]string{}, "Volumes": map[string]any{}, "ExposedPorts": map[string]any{}}}) + return + } + containerPath := "/v1.45/containers/" + strings.Repeat("c", 64) + if path == "/v1.45/containers/create" && r.Method == http.MethodPost { + var payload map[string]any + if json.NewDecoder(io.LimitReader(r.Body, 2<<20)).Decode(&payload) != nil || r.URL.Query().Get("name") != f.workerName { + f.markUnexpected() + writeBridgeJSON(w, http.StatusBadRequest, nil) + return + } + env, ok := payload["Env"].([]any) + if !ok || len(env) != 1 { + f.markUnexpected() + writeBridgeJSON(w, http.StatusBadRequest, nil) + return + } + host := payload["HostConfig"] + delete(payload, "HostConfig") + payload["Env"] = append([]any{"PATH=/usr/bin"}, env...) + f.container = map[string]any{"Id": strings.Repeat("c", 64), "Name": "/" + f.workerName, "Image": f.workerImageID, "Path": "/home/runner/bin/Runner.Listener", "Args": []string{"run", "--once"}, "Config": payload, "HostConfig": host, "Mounts": []any{}, "State": map[string]any{"Status": "created", "Running": false, "Paused": false, "Restarting": false, "Dead": false, "ExitCode": 0}, "NetworkSettings": map[string]any{"Networks": map[string]any{"bridge": map[string]any{}}}} + f.createCalls++ + writeBridgeJSON(w, http.StatusCreated, map[string]any{"Id": strings.Repeat("c", 64), "Warnings": []any{}}) + return + } + if path == containerPath+"/start" && r.Method == http.MethodPost { + if f.container == nil { + f.markUnexpected() + writeBridgeJSON(w, http.StatusNotFound, nil) + return + } + f.containerStarted = true + f.startCalls++ + state := f.container["State"].(map[string]any) + state["Status"], state["Running"] = "running", true + writeBridgeJSON(w, http.StatusNoContent, nil) + return + } + if path == containerPath+"/json" && r.Method == http.MethodGet { + if f.containerDeleted || f.container == nil { + f.workerAbsenceCalls++ + writeBridgeJSON(w, http.StatusNotFound, map[string]string{"message": "container absent"}) + return + } + if f.containerStarted && f.polls >= 2 { + state := f.container["State"].(map[string]any) + state["Status"], state["Running"] = "exited", false + } + writeBridgeJSON(w, http.StatusOK, f.container) + return + } + if path == containerPath && r.Method == http.MethodDelete { + if r.URL.Query().Get("force") != "false" || r.URL.Query().Get("v") != "false" || f.container == nil { + f.markUnexpected() + writeBridgeJSON(w, http.StatusForbidden, nil) + return + } + f.containerDeleted, f.workerDeleteCalls = true, f.workerDeleteCalls+1 + writeBridgeJSON(w, http.StatusNoContent, nil) + return + } + f.markUnexpected() + writeBridgeJSON(w, http.StatusForbidden, nil) +} + +func bridgeCA(t *testing.T, server *httptest.Server) string { + t.Helper() + return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw})) +} + +func bridgeRepoRoot(t *testing.T) string { + t.Helper() + _, file, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("bridge test path") + } + return filepath.Clean(filepath.Join(filepath.Dir(file), "../..")) +} + +func buildPairedG01Binary(t *testing.T) (string, string, string) { + t.Helper() + repo := bridgeRepoRoot(t) + command := exec.Command("git", "rev-parse", "HEAD") + command.Dir = repo + harnessBytes, err := command.Output() + if err != nil { + t.Fatal("read reviewed bridge head") + } + harness := strings.TrimSpace(string(harnessBytes)) + if len(harness) != 40 { + t.Fatal("invalid reviewed bridge head") + } + out := filepath.Join(t.TempDir(), "g01-live") + command = exec.Command("go", "build", "-tags", "g01_live,g01_pair_fixture", "-o", out, "./cmd/g01-live") + command.Dir = filepath.Join(repo, "experiments", "g01-scaleset") + command.Env = append(os.Environ(), "GOTOOLCHAIN=go1.26.8") + if output, err := command.CombinedOutput(); err != nil { + _ = output + t.Fatal("build reviewed g01 bridge binary") + } + if err := os.Chmod(out, 0500); err != nil { + t.Fatal("pin reviewed bridge binary mode") + } + data, err := os.ReadFile(out) + if err != nil { + t.Fatal("read reviewed bridge binary") + } + digest := sha256.Sum256(data) + return out, harness, hexDigest(digest[:]) +} + +func hexDigest(data []byte) string { + const hex = "0123456789abcdef" + out := make([]byte, len(data)*2) + for i, value := range data { + out[2*i], out[2*i+1] = hex[value>>4], hex[value&15] + } + return string(out) +} + +func runBridgeCommand(t *testing.T, ctx context.Context, binary string, args []string, input []byte) []byte { + t.Helper() + command := exec.CommandContext(ctx, binary, args...) + command.Env = []string{"LANG=C", "LC_ALL=C"} + command.Stdin = bytes.NewReader(input) + output, err := command.CombinedOutput() + if err != nil { + _ = output + t.Fatal("reviewed g01 bridge command failed") + } + return output +} + +func writePrivateBridgeJSON(t *testing.T, path string, value any) []byte { + t.Helper() + data, err := json.Marshal(value) + if err != nil || os.WriteFile(path, data, 0600) != nil { + t.Fatal("private bridge JSON") + } + return data +} + +func pairedBridgeBinding(t *testing.T, controllerPath, controllerState, workerPath, workerState string) map[string]any { + t.Helper() + identity := func(path string) (uint64, uint64) { + info, err := os.Stat(path) + if err != nil { + t.Fatal("bridge identity") + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + t.Fatal("bridge identity type") + } + return uint64(stat.Dev), stat.Ino + } + digest := func(path string) string { + data, err := os.ReadFile(path) + if err != nil { + t.Fatal("bridge approval digest") + } + h := sha256.Sum256(data) + return hexDigest(h[:]) + } + controllerDevice, controllerInode := identity(controllerPath) + controllerStateDevice, controllerStateInode := identity(controllerState) + workerDevice, workerInode := identity(workerPath) + workerStateDevice, workerStateInode := identity(workerState) + return map[string]any{"controller_approval_sha256": digest(controllerPath), "controller_approval_device": controllerDevice, "controller_approval_inode": controllerInode, "controller_state_device": controllerStateDevice, "controller_state_inode": controllerStateInode, "worker_approval_sha256": digest(workerPath), "worker_approval_device": workerDevice, "worker_approval_inode": workerInode, "worker_state_device": workerStateDevice, "worker_state_inode": workerStateInode} +} + +func TestPairedBrokerChainsRealControllerCreatePreparationAndTerminal(t *testing.T) { + if runtime.GOOS != "darwin" && runtime.GOOS != "linux" { + t.Skip("private Unix fixture requires a Unix host") + } + binaryPath, harness, binaryDigest := buildPairedG01Binary(t) + bridge := newPairedBrokerBridge(t) + parent := t.TempDir() + if err := os.Chmod(parent, 0700); err != nil { + t.Fatal("private bridge state root") + } + controllerState := filepath.Join(parent, "controller-state") + workerState := filepath.Join(parent, "worker-state") + if err := os.Mkdir(controllerState, 0700); err != nil { + t.Fatal("controller bridge state") + } + if err := os.Mkdir(workerState, 0700); err != nil { + t.Fatal("worker bridge state") + } + admissionConfig := map[string]string{"base_url": bridge.server.URL, "ca_pem": bridgeCA(t, bridge.server), "admission_directory": ""} + _, candidate, api, brokerFixture, attempt := newBrokerFixture(t) + admissionConfig["admission_directory"] = brokerFixture.admissionRoot + configPath := filepath.Join(controllerState, "paired-fixture.json") + writePrivateBridgeJSON(t, configPath, admissionConfig) + + now := time.Now() + expires := now.Add(20 * time.Minute) + workflow := strings.Repeat("b", 40) + controller := controllerApproval{AppID: 71, InstallationID: 201, Organization: bridge.organization, Repository: bridge.repository, RepositoryID: 501, RunnerGroupID: bridge.runnerGroupID, OwnerNonce: strings.Repeat("a", 32), HarnessSHA: harness, WorkflowSHA: workflow, WorkflowPath: bridge.workflowPath, WorkflowRunID: bridge.workflowRunID, Controller: "trusted-controller", ExpiresAt: expires, ActionsHosts: []string{"fixture.actions.githubusercontent.com"}, Phases: []string{"create", "before-ack", "after-ack", "before-acquire", "inspect", "cleanup"}} + controllerPath := filepath.Join(parent, "controller-approval.json") + controllerData := writePrivateBridgeJSON(t, controllerPath, controller) + credentials := map[string]any{"installation_token": bridge.installationToken, "verification_token": bridge.verificationToken, "app_id": 71, "installation_id": 201, "organization": bridge.organization, "expires_at": expires, "organization_self_hosted_runners": "write", "metadata": "read"} + credentialData, err := json.Marshal(credentials) + if err != nil { + t.Fatal("controller credentials") + } + createCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + runBridgeCommand(t, createCtx, binaryPath, []string{"--execute-approved-canary", "--approval", controllerPath, "--state-dir", controllerState, "--phase", "create"}, credentialData) + cancel() + journalData, err := os.ReadFile(filepath.Join(controllerState, "journal.jsonl")) + if err != nil { + t.Fatal("controller journal") + } + var operations []string + for _, line := range strings.Split(strings.TrimSpace(string(journalData)), "\n")[1:] { + var event struct { + Kind string `json:"kind"` + Operation string `json:"operation"` + ID int `json:"id"` + } + if json.Unmarshal([]byte(line), &event) != nil { + t.Fatal("controller journal event") + } + if event.Kind == "phase" || event.Kind == "inventory" || event.Kind == "intent" || event.Kind == "result" { + operations = append(operations, event.Kind+":"+event.Operation) + } + } + want := []string{"phase:create", "inventory:", "intent:observe-discovery", "result:observe-discovery", "intent:create", "result:create"} + if len(operations) != len(want) { + t.Fatalf("controller create did not produce canonical history: %v", operations) + } + for i := range want { + if operations[i] != want[i] { + t.Fatalf("controller create history[%d]=%q want %q", i, operations[i], want[i]) + } + } + if _, err := os.Stat(filepath.Join(brokerFixture.admissionRoot, "admission.json")); err != nil { + t.Fatal("controller admission claim") + } + + worker := pairedWorkerApproval{RunnerUpdatesDisabled: true, HarnessSHA: harness, WorkflowSHA: workflow, OwnerNonce: controller.OwnerNonce, Controller: controller.Controller, Endpoint: bridge.dockerEndpoint, DaemonID: bridge.workerDaemonID, ImageID: bridge.workerImageID, Image: pairedWorkerImage, ExpiresAt: expires, Phases: []string{"create", "start", "inspect", "cleanup"}} + workerPath := filepath.Join(parent, "worker-approval.json") + writePrivateBridgeJSON(t, workerPath, worker) + brokerApproval := brokerApprovalFixture() + brokerApproval.Mode, brokerApproval.Phase = "paired-terminal", "paired-terminal" + brokerApproval.AllowVerificationAuthority = true + brokerApproval.ExpiresAt = expires + brokerApproval.OwnerNonce = controller.OwnerNonce + brokerApproval.ControllerHarnessSHA = harness + brokerApproval.ControllerBinarySHA256 = binaryDigest + controllerDigest := sha256.Sum256(controllerData) + brokerApproval.ControllerApprovalSHA256 = hexDigest(controllerDigest[:]) + approvalPath := filepath.Join(parent, "broker-approval.json") + writePrivateBridgeJSON(t, approvalPath, brokerApproval) + inputPath := filepath.Join(parent, "broker-input.json") + inputData := writePrivateBridgeJSON(t, inputPath, brokerInput{PEM: string(candidate.PEM), VerificationToken: bridge.verificationToken}) + + oldOpener := brokerBinaryOpener + brokerBinaryOpener = func(path string, a BrokerApproval) (*verifiedBrokerBinary, error) { + file, err := os.Open(path) + if err != nil { + return nil, errBroker + } + return &verifiedBrokerBinary{path: path, file: file, digest: a.ControllerBinarySHA256}, nil + } + defer func() { brokerBinaryOpener = oldOpener }() + input, err := os.Open(inputPath) + if err != nil { + t.Fatal("broker input") + } + brokerCtx, brokerCancel := context.WithTimeout(context.Background(), 3*time.Minute) + result, err := runBrokerWithAPI(brokerCtx, BrokerFiles{ApprovalPath: approvalPath, StateDirectory: attempt, ControllerBinary: binaryPath, ControllerApproval: controllerPath, ControllerStateDirectory: controllerState, WorkerApproval: workerPath, WorkerStateDirectory: workerState}, input, api) + brokerCancel() + if err != nil || result.Status != "paired_terminal_completed" { + t.Fatalf("real paired bridge did not complete: status=%q err=%v", result.Status, err) + } + bridge.mu.Lock() + counts := []int{bridge.createCalls, bridge.startCalls, bridge.jitCalls, bridge.acquireCalls, bridge.ackCalls, bridge.sessionOpenCalls, bridge.sessionCloseCalls, bridge.workerDeleteCalls, bridge.workerAbsenceCalls, bridge.setCreateCalls, bridge.setDeleteCalls, bridge.setAbsenceCalls, bridge.rosterCalls, bridge.unexpected} + bridge.mu.Unlock() + if want := []int{1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 4, 0}; !slices.Equal(counts, want) { + t.Fatalf("real paired effect counts=%v want=%v", counts, want) + } + if brokerFixture.tokenCalls != 1 { + t.Fatalf("real paired bridge minted %d tokens", brokerFixture.tokenCalls) + } + ledger, err := os.ReadFile(filepath.Join(brokerFixture.admissionRoot, "broker-admission.jsonl")) + if err != nil || !strings.Contains(string(ledger), `"slot":"paired-terminal"`) || !strings.Contains(string(ledger), `"worker"`) { + t.Fatal("paired broker ledger missing controller/worker claim") + } + for _, root := range []string{attempt, controllerState, workerState, brokerFixture.admissionRoot} { + assertNoSecretFiles(t, root, string(candidate.PEM), "PRIVATE KEY", bridge.installationToken, bridge.verificationToken, pairedBridgeJIT, "synthetic-private-registration-token", "synthetic-private-admin-token", bridge.queueToken) + } + if !strings.Contains(string(journalData), `"kind":"baseline"`) { + t.Fatal("terminal child did not append to the original controller journal") + } + workerJournal, err := os.ReadFile(filepath.Join(workerState, "journal.jsonl")) + if err != nil || !strings.Contains(string(workerJournal), `"kind":"paired"`) { + t.Fatal("terminal child did not create the worker paired journal") + } + + // A completed paired claim is one-shot. Reopening the real broker entrypoint + // must stop before the second mint or any terminal effect. + retryPath := filepath.Join(parent, "broker-input-retry.json") + if err := os.WriteFile(retryPath, inputData, 0600); err != nil { + t.Fatal("retry input") + } + retryInput, err := os.Open(retryPath) + if err != nil { + t.Fatal("retry input open") + } + _, retryErr := runBrokerWithAPI(context.Background(), BrokerFiles{ApprovalPath: approvalPath, StateDirectory: attempt, ControllerBinary: binaryPath, ControllerApproval: controllerPath, ControllerStateDirectory: controllerState, WorkerApproval: workerPath, WorkerStateDirectory: workerState}, retryInput, api) + if retryErr == nil || brokerFixture.tokenCalls != 1 { + t.Fatal("completed paired claim replayed effects") + } +} diff --git a/experiments/g02-auth/broker_plan.go b/experiments/g02-auth/broker_plan.go index 3e6ca6b..6149ef6 100644 --- a/experiments/g02-auth/broker_plan.go +++ b/experiments/g02-auth/broker_plan.go @@ -27,7 +27,7 @@ type pairedWorkerApproval struct { } func (a pairedWorkerApproval) validate(now time.Time) error { - if !a.RunnerUpdatesDisabled || !brokerSHA40.MatchString(a.HarnessSHA) || !brokerSHA40.MatchString(a.WorkflowSHA) || !brokerNonce.MatchString(a.OwnerNonce) || !brokerComponent.MatchString(a.Controller) || !brokerComponent.MatchString(a.DaemonID) || !strings.HasPrefix(a.ImageID, "sha256:") || !brokerSHA256.MatchString(strings.TrimPrefix(a.ImageID, "sha256:")) || a.Image != pairedWorkerImage || !filepath.IsAbs(a.Endpoint) || filepath.Clean(a.Endpoint) != a.Endpoint || len(a.Endpoint) > 103 || strings.ContainsAny(a.Endpoint, "\x00\r\n") || !a.ExpiresAt.After(now) || a.ExpiresAt.After(now.Add(24*time.Hour)) || len(a.Phases) != 4 { + if !a.RunnerUpdatesDisabled || !brokerSHA40.MatchString(a.HarnessSHA) || !brokerSHA40.MatchString(a.WorkflowSHA) || !brokerNonce.MatchString(a.OwnerNonce) || !brokerComponent.MatchString(a.Controller) || !brokerWorkerComponent.MatchString(a.DaemonID) || !strings.HasPrefix(a.ImageID, "sha256:") || !brokerSHA256.MatchString(strings.TrimPrefix(a.ImageID, "sha256:")) || a.Image != pairedWorkerImage || !filepath.IsAbs(a.Endpoint) || filepath.Clean(a.Endpoint) != a.Endpoint || len(a.Endpoint) > 103 || strings.ContainsAny(a.Endpoint, "\x00\r\n") || !a.ExpiresAt.After(now) || a.ExpiresAt.After(now.Add(24*time.Hour)) || len(a.Phases) != 4 { return errBroker } seen := map[string]bool{} diff --git a/experiments/g02-auth/broker_process.go b/experiments/g02-auth/broker_process.go index ab5f0d3..9b787b4 100644 --- a/experiments/g02-auth/broker_process.go +++ b/experiments/g02-auth/broker_process.go @@ -29,6 +29,20 @@ type verifiedBrokerBinary struct { // production build metadata gate. var brokerBinaryOpener = openBrokerBinary +func pairedChildDeadline(parent context.Context, now time.Time) (time.Time, error) { + if parent == nil || parent.Err() != nil { + return time.Time{}, errBroker + } + deadline := now.Add(pairedTerminalMaximumChildBudget) + if parentDeadline, ok := parent.Deadline(); ok { + deadline = minTime(deadline, parentDeadline) + } + if !deadline.After(now.Add(pairedTerminalMinimumChildBudget)) { + return time.Time{}, errBroker + } + return deadline, nil +} + func validBrokerBuild(info *debug.BuildInfo, expected string) bool { if info == nil || info.GoVersion != "go1.26.8" || info.Path != "github.com/1XP-AI/gh-runnerd/experiments/g01-scaleset/cmd/g01-live" || !brokerSHA40.MatchString(expected) { return false @@ -172,7 +186,11 @@ func invokeBrokerPairedTerminal(parent context.Context, binary *verifiedBrokerBi if err != nil || len(data) > 16384 { return errBroker } - ctx, cancel := context.WithTimeout(parent, 30*time.Second) + deadline, err := pairedChildDeadline(parent, time.Now()) + if err != nil { + return errBroker + } + ctx, cancel := context.WithDeadline(parent, deadline) defer cancel() command := exec.CommandContext(ctx, binary.path, "--execute-approved-paired-terminal", "--approval", approvalPath, "--state-dir", stateDirectory, "--worker-approval", workerApprovalPath, "--worker-state-dir", workerStateDirectory, "--paired-binding", string(bindingData)) command.Dir = workingDirectory diff --git a/experiments/g02-auth/broker_process_test.go b/experiments/g02-auth/broker_process_test.go index 2d78385..c2271da 100644 --- a/experiments/g02-auth/broker_process_test.go +++ b/experiments/g02-auth/broker_process_test.go @@ -256,6 +256,27 @@ func TestBrokerPairedChildBoundsTimeoutOverflowAndCancel(t *testing.T) { }) } } + +func TestBrokerPairedChildDeadlineIsBoundedAndLeavesCadenceMargin(t *testing.T) { + now := time.Now() + deadline, err := pairedChildDeadline(context.Background(), now) + if err != nil || !deadline.Equal(now.Add(pairedTerminalMaximumChildBudget)) { + t.Fatalf("background child deadline=%v err=%v", deadline.Sub(now), err) + } + parentDeadline := now.Add(2 * time.Minute) + parent, cancel := context.WithDeadline(context.Background(), parentDeadline) + defer cancel() + deadline, err = pairedChildDeadline(parent, now) + if err != nil || !deadline.Equal(parentDeadline) { + t.Fatalf("parent authority deadline=%v err=%v", deadline.Sub(now), err) + } + short, cancel := context.WithDeadline(context.Background(), now.Add(pairedTerminalMinimumChildBudget)) + defer cancel() + if _, err := pairedChildDeadline(short, now); err == nil { + t.Fatal("child deadline accepted without a complete cadence margin") + } +} + func TestBrokerChildBoundsAndReplacedBinaryRefuse(t *testing.T) { binary := testBrokerBinary(t) root := t.TempDir() From 413f2fa26253e7c8fd70b7cd255b4c8445193d58 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 18:03:30 +0900 Subject: [PATCH 07/37] test(g01): stamp bridge binary from clean checkout --- experiments/g02-auth/broker_paired_bridge_test.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/experiments/g02-auth/broker_paired_bridge_test.go b/experiments/g02-auth/broker_paired_bridge_test.go index caf1528..439d4b9 100644 --- a/experiments/g02-auth/broker_paired_bridge_test.go +++ b/experiments/g02-auth/broker_paired_bridge_test.go @@ -577,9 +577,19 @@ func buildPairedG01Binary(t *testing.T) (string, string, string) { if len(harness) != 40 { t.Fatal("invalid reviewed bridge head") } + // The checkout contains nested Go modules and the active worktree's .git + // indirection is intentionally not used for build stamping. Build from a + // clean temporary clone so Go records vcs.revision and vcs.modified=false + // in the executable that the broker verifies. + cloneRoot := filepath.Join(t.TempDir(), "repo") + command = exec.Command("git", "clone", "--no-hardlinks", "--quiet", repo, cloneRoot) + if output, err := command.CombinedOutput(); err != nil { + _ = output + t.Fatal("clone reviewed g01 bridge source") + } out := filepath.Join(t.TempDir(), "g01-live") - command = exec.Command("go", "build", "-tags", "g01_live,g01_pair_fixture", "-o", out, "./cmd/g01-live") - command.Dir = filepath.Join(repo, "experiments", "g01-scaleset") + command = exec.Command("go", "build", "-buildvcs=true", "-tags", "g01_live,g01_pair_fixture", "-o", out, "./cmd/g01-live") + command.Dir = filepath.Join(cloneRoot, "experiments", "g01-scaleset") command.Env = append(os.Environ(), "GOTOOLCHAIN=go1.26.8") if output, err := command.CombinedOutput(); err != nil { _ = output From 9b4f86ce78e3a2efeeced4e4d945101fb09154f1 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 18:05:09 +0900 Subject: [PATCH 08/37] test(g01): retain safe bridge failure category --- experiments/g02-auth/broker_paired_bridge_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/experiments/g02-auth/broker_paired_bridge_test.go b/experiments/g02-auth/broker_paired_bridge_test.go index 439d4b9..e62439a 100644 --- a/experiments/g02-auth/broker_paired_bridge_test.go +++ b/experiments/g02-auth/broker_paired_bridge_test.go @@ -622,8 +622,10 @@ func runBridgeCommand(t *testing.T, ctx context.Context, binary string, args []s command.Stdin = bytes.NewReader(input) output, err := command.CombinedOutput() if err != nil { - _ = output - t.Fatal("reviewed g01 bridge command failed") + // g01-live's output boundary is fixed text; retaining it here makes a + // local fixture failure diagnosable without exposing credentials or SDK + // response bodies. + t.Fatalf("reviewed g01 bridge command failed: %q", strings.TrimSpace(string(output))) } return output } From 45c8447802d1044b34285a19ed52bf5d4b107d5d Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 18:05:46 +0900 Subject: [PATCH 09/37] test(g01): expose bridge protocol counters on failure --- experiments/g02-auth/broker_paired_bridge_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/experiments/g02-auth/broker_paired_bridge_test.go b/experiments/g02-auth/broker_paired_bridge_test.go index e62439a..fa2c450 100644 --- a/experiments/g02-auth/broker_paired_bridge_test.go +++ b/experiments/g02-auth/broker_paired_bridge_test.go @@ -770,6 +770,10 @@ func TestPairedBrokerChainsRealControllerCreatePreparationAndTerminal(t *testing result, err := runBrokerWithAPI(brokerCtx, BrokerFiles{ApprovalPath: approvalPath, StateDirectory: attempt, ControllerBinary: binaryPath, ControllerApproval: controllerPath, ControllerStateDirectory: controllerState, WorkerApproval: workerPath, WorkerStateDirectory: workerState}, input, api) brokerCancel() if err != nil || result.Status != "paired_terminal_completed" { + bridge.mu.Lock() + t.Logf("bridge counters: registration=%d exchange=%d inventory=%d setCreate=%d sessionOpen=%d jit=%d acquire=%d ack=%d create=%d start=%d polls=%d unexpected=%d", bridge.registrationCalls, bridge.exchangeCalls, bridge.controllerInventory, bridge.setCreateCalls, bridge.sessionOpenCalls, bridge.jitCalls, bridge.acquireCalls, bridge.ackCalls, bridge.createCalls, bridge.startCalls, bridge.polls, bridge.unexpected) + bridge.mu.Unlock() + t.Logf("broker API calls: %v", brokerFixture.calls) t.Fatalf("real paired bridge did not complete: status=%q err=%v", result.Status, err) } bridge.mu.Lock() From c440f0e4d6a5b283f1bd4c800b0dcab6b7e1f787 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 18:07:28 +0900 Subject: [PATCH 10/37] test(g01): probe paired preparation bridge --- experiments/g02-auth/broker_paired_bridge_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/experiments/g02-auth/broker_paired_bridge_test.go b/experiments/g02-auth/broker_paired_bridge_test.go index fa2c450..3024849 100644 --- a/experiments/g02-auth/broker_paired_bridge_test.go +++ b/experiments/g02-auth/broker_paired_bridge_test.go @@ -735,6 +735,12 @@ func TestPairedBrokerChainsRealControllerCreatePreparationAndTerminal(t *testing if _, err := os.Stat(filepath.Join(brokerFixture.admissionRoot, "admission.json")); err != nil { t.Fatal("controller admission claim") } + // Diagnostic preflight while narrowing the real child boundary; this is + // credential-free and does not alter the already-completed controller + // history. Remove once the broker-launched preparation path is green. + prepCtx, prepCancel := context.WithTimeout(context.Background(), time.Minute) + runBridgeCommand(t, prepCtx, binaryPath, []string{"--prepare-approved-paired-journal", "--approval", controllerPath, "--state-dir", controllerState}, nil) + prepCancel() worker := pairedWorkerApproval{RunnerUpdatesDisabled: true, HarnessSHA: harness, WorkflowSHA: workflow, OwnerNonce: controller.OwnerNonce, Controller: controller.Controller, Endpoint: bridge.dockerEndpoint, DaemonID: bridge.workerDaemonID, ImageID: bridge.workerImageID, Image: pairedWorkerImage, ExpiresAt: expires, Phases: []string{"create", "start", "inspect", "cleanup"}} workerPath := filepath.Join(parent, "worker-approval.json") From b5a5a1b136d8257bea41bd15362b029e669f0cdb Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 18:08:43 +0900 Subject: [PATCH 11/37] test(g01): classify offline paired bridge failure --- .../g01-scaleset/cmd/g01-live/fixture_support.go | 16 +++++++++++++++- .../g02-auth/broker_paired_bridge_test.go | 3 +++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/experiments/g01-scaleset/cmd/g01-live/fixture_support.go b/experiments/g01-scaleset/cmd/g01-live/fixture_support.go index b9a646d..e9f6928 100644 --- a/experiments/g01-scaleset/cmd/g01-live/fixture_support.go +++ b/experiments/g01-scaleset/cmd/g01-live/fixture_support.go @@ -4,6 +4,7 @@ package main import ( "context" + "errors" "io" "os" "path/filepath" @@ -65,6 +66,19 @@ func init() { if err != nil { return err } - return livecanary.RunPairedTerminalForFixture(ctx, files, c, config.BaseURL, []byte(config.CAPEM), config.AdmissionDirectory) + err = livecanary.RunPairedTerminalForFixture(ctx, files, c, config.BaseURL, []byte(config.CAPEM), config.AdmissionDirectory) + // Temporary fixture-only classification for local bridge diagnosis; the + // production command still emits only its fixed redacted failure text. + category := "other" + switch { + case errors.Is(err, livecanary.ErrApproval): + category = "approval" + case errors.Is(err, livecanary.ErrJournal): + category = "journal" + case errors.Is(err, livecanary.ErrQuarantine): + category = "quarantine" + } + _ = os.WriteFile(filepath.Join(files.ControllerStateDirectory, "paired-fixture-result"), []byte(category), 0600) + return err } } diff --git a/experiments/g02-auth/broker_paired_bridge_test.go b/experiments/g02-auth/broker_paired_bridge_test.go index 3024849..a95a877 100644 --- a/experiments/g02-auth/broker_paired_bridge_test.go +++ b/experiments/g02-auth/broker_paired_bridge_test.go @@ -780,6 +780,9 @@ func TestPairedBrokerChainsRealControllerCreatePreparationAndTerminal(t *testing t.Logf("bridge counters: registration=%d exchange=%d inventory=%d setCreate=%d sessionOpen=%d jit=%d acquire=%d ack=%d create=%d start=%d polls=%d unexpected=%d", bridge.registrationCalls, bridge.exchangeCalls, bridge.controllerInventory, bridge.setCreateCalls, bridge.sessionOpenCalls, bridge.jitCalls, bridge.acquireCalls, bridge.ackCalls, bridge.createCalls, bridge.startCalls, bridge.polls, bridge.unexpected) bridge.mu.Unlock() t.Logf("broker API calls: %v", brokerFixture.calls) + if category, readErr := os.ReadFile(filepath.Join(controllerState, "paired-fixture-result")); readErr == nil { + t.Logf("paired child category: %q", strings.TrimSpace(string(category))) + } t.Fatalf("real paired bridge did not complete: status=%q err=%v", result.Status, err) } bridge.mu.Lock() From 48f95e2a7e444d1214d67df5c0640369daaa17a7 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 18:09:59 +0900 Subject: [PATCH 12/37] test(g01): trace offline paired bridge stages --- .../livecanary/paired_fixture_runtime.go | 43 +++++++++++++++++-- .../g02-auth/broker_paired_bridge_test.go | 3 ++ 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/experiments/g01-scaleset/livecanary/paired_fixture_runtime.go b/experiments/g01-scaleset/livecanary/paired_fixture_runtime.go index 3592797..842b5c2 100644 --- a/experiments/g01-scaleset/livecanary/paired_fixture_runtime.go +++ b/experiments/g01-scaleset/livecanary/paired_fixture_runtime.go @@ -115,6 +115,10 @@ func fixtureFastPairCadence() pairedBaselineCadence { } } +func fixtureStage(path, stage string) { + _ = os.WriteFile(filepath.Join(path, "paired-fixture-stage"), []byte(stage), 0600) +} + // RunPairedTerminalForFixture calls the exported production entrypoint with // only generated private roots and a loopback TLS endpoint. The fast cadence // is a test-only clock seam; it still retains every production HTTP, journal, @@ -129,18 +133,49 @@ func RunPairedTerminalForFixture(ctx context.Context, files PairedTerminalFiles, if !filepath.IsAbs(controllerAdmissionDirectory) || filepath.Clean(controllerAdmissionDirectory) != controllerAdmissionDirectory { return ErrApproval } + fixtureStage(files.ControllerStateDirectory, "start") oldAdapters, oldCadence := pairedTerminalFixtureAdapters, pairedTerminalFixtureCadence pairedTerminalFixtureAdapters = &pairedTerminalAdapters{ openController: func(path string, a Approval) (*FileJournal, error) { - return OpenJournalForPairedFixtureAt(path, a, controllerAdmissionDirectory) + fixtureStage(files.ControllerStateDirectory, "open-controller") + j, err := OpenJournalForPairedFixtureAt(path, a, controllerAdmissionDirectory) + if err != nil { + fixtureStage(files.ControllerStateDirectory, "open-controller-error") + } else { + fixtureStage(files.ControllerStateDirectory, "open-controller-ok") + } + return j, err }, openWorker: func(path string, a liveworker.Approval) (*liveworker.FileJournal, error) { - return liveworker.OpenJournalForPairedFixture(path, a, path) + fixtureStage(files.ControllerStateDirectory, "open-worker") + j, err := liveworker.OpenJournalForPairedFixture(path, a, path) + if err != nil { + fixtureStage(files.ControllerStateDirectory, "open-worker-error") + } else { + fixtureStage(files.ControllerStateDirectory, "open-worker-ok") + } + return j, err }, newAPI: func(a Approval, credentials Credentials) (*SDKAPI, error) { - return NewSDKAPIForPairedFixture(a, credentials, baseURL, caPEM) + fixtureStage(files.ControllerStateDirectory, "new-api") + api, err := NewSDKAPIForPairedFixture(a, credentials, baseURL, caPEM) + if err != nil { + fixtureStage(files.ControllerStateDirectory, "new-api-error") + } else { + fixtureStage(files.ControllerStateDirectory, "new-api-ok") + } + return api, err + }, + newDocker: func(a liveworker.Approval) (*liveworker.Docker, error) { + fixtureStage(files.ControllerStateDirectory, "new-docker") + docker, err := liveworker.NewDocker(a) + if err != nil { + fixtureStage(files.ControllerStateDirectory, "new-docker-error") + } else { + fixtureStage(files.ControllerStateDirectory, "new-docker-ok") + } + return docker, err }, - newDocker: liveworker.NewDocker, } pairedTerminalFixtureCadence = fixtureFastPairCadence defer func() { diff --git a/experiments/g02-auth/broker_paired_bridge_test.go b/experiments/g02-auth/broker_paired_bridge_test.go index a95a877..5a3bc3c 100644 --- a/experiments/g02-auth/broker_paired_bridge_test.go +++ b/experiments/g02-auth/broker_paired_bridge_test.go @@ -783,6 +783,9 @@ func TestPairedBrokerChainsRealControllerCreatePreparationAndTerminal(t *testing if category, readErr := os.ReadFile(filepath.Join(controllerState, "paired-fixture-result")); readErr == nil { t.Logf("paired child category: %q", strings.TrimSpace(string(category))) } + if stage, readErr := os.ReadFile(filepath.Join(controllerState, "paired-fixture-stage")); readErr == nil { + t.Logf("paired child stage: %q", strings.TrimSpace(string(stage))) + } t.Fatalf("real paired bridge did not complete: status=%q err=%v", result.Status, err) } bridge.mu.Lock() From 1ad9aec5f397ed7c60de50c5533eea1171c9a13f Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 18:11:13 +0900 Subject: [PATCH 13/37] test(g01): isolate worker fixture admission root --- .../g01-scaleset/livecanary/paired_fixture_runtime.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/experiments/g01-scaleset/livecanary/paired_fixture_runtime.go b/experiments/g01-scaleset/livecanary/paired_fixture_runtime.go index 842b5c2..24a2888 100644 --- a/experiments/g01-scaleset/livecanary/paired_fixture_runtime.go +++ b/experiments/g01-scaleset/livecanary/paired_fixture_runtime.go @@ -133,6 +133,10 @@ func RunPairedTerminalForFixture(ctx context.Context, files PairedTerminalFiles, if !filepath.IsAbs(controllerAdmissionDirectory) || filepath.Clean(controllerAdmissionDirectory) != controllerAdmissionDirectory { return ErrApproval } + workerAdmissionDirectory := filepath.Join(filepath.Dir(files.WorkerStateDirectory), "worker-admission") + if err := os.Mkdir(workerAdmissionDirectory, 0700); err != nil && !os.IsExist(err) { + return ErrJournal + } fixtureStage(files.ControllerStateDirectory, "start") oldAdapters, oldCadence := pairedTerminalFixtureAdapters, pairedTerminalFixtureCadence pairedTerminalFixtureAdapters = &pairedTerminalAdapters{ @@ -148,7 +152,7 @@ func RunPairedTerminalForFixture(ctx context.Context, files PairedTerminalFiles, }, openWorker: func(path string, a liveworker.Approval) (*liveworker.FileJournal, error) { fixtureStage(files.ControllerStateDirectory, "open-worker") - j, err := liveworker.OpenJournalForPairedFixture(path, a, path) + j, err := liveworker.OpenJournalForPairedFixture(path, a, workerAdmissionDirectory) if err != nil { fixtureStage(files.ControllerStateDirectory, "open-worker-error") } else { From 6ef7f334fb60b14700e90978da6b65a5721e1a53 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 18:12:29 +0900 Subject: [PATCH 14/37] test(g01): diagnose worker fixture journal opening --- .../liveworker/paired_fixture_admission.go | 32 +++++++++++++++++-- .../g02-auth/broker_paired_bridge_test.go | 3 ++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/experiments/g01-scaleset/liveworker/paired_fixture_admission.go b/experiments/g01-scaleset/liveworker/paired_fixture_admission.go index 2cb8ccc..6a2c176 100644 --- a/experiments/g01-scaleset/liveworker/paired_fixture_admission.go +++ b/experiments/g01-scaleset/liveworker/paired_fixture_admission.go @@ -2,12 +2,40 @@ package liveworker -import "os" +import ( + "os" + "path/filepath" + "syscall" + "time" +) + +func fixtureWorkerStage(directory, stage string) { + _ = os.WriteFile(filepath.Join(directory, "paired-fixture-worker-stage"), []byte(stage), 0600) +} // OpenJournalForPairedFixture is a private test-only seam. It accepts only a // generated fixture admission root and is unavailable from ordinary builds; // production OpenJournal continues to derive its permanent root from the // native account. func OpenJournalForPairedFixture(directory string, a Approval, admissionDirectory string) (*FileJournal, error) { - return openJournalAtAdmission(directory, a, admissionDirectory, func(file *os.File) error { return file.Sync() }) + fixtureWorkerStage(directory, "start") + info, err := os.Lstat(directory) + if err != nil || !info.IsDir() || info.Mode().Perm() != 0700 { + fixtureWorkerStage(directory, "state-dir") + } else if stat, ok := info.Sys().(*syscall.Stat_t); !ok || int(stat.Uid) != os.Geteuid() { + fixtureWorkerStage(directory, "state-owner") + } else if a.Validate(time.Now()) != nil { + fixtureWorkerStage(directory, "approval") + } else if admissionInfo, admissionErr := os.Lstat(admissionDirectory); admissionErr != nil || !admissionInfo.IsDir() || admissionInfo.Mode().Perm() != 0700 { + fixtureWorkerStage(directory, "admission-dir") + } else if parentInfo, parentErr := os.Lstat(filepath.Dir(admissionDirectory)); parentErr != nil || parentInfo.Mode().Perm()&0022 != 0 { + fixtureWorkerStage(directory, "admission-parent") + } + j, err := openJournalAtAdmission(directory, a, admissionDirectory, func(file *os.File) error { return file.Sync() }) + if err != nil { + fixtureWorkerStage(directory, "underlying") + } else { + fixtureWorkerStage(directory, "ok") + } + return j, err } diff --git a/experiments/g02-auth/broker_paired_bridge_test.go b/experiments/g02-auth/broker_paired_bridge_test.go index 5a3bc3c..58f66d4 100644 --- a/experiments/g02-auth/broker_paired_bridge_test.go +++ b/experiments/g02-auth/broker_paired_bridge_test.go @@ -786,6 +786,9 @@ func TestPairedBrokerChainsRealControllerCreatePreparationAndTerminal(t *testing if stage, readErr := os.ReadFile(filepath.Join(controllerState, "paired-fixture-stage")); readErr == nil { t.Logf("paired child stage: %q", strings.TrimSpace(string(stage))) } + if stage, readErr := os.ReadFile(filepath.Join(workerState, "paired-fixture-worker-stage")); readErr == nil { + t.Logf("paired worker journal stage: %q", strings.TrimSpace(string(stage))) + } t.Fatalf("real paired bridge did not complete: status=%q err=%v", result.Status, err) } bridge.mu.Lock() From 8c298d03573977e1ab968f658072813d66fc3377 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 18:13:53 +0900 Subject: [PATCH 15/37] test(g01): trace worker fixture admission failure --- experiments/g01-scaleset/liveworker/journal.go | 14 ++++++++++++++ .../liveworker/paired_fixture_admission.go | 3 +++ 2 files changed, 17 insertions(+) diff --git a/experiments/g01-scaleset/liveworker/journal.go b/experiments/g01-scaleset/liveworker/journal.go index 78a63cb..301842a 100644 --- a/experiments/g01-scaleset/liveworker/journal.go +++ b/experiments/g01-scaleset/liveworker/journal.go @@ -37,6 +37,16 @@ type FileJournal struct { recordSync func(*os.File) error } +// The explicit tagged fixture may install a bounded diagnostic callback while +// production leaves this nil, preserving its fixed redacted error boundary. +var pairedFixtureJournalTrace func(string, string) + +func tracePairedFixtureJournal(directory, stage string) { + if pairedFixtureJournalTrace != nil { + pairedFixtureJournalTrace(directory, stage) + } +} + func privateFile(info os.FileInfo, mode os.FileMode) bool { s, ok := info.Sys().(*syscall.Stat_t) return ok && int(s.Uid) == os.Geteuid() && info.Mode().Perm() == mode && info.Mode().IsRegular() && s.Nlink == 1 @@ -234,18 +244,22 @@ func openJournalAtAdmission(directory string, a Approval, admissionDirectory str // valid header. File contents alone never prove its entry survived a crash. dir, err := root.Open(".") if err != nil { + tracePairedFixtureJournal(directory, "state-open") return nil, ErrState } err = syncDirectory(dir) dir.Close() if err != nil { + tracePairedFixtureJournal(directory, "state-sync") return nil, ErrState } j.claim, err = openAdmission(admissionDirectory, j, syncDirectory) if err != nil { + tracePairedFixtureJournal(directory, "admission-open") return nil, err } if j.paired.binding != nil && j.paired.binding.Worker != j.pairedIdentity() { + tracePairedFixtureJournal(directory, "binding") _ = j.claim.close() return nil, ErrState } diff --git a/experiments/g01-scaleset/liveworker/paired_fixture_admission.go b/experiments/g01-scaleset/liveworker/paired_fixture_admission.go index 6a2c176..a7e021d 100644 --- a/experiments/g01-scaleset/liveworker/paired_fixture_admission.go +++ b/experiments/g01-scaleset/liveworker/paired_fixture_admission.go @@ -18,6 +18,9 @@ func fixtureWorkerStage(directory, stage string) { // production OpenJournal continues to derive its permanent root from the // native account. func OpenJournalForPairedFixture(directory string, a Approval, admissionDirectory string) (*FileJournal, error) { + oldTrace := pairedFixtureJournalTrace + pairedFixtureJournalTrace = fixtureWorkerStage + defer func() { pairedFixtureJournalTrace = oldTrace }() fixtureWorkerStage(directory, "start") info, err := os.Lstat(directory) if err != nil || !info.IsDir() || info.Mode().Perm() != 0700 { From a9f2c4381297324aa4c735164bfe6a915103f070 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 18:14:19 +0900 Subject: [PATCH 16/37] test(g01): preserve worker journal failure stage --- .../g01-scaleset/liveworker/paired_fixture_admission.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/experiments/g01-scaleset/liveworker/paired_fixture_admission.go b/experiments/g01-scaleset/liveworker/paired_fixture_admission.go index a7e021d..6030029 100644 --- a/experiments/g01-scaleset/liveworker/paired_fixture_admission.go +++ b/experiments/g01-scaleset/liveworker/paired_fixture_admission.go @@ -36,7 +36,10 @@ func OpenJournalForPairedFixture(directory string, a Approval, admissionDirector } j, err := openJournalAtAdmission(directory, a, admissionDirectory, func(file *os.File) error { return file.Sync() }) if err != nil { - fixtureWorkerStage(directory, "underlying") + stage, readErr := os.ReadFile(filepath.Join(directory, "paired-fixture-worker-stage")) + if readErr != nil || string(stage) == "start" { + fixtureWorkerStage(directory, "underlying") + } } else { fixtureWorkerStage(directory, "ok") } From 0276d2dce43ef2450ed9049d24020d1408736033 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 18:15:09 +0900 Subject: [PATCH 17/37] test(g01): trace worker admission fixture failure --- experiments/g01-scaleset/liveworker/admission.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/experiments/g01-scaleset/liveworker/admission.go b/experiments/g01-scaleset/liveworker/admission.go index be1e196..3bb9bb0 100644 --- a/experiments/g01-scaleset/liveworker/admission.go +++ b/experiments/g01-scaleset/liveworker/admission.go @@ -63,31 +63,38 @@ type admissionClaim struct { func openAdmission(directory string, j *FileJournal, syncDirectory func(*os.File) error) (*admissionClaim, error) { if !filepath.IsAbs(directory) || filepath.Clean(directory) != directory { + tracePairedFixtureJournal(j.directory, "admission-path") return nil, ErrState } canonical, err := filepath.EvalSymlinks(directory) if err != nil || canonical != directory { + tracePairedFixtureJournal(j.directory, "admission-real") return nil, ErrState } info, err := os.Lstat(directory) if err != nil || !ownedDirectory(info, true) { + tracePairedFixtureJournal(j.directory, "admission-stat") return nil, ErrState } parentInfo, err := os.Lstat(filepath.Dir(directory)) if err != nil || !ownedDirectory(parentInfo, false) { + tracePairedFixtureJournal(j.directory, "admission-parent-stat") return nil, ErrState } parent, err := os.Open(filepath.Dir(directory)) if err != nil { + tracePairedFixtureJournal(j.directory, "admission-parent-open") return nil, ErrState } defer parent.Close() capturedParent, err := parent.Stat() if err != nil || !os.SameFile(parentInfo, capturedParent) || syncDirectory(parent) != nil { + tracePairedFixtureJournal(j.directory, "admission-parent-sync") return nil, ErrState } root, err := os.OpenRoot(directory) if err != nil { + tracePairedFixtureJournal(j.directory, "admission-root-open") return nil, ErrState } kept := false @@ -98,6 +105,7 @@ func openAdmission(directory string, j *FileJournal, syncDirectory func(*os.File }() captured, err := root.Stat(".") if err != nil || !os.SameFile(info, captured) { + tracePairedFixtureJournal(j.directory, "admission-root-stat") return nil, ErrState } // Serialize the empty-file creation window before taking the claim's @@ -105,11 +113,13 @@ func openAdmission(directory string, j *FileJournal, syncDirectory func(*os.File // claim, causing both contenders to refuse and strand an empty claim. dir, err := root.Open(".") if err != nil { + tracePairedFixtureJournal(j.directory, "admission-dir-open") return nil, ErrState } defer dir.Close() lockedDirectory, err := dir.Stat() if err != nil || !os.SameFile(info, lockedDirectory) || syscall.Flock(int(dir.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) != nil { + tracePairedFixtureJournal(j.directory, "admission-dir-lock") return nil, ErrState } created := true @@ -119,6 +129,7 @@ func openAdmission(directory string, j *FileJournal, syncDirectory func(*os.File file, err = root.OpenFile("admission.json", os.O_RDWR|syscall.O_NOFOLLOW, 0) } if err != nil { + tracePairedFixtureJournal(j.directory, "admission-file-open") return nil, ErrState } defer func() { @@ -128,24 +139,29 @@ func openAdmission(directory string, j *FileJournal, syncDirectory func(*os.File }() fileInfo, err := file.Stat() if err != nil || !privateFile(fileInfo, 0600) || fileInfo.Size() > 4096 || syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) != nil { + tracePairedFixtureJournal(j.directory, "admission-file-lock") return nil, ErrState } want := admissionFor(j) if created { data, err := json.Marshal(want) if err != nil { + tracePairedFixtureJournal(j.directory, "admission-marshal") return nil, ErrState } data = append(data, '\n') if n, err := file.Write(data); err != nil || n != len(data) { + tracePairedFixtureJournal(j.directory, "admission-write") return nil, ErrState } } claim := &admissionClaim{file, root, directory, info, fileInfo} if !claim.matches(j) || file.Sync() != nil { + tracePairedFixtureJournal(j.directory, "admission-claim") return nil, ErrState } if syncDirectory(dir) != nil { + tracePairedFixtureJournal(j.directory, "admission-sync") return nil, ErrState } kept = true From 0585f52b84b68b49eb28682c1dd4d1db4b7348c7 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 18:15:33 +0900 Subject: [PATCH 18/37] test(g01): retain detailed worker admission stage --- experiments/g01-scaleset/liveworker/journal.go | 1 - 1 file changed, 1 deletion(-) diff --git a/experiments/g01-scaleset/liveworker/journal.go b/experiments/g01-scaleset/liveworker/journal.go index 301842a..3d1bb54 100644 --- a/experiments/g01-scaleset/liveworker/journal.go +++ b/experiments/g01-scaleset/liveworker/journal.go @@ -255,7 +255,6 @@ func openJournalAtAdmission(directory string, a Approval, admissionDirectory str } j.claim, err = openAdmission(admissionDirectory, j, syncDirectory) if err != nil { - tracePairedFixtureJournal(directory, "admission-open") return nil, err } if j.paired.binding != nil && j.paired.binding.Worker != j.pairedIdentity() { From aa99d5fcab9a2dc847f8e3d118f92eed15ea0ab4 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 18:16:02 +0900 Subject: [PATCH 19/37] test(g01): canonicalize worker fixture admission path --- .../g01-scaleset/livecanary/paired_fixture_runtime.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/experiments/g01-scaleset/livecanary/paired_fixture_runtime.go b/experiments/g01-scaleset/livecanary/paired_fixture_runtime.go index 24a2888..27d8957 100644 --- a/experiments/g01-scaleset/livecanary/paired_fixture_runtime.go +++ b/experiments/g01-scaleset/livecanary/paired_fixture_runtime.go @@ -133,7 +133,11 @@ func RunPairedTerminalForFixture(ctx context.Context, files PairedTerminalFiles, if !filepath.IsAbs(controllerAdmissionDirectory) || filepath.Clean(controllerAdmissionDirectory) != controllerAdmissionDirectory { return ErrApproval } - workerAdmissionDirectory := filepath.Join(filepath.Dir(files.WorkerStateDirectory), "worker-admission") + workerStateReal, err := filepath.EvalSymlinks(files.WorkerStateDirectory) + if err != nil { + return ErrJournal + } + workerAdmissionDirectory := filepath.Join(filepath.Dir(workerStateReal), "worker-admission") if err := os.Mkdir(workerAdmissionDirectory, 0700); err != nil && !os.IsExist(err) { return ErrJournal } From 898c7c1f061dccc6f586870174baa6a19677b967 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 18:16:31 +0900 Subject: [PATCH 20/37] test(g01): record bounded bridge route sequence --- experiments/g02-auth/broker_paired_bridge_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/experiments/g02-auth/broker_paired_bridge_test.go b/experiments/g02-auth/broker_paired_bridge_test.go index 58f66d4..c4ba906 100644 --- a/experiments/g02-auth/broker_paired_bridge_test.go +++ b/experiments/g02-auth/broker_paired_bridge_test.go @@ -52,6 +52,7 @@ type pairedBrokerBridge struct { workerImage string workerImageID string workerDaemonID string + calls []string setExists bool setCreated bool setDeleted bool @@ -224,6 +225,7 @@ func (f *pairedBrokerBridge) handleGitHub(w http.ResponseWriter, r *http.Request f.mu.Lock() defer f.mu.Unlock() path := f.actionPath(r.URL.Path) + f.calls = append(f.calls, r.Method+" "+path) // These are the only REST calls that carry the temporary installation or // verification authorities. The bridge compares them but never records them. @@ -476,6 +478,7 @@ func (f *pairedBrokerBridge) handleDocker(w http.ResponseWriter, r *http.Request f.mu.Lock() defer f.mu.Unlock() path := r.URL.Path + f.calls = append(f.calls, r.Method+" "+path) if path == "/version" && r.Method == http.MethodGet { writeBridgeJSON(w, http.StatusOK, map[string]string{"ApiVersion": "1.51", "MinAPIVersion": "1.24"}) return @@ -778,6 +781,7 @@ func TestPairedBrokerChainsRealControllerCreatePreparationAndTerminal(t *testing if err != nil || result.Status != "paired_terminal_completed" { bridge.mu.Lock() t.Logf("bridge counters: registration=%d exchange=%d inventory=%d setCreate=%d sessionOpen=%d jit=%d acquire=%d ack=%d create=%d start=%d polls=%d unexpected=%d", bridge.registrationCalls, bridge.exchangeCalls, bridge.controllerInventory, bridge.setCreateCalls, bridge.sessionOpenCalls, bridge.jitCalls, bridge.acquireCalls, bridge.ackCalls, bridge.createCalls, bridge.startCalls, bridge.polls, bridge.unexpected) + t.Logf("bridge calls: %v", bridge.calls) bridge.mu.Unlock() t.Logf("broker API calls: %v", brokerFixture.calls) if category, readErr := os.ReadFile(filepath.Join(controllerState, "paired-fixture-result")); readErr == nil { From 12ae09c253e538fa99585bba0f99a967a672e929 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 18:18:01 +0900 Subject: [PATCH 21/37] test(g01): align bridge session statistics --- experiments/g02-auth/broker_paired_bridge_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/experiments/g02-auth/broker_paired_bridge_test.go b/experiments/g02-auth/broker_paired_bridge_test.go index c4ba906..0459bb6 100644 --- a/experiments/g02-auth/broker_paired_bridge_test.go +++ b/experiments/g02-auth/broker_paired_bridge_test.go @@ -439,7 +439,7 @@ func (f *pairedBrokerBridge) handleActions(w http.ResponseWriter, r *http.Reques } if strings.HasSuffix(path, "/runnerscalesets/7/sessions") && r.Method == http.MethodPost { f.sessionOpenCalls++ - writeBridgeJSON(w, http.StatusOK, map[string]any{"sessionId": "00000000-0000-4000-8000-000000000001", "ownerName": f.setName, "messageQueueUrl": f.server.URL + "/queue", "messageQueueAccessToken": f.queueToken, "statistics": map[string]int{}}) + writeBridgeJSON(w, http.StatusOK, map[string]any{"sessionId": "00000000-0000-4000-8000-000000000001", "ownerName": f.setName, "messageQueueUrl": f.server.URL + "/queue", "messageQueueAccessToken": f.queueToken, "statistics": map[string]int{"totalAvailableJobs": 0, "totalAcquiredJobs": 0, "totalAssignedJobs": 1, "totalRunningJobs": 0, "totalRegisteredRunners": 0, "totalBusyRunners": 0, "totalIdleRunners": 0}}) return } if strings.Contains(path, "/runnerscalesets/7/sessions/") && r.Method == http.MethodDelete { From aac1127f6e19018570027a2273327d027ddee5f8 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 18:19:20 +0900 Subject: [PATCH 22/37] test(g01): identify unexpected bridge route --- experiments/g02-auth/broker_paired_bridge_test.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/experiments/g02-auth/broker_paired_bridge_test.go b/experiments/g02-auth/broker_paired_bridge_test.go index 0459bb6..80488cb 100644 --- a/experiments/g02-auth/broker_paired_bridge_test.go +++ b/experiments/g02-auth/broker_paired_bridge_test.go @@ -53,6 +53,8 @@ type pairedBrokerBridge struct { workerImageID string workerDaemonID string calls []string + unexpectedCalls []string + lastCall string setExists bool setCreated bool setDeleted bool @@ -137,6 +139,7 @@ func newPairedBrokerBridge(t *testing.T) *pairedBrokerBridge { func (f *pairedBrokerBridge) markUnexpected() { f.unexpected++ + f.unexpectedCalls = append(f.unexpectedCalls, f.lastCall) } func writeBridgeJSON(w http.ResponseWriter, status int, value any) { @@ -225,7 +228,8 @@ func (f *pairedBrokerBridge) handleGitHub(w http.ResponseWriter, r *http.Request f.mu.Lock() defer f.mu.Unlock() path := f.actionPath(r.URL.Path) - f.calls = append(f.calls, r.Method+" "+path) + f.lastCall = r.Method + " " + path + f.calls = append(f.calls, f.lastCall) // These are the only REST calls that carry the temporary installation or // verification authorities. The bridge compares them but never records them. @@ -478,7 +482,8 @@ func (f *pairedBrokerBridge) handleDocker(w http.ResponseWriter, r *http.Request f.mu.Lock() defer f.mu.Unlock() path := r.URL.Path - f.calls = append(f.calls, r.Method+" "+path) + f.lastCall = r.Method + " " + path + f.calls = append(f.calls, f.lastCall) if path == "/version" && r.Method == http.MethodGet { writeBridgeJSON(w, http.StatusOK, map[string]string{"ApiVersion": "1.51", "MinAPIVersion": "1.24"}) return @@ -782,6 +787,7 @@ func TestPairedBrokerChainsRealControllerCreatePreparationAndTerminal(t *testing bridge.mu.Lock() t.Logf("bridge counters: registration=%d exchange=%d inventory=%d setCreate=%d sessionOpen=%d jit=%d acquire=%d ack=%d create=%d start=%d polls=%d unexpected=%d", bridge.registrationCalls, bridge.exchangeCalls, bridge.controllerInventory, bridge.setCreateCalls, bridge.sessionOpenCalls, bridge.jitCalls, bridge.acquireCalls, bridge.ackCalls, bridge.createCalls, bridge.startCalls, bridge.polls, bridge.unexpected) t.Logf("bridge calls: %v", bridge.calls) + t.Logf("bridge unexpected calls: %v", bridge.unexpectedCalls) bridge.mu.Unlock() t.Logf("broker API calls: %v", brokerFixture.calls) if category, readErr := os.ReadFile(filepath.Join(controllerState, "paired-fixture-result")); readErr == nil { From cca479af1f523d5dda12d9e8e236bab81baf3679 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 18:19:56 +0900 Subject: [PATCH 23/37] test(g01): accept queue authority for job acquisition --- experiments/g02-auth/broker_paired_bridge_test.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/experiments/g02-auth/broker_paired_bridge_test.go b/experiments/g02-auth/broker_paired_bridge_test.go index 80488cb..6b6f1e1 100644 --- a/experiments/g02-auth/broker_paired_bridge_test.go +++ b/experiments/g02-auth/broker_paired_bridge_test.go @@ -357,7 +357,8 @@ func (f *pairedBrokerBridge) handleGitHub(w http.ResponseWriter, r *http.Request // All remaining routes are Actions service calls. The SDK pins the admin // token in the generated private client and never follows redirects. if strings.HasPrefix(path, "/_apis/") || path == "/queue" || strings.HasPrefix(path, "/queue/") { - if f.adminToken != "" && !f.auth(r, "Bearer "+f.adminToken) && path != "/queue" && !strings.HasPrefix(path, "/queue/") { + queueCredentialPath := strings.HasSuffix(path, "/acquirejobs") + if f.adminToken != "" && !f.auth(r, "Bearer "+f.adminToken) && !queueCredentialPath && path != "/queue" && !strings.HasPrefix(path, "/queue/") { f.markUnexpected() writeBridgeJSON(w, http.StatusForbidden, nil) return @@ -457,6 +458,11 @@ func (f *pairedBrokerBridge) handleActions(w http.ResponseWriter, r *http.Reques return } if strings.HasSuffix(path, "/acquirejobs") && r.Method == http.MethodPost { + if !f.auth(r, "Bearer "+f.queueToken) { + f.markUnexpected() + writeBridgeJSON(w, http.StatusForbidden, nil) + return + } f.acquireCalls++ writeBridgeJSON(w, http.StatusOK, map[string]any{"count": 1, "value": []int64{42}}) return From c61dc220f31ed95a0937f928bfdec22d4783d17b Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 18:29:39 +0900 Subject: [PATCH 24/37] test(g01): finalize private paired bridge fixture --- .../cmd/g01-live/fixture_support.go | 16 +---- .../livecanary/paired_fixture_runtime.go | 41 ++---------- .../g01-scaleset/liveworker/admission.go | 16 ----- .../g01-scaleset/liveworker/journal.go | 13 ---- .../liveworker/paired_fixture_admission.go | 34 +--------- .../g02-auth/broker_paired_bridge_test.go | 67 ++----------------- 6 files changed, 13 insertions(+), 174 deletions(-) diff --git a/experiments/g01-scaleset/cmd/g01-live/fixture_support.go b/experiments/g01-scaleset/cmd/g01-live/fixture_support.go index e9f6928..b9a646d 100644 --- a/experiments/g01-scaleset/cmd/g01-live/fixture_support.go +++ b/experiments/g01-scaleset/cmd/g01-live/fixture_support.go @@ -4,7 +4,6 @@ package main import ( "context" - "errors" "io" "os" "path/filepath" @@ -66,19 +65,6 @@ func init() { if err != nil { return err } - err = livecanary.RunPairedTerminalForFixture(ctx, files, c, config.BaseURL, []byte(config.CAPEM), config.AdmissionDirectory) - // Temporary fixture-only classification for local bridge diagnosis; the - // production command still emits only its fixed redacted failure text. - category := "other" - switch { - case errors.Is(err, livecanary.ErrApproval): - category = "approval" - case errors.Is(err, livecanary.ErrJournal): - category = "journal" - case errors.Is(err, livecanary.ErrQuarantine): - category = "quarantine" - } - _ = os.WriteFile(filepath.Join(files.ControllerStateDirectory, "paired-fixture-result"), []byte(category), 0600) - return err + return livecanary.RunPairedTerminalForFixture(ctx, files, c, config.BaseURL, []byte(config.CAPEM), config.AdmissionDirectory) } } diff --git a/experiments/g01-scaleset/livecanary/paired_fixture_runtime.go b/experiments/g01-scaleset/livecanary/paired_fixture_runtime.go index 27d8957..81c0d4b 100644 --- a/experiments/g01-scaleset/livecanary/paired_fixture_runtime.go +++ b/experiments/g01-scaleset/livecanary/paired_fixture_runtime.go @@ -115,10 +115,6 @@ func fixtureFastPairCadence() pairedBaselineCadence { } } -func fixtureStage(path, stage string) { - _ = os.WriteFile(filepath.Join(path, "paired-fixture-stage"), []byte(stage), 0600) -} - // RunPairedTerminalForFixture calls the exported production entrypoint with // only generated private roots and a loopback TLS endpoint. The fast cadence // is a test-only clock seam; it still retains every production HTTP, journal, @@ -141,48 +137,19 @@ func RunPairedTerminalForFixture(ctx context.Context, files PairedTerminalFiles, if err := os.Mkdir(workerAdmissionDirectory, 0700); err != nil && !os.IsExist(err) { return ErrJournal } - fixtureStage(files.ControllerStateDirectory, "start") oldAdapters, oldCadence := pairedTerminalFixtureAdapters, pairedTerminalFixtureCadence pairedTerminalFixtureAdapters = &pairedTerminalAdapters{ openController: func(path string, a Approval) (*FileJournal, error) { - fixtureStage(files.ControllerStateDirectory, "open-controller") - j, err := OpenJournalForPairedFixtureAt(path, a, controllerAdmissionDirectory) - if err != nil { - fixtureStage(files.ControllerStateDirectory, "open-controller-error") - } else { - fixtureStage(files.ControllerStateDirectory, "open-controller-ok") - } - return j, err + return OpenJournalForPairedFixtureAt(path, a, controllerAdmissionDirectory) }, openWorker: func(path string, a liveworker.Approval) (*liveworker.FileJournal, error) { - fixtureStage(files.ControllerStateDirectory, "open-worker") - j, err := liveworker.OpenJournalForPairedFixture(path, a, workerAdmissionDirectory) - if err != nil { - fixtureStage(files.ControllerStateDirectory, "open-worker-error") - } else { - fixtureStage(files.ControllerStateDirectory, "open-worker-ok") - } - return j, err + return liveworker.OpenJournalForPairedFixture(path, a, workerAdmissionDirectory) }, newAPI: func(a Approval, credentials Credentials) (*SDKAPI, error) { - fixtureStage(files.ControllerStateDirectory, "new-api") - api, err := NewSDKAPIForPairedFixture(a, credentials, baseURL, caPEM) - if err != nil { - fixtureStage(files.ControllerStateDirectory, "new-api-error") - } else { - fixtureStage(files.ControllerStateDirectory, "new-api-ok") - } - return api, err + return NewSDKAPIForPairedFixture(a, credentials, baseURL, caPEM) }, newDocker: func(a liveworker.Approval) (*liveworker.Docker, error) { - fixtureStage(files.ControllerStateDirectory, "new-docker") - docker, err := liveworker.NewDocker(a) - if err != nil { - fixtureStage(files.ControllerStateDirectory, "new-docker-error") - } else { - fixtureStage(files.ControllerStateDirectory, "new-docker-ok") - } - return docker, err + return liveworker.NewDocker(a) }, } pairedTerminalFixtureCadence = fixtureFastPairCadence diff --git a/experiments/g01-scaleset/liveworker/admission.go b/experiments/g01-scaleset/liveworker/admission.go index 3bb9bb0..be1e196 100644 --- a/experiments/g01-scaleset/liveworker/admission.go +++ b/experiments/g01-scaleset/liveworker/admission.go @@ -63,38 +63,31 @@ type admissionClaim struct { func openAdmission(directory string, j *FileJournal, syncDirectory func(*os.File) error) (*admissionClaim, error) { if !filepath.IsAbs(directory) || filepath.Clean(directory) != directory { - tracePairedFixtureJournal(j.directory, "admission-path") return nil, ErrState } canonical, err := filepath.EvalSymlinks(directory) if err != nil || canonical != directory { - tracePairedFixtureJournal(j.directory, "admission-real") return nil, ErrState } info, err := os.Lstat(directory) if err != nil || !ownedDirectory(info, true) { - tracePairedFixtureJournal(j.directory, "admission-stat") return nil, ErrState } parentInfo, err := os.Lstat(filepath.Dir(directory)) if err != nil || !ownedDirectory(parentInfo, false) { - tracePairedFixtureJournal(j.directory, "admission-parent-stat") return nil, ErrState } parent, err := os.Open(filepath.Dir(directory)) if err != nil { - tracePairedFixtureJournal(j.directory, "admission-parent-open") return nil, ErrState } defer parent.Close() capturedParent, err := parent.Stat() if err != nil || !os.SameFile(parentInfo, capturedParent) || syncDirectory(parent) != nil { - tracePairedFixtureJournal(j.directory, "admission-parent-sync") return nil, ErrState } root, err := os.OpenRoot(directory) if err != nil { - tracePairedFixtureJournal(j.directory, "admission-root-open") return nil, ErrState } kept := false @@ -105,7 +98,6 @@ func openAdmission(directory string, j *FileJournal, syncDirectory func(*os.File }() captured, err := root.Stat(".") if err != nil || !os.SameFile(info, captured) { - tracePairedFixtureJournal(j.directory, "admission-root-stat") return nil, ErrState } // Serialize the empty-file creation window before taking the claim's @@ -113,13 +105,11 @@ func openAdmission(directory string, j *FileJournal, syncDirectory func(*os.File // claim, causing both contenders to refuse and strand an empty claim. dir, err := root.Open(".") if err != nil { - tracePairedFixtureJournal(j.directory, "admission-dir-open") return nil, ErrState } defer dir.Close() lockedDirectory, err := dir.Stat() if err != nil || !os.SameFile(info, lockedDirectory) || syscall.Flock(int(dir.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) != nil { - tracePairedFixtureJournal(j.directory, "admission-dir-lock") return nil, ErrState } created := true @@ -129,7 +119,6 @@ func openAdmission(directory string, j *FileJournal, syncDirectory func(*os.File file, err = root.OpenFile("admission.json", os.O_RDWR|syscall.O_NOFOLLOW, 0) } if err != nil { - tracePairedFixtureJournal(j.directory, "admission-file-open") return nil, ErrState } defer func() { @@ -139,29 +128,24 @@ func openAdmission(directory string, j *FileJournal, syncDirectory func(*os.File }() fileInfo, err := file.Stat() if err != nil || !privateFile(fileInfo, 0600) || fileInfo.Size() > 4096 || syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) != nil { - tracePairedFixtureJournal(j.directory, "admission-file-lock") return nil, ErrState } want := admissionFor(j) if created { data, err := json.Marshal(want) if err != nil { - tracePairedFixtureJournal(j.directory, "admission-marshal") return nil, ErrState } data = append(data, '\n') if n, err := file.Write(data); err != nil || n != len(data) { - tracePairedFixtureJournal(j.directory, "admission-write") return nil, ErrState } } claim := &admissionClaim{file, root, directory, info, fileInfo} if !claim.matches(j) || file.Sync() != nil { - tracePairedFixtureJournal(j.directory, "admission-claim") return nil, ErrState } if syncDirectory(dir) != nil { - tracePairedFixtureJournal(j.directory, "admission-sync") return nil, ErrState } kept = true diff --git a/experiments/g01-scaleset/liveworker/journal.go b/experiments/g01-scaleset/liveworker/journal.go index 3d1bb54..78a63cb 100644 --- a/experiments/g01-scaleset/liveworker/journal.go +++ b/experiments/g01-scaleset/liveworker/journal.go @@ -37,16 +37,6 @@ type FileJournal struct { recordSync func(*os.File) error } -// The explicit tagged fixture may install a bounded diagnostic callback while -// production leaves this nil, preserving its fixed redacted error boundary. -var pairedFixtureJournalTrace func(string, string) - -func tracePairedFixtureJournal(directory, stage string) { - if pairedFixtureJournalTrace != nil { - pairedFixtureJournalTrace(directory, stage) - } -} - func privateFile(info os.FileInfo, mode os.FileMode) bool { s, ok := info.Sys().(*syscall.Stat_t) return ok && int(s.Uid) == os.Geteuid() && info.Mode().Perm() == mode && info.Mode().IsRegular() && s.Nlink == 1 @@ -244,13 +234,11 @@ func openJournalAtAdmission(directory string, a Approval, admissionDirectory str // valid header. File contents alone never prove its entry survived a crash. dir, err := root.Open(".") if err != nil { - tracePairedFixtureJournal(directory, "state-open") return nil, ErrState } err = syncDirectory(dir) dir.Close() if err != nil { - tracePairedFixtureJournal(directory, "state-sync") return nil, ErrState } j.claim, err = openAdmission(admissionDirectory, j, syncDirectory) @@ -258,7 +246,6 @@ func openJournalAtAdmission(directory string, a Approval, admissionDirectory str return nil, err } if j.paired.binding != nil && j.paired.binding.Worker != j.pairedIdentity() { - tracePairedFixtureJournal(directory, "binding") _ = j.claim.close() return nil, ErrState } diff --git a/experiments/g01-scaleset/liveworker/paired_fixture_admission.go b/experiments/g01-scaleset/liveworker/paired_fixture_admission.go index 6030029..4ae7ce6 100644 --- a/experiments/g01-scaleset/liveworker/paired_fixture_admission.go +++ b/experiments/g01-scaleset/liveworker/paired_fixture_admission.go @@ -4,44 +4,12 @@ package liveworker import ( "os" - "path/filepath" - "syscall" - "time" ) -func fixtureWorkerStage(directory, stage string) { - _ = os.WriteFile(filepath.Join(directory, "paired-fixture-worker-stage"), []byte(stage), 0600) -} - // OpenJournalForPairedFixture is a private test-only seam. It accepts only a // generated fixture admission root and is unavailable from ordinary builds; // production OpenJournal continues to derive its permanent root from the // native account. func OpenJournalForPairedFixture(directory string, a Approval, admissionDirectory string) (*FileJournal, error) { - oldTrace := pairedFixtureJournalTrace - pairedFixtureJournalTrace = fixtureWorkerStage - defer func() { pairedFixtureJournalTrace = oldTrace }() - fixtureWorkerStage(directory, "start") - info, err := os.Lstat(directory) - if err != nil || !info.IsDir() || info.Mode().Perm() != 0700 { - fixtureWorkerStage(directory, "state-dir") - } else if stat, ok := info.Sys().(*syscall.Stat_t); !ok || int(stat.Uid) != os.Geteuid() { - fixtureWorkerStage(directory, "state-owner") - } else if a.Validate(time.Now()) != nil { - fixtureWorkerStage(directory, "approval") - } else if admissionInfo, admissionErr := os.Lstat(admissionDirectory); admissionErr != nil || !admissionInfo.IsDir() || admissionInfo.Mode().Perm() != 0700 { - fixtureWorkerStage(directory, "admission-dir") - } else if parentInfo, parentErr := os.Lstat(filepath.Dir(admissionDirectory)); parentErr != nil || parentInfo.Mode().Perm()&0022 != 0 { - fixtureWorkerStage(directory, "admission-parent") - } - j, err := openJournalAtAdmission(directory, a, admissionDirectory, func(file *os.File) error { return file.Sync() }) - if err != nil { - stage, readErr := os.ReadFile(filepath.Join(directory, "paired-fixture-worker-stage")) - if readErr != nil || string(stage) == "start" { - fixtureWorkerStage(directory, "underlying") - } - } else { - fixtureWorkerStage(directory, "ok") - } - return j, err + return openJournalAtAdmission(directory, a, admissionDirectory, func(file *os.File) error { return file.Sync() }) } diff --git a/experiments/g02-auth/broker_paired_bridge_test.go b/experiments/g02-auth/broker_paired_bridge_test.go index 6b6f1e1..d07ab50 100644 --- a/experiments/g02-auth/broker_paired_bridge_test.go +++ b/experiments/g02-auth/broker_paired_bridge_test.go @@ -19,7 +19,6 @@ import ( "strconv" "strings" "sync" - "syscall" "testing" "time" ) @@ -52,9 +51,6 @@ type pairedBrokerBridge struct { workerImage string workerImageID string workerDaemonID string - calls []string - unexpectedCalls []string - lastCall string setExists bool setCreated bool setDeleted bool @@ -139,7 +135,6 @@ func newPairedBrokerBridge(t *testing.T) *pairedBrokerBridge { func (f *pairedBrokerBridge) markUnexpected() { f.unexpected++ - f.unexpectedCalls = append(f.unexpectedCalls, f.lastCall) } func writeBridgeJSON(w http.ResponseWriter, status int, value any) { @@ -228,8 +223,6 @@ func (f *pairedBrokerBridge) handleGitHub(w http.ResponseWriter, r *http.Request f.mu.Lock() defer f.mu.Unlock() path := f.actionPath(r.URL.Path) - f.lastCall = r.Method + " " + path - f.calls = append(f.calls, f.lastCall) // These are the only REST calls that carry the temporary installation or // verification authorities. The bridge compares them but never records them. @@ -470,7 +463,9 @@ func (f *pairedBrokerBridge) handleActions(w http.ResponseWriter, r *http.Reques if strings.HasSuffix(path, "/agents/81") && r.Method == http.MethodGet { f.sdkRunnerCalls++ if f.polls >= 2 { - writeBridgeJSON(w, http.StatusNotFound, map[string]string{"message": "runner absent"}) + // The real SDK only classifies this as RunnerNotFoundError when the + // service error includes its canonical typeName. + writeBridgeJSON(w, http.StatusNotFound, map[string]string{"typeName": "AgentNotFoundException", "message": "synthetic missing runner"}) return } writeBridgeJSON(w, http.StatusOK, map[string]any{"id": 81, "name": f.workerName, "runnerScaleSetId": 7}) @@ -488,8 +483,6 @@ func (f *pairedBrokerBridge) handleDocker(w http.ResponseWriter, r *http.Request f.mu.Lock() defer f.mu.Unlock() path := r.URL.Path - f.lastCall = r.Method + " " + path - f.calls = append(f.calls, f.lastCall) if path == "/version" && r.Method == http.MethodGet { writeBridgeJSON(w, http.StatusOK, map[string]string{"ApiVersion": "1.51", "MinAPIVersion": "1.24"}) return @@ -653,34 +646,6 @@ func writePrivateBridgeJSON(t *testing.T, path string, value any) []byte { return data } -func pairedBridgeBinding(t *testing.T, controllerPath, controllerState, workerPath, workerState string) map[string]any { - t.Helper() - identity := func(path string) (uint64, uint64) { - info, err := os.Stat(path) - if err != nil { - t.Fatal("bridge identity") - } - stat, ok := info.Sys().(*syscall.Stat_t) - if !ok { - t.Fatal("bridge identity type") - } - return uint64(stat.Dev), stat.Ino - } - digest := func(path string) string { - data, err := os.ReadFile(path) - if err != nil { - t.Fatal("bridge approval digest") - } - h := sha256.Sum256(data) - return hexDigest(h[:]) - } - controllerDevice, controllerInode := identity(controllerPath) - controllerStateDevice, controllerStateInode := identity(controllerState) - workerDevice, workerInode := identity(workerPath) - workerStateDevice, workerStateInode := identity(workerState) - return map[string]any{"controller_approval_sha256": digest(controllerPath), "controller_approval_device": controllerDevice, "controller_approval_inode": controllerInode, "controller_state_device": controllerStateDevice, "controller_state_inode": controllerStateInode, "worker_approval_sha256": digest(workerPath), "worker_approval_device": workerDevice, "worker_approval_inode": workerInode, "worker_state_device": workerStateDevice, "worker_state_inode": workerStateInode} -} - func TestPairedBrokerChainsRealControllerCreatePreparationAndTerminal(t *testing.T) { if runtime.GOOS != "darwin" && runtime.GOOS != "linux" { t.Skip("private Unix fixture requires a Unix host") @@ -749,13 +714,6 @@ func TestPairedBrokerChainsRealControllerCreatePreparationAndTerminal(t *testing if _, err := os.Stat(filepath.Join(brokerFixture.admissionRoot, "admission.json")); err != nil { t.Fatal("controller admission claim") } - // Diagnostic preflight while narrowing the real child boundary; this is - // credential-free and does not alter the already-completed controller - // history. Remove once the broker-launched preparation path is green. - prepCtx, prepCancel := context.WithTimeout(context.Background(), time.Minute) - runBridgeCommand(t, prepCtx, binaryPath, []string{"--prepare-approved-paired-journal", "--approval", controllerPath, "--state-dir", controllerState}, nil) - prepCancel() - worker := pairedWorkerApproval{RunnerUpdatesDisabled: true, HarnessSHA: harness, WorkflowSHA: workflow, OwnerNonce: controller.OwnerNonce, Controller: controller.Controller, Endpoint: bridge.dockerEndpoint, DaemonID: bridge.workerDaemonID, ImageID: bridge.workerImageID, Image: pairedWorkerImage, ExpiresAt: expires, Phases: []string{"create", "start", "inspect", "cleanup"}} workerPath := filepath.Join(parent, "worker-approval.json") writePrivateBridgeJSON(t, workerPath, worker) @@ -790,23 +748,12 @@ func TestPairedBrokerChainsRealControllerCreatePreparationAndTerminal(t *testing result, err := runBrokerWithAPI(brokerCtx, BrokerFiles{ApprovalPath: approvalPath, StateDirectory: attempt, ControllerBinary: binaryPath, ControllerApproval: controllerPath, ControllerStateDirectory: controllerState, WorkerApproval: workerPath, WorkerStateDirectory: workerState}, input, api) brokerCancel() if err != nil || result.Status != "paired_terminal_completed" { - bridge.mu.Lock() - t.Logf("bridge counters: registration=%d exchange=%d inventory=%d setCreate=%d sessionOpen=%d jit=%d acquire=%d ack=%d create=%d start=%d polls=%d unexpected=%d", bridge.registrationCalls, bridge.exchangeCalls, bridge.controllerInventory, bridge.setCreateCalls, bridge.sessionOpenCalls, bridge.jitCalls, bridge.acquireCalls, bridge.ackCalls, bridge.createCalls, bridge.startCalls, bridge.polls, bridge.unexpected) - t.Logf("bridge calls: %v", bridge.calls) - t.Logf("bridge unexpected calls: %v", bridge.unexpectedCalls) - bridge.mu.Unlock() - t.Logf("broker API calls: %v", brokerFixture.calls) - if category, readErr := os.ReadFile(filepath.Join(controllerState, "paired-fixture-result")); readErr == nil { - t.Logf("paired child category: %q", strings.TrimSpace(string(category))) - } - if stage, readErr := os.ReadFile(filepath.Join(controllerState, "paired-fixture-stage")); readErr == nil { - t.Logf("paired child stage: %q", strings.TrimSpace(string(stage))) - } - if stage, readErr := os.ReadFile(filepath.Join(workerState, "paired-fixture-worker-stage")); readErr == nil { - t.Logf("paired worker journal stage: %q", strings.TrimSpace(string(stage))) - } t.Fatalf("real paired bridge did not complete: status=%q err=%v", result.Status, err) } + journalData, err = os.ReadFile(filepath.Join(controllerState, "journal.jsonl")) + if err != nil { + t.Fatal("controller journal after paired terminal") + } bridge.mu.Lock() counts := []int{bridge.createCalls, bridge.startCalls, bridge.jitCalls, bridge.acquireCalls, bridge.ackCalls, bridge.sessionOpenCalls, bridge.sessionCloseCalls, bridge.workerDeleteCalls, bridge.workerAbsenceCalls, bridge.setCreateCalls, bridge.setDeleteCalls, bridge.setAbsenceCalls, bridge.rosterCalls, bridge.unexpected} bridge.mu.Unlock() From caa29f7eeb5cbf0937101de250218521d0ea8188 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 18:35:35 +0900 Subject: [PATCH 25/37] test(g02): retain failed paired claim before inspect --- experiments/g02-auth/broker_paired_test.go | 94 +++++++++++++++++++++- 1 file changed, 92 insertions(+), 2 deletions(-) diff --git a/experiments/g02-auth/broker_paired_test.go b/experiments/g02-auth/broker_paired_test.go index 0ef7ed8..b9eb7ca 100644 --- a/experiments/g02-auth/broker_paired_test.go +++ b/experiments/g02-auth/broker_paired_test.go @@ -91,9 +91,9 @@ func TestPairedWorkerApprovalMismatchRefusesBeforeBinding(t *testing.T) { func TestPairedWorkerDaemonIDMatchesCanonicalBoundaries(t *testing.T) { for _, tc := range []struct { - name string + name string daemon string - valid bool + valid bool }{ {name: "colon and 128 bytes", daemon: "a:" + strings.Repeat("d", 126), valid: true}, {name: "129 bytes", daemon: "a" + strings.Repeat("d", 128), valid: false}, @@ -208,6 +208,96 @@ func TestPairedBrokerBindsWorkerBeforeWorkflowVerifiedHandoff(t *testing.T) { } } +func TestPairedFailureAllowsAuthorizedInspectWithoutPairedRetry(t *testing.T) { + a, candidate, api, fixture, attempt := newBrokerFixture(t) + a.Mode, a.Phase, a.AllowVerificationAuthority = "paired-terminal", "paired-terminal", true + parent := filepath.Dir(attempt) + pairedLaunches := 0 + pairedPlan := brokerTestPlan(t, &a, parent, func(context.Context, []byte, string) error { + pairedLaunches++ + return errBroker + }) + pairedPlan.controller.Phases = []string{"create", "before-ack", "after-ack", "before-acquire", "inspect", "cleanup"} + pairedPlan.controller.WorkflowRunID = 7 + pairedPlan.raw, _ = json.Marshal(pairedPlan.controller) + a.ControllerApprovalSHA256 = brokerBytesDigest(pairedPlan.raw) + pairedPlan.approval = a + workerState := filepath.Join(parent, "failed-paired-worker-state") + if err := os.Mkdir(workerState, 0700); err != nil { + t.Fatal("worker state") + } + worker := pairedWorkerApproval{RunnerUpdatesDisabled: true, HarnessSHA: pairedPlan.controller.HarnessSHA, WorkflowSHA: pairedPlan.controller.WorkflowSHA, OwnerNonce: pairedPlan.controller.OwnerNonce, Controller: pairedPlan.controller.Controller, Endpoint: "/tmp/g01-paired-failure.sock", DaemonID: "fixture-daemon", ImageID: "sha256:" + strings.Repeat("d", 64), Image: pairedWorkerImage, ExpiresAt: pairedPlan.controller.ExpiresAt, Phases: []string{"create", "start", "inspect", "cleanup"}} + workerData, err := json.Marshal(worker) + if err != nil { + t.Fatal("worker approval") + } + workerPath := filepath.Join(parent, "failed-paired-worker-approval.json") + if err := os.WriteFile(workerPath, workerData, 0600); err != nil { + t.Fatal("worker approval file") + } + pairedPlan.worker, err = openBrokerWorkerPlan(workerPath, workerState, pairedPlan.statePath, a, pairedPlan.controller) + if err != nil { + t.Fatal("paired worker plan") + } + if _, err := brokerExecute(context.Background(), a, brokerInput{PEM: string(candidate.PEM), VerificationToken: "synthetic-private-workflow-token"}, attempt, api, pairedPlan); err == nil || fixture.tokenCalls != 1 || pairedLaunches != 1 { + t.Fatalf("failed paired attempt was accepted or retried: err=%v mints=%d launches=%d", err, fixture.tokenCalls, pairedLaunches) + } + + // Inspect is a distinct, explicitly authorized controller slot. It may + // collect its own authenticated evidence, but it must not replay the + // incomplete paired claim or invoke the worker handoff again. + a.Mode, a.Phase = "controller", "inspect" + inspectLaunches := 0 + inspectPlan := brokerTestPlan(t, &a, parent, func(context.Context, []byte, string) error { + inspectLaunches++ + return nil + }) + inspectPlan.controller.Phases = append([]string(nil), pairedPlan.controller.Phases...) + inspectPlan.controller.WorkflowRunID = pairedPlan.controller.WorkflowRunID + inspectPlan.raw, _ = json.Marshal(inspectPlan.controller) + a.ControllerApprovalSHA256 = brokerBytesDigest(inspectPlan.raw) + inspectPlan.approval = a + _ = inspectPlan.state.Close() + inspectPlan.state, err = openBrokerPrivateDirectory(pairedPlan.statePath) + if err != nil { + t.Fatal("reopen controller state") + } + t.Cleanup(func() { _ = inspectPlan.state.Close() }) + inspectPlan.statePath = pairedPlan.statePath + inspectPlan.stateInfo, err = inspectPlan.state.Stat(".") + if err != nil { + t.Fatal("controller state identity") + } + result, err := brokerExecute(context.Background(), a, brokerInput{PEM: string(candidate.PEM), VerificationToken: "synthetic-private-workflow-token"}, filepath.Join(parent, "inspect-attempt"), api, inspectPlan) + if err != nil || result.Status != "controller_completed" || fixture.tokenCalls != 2 || pairedLaunches != 1 || inspectLaunches != 1 { + t.Fatalf("authorized inspect did not remain distinct from failed pair: result=%+v err=%v mints=%d paired=%d inspect=%d", result, err, fixture.tokenCalls, pairedLaunches, inspectLaunches) + } + // The inspect slot is also one-shot; reopening it cannot mint or launch. + retryPlan := brokerTestPlan(t, &a, parent, func(context.Context, []byte, string) error { + t.Fatal("inspect retry launched") + return nil + }) + retryPlan.controller.Phases = append([]string(nil), pairedPlan.controller.Phases...) + retryPlan.controller.WorkflowRunID = pairedPlan.controller.WorkflowRunID + retryPlan.raw, _ = json.Marshal(retryPlan.controller) + a.ControllerApprovalSHA256 = brokerBytesDigest(retryPlan.raw) + retryPlan.approval = a + _ = retryPlan.state.Close() + retryPlan.state, err = openBrokerPrivateDirectory(pairedPlan.statePath) + if err != nil { + t.Fatal("reopen controller state for retry") + } + t.Cleanup(func() { _ = retryPlan.state.Close() }) + retryPlan.statePath = pairedPlan.statePath + retryPlan.stateInfo, err = retryPlan.state.Stat(".") + if err != nil { + t.Fatal("controller state retry identity") + } + if _, err := brokerExecute(context.Background(), a, brokerInput{PEM: string(candidate.PEM), VerificationToken: "synthetic-private-workflow-token"}, filepath.Join(parent, "inspect-retry"), api, retryPlan); err == nil || fixture.tokenCalls != 2 || pairedLaunches != 1 || inspectLaunches != 1 { + t.Fatalf("inspect retry replayed paired or inspect effects: err=%v mints=%d paired=%d inspect=%d", err, fixture.tokenCalls, pairedLaunches, inspectLaunches) + } +} + // This is intentionally an entrypoint-level fixture. It uses the real // BrokerFiles loader, paired preparation process and brokerExecute closure, so // the canonical preparation receipt and fixed child handoff are both exercised. From 895a478eb1ed894710c75b3426e23cb3b1bceac9 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 18:36:16 +0900 Subject: [PATCH 26/37] test(g02): fence paired mint on short parent authority --- experiments/g02-auth/broker_paired_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/experiments/g02-auth/broker_paired_test.go b/experiments/g02-auth/broker_paired_test.go index b9eb7ca..3910dbf 100644 --- a/experiments/g02-auth/broker_paired_test.go +++ b/experiments/g02-auth/broker_paired_test.go @@ -132,6 +132,21 @@ func TestPairedApprovalRejectsInsufficientTerminalAuthority(t *testing.T) { } } +func TestPairedBrokerParentAuthorityStopsBeforeMintOrLaunch(t *testing.T) { + a, candidate, api, fixture, attempt := newBrokerFixture(t) + a.Mode, a.Phase = "paired-terminal", "paired-terminal" + launches := 0 + plan := brokerTestPlan(t, &a, filepath.Dir(attempt), func(context.Context, []byte, string) error { + launches++ + return nil + }) + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + if _, err := brokerExecute(ctx, a, brokerInput{PEM: string(candidate.PEM)}, attempt, api, plan); err == nil || fixture.tokenCalls != 0 || len(fixture.calls) != 0 || launches != 0 { + t.Fatalf("insufficient parent authority reached effects: err=%v mints=%d calls=%d launches=%d", err, fixture.tokenCalls, len(fixture.calls), launches) + } +} + func TestPairedBrokerBindsWorkerBeforeWorkflowVerifiedHandoff(t *testing.T) { a, candidate, api, fixture, attempt := newBrokerFixture(t) a.Mode, a.Phase, a.AllowVerificationAuthority = "paired-terminal", "paired-terminal", true From 811e6bce60102f24d261a5a39d831a94737c813f Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 18:41:02 +0900 Subject: [PATCH 27/37] docs(g01): record paired broker second-fix evidence --- docs/evidence/g01-paired-broker.md | 240 ++++++++++++++++++----------- 1 file changed, 151 insertions(+), 89 deletions(-) diff --git a/docs/evidence/g01-paired-broker.md b/docs/evidence/g01-paired-broker.md index 178e5ea..9c7c86e 100644 --- a/docs/evidence/g01-paired-broker.md +++ b/docs/evidence/g01-paired-broker.md @@ -1,117 +1,179 @@ # G01g: paired terminal executable and bounded broker handoff -Issue [60](https://github.com/1XP-AI/gh-runnerd/issues/60) connects the reviewed -paired terminal sequence to one tagged `g01-live` executable and a dedicated +Issue [60](https://github.com/1XP-AI/gh-runnerd/issues/60) and PR +[62](https://github.com/1XP-AI/gh-runnerd/pull/62) connect the reviewed paired +terminal sequence to one tagged `g01-live` executable and a dedicated `g01-broker` mode. This is an offline experiment continuation, not a live authorization, production daemon, or closure of G01/G02. -## Implementation boundary - -The paired broker approval now requires the explicit `paired-terminal` phase. -The canonical `runBrokerWithAPI` path invokes a dedicated fixed -`--prepare-approved-paired-journal` child contract and accepts only its -paired-terminal preparation receipt. The preparation contract proves a fresh -controller journal and admission claim under controller authority; it does -not borrow cleanup authority, read credentials, contact a remote service, or -authorize worker effects. - -After preparation, the broker captures the exact controller snapshot and -worker approval byte hashes plus controller/worker approval and state-root -device/inode identities. The fixed child argv carries only those paths and a -bounded credential-free binding; the controller-only credential payload is -bounded and contains no PEM. The child validates the binding before reading -controller credentials and before constructing SDK/Docker adapters, compares -the argv binding with the broker-supplied payload binding, and revalidates the -same identities throughout the terminal sequence and before completion. The -canonical controller approval and journal-derived `PairInput` remain the only -pairing authority; the binding is identity evidence, not a second pairing -manifest. - -The exported `livecanary.RunPairedTerminal` path now owns paired journal/API/ -Unix adapter construction. A private `g01_pair_fixture` seam redirects only -generated temporary journal/admission roots, the synthetic private TLS API, -and the Unix fixture; it does not expose runtime authority or alter account, -Keychain, runner, Docker, or service state. The tagged fixture calls the -exported entrypoint and verifies two session acknowledgements, one acquire, -JIT, create/start, original-session close, non-force worker deletion plus -absence, owned-set deletion plus absence, complete rosters, real journal/lease -behavior, and secret-free journals. Cancellation, lost response, reopened -history, changed approval bytes/inodes, changed state roots, and symlinked -roots fail before new effects. - -Paired child execution has a fixed argv and `LANG=C`/`LC_ALL=C` environment, -bounded output, a 30-second child timeout, cancellation handling, no retry, -and one logical controller stdin consumption. Existing controller-only, -discovery, and separate-worker paths retain their prior refusal and authority -boundaries. - -## TDD evidence and checks - -The immutable review baseline retained meaningful red behavior in commit -`3acd5ab`: +## Reviewed baseline and finding ledger + +The required first step was a normal fetch and merge of reviewed +`origin/main` at `8dd64adc551ba5174892807a678e8bc614d0a474` (the merged CI fix). +It produced merge commit `a5fcffd`; no rebase, amend, force update, workflow +replay, or live operation was performed. The implementation and focused tests +were then completed through source head +`895a478eb1ed894710c75b3426e23cb3b1bceac9` (the evidence-only commit follows +this tested source head). + +Both settled Luna/max reports were read: `/tmp/g01-paired-broker-review-ad7c2cf.md` +and `/tmp/g01-paired-review-ad7c2cf.md`. + +The exact-head Codex surfaces were also read, including the stale inline +finding, all current inline findings, review summaries, and the prior issue +comment: + +- stale phase-receipt finding: [discussion 3955069674](https://github.com/1XP-AI/gh-runnerd/pull/62#discussion_r3955069674) +- historical ledger mode: [discussion 3955069682](https://github.com/1XP-AI/gh-runnerd/pull/62#discussion_r3955069682) +- daemon-ID contract: [discussion 3955069689](https://github.com/1XP-AI/gh-runnerd/pull/62#discussion_r3955069689) +- canonical prerequisite history: [discussion 3955590270](https://github.com/1XP-AI/gh-runnerd/pull/62#discussion_r3955590270) +- child deadline: [discussion 3955590276](https://github.com/1XP-AI/gh-runnerd/pull/62#discussion_r3955590276) +- old Codex review: [review 5138884210](https://github.com/1XP-AI/gh-runnerd/pull/62#pullrequestreview-5138884210) +- Codex review summary: [comment 5580386456](https://github.com/1XP-AI/gh-runnerd/pull/62#issuecomment-5580386456) +- prior integrator note: [comment 5580442642](https://github.com/1XP-AI/gh-runnerd/pull/62#issuecomment-5580442642) + +The stale phase-receipt finding is resolved by a dedicated paired preparation +phase and paired receipt validation; the newer prerequisite-history finding +was the deeper version of that contract and is resolved below. The historical +ledger finding is reproduced by +`TestBrokerPairedAdmissionAcceptsHistoricalControllerClaim` and +`TestPairedFailureAllowsAuthorizedInspectWithoutPairedRetry`; both now accept +a prior controller claim under a paired request and a failed paired claim +under an authorized controller inspect while preserving one-shot slots. The +daemon-ID finding is reproduced at the colon and 128-byte boundaries by +`TestPairedWorkerDaemonIDMatchesCanonicalBoundaries`. The history and deadline +findings were reproduced by the red tests in `a0df276` and `278d8e9`, then fixed +in `419f9cd` and subsequent focused commits. -```text -GOTOOLCHAIN=go1.26.8 go test -race -count=1 -timeout=60s -tags=g01_live -run '^TestPairedTerminalMode' ./cmd/g01-live -exit 1: paired mode refused before its credential input gate (reads=0) - -GOTOOLCHAIN=go1.26.8 go test -race -count=1 -timeout=60s -run '^TestPairedTerminalBrokerApprovalUsesDedicatedMode$' ./... -exit 1: paired terminal broker approval was refused -``` +## Implementation boundary -Before implementing the canonical entrypoint fix, the new behavioral -regression test was run against the frozen implementation: +`PreparePairedJournal` now requires the exact canonical controller-create +prefix: create phase, nonempty lowercase SHA-256 inventory, discovery intent +and result, create intent and successful create result. It preserves those +events byte-for-byte and only captures the intended preparation receipt under +the existing controller claim. Fresh, pending, deleted, uncertain, reserved, +previous-paired, malformed, or noncanonical histories are rejected; cleanup +authority is never borrowed and no remote effect or credential read occurs. + +Broker admission replays every historical ledger event against the mode, +phase, schema, and authority recorded in that event's slot. Current paired +mode therefore does not reject a historical controller create, and current +controller inspect/cleanup can inspect a retained failed paired claim. The +cross-identity, tamper, ownership, authority-transition, incomplete-claim, +and one-shot current-attempt checks remain in force. + +Paired approval validation reserves a complete terminal budget. The child +deadline is the minimum of parent, broker, controller, and worker authority, +then capped at ten minutes; it must leave a 35-second production cadence plus +25 seconds of child margin and a separate credential margin. Insufficient +remaining authority fails before mint/launch. Cancellation, expiry, deadline +overflow, output overflow, lost response, and retry paths remain fail-stop +with no automatic cleanup or retry; the ordinary 30-second preparation bound +is unchanged. + +The paired worker daemon ID uses the authoritative worker contract +`^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$`; unrelated controller fields retain their +narrower validators. + +The tagged offline bridge starts a fresh private TLS server and private Unix +Docker endpoint. It builds the reviewed `g01-live` executable from a clean +temporary clone with VCS metadata, then runs the real controller-create CLI, +real paired-preparation child, broker entrypoint, and exported +`RunPairedTerminal`. The fixture seam only supplies generated private roots, +loopback TLS CA, and the Unix endpoint; it cannot select production account, +Keychain, runner, Docker, service, or GitHub state. + +## End-to-end evidence + +The critical chain is +`TestPairedBrokerChainsRealControllerCreatePreparationAndTerminal` in +`experiments/g02-auth/broker_paired_bridge_test.go`. The controller-create +child first produced the same six canonical history records used by paired +execution; the broker then ran the real paired preparation contract against +that history and launched the actual tagged executable through the private +TLS/Unix bridge. The terminal child appended baseline records to the original +controller journal and created the separate worker paired journal. + +The successful run asserted these exact bridge counts: ```text -GOTOOLCHAIN=go1.26.8 go test -count=1 -run '^TestPairedBrokerRealEntrypointUsesPairedPreparationClosure$' . -exit 1: real paired entrypoint did not complete one handoff ... mints=0 +create=1 start=1 JIT=1 acquire=1 acknowledgements=2 +session-open=1 session-close=1 worker-delete=1 worker-absence=1 +set-create=1 set-delete=1 set-absence=1 complete-rosters=4 unexpected=0 +broker installation-token mints=1 ``` -The focused green checks then passed: +It also asserted the broker ledger's paired controller/worker claim, original +session close, non-force worker deletion plus absence, set deletion plus +absence, canonical journal continuation, separate worker journal, and +secret-free attempt/controller/worker/admission roots. Reopening the completed +real broker entrypoint stopped before a second mint or terminal effect. + +The failed-paired recovery test separately proves an incomplete paired claim +is retained, one explicitly authorized controller inspect can proceed in its +own slot, and a repeated inspect cannot mint or launch again. Hash/inode and +symlink replacement fences are covered by the paired binding and snapshot +tests. Existing paired terminal partitions cover cancellation, expired +authority, lost responses, journal uncertainty, reopened histories, receipt +separation, worker/set absence, and no-replay behavior. + +The fast cadence used only by the tagged bridge is a deterministic test clock; +it advances the same seven five-second waits as production. The production +cadence proof is `TestPairedDistinctIDsAndOriginalCadence`, which requires +eight rounds and seven waits of at least five seconds (at least 35 seconds). +Together with the real child bridge run and +`TestBrokerPairedChildDeadlineIsBoundedAndLeavesCadenceMargin`, this proves a +bounded child may complete beyond the old 30-second limit while retaining a +finite authority cap. No production timeout was made unbounded. + +## TDD and verification record + +The meaningful red tests were committed before implementation: ```text -GOTOOLCHAIN=go1.26.8 go test -count=1 -run '^TestPairedBrokerRealEntrypointUsesPairedPreparationClosure$|^TestBrokerPaired' . -ok github.com/1XP-AI/gh-runnerd/experiments/g02-auth 1.665s - -GOTOOLCHAIN=go1.26.8 go test -tags g01_live -count=1 ./cmd/g01-live -ok github.com/1XP-AI/gh-runnerd/experiments/g01-scaleset/cmd/g01-live 0.846s +GOTOOLCHAIN=go1.26.8 go test -count=1 -run '^TestPairedPreparationPreservesCanonicalControllerHistory|^TestPairedPreparationRejectsFreshAndNonCanonicalHistory$' ./livecanary +exit 1 on the pre-fix implementation: the canonical history contract was absent. -GOTOOLCHAIN=go1.26.8 go test -tags g01_pair_fixture -count=1 -timeout=120s -run '^TestPairedTerminalExported|^TestPairedTerminalBinding' ./livecanary -ok github.com/1XP-AI/gh-runnerd/experiments/g01-scaleset/livecanary +GOTOOLCHAIN=go1.26.8 go test -count=1 -run '^TestBrokerPairedAdmissionAcceptsHistoricalControllerClaim|^TestPairedWorkerDaemonIDMatchesCanonicalBoundaries|^TestPairedApprovalRejectsInsufficientTerminalAuthority$' . +exit 1 on the pre-fix implementation: historical mode, daemon-ID boundaries, and authority budget were wrong. ``` -Pinned verification completed without live resources: +Focused green checks on the tested source head were: ```text -GOTOOLCHAIN=go1.26.8 bash scripts/check-offline-experiments.sh -offline experiment checks passed: 2 module(s) +GOTOOLCHAIN=go1.26.8 go test -count=1 -timeout=180s ./livecanary +ok 27.089s -GOTOOLCHAIN=go1.26.8 go test -race -count=1 ./... -ok: root module +GOTOOLCHAIN=go1.26.8 go test -count=1 -timeout=120s -tags='g01_live,g01_pair_fixture' ./cmd/g01-live +ok 0.844s -GOTOOLCHAIN=go1.26.8 go vet ./... -ok: root module +GOTOOLCHAIN=go1.26.8 go test -count=1 -timeout=300s -run 'Test(Paired|BrokerPaired|BrokerChild|BrokerBuild|BrokerPipe)' . +ok 11.240s before the final recovery-only test; the added recovery and parent-authority tests also passed in 0.679s. +GOTOOLCHAIN=go1.26.8 go test -count=1 -timeout=240s -run '^TestPairedBrokerChainsRealControllerCreatePreparationAndTerminal$' . +ok 3.337s after fixture cleanup; the same test passed at 3.147s before cleanup. + +gofmt -d experiments/g01-scaleset/cmd/g01-live/main_test.go +no output git diff --check ok ``` -The offline gate covers both module race/vet suites, tagged `g01-live`/ -`g01-worker` CLI tests, tagged paired fixture partitions, and the G02 module -suites on Go 1.26.8. The command and tooling files owned by issue #61 were not -edited; their canonical command updates still need integration by that task. - -No live GitHub endpoint, App, credential, runner/group/workflow, Docker/Lima -configuration, Keychain, launchd service, reboot, or manually installed -runner was touched. The same-UID private-file model is not hostile-code -isolation, and no production daemon or G01 recovery/live completion is -claimed. +The mandated full root `make check` result must be recorded here after the +final evidence edit; it includes formatting, build, vet, root tests and race +tests, fuzz smoke, dependency/license checks, both offline experiment modules, +and the pinned vulnerability check. No claim of full-goal completion is made +until the coordinator confirms independent exact-head review and hosted CI. -## Remaining gates and rollback +## Safety limits and remaining gates -Independent Luna/max review, hosted CI, and exact-head GitHub Codex review are -still required. The coordinator owns those review, stale-finding, CI, and -merge gates; a pending or unreviewed exact head blocks merge. +All tests use disposable local files, synthetic nonsecret credentials, private +loopback TLS, and private Unix sockets. No live GitHub endpoint, App, +credential, runner/group/workflow, Docker/Lima context, Keychain, launchd +service, reboot, or manually installed runner was touched. Same-UID private +file ownership is not hostile-code isolation; same-UID races after released +short checks remain outside the proof. -Offline rollback is source-only: revert the focused issue-60 commit(s). The -tests create only temporary local TLS/Unix fixtures and require no runner, -Docker, Keychain, launchd, or GitHub cleanup. +The coordinator owns pushing evidence, requesting two independent reviews of +the exact final head, reading all inline and issue-comment findings including +stale ones, waiting for fresh Codex review and hosted CI, and merge gating. From 0ecb06c1755bc3a2f49724c9b7d5fa2bc9a0c3a9 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 18:57:50 +0900 Subject: [PATCH 28/37] docs(g01): record full root check result --- docs/evidence/g01-paired-broker.md | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/evidence/g01-paired-broker.md b/docs/evidence/g01-paired-broker.md index 9c7c86e..921bb99 100644 --- a/docs/evidence/g01-paired-broker.md +++ b/docs/evidence/g01-paired-broker.md @@ -159,11 +159,19 @@ git diff --check ok ``` -The mandated full root `make check` result must be recorded here after the -final evidence edit; it includes formatting, build, vet, root tests and race -tests, fuzz smoke, dependency/license checks, both offline experiment modules, -and the pinned vulnerability check. No claim of full-goal completion is made -until the coordinator confirms independent exact-head review and hosted CI. +The mandated full root check was run after all source and test edits, before +this evidence-only update: + +```text +make check +exit 0 +toolchain, fmt-check, build, vet, root tests, root race tests, fuzz smoke, +dependency/license checks, both offline experiment modules, and govulncheck +all passed. +``` + +No claim of full-goal completion is made until the coordinator confirms +independent exact-head review and hosted CI. ## Safety limits and remaining gates From 6a35fe731202776379a9b6020a0faaae915514b5 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 19:57:39 +0900 Subject: [PATCH 29/37] fix(g01): gate paired worker preparation before mint --- .../cmd/g01-live/fixture_support.go | 11 ++ experiments/g01-scaleset/cmd/g01-live/main.go | 28 ++++- .../livecanary/paired_fixture_runtime.go | 7 ++ .../g01-scaleset/liveworker/journal_test.go | 28 +++++ .../liveworker/paired_fixture_admission.go | 9 ++ .../g01-scaleset/liveworker/preparation.go | 107 ++++++++++++++++++ experiments/g02-auth/broker.go | 13 ++- experiments/g02-auth/broker_admission.go | 9 +- experiments/g02-auth/broker_admission_test.go | 1 + experiments/g02-auth/broker_entry.go | 5 + .../g02-auth/broker_paired_bridge_test.go | 7 +- experiments/g02-auth/broker_paired_test.go | 71 ++++++++++++ experiments/g02-auth/broker_plan.go | 47 ++++++-- experiments/g02-auth/broker_preparation.go | 7 ++ .../g02-auth/broker_preparation_test.go | 63 +++++++++++ experiments/g02-auth/broker_process.go | 38 ++++++- experiments/g02-auth/broker_process_test.go | 75 ++++++++++++ 17 files changed, 508 insertions(+), 18 deletions(-) create mode 100644 experiments/g01-scaleset/liveworker/preparation.go diff --git a/experiments/g01-scaleset/cmd/g01-live/fixture_support.go b/experiments/g01-scaleset/cmd/g01-live/fixture_support.go index b9a646d..8064dd8 100644 --- a/experiments/g01-scaleset/cmd/g01-live/fixture_support.go +++ b/experiments/g01-scaleset/cmd/g01-live/fixture_support.go @@ -10,6 +10,7 @@ import ( "syscall" "github.com/1XP-AI/gh-runnerd/experiments/g01-scaleset/livecanary" + "github.com/1XP-AI/gh-runnerd/experiments/g01-scaleset/liveworker" ) type fixtureEndpointConfig struct { @@ -53,6 +54,16 @@ func init() { } return livecanary.PreparePairedJournalForFixtureAt(stateDirectory, a, config.AdmissionDirectory) } + prepareWorkerJournalForCommand = func(stateDirectory string, a liveworker.Approval) (liveworker.PreparationReceipt, error) { + // Worker admission is a distinct disposable fixture root. It is derived + // from the worker state identity, never supplied by production approval + // or the broker's controller admission root. + admissionDirectory := filepath.Join(filepath.Dir(stateDirectory), "worker-admission") + if err := os.Mkdir(admissionDirectory, 0700); err != nil && !os.IsExist(err) { + return liveworker.PreparationReceipt{}, liveworker.ErrState + } + return livecanary.PrepareWorkerJournalForPairedFixtureAt(stateDirectory, a, admissionDirectory) + } newSDKAPIForCommand = func(a livecanary.Approval, c livecanary.Credentials, stateDirectory string) (*livecanary.SDKAPI, error) { config, err := readFixtureEndpointConfig(stateDirectory) if err != nil { diff --git a/experiments/g01-scaleset/cmd/g01-live/main.go b/experiments/g01-scaleset/cmd/g01-live/main.go index 76d41b6..80a7bf2 100644 --- a/experiments/g01-scaleset/cmd/g01-live/main.go +++ b/experiments/g01-scaleset/cmd/g01-live/main.go @@ -43,6 +43,7 @@ func buildRevision() (string, bool) { var openJournalForCommand = livecanary.OpenJournal var pairedPrepareJournalForCommand = livecanary.PreparePairedJournal +var prepareWorkerJournalForCommand = liveworker.PrepareJournal var newSDKAPIForCommand = func(a livecanary.Approval, c livecanary.Credentials, _ string) (*livecanary.SDKAPI, error) { return livecanary.NewSDKAPI(a, c) } @@ -69,6 +70,7 @@ func runWithPreparation(args []string, in io.Reader, out io.Writer, revisionForB pairedExecute := flags.Bool("execute-approved-paired-terminal", false, "") prepare := flags.Bool("prepare-approved-journal", false, "") pairedPrepare := flags.Bool("prepare-approved-paired-journal", false, "") + prepareWorker := flags.Bool("prepare-approved-paired-worker-journal", false, "") approvalPath := flags.String("approval", "", "") statePath := flags.String("state-dir", "", "") phase := flags.String("phase", "", "") @@ -83,7 +85,7 @@ func runWithPreparation(args []string, in io.Reader, out io.Writer, revisionForB return reject() } workerInputs := *workerApprovalPath != "" || *workerStatePath != "" - if *plan && !*execute && !*pairedExecute && !*prepare && !*pairedPrepare { + if *plan && !*execute && !*pairedExecute && !*prepare && !*pairedPrepare && !*prepareWorker { if *approvalPath != "" || *statePath != "" || *phase != "" || workerInputs || *pairedBinding != "" { return reject() } @@ -103,10 +105,17 @@ func runWithPreparation(args []string, in io.Reader, out io.Writer, revisionForB if *pairedPrepare { modeCount++ } + if *prepareWorker { + modeCount++ + } if modeCount != 1 || *plan || *approvalPath == "" || *statePath == "" { return reject() } - if *pairedExecute { + if *prepareWorker { + if *phase != "" || workerInputs || *pairedBinding != "" || prepareWorkerJournalForCommand == nil { + return reject() + } + } else if *pairedExecute { if *phase != "" || *workerApprovalPath == "" || *workerStatePath == "" || *pairedBinding == "" { return reject() } @@ -117,6 +126,21 @@ func runWithPreparation(args []string, in io.Reader, out io.Writer, revisionForB } else if *phase == "" || workerInputs || *pairedBinding != "" { return reject() } + if *prepareWorker { + worker, workerErr := liveworker.ReadApproval(*approvalPath) + if workerErr != nil || worker.Validate(time.Now()) != nil { + return reject() + } + revision, ok := revisionForBuild() + if !ok || revision != worker.HarnessSHA { + return reject() + } + receipt, preparationErr := prepareWorkerJournalForCommand(*statePath, worker) + if preparationErr != nil || json.NewEncoder(out).Encode(receipt) != nil { + return reject() + } + return 0 + } a, err := livecanary.ReadApproval(*approvalPath) if err != nil || a.Validate(time.Now()) != nil || (!*pairedExecute && !*pairedPrepare && !slices.Contains(a.Phases, *phase)) { return reject() diff --git a/experiments/g01-scaleset/livecanary/paired_fixture_runtime.go b/experiments/g01-scaleset/livecanary/paired_fixture_runtime.go index 81c0d4b..7a6960e 100644 --- a/experiments/g01-scaleset/livecanary/paired_fixture_runtime.go +++ b/experiments/g01-scaleset/livecanary/paired_fixture_runtime.go @@ -42,6 +42,13 @@ func PreparePairedJournalForFixtureAt(directory string, a Approval, admissionDir }) } +// PrepareWorkerJournalForPairedFixtureAt routes worker preparation through the +// canonical liveworker journal/admission parser while keeping the root inside +// the generated offline fixture. +func PrepareWorkerJournalForPairedFixtureAt(directory string, a liveworker.Approval, admissionDirectory string) (liveworker.PreparationReceipt, error) { + return liveworker.PrepareJournalForPairedFixture(directory, a, admissionDirectory) +} + func fixtureEndpoint(baseURL string, caPEM []byte) (*url.URL, *x509.CertPool, *x509.Certificate, error) { u, err := url.Parse(baseURL) if err != nil || u.Scheme != "https" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || u.Path != "" { diff --git a/experiments/g01-scaleset/liveworker/journal_test.go b/experiments/g01-scaleset/liveworker/journal_test.go index b77378d..f5d204d 100644 --- a/experiments/g01-scaleset/liveworker/journal_test.go +++ b/experiments/g01-scaleset/liveworker/journal_test.go @@ -54,6 +54,34 @@ func TestPrivateJournalLocksAndRetainsReservationAcrossRestart(t *testing.T) { } } +func TestWorkerPreparationReturnsCanonicalSnapshotAndRejectsPriorEffect(t *testing.T) { + dir := privateDir(t) + a := approval() + open := func(path string, approval Approval) (*FileJournal, error) { + return openTestJournal(t, path, approval) + } + receipt, err := prepareJournal(dir, a, open) + if err != nil { + t.Fatal("fresh worker preparation refused", err) + } + if receipt.Version != 1 || receipt.Status != "worker_journal_prepared" || receipt.Phase != "paired-worker" || receipt.ApprovalDigest != approvalDigest(a) || receipt.State.Inode == 0 || receipt.Journal.Inode == 0 || receipt.Claim.Inode == 0 || !id.MatchString(receipt.JournalDigest) || !id.MatchString(receipt.ClaimDigest) { + t.Fatalf("incomplete worker preparation receipt: %+v", receipt) + } + j, err := openTestJournal(t, dir, a) + if err != nil { + t.Fatal("reopen prepared worker journal", err) + } + if err := j.Append(Event{Kind: "intent", Operation: "create"}); err != nil { + t.Fatal("persist prior worker intent", err) + } + if err := j.Close(); err != nil { + t.Fatal("close prior worker journal", err) + } + if _, err := prepareJournal(dir, a, open); err == nil { + t.Fatal("worker preparation adopted prior effect") + } +} + func TestJournalRejectsChangedApprovalTornTailAndUnsafeFiles(t *testing.T) { for _, fault := range []string{"endpoint", "daemon", "image", "workflow", "tail", "symlink", "hardlink", "mode"} { t.Run(fault, func(t *testing.T) { diff --git a/experiments/g01-scaleset/liveworker/paired_fixture_admission.go b/experiments/g01-scaleset/liveworker/paired_fixture_admission.go index 4ae7ce6..dd6c92f 100644 --- a/experiments/g01-scaleset/liveworker/paired_fixture_admission.go +++ b/experiments/g01-scaleset/liveworker/paired_fixture_admission.go @@ -13,3 +13,12 @@ import ( func OpenJournalForPairedFixture(directory string, a Approval, admissionDirectory string) (*FileJournal, error) { return openJournalAtAdmission(directory, a, admissionDirectory, func(file *os.File) error { return file.Sync() }) } + +// PrepareJournalForPairedFixture is the explicit offline-only counterpart to +// PrepareJournal. It keeps the canonical worker parser and admission checks in +// this package while allowing generated tests to supply a disposable root. +func PrepareJournalForPairedFixture(directory string, a Approval, admissionDirectory string) (PreparationReceipt, error) { + return prepareJournal(directory, a, func(path string, approval Approval) (*FileJournal, error) { + return OpenJournalForPairedFixture(path, approval, admissionDirectory) + }) +} diff --git a/experiments/g01-scaleset/liveworker/preparation.go b/experiments/g01-scaleset/liveworker/preparation.go new file mode 100644 index 0000000..485b079 --- /dev/null +++ b/experiments/g01-scaleset/liveworker/preparation.go @@ -0,0 +1,107 @@ +package liveworker + +import ( + "crypto/sha256" + "encoding/hex" + "io" + "os" + "time" +) + +// PreparationReceipt is a credential-free snapshot of the canonical worker +// journal/admission boundary. It is returned only after OpenJournal has replayed +// the complete history and held the worker authority lease. +type PreparationReceipt struct { + Version int `json:"version"` + Status string `json:"status"` + Phase string `json:"phase"` + ApprovalDigest string `json:"approval_digest"` + State FileIdentity `json:"state"` + Journal FileIdentity `json:"journal"` + Claim FileIdentity `json:"claim"` + JournalDigest string `json:"journal_digest"` + ClaimDigest string `json:"claim_digest"` +} + +func preparationDigest(file *os.File, limit int64) (string, error) { + info, err := file.Stat() + if err != nil || info.Size() < 1 || info.Size() > limit { + return "", ErrState + } + hash := sha256.New() + read, err := io.Copy(hash, io.NewSectionReader(file, 0, limit+1)) + if err != nil || read != info.Size() { + return "", ErrState + } + return hex.EncodeToString(hash.Sum(nil)), nil +} + +func prepareJournal(directory string, a Approval, open func(string, Approval) (*FileJournal, error)) (receipt PreparationReceipt, err error) { + if open == nil || a.Validate(time.Now()) != nil { + return receipt, ErrApproval + } + j, err := open(directory, a) + if err != nil { + return receipt, ErrState + } + defer func() { + if j.Close() != nil { + receipt = PreparationReceipt{} + err = ErrState + } + }() + release, err := j.authorize(a) + if err != nil { + return receipt, ErrState + } + defer release() + // The paired broker is preparing the first worker handoff. A prior effect, + // reservation, uncertainty or paired binding is retained for inspection but + // cannot be adopted as fresh authority for another mint/launch. + for _, event := range j.Events() { + if event.Kind != "authority" { + return receipt, ErrState + } + } + if !j.ownsCurrentJournal() || j.claim == nil || !j.claim.matches(j) { + return receipt, ErrState + } + journalInfo, err := j.file.Stat() + if err != nil { + return receipt, ErrState + } + claimInfo, err := j.claim.file.Stat() + if err != nil { + return receipt, ErrState + } + journalDigest, err := preparationDigest(j.file, maxJournal) + if err != nil { + return receipt, ErrState + } + claimDigest, err := preparationDigest(j.claim.file, 4096) + if err != nil { + return receipt, ErrState + } + if !j.ownsCurrentJournal() || !j.claim.matches(j) { + return receipt, ErrState + } + return PreparationReceipt{ + Version: 1, + Status: "worker_journal_prepared", + Phase: "paired-worker", + ApprovalDigest: approvalDigest(a), + State: identityOf(j.directoryInfo), + Journal: identityOf(journalInfo), + Claim: identityOf(claimInfo), + JournalDigest: journalDigest, + ClaimDigest: claimDigest, + }, nil +} + +// PrepareJournal validates the worker approval, replays its canonical journal, +// and checks the permanent admission claim without reading credentials or +// contacting a runtime. It may initialize a new worker journal, but never +// records a worker effect. +func PrepareJournal(directory string, a Approval) (PreparationReceipt, error) { + return prepareJournal(directory, a, OpenJournal) +} diff --git a/experiments/g02-auth/broker.go b/experiments/g02-auth/broker.go index d509bdc..547a5c1 100644 --- a/experiments/g02-auth/broker.go +++ b/experiments/g02-auth/broker.go @@ -50,6 +50,14 @@ var errBroker = errors.New("broker stopped; retain private intent and review; no var brokerComponent = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]{0,99}$`) var brokerWorkerComponent = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$`) var brokerPhases = map[string]bool{"create": true, "before-ack": true, "after-ack": true, "before-acquire": true, "acquire-loss": true, "jit-loss": true, "inspect": true, "cleanup": true} +var brokerSpecialSlots = map[string]bool{"discover-actions-host": true, "paired-terminal": true} + +func brokerSlotAllowed(slot string) bool { return brokerPhases[slot] || brokerSpecialSlots[slot] } + +// One header, one claim and one completion for every finite schema slot, and +// the final empty split element from the required trailing newline. This is a +// structural bound derived from the actual slot schema, not an unbounded log. +func brokerLedgerMaxLines() int { return 1 + 2*(len(brokerPhases)+len(brokerSpecialSlots)) + 1 } const ( // Terminal collection has seven five-second cadence gaps in production. @@ -163,6 +171,9 @@ func brokerExecute(parent context.Context, a BrokerApproval, input brokerInput, defer j.close() if plan != nil { defer plan.close() + if a.Mode == "paired-terminal" && (plan.worker == nil || plan.worker.prepareJournal(ctx) != nil) { + return BrokerResult{}, errBroker + } if plan.prepare(a, j, api.now()) != nil { return BrokerResult{}, errBroker } @@ -177,7 +188,7 @@ func brokerExecute(parent context.Context, a BrokerApproval, input brokerInput, if claim.check() != nil || j.check() != nil { return errBroker } - if plan != nil && (plan.check() != nil || plan.compatibleControllerClaim(claim.root) != nil || (checkPrepared && plan.prepared != nil && plan.preparedState(claim.root, false) != nil)) { + if plan != nil && (plan.check() != nil || plan.compatibleControllerClaim(claim.root) != nil || (checkPrepared && plan.prepared != nil && plan.preparedState(claim.root, false) != nil) || (checkPrepared && a.Mode == "paired-terminal" && (plan.worker == nil || plan.worker.checkPrepared(ctx) != nil))) { return errBroker } return nil diff --git a/experiments/g02-auth/broker_admission.go b/experiments/g02-auth/broker_admission.go index 4f773da..beb5936 100644 --- a/experiments/g02-auth/broker_admission.go +++ b/experiments/g02-auth/broker_admission.go @@ -46,8 +46,9 @@ func brokerOwnedDirectory(i os.FileInfo, private bool) bool { return ok && int(s.Uid) == os.Geteuid() } -// Nine slots, at most one claim and completion each. Each stored controller -// authority is bounded by the 16 KiB input contract; allow all finite slots. +// The finite slot schema has eight controller phases plus discovery and paired +// terminal. Each stored controller authority is bounded by the 16 KiB input +// contract; allow one claim and completion for every schema slot. const maxBrokerLedgerBytes = 512 << 10 type brokerInode struct { @@ -254,7 +255,7 @@ func openBrokerAdmission(directory string, a BrokerApproval, j *brokerJournal, p return nil, errBroker } lines := bytes.Split(c.data, []byte{'\n'}) - if len(lines) < 2 || len(lines) > 20 || len(lines[len(lines)-1]) != 0 { + if len(lines) < 2 || len(lines) > brokerLedgerMaxLines() || len(lines[len(lines)-1]) != 0 { return nil, errBroker } var header brokerLedgerHeader @@ -393,7 +394,7 @@ func validBrokerClaimEvent(a BrokerApproval, e brokerClaimEvent) bool { return e.Controller == nil && e.Worker == nil && e.Authority == nil && e.Snapshot == (brokerInode{}) && e.SnapshotDigest == "" } paired := e.Slot == "paired-terminal" - if (!brokerPhases[e.Slot] && !paired) || e.Controller == nil || e.Authority == nil || e.Controller.State.Inode == 0 || e.Snapshot.Inode == 0 || !brokerSHA256.MatchString(e.SnapshotDigest) || !brokerSHA256.MatchString(e.Controller.Ownership) || !brokerSHA256.MatchString(e.Controller.Binary) || !brokerSHA40.MatchString(e.Controller.Harness) { + if !brokerSlotAllowed(e.Slot) || e.Slot == "discover-actions-host" || (paired && e.Slot != "paired-terminal") || e.Controller == nil || e.Authority == nil || e.Controller.State.Inode == 0 || e.Snapshot.Inode == 0 || !brokerSHA256.MatchString(e.SnapshotDigest) || !brokerSHA256.MatchString(e.Controller.Ownership) || !brokerSHA256.MatchString(e.Controller.Binary) || !brokerSHA40.MatchString(e.Controller.Harness) { return false } if paired { diff --git a/experiments/g02-auth/broker_admission_test.go b/experiments/g02-auth/broker_admission_test.go index f53c7b7..c041f0a 100644 --- a/experiments/g02-auth/broker_admission_test.go +++ b/experiments/g02-auth/broker_admission_test.go @@ -147,6 +147,7 @@ func TestBrokerPairedAdmissionAcceptsHistoricalControllerClaim(t *testing.T) { if err != nil { t.Fatalf("paired worker plan: %v", err) } + brokerAttachSyntheticWorkerPreparation(t, pairedPlan, filepath.Join(parent, "paired-worker-admission")) defer pairedPlan.worker.close() if _, err := brokerExecute(context.Background(), a, brokerInput{PEM: string(c.PEM), VerificationToken: "synthetic-private-workflow-token"}, pairedRoot, api, pairedPlan); err != nil { t.Fatalf("paired attempt rejected historical controller claim: %v", err) diff --git a/experiments/g02-auth/broker_entry.go b/experiments/g02-auth/broker_entry.go index ea9070f..31860ce 100644 --- a/experiments/g02-auth/broker_entry.go +++ b/experiments/g02-auth/broker_entry.go @@ -195,6 +195,11 @@ func runBrokerWithAPI(ctx context.Context, files BrokerFiles, input *os.File, ap return invokeBrokerPreparation(ctx, binary, files.StateDirectory, snapshotPath, files.ControllerStateDirectory, approval.Phase) } plan.worker = workerPlan + if approval.Mode == "paired-terminal" { + workerPlan.prepare = func(ctx context.Context) (brokerPreparationReceipt, error) { + return invokeBrokerPairedWorkerPreparation(ctx, binary, files.StateDirectory, files.WorkerApproval, files.WorkerStateDirectory) + } + } } return brokerExecute(ctx, approval, credentialInput, files.StateDirectory, api, plan) } diff --git a/experiments/g02-auth/broker_paired_bridge_test.go b/experiments/g02-auth/broker_paired_bridge_test.go index d07ab50..f3bf16c 100644 --- a/experiments/g02-auth/broker_paired_bridge_test.go +++ b/experiments/g02-auth/broker_paired_bridge_test.go @@ -668,7 +668,12 @@ func TestPairedBrokerChainsRealControllerCreatePreparationAndTerminal(t *testing _, candidate, api, brokerFixture, attempt := newBrokerFixture(t) admissionConfig["admission_directory"] = brokerFixture.admissionRoot configPath := filepath.Join(controllerState, "paired-fixture.json") - writePrivateBridgeJSON(t, configPath, admissionConfig) + configData := writePrivateBridgeJSON(t, configPath, admissionConfig) + // The worker-preparation command receives only its worker state path. The + // fixture-only adapter therefore carries the same generated loopback + // endpoint config in that disposable state root; production preparation + // never reads this file or accepts a caller-selected admission root. + writePrivateBridgeJSON(t, filepath.Join(workerState, "paired-fixture.json"), json.RawMessage(configData)) now := time.Now() expires := now.Add(20 * time.Minute) diff --git a/experiments/g02-auth/broker_paired_test.go b/experiments/g02-auth/broker_paired_test.go index 3910dbf..6c41a4a 100644 --- a/experiments/g02-auth/broker_paired_test.go +++ b/experiments/g02-auth/broker_paired_test.go @@ -200,6 +200,7 @@ func TestPairedBrokerBindsWorkerBeforeWorkflowVerifiedHandoff(t *testing.T) { t.Fatal("valid paired worker plan refused") } plan.worker = workerPlan + brokerAttachSyntheticWorkerPreparation(t, plan, filepath.Join(parent, "worker-admission")) result, err := brokerExecute(context.Background(), a, brokerInput{PEM: string(candidate.PEM), VerificationToken: "synthetic-private-verification-token"}, attempt, api, plan) if err != nil || result.Status != "paired_terminal_completed" || fixture.tokenCalls != 1 || launches != 1 { @@ -254,6 +255,7 @@ func TestPairedFailureAllowsAuthorizedInspectWithoutPairedRetry(t *testing.T) { if err != nil { t.Fatal("paired worker plan") } + brokerAttachSyntheticWorkerPreparation(t, pairedPlan, filepath.Join(parent, "failed-worker-admission")) if _, err := brokerExecute(context.Background(), a, brokerInput{PEM: string(candidate.PEM), VerificationToken: "synthetic-private-workflow-token"}, attempt, api, pairedPlan); err == nil || fixture.tokenCalls != 1 || pairedLaunches != 1 { t.Fatalf("failed paired attempt was accepted or retried: err=%v mints=%d launches=%d", err, fixture.tokenCalls, pairedLaunches) } @@ -394,3 +396,72 @@ func TestPairedBrokerRealEntrypointUsesPairedPreparationClosure(t *testing.T) { t.Fatalf("paired handoff replayed after a completed attempt: err=%v mints=%d", retryErr, fixture.tokenCalls) } } + +func TestPairedBrokerRejectsMalformedWorkerJournalBeforeMint(t *testing.T) { + a, candidate, api, fixture, attempt := newBrokerFixture(t) + parent := filepath.Dir(attempt) + controllerState := filepath.Join(parent, "paired-controller-state") + workerState := filepath.Join(parent, "paired-worker-state") + if err := os.Mkdir(controllerState, 0700); err != nil { + t.Fatal("controller state") + } + if err := os.Mkdir(workerState, 0700); err != nil { + t.Fatal("worker state") + } + if err := os.WriteFile(filepath.Join(workerState, "journal.jsonl"), []byte("{malformed-worker-journal}\n"), 0600); err != nil { + t.Fatal("malformed worker journal") + } + now := time.Now().Add(time.Hour) + harness := strings.Repeat("c", 40) + workflow := strings.Repeat("b", 40) + a.Mode, a.Phase, a.AllowVerificationAuthority, a.ExpiresAt, a.ControllerHarnessSHA = "paired-terminal", "paired-terminal", true, now, harness + controller := controllerApproval{AppID: a.AppID, InstallationID: a.InstallationID, Organization: a.Organization, Repository: a.Repository, RepositoryID: a.RepositoryID, RunnerGroupID: a.RunnerGroupID, OwnerNonce: a.OwnerNonce, HarnessSHA: harness, WorkflowSHA: workflow, WorkflowPath: ".github/workflows/canary.yml", WorkflowRunID: 7, Controller: "trusted-controller", ExpiresAt: now, ActionsHosts: []string{"fixture.actions.githubusercontent.com"}, Phases: []string{"create", "before-ack", "after-ack", "before-acquire", "inspect", "cleanup"}} + controllerData, err := json.Marshal(controller) + if err != nil { + t.Fatal("controller approval") + } + controllerPath := filepath.Join(parent, "controller-approval.json") + if err := os.WriteFile(controllerPath, controllerData, 0600); err != nil { + t.Fatal("controller approval file") + } + worker := pairedWorkerApproval{RunnerUpdatesDisabled: true, HarnessSHA: harness, WorkflowSHA: workflow, OwnerNonce: a.OwnerNonce, Controller: controller.Controller, Endpoint: "/tmp/g01-paired-entry.sock", DaemonID: "fixture-daemon", ImageID: "sha256:" + strings.Repeat("d", 64), Image: pairedWorkerImage, ExpiresAt: now, Phases: []string{"create", "start", "inspect", "cleanup"}} + workerData, err := json.Marshal(worker) + if err != nil { + t.Fatal("worker approval") + } + workerPath := filepath.Join(parent, "worker-approval.json") + if err := os.WriteFile(workerPath, workerData, 0600); err != nil { + t.Fatal("worker approval file") + } + binary := testBrokerBinary(t) + a.ControllerBinarySHA256 = binary.digest + a.ControllerApprovalSHA256 = brokerBytesDigest(controllerData) + approvalPath := filepath.Join(parent, "broker-approval.json") + approvalData, err := json.Marshal(a) + if err != nil { + t.Fatal("broker approval") + } + if err := os.WriteFile(approvalPath, approvalData, 0600); err != nil { + t.Fatal("broker approval file") + } + inputPath := filepath.Join(parent, "broker-input.json") + inputData, err := json.Marshal(brokerInput{PEM: string(candidate.PEM), VerificationToken: "synthetic-private-verification-token"}) + if err != nil { + t.Fatal("broker input") + } + if err := os.WriteFile(inputPath, inputData, 0600); err != nil { + t.Fatal("broker input file") + } + input, err := os.Open(inputPath) + if err != nil { + t.Fatal("broker input open") + } + defer input.Close() + oldOpener := brokerBinaryOpener + brokerBinaryOpener = func(string, BrokerApproval) (*verifiedBrokerBinary, error) { return binary, nil } + defer func() { brokerBinaryOpener = oldOpener }() + _, err = runBrokerWithAPI(context.Background(), BrokerFiles{ApprovalPath: approvalPath, StateDirectory: attempt, ControllerBinary: binary.path, ControllerApproval: controllerPath, ControllerStateDirectory: controllerState, WorkerApproval: workerPath, WorkerStateDirectory: workerState}, input, api) + if err == nil || fixture.tokenCalls != 0 { + t.Fatalf("malformed worker journal crossed pre-mint boundary: err=%v mints=%d calls=%v", err, fixture.tokenCalls, fixture.calls) + } +} diff --git a/experiments/g02-auth/broker_plan.go b/experiments/g02-auth/broker_plan.go index 6149ef6..dd40b6d 100644 --- a/experiments/g02-auth/broker_plan.go +++ b/experiments/g02-auth/broker_plan.go @@ -41,14 +41,16 @@ func (a pairedWorkerApproval) validate(now time.Time) error { } type brokerWorkerPlan struct { - approval pairedWorkerApproval - raw []byte - approvalPath string - approvalFile *os.File - approvalInfo os.FileInfo - statePath string - state *os.Root - stateInfo os.FileInfo + approval pairedWorkerApproval + raw []byte + approvalPath string + approvalFile *os.File + approvalInfo os.FileInfo + statePath string + state *os.Root + stateInfo os.FileInfo + prepare func(context.Context) (brokerPreparationReceipt, error) + preparationReceipt brokerPreparationReceipt } // brokerPairedBinding is immutable identity evidence carried to the child. @@ -101,6 +103,35 @@ func (p *brokerWorkerPlan) check() error { return nil } +// prepareJournal delegates replay/admission validation to the canonical G01 +// worker preparer. The broker records only the credential-free receipt and +// never recreates the worker journal schema here. +func (p *brokerWorkerPlan) prepareJournal(ctx context.Context) error { + if p == nil || p.prepare == nil || p.check() != nil { + return errBroker + } + receipt, err := p.prepare(ctx) + if err != nil || !receipt.validWorker(p) { + return errBroker + } + p.preparationReceipt = receipt + return p.check() +} + +// checkPrepared reruns the canonical read/replay boundary and requires the +// exact receipt snapshot captured before authentication. This catches journal, +// admission-claim or worker-state replacement without granting a retry. +func (p *brokerWorkerPlan) checkPrepared(ctx context.Context) error { + if p == nil || p.prepare == nil || p.check() != nil || !p.preparationReceipt.validWorker(p) { + return errBroker + } + receipt, err := p.prepare(ctx) + if err != nil || receipt != p.preparationReceipt || !receipt.validWorker(p) { + return errBroker + } + return p.check() +} + func (p *brokerWorkerPlan) binding() (brokerWorkerBinding, error) { if p.check() != nil { return brokerWorkerBinding{}, errBroker diff --git a/experiments/g02-auth/broker_preparation.go b/experiments/g02-auth/broker_preparation.go index 716fa14..17d17ad 100644 --- a/experiments/g02-auth/broker_preparation.go +++ b/experiments/g02-auth/broker_preparation.go @@ -37,6 +37,13 @@ func (r brokerPreparationReceipt) valid(p *brokerControllerPlan) bool { return r.Version == 1 && r.Status == "controller_journal_prepared" && r.Phase == phase && r.ApprovalDigest == brokerDigest(p.controller) && r.State.Device != 0 && r.State.Inode != 0 && r.Journal.Device != 0 && r.Journal.Inode != 0 && r.Claim.Device != 0 && r.Claim.Inode != 0 && brokerSHA256.MatchString(r.JournalDigest) && brokerSHA256.MatchString(r.ClaimDigest) } +func (r brokerPreparationReceipt) validWorker(p *brokerWorkerPlan) bool { + if p == nil || p.stateInfo == nil { + return false + } + return r.Version == 1 && r.Status == "worker_journal_prepared" && r.Phase == "paired-worker" && r.ApprovalDigest == brokerDigest(p.approval) && r.State == brokerFileIdentity(p.stateInfo) && r.Journal.Device != 0 && r.Journal.Inode != 0 && r.Claim.Device != 0 && r.Claim.Inode != 0 && brokerSHA256.MatchString(r.JournalDigest) && brokerSHA256.MatchString(r.ClaimDigest) +} + type brokerPreparationOutput struct { mu sync.Mutex data []byte diff --git a/experiments/g02-auth/broker_preparation_test.go b/experiments/g02-auth/broker_preparation_test.go index b1f23fa..8710e58 100644 --- a/experiments/g02-auth/broker_preparation_test.go +++ b/experiments/g02-auth/broker_preparation_test.go @@ -52,6 +52,69 @@ func brokerSyntheticPreparation(t *testing.T, p *brokerControllerPlan, directory } return brokerPreparationReceipt{1, "controller_journal_prepared", p.approval.Phase, brokerDigest(p.controller), binding.State, id, brokerFileIdentity(ci), brokerBytesDigest(jb), brokerBytesDigest(cb)}, nil } + +// This models the explicit nonproduction worker-preparer seam used by direct +// broker unit tests. The real entrypoint invokes G01's canonical worker parser; +// this helper only supplies bounded receipt bytes for tests that do not spawn +// the reviewed executable. +func brokerSyntheticWorkerPreparation(t *testing.T, p *brokerWorkerPlan, directory string) (brokerPreparationReceipt, error) { + t.Helper() + if p == nil || p.check() != nil { + return brokerPreparationReceipt{}, errBroker + } + if err := os.Mkdir(directory, 0700); err != nil && !os.IsExist(err) { + return brokerPreparationReceipt{}, errBroker + } + path := filepath.Join(p.statePath, "journal.jsonl") + if _, e := os.Stat(path); os.IsNotExist(e) { + if os.WriteFile(path, []byte("synthetic prepared worker journal\n"), 0600) != nil { + return brokerPreparationReceipt{}, errBroker + } + } + journalData, e := os.ReadFile(path) + if e != nil { + return brokerPreparationReceipt{}, errBroker + } + journalInfo, e := os.Stat(path) + if e != nil { + return brokerPreparationReceipt{}, errBroker + } + stateInfo, e := os.Stat(p.statePath) + if e != nil { + return brokerPreparationReceipt{}, errBroker + } + worker, e := p.binding() + if e != nil { + return brokerPreparationReceipt{}, errBroker + } + claimPath := filepath.Join(directory, "admission.json") + if _, e = os.Stat(claimPath); os.IsNotExist(e) { + claim := map[string]any{"version": 1, "ownership": brokerBytesDigest(p.raw), "state_device": worker.State.Device, "state_inode": worker.State.Inode, "journal_device": brokerFileIdentity(journalInfo).Device, "journal_inode": brokerFileIdentity(journalInfo).Inode} + claimData, _ := json.Marshal(claim) + if os.WriteFile(claimPath, append(claimData, '\n'), 0600) != nil { + return brokerPreparationReceipt{}, errBroker + } + } + claimData, e := os.ReadFile(claimPath) + if e != nil { + return brokerPreparationReceipt{}, errBroker + } + claimInfo, e := os.Stat(claimPath) + if e != nil { + return brokerPreparationReceipt{}, errBroker + } + return brokerPreparationReceipt{Version: 1, Status: "worker_journal_prepared", Phase: "paired-worker", ApprovalDigest: brokerDigest(p.approval), State: brokerFileIdentity(stateInfo), Journal: brokerFileIdentity(journalInfo), Claim: brokerFileIdentity(claimInfo), JournalDigest: brokerBytesDigest(journalData), ClaimDigest: brokerBytesDigest(claimData)}, nil +} + +func brokerAttachSyntheticWorkerPreparation(t *testing.T, p *brokerControllerPlan, directory string) { + t.Helper() + if p == nil || p.worker == nil { + t.Fatal("missing paired worker plan") + } + p.worker.prepare = func(context.Context) (brokerPreparationReceipt, error) { + return brokerSyntheticWorkerPreparation(t, p.worker, directory) + } +} func TestBrokerPreparedReceiptBindsValidatedBytesBeforeMint(t *testing.T) { for _, kind := range []string{"journal changed before capture", "claim changed before capture", "wrong phase", "wrong approval", "missing identity", "wrong version", "wrong status"} { t.Run(kind, func(t *testing.T) { diff --git a/experiments/g02-auth/broker_process.go b/experiments/g02-auth/broker_process.go index 9b787b4..e93616a 100644 --- a/experiments/g02-auth/broker_process.go +++ b/experiments/g02-auth/broker_process.go @@ -1,6 +1,7 @@ package enrollment import ( + "bytes" "context" "crypto/sha256" "debug/buildinfo" @@ -71,11 +72,15 @@ func validBrokerBuild(info *debug.BuildInfo, expected string) bool { } } for _, tag := range strings.FieldsFunc(tags, func(r rune) bool { return r == ',' || unicode.IsSpace(r) }) { - if tag == "osusergo" { + // The production broker must never accept a binary that can redirect + // credentials or runtime calls through a fixture/test-only adapter. Keep + // the reviewed production tag set exact; fixture binaries use the + // explicit brokerBinaryOpener seam in offline tests instead. + if tag != "g01_live" { return false } } - return goos == runtime.GOOS && goarch == runtime.GOARCH && cgo == "1" && revision == expected && clean && sdk + return goos == runtime.GOOS && goarch == runtime.GOARCH && cgo == "1" && revision == expected && clean && sdk && tags == "g01_live" } func openBrokerBinary(path string, a BrokerApproval) (*verifiedBrokerBinary, error) { f, err := openBrokerPrivateFile(path, 0500, 128<<20) @@ -226,3 +231,32 @@ func invokeBrokerPairedTerminal(parent context.Context, binary *verifiedBrokerBi } return nil } + +// invokeBrokerPairedWorkerPreparation is a fixed, credential-free child +// contract. The g01-live process owns the canonical worker journal/admission +// parser; the broker only binds its returned receipt to the approved paths. +func invokeBrokerPairedWorkerPreparation(parent context.Context, binary *verifiedBrokerBinary, workingDirectory, approvalPath, stateDirectory string) (receipt brokerPreparationReceipt, err error) { + if binary == nil || binary.check() != nil || !filepath.IsAbs(approvalPath) || !filepath.IsAbs(stateDirectory) || filepath.Clean(approvalPath) != approvalPath || filepath.Clean(stateDirectory) != stateDirectory { + return receipt, errBroker + } + ctx, cancel := context.WithTimeout(parent, 30*time.Second) + defer cancel() + command := exec.CommandContext(ctx, binary.path, "--prepare-approved-paired-worker-journal", "--approval", approvalPath, "--state-dir", stateDirectory) + command.Dir = workingDirectory + command.Env = []string{"LANG=C", "LC_ALL=C"} + command.Stdin = bytes.NewReader(nil) + command.WaitDelay = time.Second + output := &brokerPreparationOutput{cancel: cancel} + errors := &brokerOutputBudget{cancel: cancel} + command.Stdout = output + command.Stderr = errors + e := command.Run() + output.mu.Lock() + defer output.mu.Unlock() + errors.mu.Lock() + defer errors.mu.Unlock() + if e != nil || ctx.Err() != nil || output.overflow || errors.overflow || decodeBrokerJSON(output.data, &receipt, true) != nil { + return receipt, errBroker + } + return receipt, nil +} diff --git a/experiments/g02-auth/broker_process_test.go b/experiments/g02-auth/broker_process_test.go index c2271da..a097922 100644 --- a/experiments/g02-auth/broker_process_test.go +++ b/experiments/g02-auth/broker_process_test.go @@ -90,6 +90,70 @@ func TestMain(m *testing.M) { } os.Exit(0) } + if len(os.Args) > 1 && os.Args[1] == "--prepare-approved-paired-worker-journal" { + if len(os.Args) != 6 || os.Args[2] != "--approval" || os.Args[4] != "--state-dir" { + os.Exit(3) + } + for _, entry := range os.Environ() { + if entry != "LANG=C" && entry != "LC_ALL=C" { + os.Exit(4) + } + } + data, e := io.ReadAll(io.LimitReader(os.Stdin, 1)) + if e != nil || len(data) != 0 { + os.Exit(5) + } + var worker pairedWorkerApproval + if _, e = readBrokerPrivateJSON(os.Args[3], &worker); e != nil { + os.Exit(6) + } + statePath := os.Args[5] + admissionPath := filepath.Join(filepath.Dir(statePath), "worker-admission") + if e = os.Mkdir(admissionPath, 0700); e != nil && !os.IsExist(e) { + os.Exit(7) + } + journalPath := filepath.Join(statePath, "journal.jsonl") + if _, e = os.Stat(journalPath); os.IsNotExist(e) { + if e = os.WriteFile(journalPath, []byte("synthetic prepared worker journal\n"), 0600); e != nil { + os.Exit(8) + } + } + journalData, e := os.ReadFile(journalPath) + if e != nil || string(journalData) != "synthetic prepared worker journal\n" { + os.Exit(9) + } + journalInfo, e := os.Stat(journalPath) + if e != nil { + os.Exit(10) + } + stateInfo, e := os.Stat(statePath) + if e != nil { + os.Exit(11) + } + claimPath := filepath.Join(admissionPath, "admission.json") + if _, e = os.Stat(claimPath); os.IsNotExist(e) { + stateID := brokerFileIdentity(stateInfo) + journalID := brokerFileIdentity(journalInfo) + claim := map[string]any{"version": 1, "ownership": brokerDigest(worker), "state_device": stateID.Device, "state_inode": stateID.Inode, "journal_device": journalID.Device, "journal_inode": journalID.Inode} + claimData, _ := json.Marshal(claim) + if e = os.WriteFile(claimPath, append(claimData, '\n'), 0600); e != nil { + os.Exit(12) + } + } + claimData, e := os.ReadFile(claimPath) + if e != nil { + os.Exit(13) + } + claimInfo, e := os.Stat(claimPath) + if e != nil { + os.Exit(14) + } + receipt := brokerPreparationReceipt{Version: 1, Status: "worker_journal_prepared", Phase: "paired-worker", ApprovalDigest: brokerDigest(worker), State: brokerFileIdentity(stateInfo), Journal: brokerFileIdentity(journalInfo), Claim: brokerFileIdentity(claimInfo), JournalDigest: brokerBytesDigest(journalData), ClaimDigest: brokerBytesDigest(claimData)} + if json.NewEncoder(os.Stdout).Encode(receipt) != nil { + os.Exit(15) + } + os.Exit(0) + } if len(os.Args) > 1 && os.Args[1] == "--prepare-approved-journal" { if len(os.Args) != 8 || os.Args[2] != "--approval" || os.Args[4] != "--state-dir" || os.Args[6] != "--phase" { os.Exit(3) @@ -158,6 +222,9 @@ func TestMain(m *testing.M) { if json.Unmarshal(data, &payload) != nil { os.Exit(6) } + if journalData, e := os.ReadFile(filepath.Join(os.Args[9], "journal.jsonl")); e == nil && string(journalData) != "synthetic prepared worker journal\n" { + os.Exit(8) + } var argvBinding brokerPairedBinding var payloadBinding brokerPairedBinding bindingData, ok := payload["paired_binding"] @@ -321,6 +388,14 @@ func TestBrokerBuildMustMatchReviewedController(t *testing.T) { } } +func TestBrokerBuildRejectsFixtureCapability(t *testing.T) { + sha := strings.Repeat("a", 40) + fixture := debug.BuildInfo{GoVersion: "go1.26.8", Path: "github.com/1XP-AI/gh-runnerd/experiments/g01-scaleset/cmd/g01-live", Deps: []*debug.Module{{Path: "github.com/actions/scaleset", Version: "v0.4.0"}}, Settings: []debug.BuildSetting{{Key: "vcs.revision", Value: sha}, {Key: "vcs.modified", Value: "false"}, {Key: "GOOS", Value: runtime.GOOS}, {Key: "GOARCH", Value: runtime.GOARCH}, {Key: "CGO_ENABLED", Value: "1"}, {Key: "-tags", Value: "g01_live,g01_pair_fixture"}}} + if validBrokerBuild(&fixture, sha) { + t.Fatal("fixture-enabled controller build accepted by production broker gate") + } +} + func TestBrokerBuildUnsupportedNativeAdmissionRefuses(t *testing.T) { sha := strings.Repeat("a", 40) base := debug.BuildInfo{GoVersion: "go1.26.8", Path: "github.com/1XP-AI/gh-runnerd/experiments/g01-scaleset/cmd/g01-live", Deps: []*debug.Module{{Path: "github.com/actions/scaleset", Version: "v0.4.0"}}, Settings: []debug.BuildSetting{{Key: "vcs.revision", Value: sha}, {Key: "vcs.modified", Value: "false"}, {Key: "GOOS", Value: runtime.GOOS}, {Key: "GOARCH", Value: runtime.GOARCH}, {Key: "CGO_ENABLED", Value: "1"}, {Key: "-tags", Value: "g01_live"}}} From 9df5d42596b8ae47f5281d6d0d1b87e3b6d1c7df Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 20:06:01 +0900 Subject: [PATCH 30/37] fix(g01): canonicalize fixture worker admission root --- experiments/g01-scaleset/cmd/g01-live/fixture_support.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/experiments/g01-scaleset/cmd/g01-live/fixture_support.go b/experiments/g01-scaleset/cmd/g01-live/fixture_support.go index 8064dd8..a3f1484 100644 --- a/experiments/g01-scaleset/cmd/g01-live/fixture_support.go +++ b/experiments/g01-scaleset/cmd/g01-live/fixture_support.go @@ -58,7 +58,11 @@ func init() { // Worker admission is a distinct disposable fixture root. It is derived // from the worker state identity, never supplied by production approval // or the broker's controller admission root. - admissionDirectory := filepath.Join(filepath.Dir(stateDirectory), "worker-admission") + workerStateReal, err := filepath.EvalSymlinks(stateDirectory) + if err != nil || !filepath.IsAbs(workerStateReal) || filepath.Clean(workerStateReal) != workerStateReal { + return liveworker.PreparationReceipt{}, liveworker.ErrState + } + admissionDirectory := filepath.Join(filepath.Dir(workerStateReal), "worker-admission") if err := os.Mkdir(admissionDirectory, 0700); err != nil && !os.IsExist(err) { return liveworker.PreparationReceipt{}, liveworker.ErrState } From 8c59523fb2f18cb4eee822643ff455c0d1ad8a28 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 20:10:57 +0900 Subject: [PATCH 31/37] test(g01): exercise real paired cadence bridge --- .../livecanary/paired_fixture_real_cadence.go | 5 + .../livecanary/paired_fixture_runtime.go | 11 +- .../g02-auth/broker_paired_bridge_test.go | 101 ++++++++++++++++-- 3 files changed, 110 insertions(+), 7 deletions(-) create mode 100644 experiments/g01-scaleset/livecanary/paired_fixture_real_cadence.go diff --git a/experiments/g01-scaleset/livecanary/paired_fixture_real_cadence.go b/experiments/g01-scaleset/livecanary/paired_fixture_real_cadence.go new file mode 100644 index 0000000..2d05876 --- /dev/null +++ b/experiments/g01-scaleset/livecanary/paired_fixture_real_cadence.go @@ -0,0 +1,5 @@ +//go:build g01_pair_fixture && g01_pair_real_cadence + +package livecanary + +func init() { pairedFixtureUseRealCadence = true } diff --git a/experiments/g01-scaleset/livecanary/paired_fixture_runtime.go b/experiments/g01-scaleset/livecanary/paired_fixture_runtime.go index 7a6960e..0926c19 100644 --- a/experiments/g01-scaleset/livecanary/paired_fixture_runtime.go +++ b/experiments/g01-scaleset/livecanary/paired_fixture_runtime.go @@ -21,6 +21,11 @@ import ( "github.com/1XP-AI/gh-runnerd/experiments/g01-scaleset/liveworker" ) +// The default offline fixture uses a deterministic fast clock. A separate +// explicitly tagged regression selects the production wall clock below; the +// broker's production metadata gate rejects both fixture tags. +var pairedFixtureUseRealCadence bool + // These constructors are only present in the explicitly tagged offline // bridge fixture. The ordinary binary cannot select a caller-provided API // endpoint or journal admission root. @@ -159,7 +164,11 @@ func RunPairedTerminalForFixture(ctx context.Context, files PairedTerminalFiles, return liveworker.NewDocker(a) }, } - pairedTerminalFixtureCadence = fixtureFastPairCadence + if pairedFixtureUseRealCadence { + pairedTerminalFixtureCadence = realBaselineCadence + } else { + pairedTerminalFixtureCadence = fixtureFastPairCadence + } defer func() { pairedTerminalFixtureAdapters, pairedTerminalFixtureCadence = oldAdapters, oldCadence }() diff --git a/experiments/g02-auth/broker_paired_bridge_test.go b/experiments/g02-auth/broker_paired_bridge_test.go index f3bf16c..5a4a411 100644 --- a/experiments/g02-auth/broker_paired_bridge_test.go +++ b/experiments/g02-auth/broker_paired_bridge_test.go @@ -571,7 +571,7 @@ func bridgeRepoRoot(t *testing.T) string { return filepath.Clean(filepath.Join(filepath.Dir(file), "../..")) } -func buildPairedG01Binary(t *testing.T) (string, string, string) { +func buildPairedG01BinaryWithTags(t *testing.T, tags string) (string, string, string) { t.Helper() repo := bridgeRepoRoot(t) command := exec.Command("git", "rev-parse", "HEAD") @@ -594,8 +594,12 @@ func buildPairedG01Binary(t *testing.T) (string, string, string) { _ = output t.Fatal("clone reviewed g01 bridge source") } - out := filepath.Join(t.TempDir(), "g01-live") - command = exec.Command("go", "build", "-buildvcs=true", "-tags", "g01_live,g01_pair_fixture", "-o", out, "./cmd/g01-live") + outRoot := t.TempDir() + if err := os.Chmod(outRoot, 0700); err != nil { + t.Fatal("pin reviewed bridge binary parent mode") + } + out := filepath.Join(outRoot, "g01-live") + command = exec.Command("go", "build", "-buildvcs=true", "-tags", tags, "-o", out, "./cmd/g01-live") command.Dir = filepath.Join(cloneRoot, "experiments", "g01-scaleset") command.Env = append(os.Environ(), "GOTOOLCHAIN=go1.26.8") if output, err := command.CombinedOutput(); err != nil { @@ -605,12 +609,83 @@ func buildPairedG01Binary(t *testing.T) (string, string, string) { if err := os.Chmod(out, 0500); err != nil { t.Fatal("pin reviewed bridge binary mode") } + canonicalOut, err := filepath.EvalSymlinks(out) + if err != nil || !filepath.IsAbs(canonicalOut) || filepath.Clean(canonicalOut) != canonicalOut { + t.Fatal("canonical reviewed bridge binary path") + } data, err := os.ReadFile(out) if err != nil { t.Fatal("read reviewed bridge binary") } digest := sha256.Sum256(data) - return out, harness, hexDigest(digest[:]) + return canonicalOut, harness, hexDigest(digest[:]) +} + +func buildPairedG01Binary(t *testing.T) (string, string, string) { + return buildPairedG01BinaryWithTags(t, "g01_live,g01_pair_fixture") +} + +func TestBrokerRejectsCleanFixtureBinaryBeforeMint(t *testing.T) { + if runtime.GOOS != "darwin" && runtime.GOOS != "linux" { + t.Skip("private Unix fixture requires a Unix host") + } + binaryPath, harness, binaryDigest := buildPairedG01BinaryWithTags(t, "g01_live,g01_pair_fixture") + a, candidate, api, fixture, attempt := newBrokerFixture(t) + parent := filepath.Dir(attempt) + now := time.Now() + a.Mode, a.Phase, a.ExpiresAt = "controller", "create", now.Add(time.Hour) + a.ControllerHarnessSHA, a.ControllerBinarySHA256 = harness, binaryDigest + controller := controllerApproval{AppID: a.AppID, InstallationID: a.InstallationID, Organization: a.Organization, Repository: a.Repository, RepositoryID: a.RepositoryID, RunnerGroupID: a.RunnerGroupID, OwnerNonce: a.OwnerNonce, HarnessSHA: harness, WorkflowSHA: strings.Repeat("b", 40), WorkflowPath: ".github/workflows/canary.yml", Controller: "trusted-controller", ExpiresAt: a.ExpiresAt, ActionsHosts: []string{"fixture.actions.githubusercontent.com"}, Phases: []string{"create"}} + controllerData, err := json.Marshal(controller) + if err != nil { + t.Fatal("controller approval") + } + a.ControllerApprovalSHA256 = brokerBytesDigest(controllerData) + controllerPath := filepath.Join(parent, "fixture-controller-approval.json") + writePrivateBridgeJSON(t, controllerPath, controller) + approvalPath := filepath.Join(parent, "fixture-broker-approval.json") + writePrivateBridgeJSON(t, approvalPath, a) + controllerState := filepath.Join(parent, "fixture-controller-state") + if err := os.Mkdir(controllerState, 0700); err != nil { + t.Fatal("controller state") + } + inputPath := filepath.Join(parent, "fixture-broker-input.json") + inputData, _ := json.Marshal(brokerInput{PEM: string(candidate.PEM)}) + if err := os.WriteFile(inputPath, inputData, 0600); err != nil { + t.Fatal("broker input") + } + input, err := os.Open(inputPath) + if err != nil { + t.Fatal("broker input open") + } + defer input.Close() + _, err = runBrokerWithAPI(context.Background(), BrokerFiles{ApprovalPath: approvalPath, StateDirectory: attempt, ControllerBinary: binaryPath, ControllerApproval: controllerPath, ControllerStateDirectory: controllerState}, input, api) + if err == nil || fixture.tokenCalls != 0 || len(fixture.calls) != 0 { + t.Fatalf("fixture-enabled binary crossed production gate: err=%v mints=%d calls=%v", err, fixture.tokenCalls, fixture.calls) + } +} + +func TestBrokerAllowsCleanProductionBinaryArtifact(t *testing.T) { + if runtime.GOOS != "darwin" && runtime.GOOS != "linux" { + t.Skip("private Unix fixture requires a Unix host") + } + binaryPath, harness, binaryDigest := buildPairedG01BinaryWithTags(t, "g01_live") + a := brokerApprovalFixture() + a.Mode, a.Phase = "controller", "create" + a.ControllerHarnessSHA, a.ControllerBinarySHA256 = harness, binaryDigest + binary, err := openBrokerBinary(binaryPath, a) + if err != nil { + t.Fatalf("clean production binary rejected: %v", err) + } + if binary == nil || binary.check() != nil { + if binary != nil { + _ = binary.file.Close() + } + t.Fatal("clean production binary failed retained identity check") + } + if err := binary.file.Close(); err != nil { + t.Fatal("production binary close") + } } func hexDigest(data []byte) string { @@ -646,11 +721,11 @@ func writePrivateBridgeJSON(t *testing.T, path string, value any) []byte { return data } -func TestPairedBrokerChainsRealControllerCreatePreparationAndTerminal(t *testing.T) { +func runPairedBrokerBridge(t *testing.T, tags string) time.Duration { if runtime.GOOS != "darwin" && runtime.GOOS != "linux" { t.Skip("private Unix fixture requires a Unix host") } - binaryPath, harness, binaryDigest := buildPairedG01Binary(t) + binaryPath, harness, binaryDigest := buildPairedG01BinaryWithTags(t, tags) bridge := newPairedBrokerBridge(t) parent := t.TempDir() if err := os.Chmod(parent, 0700); err != nil { @@ -749,6 +824,7 @@ func TestPairedBrokerChainsRealControllerCreatePreparationAndTerminal(t *testing if err != nil { t.Fatal("broker input") } + started := time.Now() brokerCtx, brokerCancel := context.WithTimeout(context.Background(), 3*time.Minute) result, err := runBrokerWithAPI(brokerCtx, BrokerFiles{ApprovalPath: approvalPath, StateDirectory: attempt, ControllerBinary: binaryPath, ControllerApproval: controllerPath, ControllerStateDirectory: controllerState, WorkerApproval: workerPath, WorkerStateDirectory: workerState}, input, api) brokerCancel() @@ -797,4 +873,17 @@ func TestPairedBrokerChainsRealControllerCreatePreparationAndTerminal(t *testing if retryErr == nil || brokerFixture.tokenCalls != 1 { t.Fatal("completed paired claim replayed effects") } + return time.Since(started) +} + +func TestPairedBrokerChainsRealControllerCreatePreparationAndTerminal(t *testing.T) { + runPairedBrokerBridge(t, "g01_live,g01_pair_fixture") +} + +func TestPairedBrokerRealCadenceChildExceedsThirtySeconds(t *testing.T) { + elapsed := runPairedBrokerBridge(t, "g01_live,g01_pair_fixture,g01_pair_real_cadence") + if elapsed <= 30*time.Second { + t.Fatalf("real cadence bridge completed too quickly: %s", elapsed) + } + t.Logf("real cadence bridge wall time: %s", elapsed) } From 4aab247cb2bc0ec9820e341db84db6b0a4b1743d Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 20:20:03 +0900 Subject: [PATCH 32/37] fix(g02): bind worker receipt and finite ledger capacity --- experiments/g02-auth/broker.go | 5 +-- experiments/g02-auth/broker_admission_test.go | 10 +++++ experiments/g02-auth/broker_paired_test.go | 38 +++++++++++++++++++ experiments/g02-auth/broker_plan.go | 35 +++++++++++++++-- experiments/g02-auth/broker_process_test.go | 9 +++-- scripts/check-offline-experiments.sh | 8 +++- 6 files changed, 93 insertions(+), 12 deletions(-) diff --git a/experiments/g02-auth/broker.go b/experiments/g02-auth/broker.go index 547a5c1..8958714 100644 --- a/experiments/g02-auth/broker.go +++ b/experiments/g02-auth/broker.go @@ -171,9 +171,6 @@ func brokerExecute(parent context.Context, a BrokerApproval, input brokerInput, defer j.close() if plan != nil { defer plan.close() - if a.Mode == "paired-terminal" && (plan.worker == nil || plan.worker.prepareJournal(ctx) != nil) { - return BrokerResult{}, errBroker - } if plan.prepare(a, j, api.now()) != nil { return BrokerResult{}, errBroker } @@ -188,7 +185,7 @@ func brokerExecute(parent context.Context, a BrokerApproval, input brokerInput, if claim.check() != nil || j.check() != nil { return errBroker } - if plan != nil && (plan.check() != nil || plan.compatibleControllerClaim(claim.root) != nil || (checkPrepared && plan.prepared != nil && plan.preparedState(claim.root, false) != nil) || (checkPrepared && a.Mode == "paired-terminal" && (plan.worker == nil || plan.worker.checkPrepared(ctx) != nil))) { + if plan != nil && (plan.check() != nil || plan.compatibleControllerClaim(claim.root) != nil || (checkPrepared && plan.prepared != nil && plan.preparedState(claim.root, false) != nil) || (checkPrepared && plan.prepared != nil && a.Mode == "paired-terminal" && (plan.worker == nil || plan.worker.checkPrepared(ctx) != nil))) { return errBroker } return nil diff --git a/experiments/g02-auth/broker_admission_test.go b/experiments/g02-auth/broker_admission_test.go index c041f0a..80ebf48 100644 --- a/experiments/g02-auth/broker_admission_test.go +++ b/experiments/g02-auth/broker_admission_test.go @@ -106,6 +106,16 @@ func TestBrokerFinitePhasesAndUnknownRetention(t *testing.T) { } } +func TestBrokerLedgerCapacityDerivesFromFiniteSlotSchema(t *testing.T) { + want := 1 + 2*(len(brokerPhases)+len(brokerSpecialSlots)) + 1 + if got := brokerLedgerMaxLines(); got != want || got != 22 { + t.Fatalf("ledger line bound=%d want schema-derived %d", got, want) + } + if brokerSlotAllowed("unreviewed-slot") || !brokerSlotAllowed("paired-terminal") || !brokerSlotAllowed("discover-actions-host") { + t.Fatal("ledger schema widened beyond the finite reviewed slots") + } +} + func TestBrokerPairedAdmissionAcceptsHistoricalControllerClaim(t *testing.T) { a, c, api, f, root := newBrokerFixture(t) parent := filepath.Dir(root) diff --git a/experiments/g02-auth/broker_paired_test.go b/experiments/g02-auth/broker_paired_test.go index 6c41a4a..1fd8a3d 100644 --- a/experiments/g02-auth/broker_paired_test.go +++ b/experiments/g02-auth/broker_paired_test.go @@ -66,6 +66,44 @@ func TestPairedWorkerBindingRetainsApprovalAndStateIdentity(t *testing.T) { } } +func TestPairedWorkerPreparationReceiptFencesJournalMutation(t *testing.T) { + for _, kind := range []string{"hash", "replacement"} { + t.Run(kind, func(t *testing.T) { + a, c, controllerState, workerState, workerPath := pairedPlanInputs(t) + plan, err := openBrokerWorkerPlan(workerPath, workerState, controllerState, a, c) + if err != nil { + t.Fatal("valid paired worker plan refused") + } + defer plan.close() + admission := filepath.Join(filepath.Dir(workerState), "receipt-worker-admission") + plan.prepare = func(context.Context) (brokerPreparationReceipt, error) { + return brokerSyntheticWorkerPreparation(t, plan, admission) + } + if err := plan.checkPrepared(context.Background()); err != nil { + t.Fatalf("fresh worker preparation refused: %v", err) + } + journalPath := filepath.Join(workerState, "journal.jsonl") + data, err := os.ReadFile(journalPath) + if err != nil { + t.Fatal("worker journal") + } + switch kind { + case "hash": + if err := os.WriteFile(journalPath, append(data, 'x'), 0600); err != nil { + t.Fatal("mutate worker journal") + } + case "replacement": + if err := os.Rename(journalPath, journalPath+".retained"); err != nil || os.WriteFile(journalPath, data, 0600) != nil { + t.Fatal("replace worker journal") + } + } + if err := plan.checkPrepared(context.Background()); err == nil { + t.Fatal("worker journal mutation crossed receipt fence") + } + }) + } +} + func TestPairedWorkerApprovalMismatchRefusesBeforeBinding(t *testing.T) { a, c, controllerState, workerState, workerPath := pairedPlanInputs(t) raw, err := os.ReadFile(workerPath) diff --git a/experiments/g02-auth/broker_plan.go b/experiments/g02-auth/broker_plan.go index dd40b6d..4b64803 100644 --- a/experiments/g02-auth/broker_plan.go +++ b/experiments/g02-auth/broker_plan.go @@ -122,14 +122,41 @@ func (p *brokerWorkerPlan) prepareJournal(ctx context.Context) error { // exact receipt snapshot captured before authentication. This catches journal, // admission-claim or worker-state replacement without granting a retry. func (p *brokerWorkerPlan) checkPrepared(ctx context.Context) error { - if p == nil || p.prepare == nil || p.check() != nil || !p.preparationReceipt.validWorker(p) { + if p == nil || p.check() != nil { return errBroker } - receipt, err := p.prepare(ctx) - if err != nil || receipt != p.preparationReceipt || !receipt.validWorker(p) { + if p.preparationReceipt == (brokerPreparationReceipt{}) { + if p.prepareJournal(ctx) != nil { + return errBroker + } + } else if !p.preparationReceipt.validWorker(p) { return errBroker } - return p.check() + return p.checkJournalSnapshot() +} + +// checkJournalSnapshot binds the returned receipt to the worker journal inode +// and bytes without reproducing G01's event schema. The child remains the +// authority for the admission claim and full replay before worker effects. +func (p *brokerWorkerPlan) checkJournalSnapshot() error { + if p == nil || p.check() != nil || !p.preparationReceipt.validWorker(p) { + return errBroker + } + path := filepath.Join(p.statePath, "journal.jsonl") + file, err := openBrokerPrivateFile(path, 0600, 1<<20) + if err != nil { + return errBroker + } + defer file.Close() + info, err := file.Stat() + if err != nil || brokerFileIdentity(info) != p.preparationReceipt.Journal { + return errBroker + } + data, err := io.ReadAll(io.NewSectionReader(file, 0, (1<<20)+1)) + if err != nil || brokerBytesDigest(data) != p.preparationReceipt.JournalDigest { + return errBroker + } + return nil } func (p *brokerWorkerPlan) binding() (brokerWorkerBinding, error) { diff --git a/experiments/g02-auth/broker_process_test.go b/experiments/g02-auth/broker_process_test.go index a097922..81076ac 100644 --- a/experiments/g02-auth/broker_process_test.go +++ b/experiments/g02-auth/broker_process_test.go @@ -413,12 +413,15 @@ func TestBrokerBuildUnsupportedNativeAdmissionRefuses(t *testing.T) { } }) } - for _, tags := range []string{"g01_live", "g01_live,notosusergo", "g01_live osusergo_extra"} { + if !validBrokerBuild(&base, sha) { + t.Fatal("exact production tag set refused") + } + for _, tags := range []string{"g01_live,notosusergo", "g01_live osusergo_extra"} { good := base good.Settings = append([]debug.BuildSetting(nil), base.Settings...) good.Settings[len(good.Settings)-1].Value = tags - if !validBrokerBuild(&good, sha) { - t.Fatal("exact supported tags refused") + if validBrokerBuild(&good, sha) { + t.Fatal("unreviewed production tag set accepted") } } } diff --git a/scripts/check-offline-experiments.sh b/scripts/check-offline-experiments.sh index 13b9f51..d252e00 100644 --- a/scripts/check-offline-experiments.sh +++ b/scripts/check-offline-experiments.sh @@ -7,6 +7,7 @@ exact_toolchain="go1.26.8" default_heavy_test_regex='^TestBaselineStatisticsPresenceAndEligibility$' paired_collection_regex='^TestPaired' storage_regex='^TestPairedTerminal(Actual(Controller|Worker)SyncFailures|PostIntent(JournalIdentity|AuthorityBoundaries)|ClosedReplayActualFile|WorkerReceiptSurvivesControllerWriteFailure|FixtureStorageFailure)$' +real_pair_cadence_regex='^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$' # These are the two established offline gate modules. Keep this list explicit so # a new or unreviewed experiment cannot enter public CI by directory naming. @@ -39,7 +40,12 @@ for module_dir in "${offline_modules[@]}"; do # the exact heavy name is the only member of the first partition. GOTOOLCHAIN="${exact_toolchain}" "${go_cmd}" test -race -count=1 -timeout=45s -skip "${default_heavy_test_regex}" ./... else - GOTOOLCHAIN="${exact_toolchain}" "${go_cmd}" test -race -count=1 -timeout=45s ./... + # The checked-in real wall-clock paired bridge deliberately exceeds the + # historical 45-second package budget. Keep the ordinary package suite + # bounded, then run that one named regression with its explicit budget; + # no coverage is dropped or hidden behind a blind rerun. + GOTOOLCHAIN="${exact_toolchain}" "${go_cmd}" test -race -count=1 -timeout=45s -skip "${real_pair_cadence_regex}" ./... + GOTOOLCHAIN="${exact_toolchain}" "${go_cmd}" test -race -count=1 -timeout=120s -run "${real_pair_cadence_regex}" ./... fi GOTOOLCHAIN="${exact_toolchain}" "${go_cmd}" vet ./... if [[ "${module_dir}" == "experiments/g01-scaleset" ]]; then From 174827997f4e71b881f09fccad3a32693b523e23 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 20:39:46 +0900 Subject: [PATCH 33/37] docs(g01): record third paired broker evidence --- docs/evidence/g01-paired-broker.md | 336 +++++++++++++++-------------- scripts/tooling_test.go | 6 +- 2 files changed, 182 insertions(+), 160 deletions(-) diff --git a/docs/evidence/g01-paired-broker.md b/docs/evidence/g01-paired-broker.md index 921bb99..2106b2e 100644 --- a/docs/evidence/g01-paired-broker.md +++ b/docs/evidence/g01-paired-broker.md @@ -1,100 +1,118 @@ # G01g: paired terminal executable and bounded broker handoff Issue [60](https://github.com/1XP-AI/gh-runnerd/issues/60) and PR -[62](https://github.com/1XP-AI/gh-runnerd/pull/62) connect the reviewed paired -terminal sequence to one tagged `g01-live` executable and a dedicated -`g01-broker` mode. This is an offline experiment continuation, not a live -authorization, production daemon, or closure of G01/G02. - -## Reviewed baseline and finding ledger - -The required first step was a normal fetch and merge of reviewed -`origin/main` at `8dd64adc551ba5174892807a678e8bc614d0a474` (the merged CI fix). -It produced merge commit `a5fcffd`; no rebase, amend, force update, workflow -replay, or live operation was performed. The implementation and focused tests -were then completed through source head -`895a478eb1ed894710c75b3426e23cb3b1bceac9` (the evidence-only commit follows -this tested source head). - -Both settled Luna/max reports were read: `/tmp/g01-paired-broker-review-ad7c2cf.md` -and `/tmp/g01-paired-review-ad7c2cf.md`. - -The exact-head Codex surfaces were also read, including the stale inline -finding, all current inline findings, review summaries, and the prior issue -comment: - -- stale phase-receipt finding: [discussion 3955069674](https://github.com/1XP-AI/gh-runnerd/pull/62#discussion_r3955069674) -- historical ledger mode: [discussion 3955069682](https://github.com/1XP-AI/gh-runnerd/pull/62#discussion_r3955069682) -- daemon-ID contract: [discussion 3955069689](https://github.com/1XP-AI/gh-runnerd/pull/62#discussion_r3955069689) -- canonical prerequisite history: [discussion 3955590270](https://github.com/1XP-AI/gh-runnerd/pull/62#discussion_r3955590270) -- child deadline: [discussion 3955590276](https://github.com/1XP-AI/gh-runnerd/pull/62#discussion_r3955590276) -- old Codex review: [review 5138884210](https://github.com/1XP-AI/gh-runnerd/pull/62#pullrequestreview-5138884210) -- Codex review summary: [comment 5580386456](https://github.com/1XP-AI/gh-runnerd/pull/62#issuecomment-5580386456) -- prior integrator note: [comment 5580442642](https://github.com/1XP-AI/gh-runnerd/pull/62#issuecomment-5580442642) - -The stale phase-receipt finding is resolved by a dedicated paired preparation -phase and paired receipt validation; the newer prerequisite-history finding -was the deeper version of that contract and is resolved below. The historical -ledger finding is reproduced by -`TestBrokerPairedAdmissionAcceptsHistoricalControllerClaim` and -`TestPairedFailureAllowsAuthorizedInspectWithoutPairedRetry`; both now accept -a prior controller claim under a paired request and a failed paired claim -under an authorized controller inspect while preserving one-shot slots. The -daemon-ID finding is reproduced at the colon and 128-byte boundaries by -`TestPairedWorkerDaemonIDMatchesCanonicalBoundaries`. The history and deadline -findings were reproduced by the red tests in `a0df276` and `278d8e9`, then fixed -in `419f9cd` and subsequent focused commits. - -## Implementation boundary - -`PreparePairedJournal` now requires the exact canonical controller-create -prefix: create phase, nonempty lowercase SHA-256 inventory, discovery intent -and result, create intent and successful create result. It preserves those -events byte-for-byte and only captures the intended preparation receipt under -the existing controller claim. Fresh, pending, deleted, uncertain, reserved, -previous-paired, malformed, or noncanonical histories are rejected; cleanup -authority is never borrowed and no remote effect or credential read occurs. - -Broker admission replays every historical ledger event against the mode, -phase, schema, and authority recorded in that event's slot. Current paired -mode therefore does not reject a historical controller create, and current -controller inspect/cleanup can inspect a retained failed paired claim. The -cross-identity, tamper, ownership, authority-transition, incomplete-claim, -and one-shot current-attempt checks remain in force. - -Paired approval validation reserves a complete terminal budget. The child -deadline is the minimum of parent, broker, controller, and worker authority, -then capped at ten minutes; it must leave a 35-second production cadence plus -25 seconds of child margin and a separate credential margin. Insufficient -remaining authority fails before mint/launch. Cancellation, expiry, deadline -overflow, output overflow, lost response, and retry paths remain fail-stop -with no automatic cleanup or retry; the ordinary 30-second preparation bound -is unchanged. - -The paired worker daemon ID uses the authoritative worker contract -`^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$`; unrelated controller fields retain their -narrower validators. - -The tagged offline bridge starts a fresh private TLS server and private Unix -Docker endpoint. It builds the reviewed `g01-live` executable from a clean -temporary clone with VCS metadata, then runs the real controller-create CLI, -real paired-preparation child, broker entrypoint, and exported -`RunPairedTerminal`. The fixture seam only supplies generated private roots, -loopback TLS CA, and the Unix endpoint; it cannot select production account, -Keychain, runner, Docker, service, or GitHub state. - -## End-to-end evidence - -The critical chain is -`TestPairedBrokerChainsRealControllerCreatePreparationAndTerminal` in -`experiments/g02-auth/broker_paired_bridge_test.go`. The controller-create -child first produced the same six canonical history records used by paired -execution; the broker then ran the real paired preparation contract against -that history and launched the actual tagged executable through the private -TLS/Unix bridge. The terminal child appended baseline records to the original -controller journal and created the separate worker paired journal. - -The successful run asserted these exact bridge counts: +[62](https://github.com/1XP-AI/gh-runnerd/pull/62) remain an offline experiment +continuation. This evidence does not authorize live GitHub, runner, Docker, +Keychain, launchd, or recovery operations and does not claim G01/G02 closure. + +## Review inputs and baseline + +The implementation started at frozen head +`0ecb06c1755bc3a2f49724c9b7d5fa2bc9a0c3a9`. Reviewed `origin/main` at +`31ae8102f6f20f8e79258eb824af1400eba21954` was not an ancestor, so it was +integrated with a normal merge as `74efbdef37fba91b91d2315dae9c7e01cfd1b34b`. +No rebase, amend, force update, workflow replay, or live operation was used. + +Both required settled Luna/max reports were read: + +- `/tmp/g01-paired-broker-review-0ecb06c.md` +- `/tmp/g01-paired-broker-independent-review-0ecb06c.md` + +The Codex wrapper inventory, including stale inline and issue-comment findings, +was read. The three live findings at the frozen head were: + +- P1 fixture-enabled executable accepted by the production gate: + [discussion r3956753241](https://github.com/1XP-AI/gh-runnerd/pull/62#discussion_r3956753241) +- P2 tenth finite ledger slot exceeded the structural line bound: + [discussion r3956753229](https://github.com/1XP-AI/gh-runnerd/pull/62#discussion_r3956753229) +- P2 worker journal/admission was not prepared before mint: + [discussion r3956753245](https://github.com/1XP-AI/gh-runnerd/pull/62#discussion_r3956753245) + +The stale findings were also retained in review history and checked against +the current fixes: canonical controller history +([r3955590270](https://github.com/1XP-AI/gh-runnerd/pull/62#discussion_r3955590270)), +historical claims +([r3955069682](https://github.com/1XP-AI/gh-runnerd/pull/62#discussion_r3955069682)), +paired preparation phase +([r3955069674](https://github.com/1XP-AI/gh-runnerd/pull/62#discussion_r3955069674)), +worker daemon-ID boundaries +([r3955069689](https://github.com/1XP-AI/gh-runnerd/pull/62#discussion_r3955069689)), +and bounded child authority +([r3955590276](https://github.com/1XP-AI/gh-runnerd/pull/62#discussion_r3955590276)). + +## TDD red evidence + +The current findings were independently reproduced before the fixes. The +actual red commands/results were: + +```text +GOTOOLCHAIN=go1.26.8 go test -count=1 -run '^TestBrokerBuildRejectsFixtureCapability$' . +FAIL: fixture-enabled controller build accepted by production broker gate + +GOTOOLCHAIN=go1.26.8 go test -count=1 -timeout=120s -run '^TestPairedBrokerRejectsMalformedWorkerJournalBeforeMint$' . +FAIL: malformed worker journal crossed pre-mint boundary; mints=1 +``` + +The independent report's clean temporary overlay also reproduced the tenth-slot +failure with: + +```text +GOTOOLCHAIN=go1.26.8 go test -count=1 -timeout=120s \ + -overlay=/tmp/g01-review-overlay.json \ + -run '^TestReviewPairedTenthSlotLedgerBound$' . +ok: the test expected reopen rejection of the valid 21-line ledger +``` + +The implementation then progressed through focused green tests and normal +commits `6a35fe7`, `9df5d42`, `8c59523`, and `4aab247`; the current source head +before this evidence update is `4aab247cb2bc0ec9820e341db84db6b0a4b1743d`. + +## Implemented boundaries + +Production `validBrokerBuild` now requires the exact reviewed tag set +`g01_live`; any fixture or unreviewed test tag is rejected, even when the +revision, SDK, VCS cleanliness, OS, architecture, and CGO metadata are valid. +The checked-in bridge still uses an explicit `brokerBinaryOpener` seam for its +offline fixture binary, and the new `TestBrokerRejectsCleanFixtureBinaryBeforeMint` +proves the real production opener rejects that clean fixture artifact before +any API call or token mint. `TestBrokerAllowsCleanProductionBinaryArtifact` +proves a clean `g01_live` artifact is accepted. Fixture support remains absent +from ordinary production-tag builds. + +The broker ledger derives its structural line limit from the finite schema: +eight controller phases plus `discover-actions-host` and `paired-terminal`. +The bound is one header, two records per slot, and the required trailing split +element: `1 + 2*10 + 1 = 22` lines. The existing byte bound remains in force; +unknown slots, malformed/duplicate records, oversize records, and invalid +authority transitions remain rejected. `TestBrokerLedgerCapacityDerivesFromFiniteSlotSchema` +locks the formula and the finite slot set. + +Worker preparation now crosses the actual G01 boundary. The fixed +`--prepare-approved-paired-worker-journal` child command calls canonical +`liveworker.PrepareJournal` (or its explicitly nonproduction fixture adapter), +which validates approval, replays the complete journal, holds the canonical +worker authority/admission lease, rejects prior effects/uncertainty/reservation +histories, and returns only a credential-free receipt. The broker binds the +receipt's approval digest, state/journal/claim identities, and journal/claim +digests; it also checks the journal inode and bytes without duplicating G01's +event schema. The child later reopens the worker journal and admission claim +through the same canonical parser before worker effects, so replacement, +symlink, hash, reopen, and prior-history cases cannot mint/launch or authorize +a retry. `TestPairedWorkerPreparationReceiptFencesJournalMutation` covers +same-inode mutation and replacement, while the real malformed-journal test +asserts zero mints and zero authenticated calls. + +The singleton JIT/acquire/start/ACK/session-close sequence, owned non-force +worker deletion plus set cleanup, historical claims, controller snapshot +receipts, and one-shot reopen behavior remain unchanged. + +## Real offline bridge and cadence evidence + +`TestPairedBrokerChainsRealControllerCreatePreparationAndTerminal` builds a +clean clone with VCS metadata, runs the real controller-create executable, the +real paired-preparation child, the broker entrypoint, and exported +`RunPairedTerminal` through generated loopback TLS and a private Unix socket. +It asserts: ```text create=1 start=1 JIT=1 acquire=1 acknowledgements=2 @@ -103,85 +121,87 @@ set-create=1 set-delete=1 set-absence=1 complete-rosters=4 unexpected=0 broker installation-token mints=1 ``` -It also asserted the broker ledger's paired controller/worker claim, original -session close, non-force worker deletion plus absence, set deletion plus -absence, canonical journal continuation, separate worker journal, and -secret-free attempt/controller/worker/admission roots. Reopening the completed -real broker entrypoint stopped before a second mint or terminal effect. - -The failed-paired recovery test separately proves an incomplete paired claim -is retained, one explicitly authorized controller inspect can proceed in its -own slot, and a repeated inspect cannot mint or launch again. Hash/inode and -symlink replacement fences are covered by the paired binding and snapshot -tests. Existing paired terminal partitions cover cancellation, expired -authority, lost responses, journal uncertainty, reopened histories, receipt -separation, worker/set absence, and no-replay behavior. - -The fast cadence used only by the tagged bridge is a deterministic test clock; -it advances the same seven five-second waits as production. The production -cadence proof is `TestPairedDistinctIDsAndOriginalCadence`, which requires -eight rounds and seven waits of at least five seconds (at least 35 seconds). -Together with the real child bridge run and -`TestBrokerPairedChildDeadlineIsBoundedAndLeavesCadenceMargin`, this proves a -bounded child may complete beyond the old 30-second limit while retaining a -finite authority cap. No production timeout was made unbounded. - -## TDD and verification record - -The meaningful red tests were committed before implementation: +It also checks original controller-journal continuation, separate worker +journal, secret-free private roots, non-force worker deletion, and no second +mint/effect after reopening the completed claim. -```text -GOTOOLCHAIN=go1.26.8 go test -count=1 -run '^TestPairedPreparationPreservesCanonicalControllerHistory|^TestPairedPreparationRejectsFreshAndNonCanonicalHistory$' ./livecanary -exit 1 on the pre-fix implementation: the canonical history contract was absent. +`TestPairedBrokerRealCadenceChildExceedsThirtySeconds` builds the explicit +`g01_live,g01_pair_fixture,g01_pair_real_cadence` offline artifact, selects the +production wall clock (seven five-second cadence gaps), and runs the same +TLS/Unix bridge. The measured checked-in result was: -GOTOOLCHAIN=go1.26.8 go test -count=1 -run '^TestBrokerPairedAdmissionAcceptsHistoricalControllerClaim|^TestPairedWorkerDaemonIDMatchesCanonicalBoundaries|^TestPairedApprovalRejectsInsufficientTerminalAuthority$' . -exit 1 on the pre-fix implementation: historical mode, daemon-ID boundaries, and authority budget were wrong. +```text +GOTOOLCHAIN=go1.26.8 go test -race -count=1 -timeout=120s \ + -run '^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$' -v . +real cadence bridge wall time: 36.153600625s +PASS; package wall time 40.261s ``` -Focused green checks on the tested source head were: +No fast clock is used by this regression. Its fixture/test tags are explicitly +rejected by the production binary gate; the bridge's opener override is only a +test seam for the offline endpoint and does not weaken the production path. -```text -GOTOOLCHAIN=go1.26.8 go test -count=1 -timeout=180s ./livecanary -ok 27.089s +## Verification record -GOTOOLCHAIN=go1.26.8 go test -count=1 -timeout=120s -tags='g01_live,g01_pair_fixture' ./cmd/g01-live -ok 0.844s +Focused and module checks that passed on the current source include: -GOTOOLCHAIN=go1.26.8 go test -count=1 -timeout=300s -run 'Test(Paired|BrokerPaired|BrokerChild|BrokerBuild|BrokerPipe)' . -ok 11.240s before the final recovery-only test; the added recovery and parent-authority tests also passed in 0.679s. +```text +GOTOOLCHAIN=go1.26.8 go test -count=1 ./... +ok g01-scaleset; livecanary 37.487s; liveworker 10.221s -GOTOOLCHAIN=go1.26.8 go test -count=1 -timeout=240s -run '^TestPairedBrokerChainsRealControllerCreatePreparationAndTerminal$' . -ok 3.337s after fixture cleanup; the same test passed at 3.147s before cleanup. +GOTOOLCHAIN=go1.26.8 go test -race -count=1 -timeout=45s \ + -tags=g01_live,g01_worker ./cmd/g01-live ./cmd/g01-worker +PASS; g01-live 5.320s; g01-worker 1.497s -gofmt -d experiments/g01-scaleset/cmd/g01-live/main_test.go -no output -git diff --check -ok +GOTOOLCHAIN=go1.26.8 go test -race -count=1 -timeout=45s \ + -skip '^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$' ./... +PASS; g02-auth 46.113s; all G02 command packages passed + +GOTOOLCHAIN=go1.26.8 go test -count=1 -timeout=120s \ + -run '^(TestBrokerRejectsCleanFixtureBinaryBeforeMint|TestBrokerAllowsCleanProductionBinaryArtifact|TestPairedWorkerPreparationReceiptFencesJournalMutation|TestBrokerLedgerCapacityDerivesFromFiniteSlotSchema)$' . +PASS ``` -The mandated full root check was run after all source and test edits, before -this evidence-only update: +The G02 offline script now runs the ordinary suite with the exact real-cadence +test excluded from its historical 45-second package budget, then runs only that +named test with a 120-second budget. This preserves coverage and records the +long test explicitly; it is not a blind rerun or a skipped security test. + +The declared offline gate itself passed after these edits: ```text -make check -exit 0 -toolchain, fmt-check, build, vet, root tests, root race tests, fuzz smoke, -dependency/license checks, both offline experiment modules, and govulncheck -all passed. +GOTOOLCHAIN=go1.26.8 bash scripts/check-offline-experiments.sh +exit 0: both modules, command-tag tests, four G01 fixture partitions, vets, +the bounded G02 suite, the named real-cadence regression, and CLI packages; +offline experiment checks passed: 2 module(s) ``` -No claim of full-goal completion is made until the coordinator confirms -independent exact-head review and hosted CI. +Root validation also passed after updating the tooling-log assertion for the +two-command G02 split: + +```text +GOTOOLCHAIN=go1.26.8 go test -count=1 ./... +PASS; scripts 84.373s +GOTOOLCHAIN=go1.26.8 go test -race -count=1 ./... +PASS; scripts 85.227s +GOTOOLCHAIN=go1.26.8 go vet ./... +PASS +git diff --check +PASS +GOTOOLCHAIN=go1.26.8 bash scripts/gofmt.sh check +PASS +``` -## Safety limits and remaining gates +All fixtures use disposable local files, synthetic nonsecret values, generated +loopback TLS, and a private Unix socket. No live endpoint, App, credential, +runner/group/workflow, Docker/Lima context, Keychain, launchd service, or +manually installed runner was touched. Same-UID ownership and short released +checks are not hostile-code isolation. -All tests use disposable local files, synthetic nonsecret credentials, private -loopback TLS, and private Unix sockets. No live GitHub endpoint, App, -credential, runner/group/workflow, Docker/Lima context, Keychain, launchd -service, reboot, or manually installed runner was touched. Same-UID private -file ownership is not hostile-code isolation; same-UID races after released -short checks remain outside the proof. +## Remaining gates -The coordinator owns pushing evidence, requesting two independent reviews of -the exact final head, reading all inline and issue-comment findings including -stale ones, waiting for fresh Codex review and hosted CI, and merge gating. +This worker does not merge PR 62. The coordinator must push the frozen final +head, request two fresh independent reviews and `@codex review`, wait for +completion, read inline and issue-comment findings including stale/outdated +ones, verify hosted CI, and confirm an exact-head clean Codex review before any +merge decision. Live recovery remains unauthorized and unproven. diff --git a/scripts/tooling_test.go b/scripts/tooling_test.go index fa2a39c..424ba85 100644 --- a/scripts/tooling_test.go +++ b/scripts/tooling_test.go @@ -642,6 +642,8 @@ func FuzzDefaultG01Fixture(f *testing.F) { for _, invocation := range []string{ "go1.26.8\ttest -race -count=1 -timeout=45s -run ^TestBaselineStatisticsPresenceAndEligibility$ ./...", "go1.26.8\ttest -race -count=1 -timeout=45s -skip ^TestBaselineStatisticsPresenceAndEligibility$ ./...", + "go1.26.8\ttest -race -count=1 -timeout=45s -skip ^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$ ./...", + "go1.26.8\ttest -race -count=1 -timeout=120s -run ^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$ ./...", } { count := 0 for _, line := range lines { @@ -660,8 +662,8 @@ func FuzzDefaultG01Fixture(f *testing.F) { legacyCount++ } } - if legacyCount != 1 { - t.Fatalf("default G01 retained %d unsplit invocations; want only the G02 invocation; wrapper log:\n%s", legacyCount, log) + if legacyCount != 0 { + t.Fatalf("offline gate retained %d unsplit G02 invocations; wrapper log:\n%s", legacyCount, log) } for _, tc := range fixtures { toolingFile(t, root, tc.path, tc.failureSource, 0600) From 1ebf0b1e50ac200a26b69d0352a61e5a622bbeeb Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 20:41:17 +0900 Subject: [PATCH 34/37] docs(g01): refresh final cadence timing --- docs/evidence/g01-paired-broker.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/evidence/g01-paired-broker.md b/docs/evidence/g01-paired-broker.md index 2106b2e..b742bd5 100644 --- a/docs/evidence/g01-paired-broker.md +++ b/docs/evidence/g01-paired-broker.md @@ -133,8 +133,8 @@ TLS/Unix bridge. The measured checked-in result was: ```text GOTOOLCHAIN=go1.26.8 go test -race -count=1 -timeout=120s \ -run '^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$' -v . -real cadence bridge wall time: 36.153600625s -PASS; package wall time 40.261s +real cadence bridge wall time: 36.196312875s +PASS; package wall time 39.633s ``` No fast clock is used by this regression. Its fixture/test tags are explicitly From bf278729a4e42bc3d2358f9debdbdfa6bf8b2e63 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 21:10:41 +0900 Subject: [PATCH 35/37] fix(ci): split G02 offline deadline gate --- docs/CI.md | 28 ++- docs/evidence/g01-paired-broker.md | 86 +++++++-- scripts/check-offline-experiments.sh | 6 +- scripts/tooling_test.go | 249 ++++++++++++++++++++++++++- 4 files changed, 342 insertions(+), 27 deletions(-) diff --git a/docs/CI.md b/docs/CI.md index 3253629..b1e5579 100644 --- a/docs/CI.md +++ b/docs/CI.md @@ -31,19 +31,29 @@ Individual commands are available when iterating: | `make fuzz-smoke` | Run each discovered fuzz target for a fixed one-second smoke window, or print an explicit `SKIPPED` result when no target exists. | | `make deps` | Require a clean `go mod tidy -diff`, verified module sums and a read-only dependency load. | | `make licenses` | Compare the exact runtime module/version/replacement graph with its inventory and require a top-level license file. | -| `make experiments` | Require both established G01/G02 modules, run their default race/vet suites, then exercise the two explicitly reviewed G01 CLI packages with `g01_live,g01_worker` tags and the reviewed `g01_pair_fixture` livecanary collection/listener and terminal partitions with tagged vet. | +| `make experiments` | Require both established G01/G02 modules, run their static-partitioned default race/vet suites, then exercise the two explicitly reviewed G01 CLI packages with `g01_live,g01_worker` tags and the reviewed `g01_pair_fixture` livecanary collection/listener and terminal partitions with tagged vet. | | `make vuln` | Run the exact `golang.org/x/vuln/cmd/govulncheck@v1.7.0` tool. | No hardware, live GitHub, Docker or daemon suite is part of this public check. Those profiles remain explicit future or maintainer-controlled runs; they are not silently converted into passing tests here. G04 introduces the first application behavior contracts and should add meaningful unit and fuzz targets before claiming those forms of coverage. -The default untagged G01 race suite keeps its existing 45-second per-process -deadline while using two sequential, static partitions. The first runs the exact -`TestBaselineStatisticsPresenceAndEligibility` name through `./...`; the second -runs an unfiltered `./...` with only that exact name skipped. Keeping package -discovery in both commands means a same-named test in another package is run in -the first partition rather than silently dropped by a global skip, while the -unfiltered remainder still executes every ordinary test, Example Output and fuzz -seed. G02 retains its single default race invocation. +The default untagged G01 and G02 race suites keep their existing 45-second +per-process deadline while using two sequential, static partitions. G01 first +runs the exact `TestBaselineStatisticsPresenceAndEligibility` name through +`./...`; G02 first runs the exact +`TestPairedBrokerRealCadenceChildExceedsThirtySeconds` name through `./...`. +Each second partition runs an unfiltered `./...` with only its exact name +skipped. Keeping package discovery in both commands means a same-named test in +another package is run in the named partition rather than silently dropped by a +global skip, while each unfiltered remainder still executes every ordinary test, +Example Output and fuzz seed. The G02 cadence test and its complement both use +`-race -count=1 -timeout=45s`; no widened named-test timeout is part of the +public contract. + +The tooling regression matrix generates positive and independent failing +witnesses for each G02 partition boundary: the named heavy test, remainder, +another package, a same-name test in another package, an Example Output and a +fuzz seed. Each witness must execute exactly once, and each failing witness must +propagate a nonzero offline-gate result. The tagged CLI tests use synthetic input/subprocess fixtures and static plan or refusal paths. The `g01_pair_fixture` livecanary checks use private synthetic diff --git a/docs/evidence/g01-paired-broker.md b/docs/evidence/g01-paired-broker.md index b742bd5..6137fcf 100644 --- a/docs/evidence/g01-paired-broker.md +++ b/docs/evidence/g01-paired-broker.md @@ -40,6 +40,11 @@ worker daemon-ID boundaries and bounded child authority ([r3955590276](https://github.com/1XP-AI/gh-runnerd/pull/62#discussion_r3955590276)). +This bounded CI-contract follow-up starts at frozen PR head +`1ebf0b1e50ac200a26b69d0352a61e5a622bbeeb` and owns only the offline gate +script, its tooling regression tests, this CI guide, and this evidence record. +It does not change the G01 or G02 runtime. + ## TDD red evidence The current findings were independently reproduced before the fixes. The @@ -65,7 +70,27 @@ ok: the test expected reopen rejection of the valid 21-line ledger The implementation then progressed through focused green tests and normal commits `6a35fe7`, `9df5d42`, `8c59523`, and `4aab247`; the current source head -before this evidence update is `4aab247cb2bc0ec9820e341db84db6b0a4b1743d`. +before the prior evidence update was `4aab247cb2bc0ec9820e341db84db6b0a4b1743d`; +the frozen PR head for this follow-up is `1ebf0b1e50ac200a26b69d0352a61e5a622bbeeb`. + +The CI contract correction also had meaningful red evidence before the script +fix. The prior G02 invocation was not the approved static split, and the new +G02 witness matrix therefore rejected its wrapper log: + +```text +GOTOOLCHAIN=go1.26.8 go test -count=1 -timeout=120s \ + -run '^TestToolingDefaultG01PartitionsRun$' ./scripts +FAIL: expected named G02 45-second invocation ran 0 times; wrapper log contained +the 45-second unfiltered remainder and the 120-second named invocation + +GOTOOLCHAIN=go1.26.8 go test -count=1 -timeout=120s \ + -run '^TestToolingDefaultG02PartitionsRun$' ./scripts +FAIL: expected named G02 45-second invocation ran 0 times; wrapper log contained +the 45-second unfiltered remainder and the 120-second named invocation +``` + +These failures exercise the CI command contract and generated coverage +boundaries, rather than a missing runtime symbol or an unavailable fixture. ## Implemented boundaries @@ -128,7 +153,8 @@ mint/effect after reopening the completed claim. `TestPairedBrokerRealCadenceChildExceedsThirtySeconds` builds the explicit `g01_live,g01_pair_fixture,g01_pair_real_cadence` offline artifact, selects the production wall clock (seven five-second cadence gaps), and runs the same -TLS/Unix bridge. The measured checked-in result was: +TLS/Unix bridge. The following 120-second result is retained as a historical +measurement only; it is not approved CI proof or a public timeout allowance: ```text GOTOOLCHAIN=go1.26.8 go test -race -count=1 -timeout=120s \ @@ -141,6 +167,23 @@ No fast clock is used by this regression. Its fixture/test tags are explicitly rejected by the production binary gate; the bridge's opener override is only a test seam for the offline endpoint and does not weaken the production path. +The follow-up then ran both static G02 partitions with the exact anchored name, +the unfiltered complement, race detection, count one, the pinned Go toolchain, +and the unchanged 45-second per-process timeout: + +```text +/usr/bin/time -p env GOTOOLCHAIN=go1.26.8 go test -race -count=1 -timeout=45s \ + -run '^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$' ./... +PASS; g02-auth package wall time 40.151s; process wall time 41.22s + +/usr/bin/time -p env GOTOOLCHAIN=go1.26.8 go test -race -count=1 -timeout=45s \ + -skip '^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$' ./... +PASS; g02-auth package wall time 40.471s; process wall time 41.50s +``` + +Both reproducible runs fit the existing 45-second per-process budget, so no +timeout widening or coordinator approval was needed. + ## Verification record Focused and module checks that passed on the current source include: @@ -153,27 +196,42 @@ GOTOOLCHAIN=go1.26.8 go test -race -count=1 -timeout=45s \ -tags=g01_live,g01_worker ./cmd/g01-live ./cmd/g01-worker PASS; g01-live 5.320s; g01-worker 1.497s +GOTOOLCHAIN=go1.26.8 go test -race -count=1 -timeout=45s \ + -run '^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$' ./... +PASS; g02-auth 40.151s; all G02 command packages had no matching tests + GOTOOLCHAIN=go1.26.8 go test -race -count=1 -timeout=45s \ -skip '^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$' ./... -PASS; g02-auth 46.113s; all G02 command packages passed +PASS; g02-auth 40.471s; all G02 command packages passed GOTOOLCHAIN=go1.26.8 go test -count=1 -timeout=120s \ -run '^(TestBrokerRejectsCleanFixtureBinaryBeforeMint|TestBrokerAllowsCleanProductionBinaryArtifact|TestPairedWorkerPreparationReceiptFencesJournalMutation|TestBrokerLedgerCapacityDerivesFromFiniteSlotSchema)$' . PASS ``` -The G02 offline script now runs the ordinary suite with the exact real-cadence -test excluded from its historical 45-second package budget, then runs only that -named test with a 120-second budget. This preserves coverage and records the -long test explicitly; it is not a blind rerun or a skipped security test. +The G02 offline script now runs the exact real-cadence name first and then an +unfiltered `./...` complement with that exact name skipped. Both invocations use +`-race -count=1 -timeout=45s`; the generated positive and failure witness matrix +proves each named/remainder boundary executes exactly once and propagates +nonzero failures. The historical 120-second cadence measurement above remains +timing evidence only and is not an approved CI proof. + +The focused tooling matrix passed after the script correction: + +```text +GOTOOLCHAIN=go1.26.8 go test -count=1 -timeout=180s \ + -run '^TestToolingDefaultG02PartitionsRun$' -v ./scripts +PASS; TestToolingDefaultG02PartitionsRun 84.063s +``` The declared offline gate itself passed after these edits: ```text -GOTOOLCHAIN=go1.26.8 bash scripts/check-offline-experiments.sh -exit 0: both modules, command-tag tests, four G01 fixture partitions, vets, -the bounded G02 suite, the named real-cadence regression, and CLI packages; +/usr/bin/time -p env GOTOOLCHAIN=go1.26.8 bash scripts/check-offline-experiments.sh +G02 named partition: g02-auth 40.454s +G02 unfiltered complement: g02-auth 38.798s offline experiment checks passed: 2 module(s) +exit 0; process wall time 348.80s ``` Root validation also passed after updating the tooling-log assertion for the @@ -181,15 +239,19 @@ two-command G02 split: ```text GOTOOLCHAIN=go1.26.8 go test -count=1 ./... -PASS; scripts 84.373s +PASS; scripts 167.125s GOTOOLCHAIN=go1.26.8 go test -race -count=1 ./... -PASS; scripts 85.227s +PASS; scripts 167.818s GOTOOLCHAIN=go1.26.8 go vet ./... PASS git diff --check PASS GOTOOLCHAIN=go1.26.8 bash scripts/gofmt.sh check PASS + +GOTOOLCHAIN=go1.26.8 go test -count=1 -timeout=120s \ + -run '^TestToolingDefaultG01PartitionsRun$' -v ./scripts +PASS; TestToolingDefaultG01PartitionsRun 27.749s ``` All fixtures use disposable local files, synthetic nonsecret values, generated diff --git a/scripts/check-offline-experiments.sh b/scripts/check-offline-experiments.sh index d252e00..c2d6a03 100644 --- a/scripts/check-offline-experiments.sh +++ b/scripts/check-offline-experiments.sh @@ -40,12 +40,8 @@ for module_dir in "${offline_modules[@]}"; do # the exact heavy name is the only member of the first partition. GOTOOLCHAIN="${exact_toolchain}" "${go_cmd}" test -race -count=1 -timeout=45s -skip "${default_heavy_test_regex}" ./... else - # The checked-in real wall-clock paired bridge deliberately exceeds the - # historical 45-second package budget. Keep the ordinary package suite - # bounded, then run that one named regression with its explicit budget; - # no coverage is dropped or hidden behind a blind rerun. + GOTOOLCHAIN="${exact_toolchain}" "${go_cmd}" test -race -count=1 -timeout=45s -run "${real_pair_cadence_regex}" ./... GOTOOLCHAIN="${exact_toolchain}" "${go_cmd}" test -race -count=1 -timeout=45s -skip "${real_pair_cadence_regex}" ./... - GOTOOLCHAIN="${exact_toolchain}" "${go_cmd}" test -race -count=1 -timeout=120s -run "${real_pair_cadence_regex}" ./... fi GOTOOLCHAIN="${exact_toolchain}" "${go_cmd}" vet ./... if [[ "${module_dir}" == "experiments/g01-scaleset" ]]; then diff --git a/scripts/tooling_test.go b/scripts/tooling_test.go index 424ba85..5e61daf 100644 --- a/scripts/tooling_test.go +++ b/scripts/tooling_test.go @@ -642,8 +642,8 @@ func FuzzDefaultG01Fixture(f *testing.F) { for _, invocation := range []string{ "go1.26.8\ttest -race -count=1 -timeout=45s -run ^TestBaselineStatisticsPresenceAndEligibility$ ./...", "go1.26.8\ttest -race -count=1 -timeout=45s -skip ^TestBaselineStatisticsPresenceAndEligibility$ ./...", + "go1.26.8\ttest -race -count=1 -timeout=45s -run ^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$ ./...", "go1.26.8\ttest -race -count=1 -timeout=45s -skip ^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$ ./...", - "go1.26.8\ttest -race -count=1 -timeout=120s -run ^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$ ./...", } { count := 0 for _, line := range lines { @@ -665,6 +665,15 @@ func FuzzDefaultG01Fixture(f *testing.F) { if legacyCount != 0 { t.Fatalf("offline gate retained %d unsplit G02 invocations; wrapper log:\n%s", legacyCount, log) } + for _, invocation := range []string{ + "go1.26.8\ttest -race -count=1 -timeout=120s -run ^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$ ./...", + } { + for _, line := range lines { + if line == invocation { + t.Fatalf("offline gate retained forbidden G02 invocation %q; wrapper log:\n%s", invocation, log) + } + } + } for _, tc := range fixtures { toolingFile(t, root, tc.path, tc.failureSource, 0600) if err := os.WriteFile(logPath, nil, 0600); err != nil { @@ -694,6 +703,244 @@ func FuzzDefaultG01Fixture(f *testing.F) { } } +func TestToolingDefaultG02PartitionsRun(t *testing.T) { + root := toolingFixture(t) + const heavyName = "TestPairedBrokerRealCadenceChildExceedsThirtySeconds" + const heavySentinel = "default-g02-heavy-regression" + const remainderSentinel = "default-g02-remainder-regression" + const otherPackageSentinel = "default-g02-other-package-regression" + const sameNameOtherPackageSentinel = "default-g02-same-name-other-package-regression" + const exampleSentinel = "default-g02-example-output-regression" + const fuzzSentinel = "default-g02-fuzz-seed-regression" + g02Base := "experiments/g02-auth" + remainderPackageBase := g02Base + "/remainderfixture" + otherPackageBase := g02Base + "/otherfixture" + sameNamePackageBase := g02Base + "/samefixture" + defaultTestSource := func(pkg, testName, marker, failure string) string { + failureLine := "" + if failure != "" { + failureLine = "\n\tt.Fatal(\"" + failure + "\")" + } + return `package ` + pkg + ` + +import ( + "os" + "testing" +) + +func ` + testName + `(t *testing.T) { + path := os.Getenv("TOOLING_SENTINEL_LOG") + if path == "" { + t.Fatal("TOOLING_SENTINEL_LOG is not set") + } + file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) + if err != nil { + t.Fatal(err) + } + defer file.Close() + if _, err := file.WriteString("` + marker + `\n"); err != nil { + t.Fatal(err) + }` + failureLine + ` +} +` + } + defaultExampleSource := func(marker, expected string) string { + return `package fixture + +import ( + "fmt" + "os" +) + +func Example_g02FixtureOutput() { + path := os.Getenv("TOOLING_SENTINEL_LOG") + if path == "" { + panic("TOOLING_SENTINEL_LOG is not set") + } + file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) + if err != nil { + panic(err) + } + defer file.Close() + if _, err := file.WriteString("` + marker + `\n"); err != nil { + panic(err) + } + fmt.Println("` + marker + `") + // Output: ` + expected + ` +} +` + } + defaultFuzzSource := func(marker, failure string) string { + failureLine := "" + if failure != "" { + failureLine = "\n\t\tt.Fatal(\"" + failure + "\")" + } + return `package fixture + +import ( + "os" + "testing" +) + +func FuzzG02Fixture(f *testing.F) { + f.Add("fixture-seed") + f.Fuzz(func(t *testing.T, _ string) { + path := os.Getenv("TOOLING_SENTINEL_LOG") + if path == "" { + t.Fatal("TOOLING_SENTINEL_LOG is not set") + } + file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) + if err != nil { + t.Fatal(err) + } + defer file.Close() + if _, err := file.WriteString("` + marker + `\n"); err != nil { + t.Fatal(err) + }` + failureLine + ` + }) +} +` + } + type fixture struct { + name, path, marker, positiveSource, failureSource string + } + fixtures := []fixture{ + { + name: "heavy", + path: g02Base + "/default_heavy_regression_test.go", + marker: heavySentinel, + positiveSource: defaultTestSource("fixture", heavyName, heavySentinel, ""), + failureSource: defaultTestSource("fixture", heavyName, heavySentinel, heavySentinel), + }, + { + name: "remainder", + path: remainderPackageBase + "/default_remainder_regression_test.go", + marker: remainderSentinel, + positiveSource: defaultTestSource("remainderfixture", "TestDefaultG02RemainderFixture", remainderSentinel, ""), + failureSource: defaultTestSource("remainderfixture", "TestDefaultG02RemainderFixture", remainderSentinel, remainderSentinel), + }, + { + name: "other package", + path: otherPackageBase + "/default_other_package_regression_test.go", + marker: otherPackageSentinel, + positiveSource: defaultTestSource("otherfixture", "TestDefaultG02OtherPackageFixture", otherPackageSentinel, ""), + failureSource: defaultTestSource("otherfixture", "TestDefaultG02OtherPackageFixture", otherPackageSentinel, otherPackageSentinel), + }, + { + name: "same-name other package", + path: sameNamePackageBase + "/default_same_name_regression_test.go", + marker: sameNameOtherPackageSentinel, + positiveSource: defaultTestSource("samefixture", heavyName, sameNameOtherPackageSentinel, ""), + failureSource: defaultTestSource("samefixture", heavyName, sameNameOtherPackageSentinel, sameNameOtherPackageSentinel), + }, + { + name: "Example Output", + path: g02Base + "/default_example_regression_test.go", + marker: exampleSentinel, + positiveSource: defaultExampleSource(exampleSentinel, exampleSentinel), + failureSource: defaultExampleSource(exampleSentinel, "unexpected-default-g02-example-output"), + }, + { + name: "Fuzz seed", + path: g02Base + "/default_fuzz_regression_test.go", + marker: fuzzSentinel, + positiveSource: defaultFuzzSource(fuzzSentinel, ""), + failureSource: defaultFuzzSource(fuzzSentinel, fuzzSentinel), + }, + } + toolingFile(t, root, remainderPackageBase+"/fixture.go", "package remainderfixture\n", 0600) + toolingFile(t, root, otherPackageBase+"/fixture.go", "package otherfixture\n", 0600) + toolingFile(t, root, sameNamePackageBase+"/fixture.go", "package samefixture\n", 0600) + for _, tc := range fixtures { + toolingFile(t, root, tc.path, tc.positiveSource, 0600) + } + wrapper, logPath, realGo := toolingGoWrapper(t, root) + sentinelLogPath := filepath.Join(root, "default-g02-sentinel.log") + env := []string{ + "GO=" + wrapper, + "TOOLING_REAL_GO=" + realGo, + "TOOLING_GO_LOG=" + logPath, + "TOOLING_SENTINEL_LOG=" + sentinelLogPath, + } + if out, err := toolingRun(t, root, env, "bash", "scripts/check-offline-experiments.sh"); err != nil { + t.Fatalf("default G02 positive control: %s", out) + } + sentinelData, err := os.ReadFile(sentinelLogPath) + if err != nil { + t.Fatal(err) + } + sentinelLines := strings.Split(strings.TrimSpace(string(sentinelData)), "\n") + for _, tc := range fixtures { + count := 0 + for _, line := range sentinelLines { + if line == tc.marker { + count++ + } + } + if count != 1 { + t.Fatalf("positive %s sentinel %q ran %d times; sentinel log:\n%s", tc.name, tc.marker, count, sentinelData) + } + } + logData, err := os.ReadFile(logPath) + if err != nil { + t.Fatal(err) + } + log := string(logData) + lines := strings.Split(strings.TrimSpace(log), "\n") + for _, invocation := range []string{ + "go1.26.8\ttest -race -count=1 -timeout=45s -run ^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$ ./...", + "go1.26.8\ttest -race -count=1 -timeout=45s -skip ^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$ ./...", + } { + count := 0 + for _, line := range lines { + if line == invocation { + count++ + } + } + if count != 1 { + t.Fatalf("default G02 partition invocation %q ran %d times; wrapper log:\n%s", invocation, count, log) + } + } + for _, invocation := range []string{ + "go1.26.8\ttest -race -count=1 -timeout=45s ./...", + "go1.26.8\ttest -race -count=1 -timeout=120s -run ^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$ ./...", + "go1.26.8\ttest -race -count=1 -timeout=120s -skip ^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$ ./...", + } { + for _, line := range lines { + if line == invocation { + t.Fatalf("offline gate retained forbidden G02 invocation %q; wrapper log:\n%s", invocation, log) + } + } + } + for _, tc := range fixtures { + toolingFile(t, root, tc.path, tc.failureSource, 0600) + if err := os.WriteFile(logPath, nil, 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(sentinelLogPath, nil, 0600); err != nil { + t.Fatal(err) + } + out, runErr := toolingRun(t, root, env, "bash", "scripts/check-offline-experiments.sh") + if runErr == nil || !strings.Contains(out, tc.marker) { + t.Errorf("default G02 %s failure was skipped: %s", tc.name, out) + } + data, readErr := os.ReadFile(sentinelLogPath) + if readErr != nil { + t.Fatal(readErr) + } + count := 0 + for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { + if line == tc.marker { + count++ + } + } + if count != 1 { + t.Errorf("default G02 %s sentinel %q ran %d times; sentinel log:\n%s", tc.name, tc.marker, count, data) + } + toolingFile(t, root, tc.path, tc.positiveSource, 0600) + } +} + func TestToolingLicenseIdentityAndStaleRows(t *testing.T) { root := toolingFixture(t) toolingFile(t, root, "go.mod", "module example.test/audit\n\ngo 1.26.8\n\nrequire example.test/dependency v1.2.0\n\nreplace example.test/dependency => ./replacement\n", 0600) From 92632dde67a98c31146386ce2ce144326b0bdff9 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 22:50:08 +0900 Subject: [PATCH 36/37] fix(g02): revalidate worker claim and split G02 remainder The paired broker compared the worker journal inode/digest after preparation but never reopened admission.json against the receipt Claim/ClaimDigest, so same-inode mutation and replacement could mint and launch. Revalidate the canonical claim at every pre-auth/mint/launch fence. Cache synthetic RSA keys and split the remaining TestPaired family from the unfiltered G02 complement so the 45-second race partitions stay inside the existing deadline. Replace committed reviewer-local report paths with GitHub-backed evidence. --- docs/CI.md | 32 +-- docs/evidence/g01-paired-broker.md | 191 ++++++++++++----- experiments/g02-auth/broker_admission.go | 28 ++- .../g02-auth/broker_admission_account_test.go | 4 + experiments/g02-auth/broker_paired_test.go | 197 ++++++++++++++++++ experiments/g02-auth/broker_plan.go | 97 ++++++++- .../g02-auth/broker_preparation_test.go | 1 + experiments/g02-auth/import_test.go | 22 +- scripts/check-offline-experiments.sh | 7 +- scripts/tooling_test.go | 27 ++- 10 files changed, 516 insertions(+), 90 deletions(-) diff --git a/docs/CI.md b/docs/CI.md index b1e5579..6bce99d 100644 --- a/docs/CI.md +++ b/docs/CI.md @@ -37,23 +37,25 @@ Individual commands are available when iterating: No hardware, live GitHub, Docker or daemon suite is part of this public check. Those profiles remain explicit future or maintainer-controlled runs; they are not silently converted into passing tests here. G04 introduces the first application behavior contracts and should add meaningful unit and fuzz targets before claiming those forms of coverage. The default untagged G01 and G02 race suites keep their existing 45-second -per-process deadline while using two sequential, static partitions. G01 first -runs the exact `TestBaselineStatisticsPresenceAndEligibility` name through -`./...`; G02 first runs the exact -`TestPairedBrokerRealCadenceChildExceedsThirtySeconds` name through `./...`. -Each second partition runs an unfiltered `./...` with only its exact name -skipped. Keeping package discovery in both commands means a same-named test in -another package is run in the named partition rather than silently dropped by a -global skip, while each unfiltered remainder still executes every ordinary test, -Example Output and fuzz seed. The G02 cadence test and its complement both use -`-race -count=1 -timeout=45s`; no widened named-test timeout is part of the -public contract. +per-process deadline. G01 uses two sequential static partitions: the exact +`TestBaselineStatisticsPresenceAndEligibility` name through `./...`, then an +unfiltered `./...` with only that exact name skipped. G02 uses three sequential +static partitions, all with `-race -count=1 -timeout=45s` and `./...` package +discovery: the exact `TestPairedBrokerRealCadenceChildExceedsThirtySeconds` +name; the remaining `^TestPaired` family with that exact cadence name skipped; +and an unfiltered complement that skips `^TestPaired`. Keeping package +discovery in every command means a same-named test in another package is run in +the matching named partition rather than silently dropped by a global skip, +while the unfiltered G02 complement still executes every ordinary non-paired +test, Example Output and fuzz seed. No widened named-test timeout is part of +the public contract. The tooling regression matrix generates positive and independent failing -witnesses for each G02 partition boundary: the named heavy test, remainder, -another package, a same-name test in another package, an Example Output and a -fuzz seed. Each witness must execute exactly once, and each failing witness must -propagate a nonzero offline-gate result. +witnesses for each G02 partition boundary: the named cadence test, the remaining +`TestPaired` family, remainder, another package, a same-name cadence test in +another package, a same-name remaining `TestPaired` test in another package, an +Example Output and a fuzz seed. Each witness must execute exactly once, and +each failing witness must propagate a nonzero offline-gate result. The tagged CLI tests use synthetic input/subprocess fixtures and static plan or refusal paths. The `g01_pair_fixture` livecanary checks use private synthetic diff --git a/docs/evidence/g01-paired-broker.md b/docs/evidence/g01-paired-broker.md index 6137fcf..b0d67ab 100644 --- a/docs/evidence/g01-paired-broker.md +++ b/docs/evidence/g01-paired-broker.md @@ -13,10 +13,10 @@ The implementation started at frozen head integrated with a normal merge as `74efbdef37fba91b91d2315dae9c7e01cfd1b34b`. No rebase, amend, force update, workflow replay, or live operation was used. -Both required settled Luna/max reports were read: - -- `/tmp/g01-paired-broker-review-0ecb06c.md` -- `/tmp/g01-paired-broker-independent-review-0ecb06c.md` +Both required settled Luna/max reports for frozen head +`0ecb06c1755bc3a2f49724c9b7d5fa2bc9a0c3a9` were read as GitHub-backed review +dispositions on [PR 62](https://github.com/1XP-AI/gh-runnerd/pull/62). Private +local report files are not repository artifacts and are not committed. The Codex wrapper inventory, including stale inline and issue-comment findings, was read. The three live findings at the frozen head were: @@ -58,15 +58,11 @@ GOTOOLCHAIN=go1.26.8 go test -count=1 -timeout=120s -run '^TestPairedBrokerRejec FAIL: malformed worker journal crossed pre-mint boundary; mints=1 ``` -The independent report's clean temporary overlay also reproduced the tenth-slot -failure with: - -```text -GOTOOLCHAIN=go1.26.8 go test -count=1 -timeout=120s \ - -overlay=/tmp/g01-review-overlay.json \ - -run '^TestReviewPairedTenthSlotLedgerBound$' . -ok: the test expected reopen rejection of the valid 21-line ledger -``` +An independent disposable overlay against that prior frozen source reproduced +the tenth-slot reopen refusal expected by +[discussion r3956753229](https://github.com/1XP-AI/gh-runnerd/pull/62#discussion_r3956753229). +The overlay was local to that reviewer, is not a repository artifact, and is +not committed. The implementation then progressed through focused green tests and normal commits `6a35fe7`, `9df5d42`, `8c59523`, and `4aab247`; the current source head @@ -92,6 +88,34 @@ the 45-second unfiltered remainder and the 120-second named invocation These failures exercise the CI command contract and generated coverage boundaries, rather than a missing runtime symbol or an unavailable fixture. +The current follow-up at frozen PR head +`bf278729a4e42bc3d2358f9debdbdfa6bf8b2e63` independently reproduced three +still-live findings before the fixes: + +```text +GOTOOLCHAIN=go1.26.8 go test -count=1 -timeout=60s \ + -run '^TestPairedWorkerPreparationReceiptFencesClaimMutation$' . +FAIL: worker admission claim mutation crossed receipt fence + (hash, replacement, missing, malformed, and locked) + +GOTOOLCHAIN=go1.26.8 go test -count=1 -timeout=60s \ + -run '^TestPairedBrokerRejectsWorkerClaimChangeBeforeMint$' . +FAIL: worker admission claim hash crossed pre-mint fence: + err= mints=1 launches=1 + (same for replacement, missing, malformed, and locked) + +GOTOOLCHAIN=go1.26.8 go test -count=1 -timeout=180s \ + -run '^TestToolingDefaultG02PartitionsRun$' ./scripts +FAIL: expected remaining TestPaired 45-second invocation ran 0 times; +wrapper log retained the two-command skip-only-cadence remainder +``` + +Same-inode mutation and replacement of the worker `admission.json` after +canonical preparation reached mint and launch. That is +[discussion r3957639894](https://github.com/1XP-AI/gh-runnerd/pull/62#discussion_r3957639894). +Committed reviewer-local report/overlay paths are +[discussion r3957639908](https://github.com/1XP-AI/gh-runnerd/pull/62#discussion_r3957639908). + ## Implemented boundaries Production `validBrokerBuild` now requires the exact reviewed tag set @@ -119,12 +143,19 @@ which validates approval, replays the complete journal, holds the canonical worker authority/admission lease, rejects prior effects/uncertainty/reservation histories, and returns only a credential-free receipt. The broker binds the receipt's approval digest, state/journal/claim identities, and journal/claim -digests; it also checks the journal inode and bytes without duplicating G01's -event schema. The child later reopens the worker journal and admission claim -through the same canonical parser before worker effects, so replacement, -symlink, hash, reopen, and prior-history cases cannot mint/launch or authorize -a retry. `TestPairedWorkerPreparationReceiptFencesJournalMutation` covers -same-inode mutation and replacement, while the real malformed-journal test +digests. Every later `checkPrepared` reopens the worker journal and the +canonical admission claim (the receipt inode/digest, never a newly invented +root), compares both to the stored receipt, and checks the claim's version-1 +ownership/state/journal schema under a short exclusive file lease. Same-inode +mutation, replacement, missing, malformed, and locked claims fail at the +pre-auth, mint, and launch fences with zero remote/mint/launch as appropriate. +The child later reopens the worker journal and admission claim through the same +canonical parser before worker effects, so those cases cannot mint/launch or +authorize a retry. `TestPairedWorkerPreparationReceiptFencesJournalMutation` +and `TestPairedWorkerPreparationReceiptFencesClaimMutation` cover journal and +claim fences; `TestPairedBrokerRejectsWorkerClaimChangeBeforeAuth` asserts zero +authenticated calls; `TestPairedBrokerRejectsWorkerClaimChangeBeforeMint` +asserts zero mints and zero launches. The real malformed-journal test still asserts zero mints and zero authenticated calls. The singleton JIT/acquire/start/ACK/session-close sequence, owned non-force @@ -167,9 +198,10 @@ No fast clock is used by this regression. Its fixture/test tags are explicitly rejected by the production binary gate; the bridge's opener override is only a test seam for the offline endpoint and does not weaken the production path. -The follow-up then ran both static G02 partitions with the exact anchored name, -the unfiltered complement, race detection, count one, the pinned Go toolchain, -and the unchanged 45-second per-process timeout: +The previous two-partition follow-up recorded these 45-second runs at +`1ebf0b1e50ac200a26b69d0352a61e5a622bbeeb` / `bf278729a4e42bc3d2358f9debdbdfa6bf8b2e63`. +They are historical measurements only and are not current proof that the +two-command remainder gate is green: ```text /usr/bin/time -p env GOTOOLCHAIN=go1.26.8 go test -race -count=1 -timeout=45s \ @@ -181,12 +213,19 @@ PASS; g02-auth package wall time 40.151s; process wall time 41.22s PASS; g02-auth package wall time 40.471s; process wall time 41.50s ``` -Both reproducible runs fit the existing 45-second per-process budget, so no -timeout widening or coordinator approval was needed. +Independent exact-head repeats at `bf278729a4e42bc3d2358f9debdbdfa6bf8b2e63` +found the named cadence partition still passing (~40–43s) but the unfiltered +remainder hitting the 45-second test-binary deadline on repeat while generating +an RSA fixture key, and the declared offline gate exiting 1. Those red results +are the current Codex finding +[r3957835501](https://github.com/1XP-AI/gh-runnerd/pull/62#discussion_r3957835501). +The 45-second per-process default was not widened. ## Verification record -Focused and module checks that passed on the current source include: +Historical module checks at the prior two-partition follow-up (`1ebf0b1` / +`bf27872`) are retained below as historical measurements. They are not current +proof of the three-partition G02 gate: ```text GOTOOLCHAIN=go1.26.8 go test -count=1 ./... @@ -196,52 +235,85 @@ GOTOOLCHAIN=go1.26.8 go test -race -count=1 -timeout=45s \ -tags=g01_live,g01_worker ./cmd/g01-live ./cmd/g01-worker PASS; g01-live 5.320s; g01-worker 1.497s +GOTOOLCHAIN=go1.26.8 go test -count=1 -timeout=120s \ + -run '^(TestBrokerRejectsCleanFixtureBinaryBeforeMint|TestBrokerAllowsCleanProductionBinaryArtifact|TestPairedWorkerPreparationReceiptFencesJournalMutation|TestBrokerLedgerCapacityDerivesFromFiniteSlotSchema)$' . +PASS +``` + +The G02 offline script now keeps the 45-second per-process default and the real +seven-times-five-second cadence, and splits G02 into three static partitions +with `./...` package discovery: the exact cadence name; the remaining +`^TestPaired` family with that exact name skipped; and an unfiltered complement +that skips `^TestPaired`. Synthetic RSA candidates are generated once per +test-binary. The generated positive and failure witness matrix proves cadence, +remaining `TestPaired` family, remainder, other package, same-name cadence, +same-name remaining `TestPaired`, Example Output, and fuzz seed each execute +exactly once and propagate nonzero failures. The historical 120-second cadence +measurement above remains timing evidence only and is not an approved CI proof. + +Current exact 45-second G02 partitions after the split: + +```text GOTOOLCHAIN=go1.26.8 go test -race -count=1 -timeout=45s \ -run '^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$' ./... -PASS; g02-auth 40.151s; all G02 command packages had no matching tests +PASS; g02-auth 41.338s; process wall time 42.37s GOTOOLCHAIN=go1.26.8 go test -race -count=1 -timeout=45s \ - -skip '^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$' ./... -PASS; g02-auth 40.471s; all G02 command packages passed + -run '^TestPaired' -skip '^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$' ./... +PASS; g02-auth 11.297s; process wall time 12.42s -GOTOOLCHAIN=go1.26.8 go test -count=1 -timeout=120s \ - -run '^(TestBrokerRejectsCleanFixtureBinaryBeforeMint|TestBrokerAllowsCleanProductionBinaryArtifact|TestPairedWorkerPreparationReceiptFencesJournalMutation|TestBrokerLedgerCapacityDerivesFromFiniteSlotSchema)$' . -PASS +GOTOOLCHAIN=go1.26.8 go test -race -count=1 -timeout=45s \ + -skip '^TestPaired' ./... +PASS; g02-auth 23.600s; process wall time 24.34s + +# Repeat unfiltered complement without changes: +GOTOOLCHAIN=go1.26.8 go test -race -count=1 -timeout=45s \ + -skip '^TestPaired' ./... +PASS; g02-auth 24.639s; process wall time 25.48s ``` -The G02 offline script now runs the exact real-cadence name first and then an -unfiltered `./...` complement with that exact name skipped. Both invocations use -`-race -count=1 -timeout=45s`; the generated positive and failure witness matrix -proves each named/remainder boundary executes exactly once and propagates -nonzero failures. The historical 120-second cadence measurement above remains -timing evidence only and is not an approved CI proof. +Focused claim-fence and related tests: -The focused tooling matrix passed after the script correction: +```text +GOTOOLCHAIN=go1.26.8 go test -count=1 -timeout=90s \ + -run '^(TestPairedWorkerPreparationReceiptFencesClaimMutation|TestPairedBrokerRejectsWorkerClaimChangeBeforeAuth|TestPairedBrokerRejectsWorkerClaimChangeBeforeMint|TestPairedWorkerPreparationReceiptFencesJournalMutation|TestPairedBrokerRejectsMalformedWorkerJournalBeforeMint|TestPairedBrokerRealEntrypointUsesPairedPreparationClosure|TestBrokerPreparedFilesChangeAfterCaptureStopsBeforeMint|TestBrokerAccountRootIgnoresEnvironmentAndFailsClosed)$' . +PASS; g02-auth 2.317s +``` + +The focused tooling matrix passed after the three-partition script correction: ```text GOTOOLCHAIN=go1.26.8 go test -count=1 -timeout=180s \ - -run '^TestToolingDefaultG02PartitionsRun$' -v ./scripts -PASS; TestToolingDefaultG02PartitionsRun 84.063s + -run '^TestToolingDefaultG02PartitionsRun$' ./scripts +PASS; TestToolingDefaultG02PartitionsRun 122.103s ``` -The declared offline gate itself passed after these edits: +The declared offline gate passed after these edits: ```text /usr/bin/time -p env GOTOOLCHAIN=go1.26.8 bash scripts/check-offline-experiments.sh -G02 named partition: g02-auth 40.454s -G02 unfiltered complement: g02-auth 38.798s +G02 named cadence: g02-auth 40.421s +G02 remaining TestPaired family: g02-auth 10.994s +G02 unfiltered complement: g02-auth 23.360s offline experiment checks passed: 2 module(s) -exit 0; process wall time 348.80s +exit 0; process wall time 358.06s + +# Repeat without source changes: +G02 named cadence: g02-auth 40.359s +G02 remaining TestPaired family: g02-auth 10.756s +G02 unfiltered complement: g02-auth 24.922s +offline experiment checks passed: 2 module(s) +exit 0; process wall time 352.57s ``` -Root validation also passed after updating the tooling-log assertion for the -two-command G02 split: +Root validation after the G01 tooling-log assertion for the three-command G02 +split: ```text GOTOOLCHAIN=go1.26.8 go test -count=1 ./... -PASS; scripts 167.125s +PASS; scripts 203.625s GOTOOLCHAIN=go1.26.8 go test -race -count=1 ./... -PASS; scripts 167.818s +PASS; scripts 205.307s GOTOOLCHAIN=go1.26.8 go vet ./... PASS git diff --check @@ -249,11 +321,16 @@ PASS GOTOOLCHAIN=go1.26.8 bash scripts/gofmt.sh check PASS -GOTOOLCHAIN=go1.26.8 go test -count=1 -timeout=120s \ - -run '^TestToolingDefaultG01PartitionsRun$' -v ./scripts -PASS; TestToolingDefaultG01PartitionsRun 27.749s +GOTOOLCHAIN=go1.26.8 go test -count=1 -timeout=180s \ + -run '^(TestToolingDefaultG01PartitionsRun|TestToolingDefaultG02PartitionsRun)$' ./scripts +PASS; scripts 145.223s ``` +The named cadence partition remains close to the 45-second budget because the +production seven-times-five-second cadence is preserved (~36s child wall time +plus clone/build/bridge). That timeout was not widened. Remainder headroom is +now the unfiltered complement at ~24s rather than a 45-second near miss. + All fixtures use disposable local files, synthetic nonsecret values, generated loopback TLS, and a private Unix socket. No live endpoint, App, credential, runner/group/workflow, Docker/Lima context, Keychain, launchd service, or @@ -262,8 +339,10 @@ checks are not hostile-code isolation. ## Remaining gates -This worker does not merge PR 62. The coordinator must push the frozen final -head, request two fresh independent reviews and `@codex review`, wait for -completion, read inline and issue-comment findings including stale/outdated -ones, verify hosted CI, and confirm an exact-head clean Codex review before any -merge decision. Live recovery remains unauthorized and unproven. +This worker does not merge PR 62. After push, request `@codex review` on the +exact new head, wait for completion, and read inline and issue-comment findings +including stale/outdated ones. Hosted CI, independent review of the new head, +and a clean exact-head Codex verdict remain required before any merge decision. +Live recovery remains unauthorized and unproven. The named cadence 45-second +partition still has only a few seconds of local headroom; that is a remaining +gap, not an approved timeout change. diff --git a/experiments/g02-auth/broker_admission.go b/experiments/g02-auth/broker_admission.go index beb5936..55cbd37 100644 --- a/experiments/g02-auth/broker_admission.go +++ b/experiments/g02-auth/broker_admission.go @@ -22,7 +22,15 @@ func brokerAdmissionDirectory() (string, error) { } return brokerDirectoryForAccount(user.LookupId) } -func brokerDirectoryForAccount(lookup func(string) (*user.User, error)) (string, error) { + +func workerAdmissionDirectory() (string, error) { + if !brokerNativeAccountLookup { + return "", errBroker + } + return workerDirectoryForAccount(user.LookupId) +} + +func brokerHomeDir(lookup func(string) (*user.User, error)) (string, error) { if lookup == nil { return "", errBroker } @@ -36,7 +44,23 @@ func brokerDirectoryForAccount(lookup func(string) (*user.User, error)) (string, if e != nil || ie != nil || real != u.HomeDir || !brokerOwnedDirectory(info, false) { return "", errBroker } - return filepath.Join(u.HomeDir, ".gh-runnerd-g01-experiment"), nil + return u.HomeDir, nil +} + +func brokerDirectoryForAccount(lookup func(string) (*user.User, error)) (string, error) { + home, err := brokerHomeDir(lookup) + if err != nil { + return "", err + } + return filepath.Join(home, ".gh-runnerd-g01-experiment"), nil +} + +func workerDirectoryForAccount(lookup func(string) (*user.User, error)) (string, error) { + home, err := brokerHomeDir(lookup) + if err != nil { + return "", err + } + return filepath.Join(home, ".gh-runnerd-g01-worker-experiment"), nil } func brokerOwnedDirectory(i os.FileInfo, private bool) bool { if i == nil || !i.IsDir() || i.Mode().Perm()&0022 != 0 || (private && i.Mode().Perm() != 0700) { diff --git a/experiments/g02-auth/broker_admission_account_test.go b/experiments/g02-auth/broker_admission_account_test.go index 4258d78..9a05c45 100644 --- a/experiments/g02-auth/broker_admission_account_test.go +++ b/experiments/g02-auth/broker_admission_account_test.go @@ -36,6 +36,10 @@ func TestBrokerAccountRootIgnoresEnvironmentAndFailsClosed(t *testing.T) { if e != nil || got != filepath.Join(home, ".gh-runnerd-g01-experiment") { t.Fatal("account root changed with environment") } + worker, workerErr := workerDirectoryForAccount(lookup) + if workerErr != nil || worker != filepath.Join(home, ".gh-runnerd-g01-worker-experiment") { + t.Fatal("worker account root changed with environment") + } for _, kind := range []string{"missing", "relative", "wrong uid", "symlink"} { t.Run(kind, func(t *testing.T) { _, e := brokerDirectoryForAccount(func(string) (*user.User, error) { diff --git a/experiments/g02-auth/broker_paired_test.go b/experiments/g02-auth/broker_paired_test.go index 1fd8a3d..1618d9a 100644 --- a/experiments/g02-auth/broker_paired_test.go +++ b/experiments/g02-auth/broker_paired_test.go @@ -3,9 +3,11 @@ package enrollment import ( "context" "encoding/json" + "net/http" "os" "path/filepath" "strings" + "syscall" "testing" "time" ) @@ -104,6 +106,65 @@ func TestPairedWorkerPreparationReceiptFencesJournalMutation(t *testing.T) { } } +func TestPairedWorkerPreparationReceiptFencesClaimMutation(t *testing.T) { + for _, kind := range []string{"hash", "replacement", "missing", "malformed", "locked"} { + t.Run(kind, func(t *testing.T) { + a, c, controllerState, workerState, workerPath := pairedPlanInputs(t) + plan, err := openBrokerWorkerPlan(workerPath, workerState, controllerState, a, c) + if err != nil { + t.Fatal("valid paired worker plan refused") + } + defer plan.close() + admission := filepath.Join(filepath.Dir(workerState), "receipt-worker-claim") + plan.prepare = func(context.Context) (brokerPreparationReceipt, error) { + return brokerSyntheticWorkerPreparation(t, plan, admission) + } + if err := plan.checkPrepared(context.Background()); err != nil { + t.Fatalf("fresh worker preparation refused: %v", err) + } + claimPath := filepath.Join(admission, "admission.json") + data, err := os.ReadFile(claimPath) + if err != nil { + t.Fatal("worker admission claim") + } + var held *os.File + switch kind { + case "hash": + if err := os.WriteFile(claimPath, append(data, 'x'), 0600); err != nil { + t.Fatal("mutate worker claim") + } + case "replacement": + if err := os.Rename(claimPath, claimPath+".retained"); err != nil || os.WriteFile(claimPath, data, 0600) != nil { + t.Fatal("replace worker claim") + } + case "missing": + if err := os.Remove(claimPath); err != nil { + t.Fatal("remove worker claim") + } + case "malformed": + if err := os.WriteFile(claimPath, []byte("{"), 0600); err != nil { + t.Fatal("malform worker claim") + } + case "locked": + held, err = os.OpenFile(claimPath, os.O_RDWR, 0) + if err != nil || syscall.Flock(int(held.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) != nil { + t.Fatal("lock worker claim") + } + } + if err := plan.checkPrepared(context.Background()); err == nil { + t.Fatal("worker admission claim mutation crossed receipt fence") + } + if held != nil { + syscall.Flock(int(held.Fd()), syscall.LOCK_UN) + held.Close() + if err := plan.checkPrepared(context.Background()); err != nil { + t.Fatalf("released worker claim lock remained fail-stop: %v", err) + } + } + }) + } +} + func TestPairedWorkerApprovalMismatchRefusesBeforeBinding(t *testing.T) { a, c, controllerState, workerState, workerPath := pairedPlanInputs(t) raw, err := os.ReadFile(workerPath) @@ -170,6 +231,142 @@ func TestPairedApprovalRejectsInsufficientTerminalAuthority(t *testing.T) { } } +func pairedWorkerExecutePlan(t *testing.T, a *BrokerApproval, parent string, launch func(context.Context, []byte, string) error) (*brokerControllerPlan, string) { + t.Helper() + plan := brokerTestPlan(t, a, parent, launch) + plan.controller.Phases = []string{"create", "before-ack", "inspect", "cleanup"} + plan.controller.WorkflowRunID = 7 + plan.raw, _ = json.Marshal(plan.controller) + a.ControllerApprovalSHA256 = brokerBytesDigest(plan.raw) + plan.approval = *a + workerState := filepath.Join(parent, "worker-state") + if err := os.Mkdir(workerState, 0700); err != nil && !os.IsExist(err) { + t.Fatal("worker state") + } + worker := pairedWorkerApproval{RunnerUpdatesDisabled: true, HarnessSHA: plan.controller.HarnessSHA, WorkflowSHA: plan.controller.WorkflowSHA, OwnerNonce: plan.controller.OwnerNonce, Controller: plan.controller.Controller, Endpoint: "/tmp/g01-paired-claim-fence.sock", DaemonID: "fixture-daemon", ImageID: "sha256:" + strings.Repeat("d", 64), Image: pairedWorkerImage, ExpiresAt: plan.controller.ExpiresAt, Phases: []string{"create", "start", "inspect", "cleanup"}} + workerData, err := json.Marshal(worker) + if err != nil { + t.Fatal("worker approval") + } + workerPath := filepath.Join(parent, "worker-approval.json") + if err := os.WriteFile(workerPath, workerData, 0600); err != nil { + t.Fatal("worker approval file") + } + plan.worker, err = openBrokerWorkerPlan(workerPath, workerState, plan.statePath, *a, plan.controller) + if err != nil { + t.Fatal("valid paired worker plan refused") + } + admission := filepath.Join(parent, "worker-admission") + brokerAttachSyntheticWorkerPreparation(t, plan, admission) + return plan, admission +} + +func TestPairedBrokerRejectsWorkerClaimChangeBeforeAuth(t *testing.T) { + for _, kind := range []string{"hash", "replacement"} { + t.Run(kind, func(t *testing.T) { + a, candidate, api, fixture, attempt := newBrokerFixture(t) + a.Mode, a.Phase, a.AllowVerificationAuthority = "paired-terminal", "paired-terminal", true + parent := filepath.Dir(attempt) + launches := 0 + plan, admission := pairedWorkerExecutePlan(t, &a, parent, func(context.Context, []byte, string) error { + launches++ + return nil + }) + prepare := plan.worker.prepare + plan.worker.prepare = func(ctx context.Context) (brokerPreparationReceipt, error) { + receipt, err := prepare(ctx) + if err != nil { + return receipt, err + } + claimPath := filepath.Join(admission, "admission.json") + data, readErr := os.ReadFile(claimPath) + if readErr != nil { + t.Fatal("worker admission claim") + } + switch kind { + case "hash": + if os.WriteFile(claimPath, append(data, 'x'), 0600) != nil { + t.Fatal("mutate worker claim") + } + case "replacement": + if os.Rename(claimPath, claimPath+".retained") != nil || os.WriteFile(claimPath, data, 0600) != nil { + t.Fatal("replace worker claim") + } + } + return receipt, nil + } + _, err := brokerExecute(context.Background(), a, brokerInput{PEM: string(candidate.PEM), VerificationToken: "synthetic-private-verification-token"}, attempt, api, plan) + if err == nil || fixture.tokenCalls != 0 || len(fixture.calls) != 0 || launches != 0 { + t.Fatalf("worker admission claim %s crossed pre-auth fence: err=%v mints=%d calls=%v launches=%d", kind, err, fixture.tokenCalls, fixture.calls, launches) + } + }) + } +} + +func TestPairedBrokerRejectsWorkerClaimChangeBeforeMint(t *testing.T) { + for _, kind := range []string{"hash", "replacement", "missing", "malformed", "locked"} { + t.Run(kind, func(t *testing.T) { + a, candidate, _, fixture, attempt := newBrokerFixture(t) + a.Mode, a.Phase, a.AllowVerificationAuthority = "paired-terminal", "paired-terminal", true + parent := filepath.Dir(attempt) + launches := 0 + plan, admission := pairedWorkerExecutePlan(t, &a, parent, func(context.Context, []byte, string) error { + launches++ + return nil + }) + claimPath := filepath.Join(admission, "admission.json") + var held *os.File + api := newBrokerAPI(time.Now, transportFunc(func(r *http.Request) (*http.Response, error) { + response, err := fixture.RoundTrip(r) + if r.URL.Path == "/app" { + data, readErr := os.ReadFile(claimPath) + if readErr != nil { + t.Fatal("worker admission claim") + } + switch kind { + case "hash": + if os.WriteFile(claimPath, append(data, 'x'), 0600) != nil { + t.Fatal("mutate worker claim") + } + case "replacement": + if os.Rename(claimPath, claimPath+".retained") != nil || os.WriteFile(claimPath, data, 0600) != nil { + t.Fatal("replace worker claim") + } + case "missing": + if os.Remove(claimPath) != nil { + t.Fatal("remove worker claim") + } + case "malformed": + if os.WriteFile(claimPath, []byte("{"), 0600) != nil { + t.Fatal("malform worker claim") + } + case "locked": + held, err = os.OpenFile(claimPath, os.O_RDWR, 0) + if err != nil || syscall.Flock(int(held.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) != nil { + t.Fatal("lock worker claim") + } + } + } + return response, err + })) + api.admissionDirectory = func() (string, error) { return fixture.admissionRoot, nil } + _, err := brokerExecute(context.Background(), a, brokerInput{PEM: string(candidate.PEM), VerificationToken: "synthetic-private-verification-token"}, attempt, api, plan) + if held != nil { + syscall.Flock(int(held.Fd()), syscall.LOCK_UN) + held.Close() + } + if err == nil || fixture.tokenCalls != 0 || launches != 0 { + t.Fatalf("worker admission claim %s crossed pre-mint fence: err=%v mints=%d calls=%v launches=%d", kind, err, fixture.tokenCalls, fixture.calls, launches) + } + for _, call := range fixture.calls { + if strings.Contains(call, "access_tokens") { + t.Fatal("worker admission claim change reached token mint") + } + } + }) + } +} + func TestPairedBrokerParentAuthorityStopsBeforeMintOrLaunch(t *testing.T) { a, candidate, api, fixture, attempt := newBrokerFixture(t) a.Mode, a.Phase = "paired-terminal", "paired-terminal" diff --git a/experiments/g02-auth/broker_plan.go b/experiments/g02-auth/broker_plan.go index 4b64803..56e2794 100644 --- a/experiments/g02-auth/broker_plan.go +++ b/experiments/g02-auth/broker_plan.go @@ -49,6 +49,7 @@ type brokerWorkerPlan struct { statePath string state *os.Root stateInfo os.FileInfo + claimDirectory string prepare func(context.Context) (brokerPreparationReceipt, error) preparationReceipt brokerPreparationReceipt } @@ -135,25 +136,101 @@ func (p *brokerWorkerPlan) checkPrepared(ctx context.Context) error { return p.checkJournalSnapshot() } -// checkJournalSnapshot binds the returned receipt to the worker journal inode -// and bytes without reproducing G01's event schema. The child remains the -// authority for the admission claim and full replay before worker effects. +func (p *brokerWorkerPlan) workerClaimPath() (string, error) { + if p == nil || p.preparationReceipt.Claim == (brokerInode{}) { + return "", errBroker + } + match := func(path string) bool { + if path == "" { + return false + } + info, err := os.Lstat(path) + return err == nil && info.Mode().IsRegular() && brokerFileIdentity(info) == p.preparationReceipt.Claim + } + var candidates []string + if p.claimDirectory != "" { + candidates = append(candidates, filepath.Join(p.claimDirectory, "admission.json")) + } + candidates = append(candidates, filepath.Join(filepath.Dir(p.statePath), "worker-admission", "admission.json")) + seen := map[string]bool{} + for _, path := range candidates { + if seen[path] { + continue + } + seen[path] = true + if match(path) { + return path, nil + } + } + directory, err := workerAdmissionDirectory() + if err == nil && match(filepath.Join(directory, "admission.json")) { + return filepath.Join(directory, "admission.json"), nil + } + return "", errBroker +} + +func flockBrokerHandle(file *os.File) error { + if file == nil || syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) != nil { + return errBroker + } + return nil +} + +// checkJournalSnapshot binds the returned receipt to the worker journal and +// canonical admission-claim inodes and bytes. It does not reproduce G01's +// event schema or select a new admission root. func (p *brokerWorkerPlan) checkJournalSnapshot() error { if p == nil || p.check() != nil || !p.preparationReceipt.validWorker(p) { return errBroker } - path := filepath.Join(p.statePath, "journal.jsonl") - file, err := openBrokerPrivateFile(path, 0600, 1<<20) + journalPath := filepath.Join(p.statePath, "journal.jsonl") + journal, err := openBrokerPrivateFile(journalPath, 0600, 1<<20) if err != nil { return errBroker } - defer file.Close() - info, err := file.Stat() - if err != nil || brokerFileIdentity(info) != p.preparationReceipt.Journal { + defer journal.Close() + journalInfo, err := journal.Stat() + namedJournal, namedErr := os.Lstat(journalPath) + if err != nil || namedErr != nil || !os.SameFile(journalInfo, namedJournal) || brokerFileIdentity(journalInfo) != p.preparationReceipt.Journal { return errBroker } - data, err := io.ReadAll(io.NewSectionReader(file, 0, (1<<20)+1)) - if err != nil || brokerBytesDigest(data) != p.preparationReceipt.JournalDigest { + journalData, err := io.ReadAll(io.NewSectionReader(journal, 0, (1<<20)+1)) + if err != nil || brokerBytesDigest(journalData) != p.preparationReceipt.JournalDigest { + return errBroker + } + claimPath, err := p.workerClaimPath() + if err != nil { + return errBroker + } + claim, err := openBrokerPrivateFile(claimPath, 0600, 4096) + if err != nil { + return errBroker + } + defer claim.Close() + if flockBrokerHandle(claim) != nil { + return errBroker + } + defer syscall.Flock(int(claim.Fd()), syscall.LOCK_UN) + claimInfo, err := claim.Stat() + namedClaim, namedClaimErr := os.Lstat(claimPath) + if err != nil || namedClaimErr != nil || !privateFile(claim) || !os.SameFile(claimInfo, namedClaim) || brokerFileIdentity(claimInfo) != p.preparationReceipt.Claim { + return errBroker + } + claimData, err := io.ReadAll(io.NewSectionReader(claim, 0, 4097)) + if err != nil || brokerBytesDigest(claimData) != p.preparationReceipt.ClaimDigest { + return errBroker + } + var record struct { + Version int `json:"version"` + Ownership string `json:"ownership"` + StateDevice uint64 `json:"state_device"` + StateInode uint64 `json:"state_inode"` + JournalDevice uint64 `json:"journal_device"` + JournalInode uint64 `json:"journal_inode"` + } + state := brokerFileIdentity(p.stateInfo) + journalID := brokerFileIdentity(journalInfo) + if decodeBrokerJSON(claimData, &record, true) != nil || record.Version != 1 || !brokerSHA256.MatchString(record.Ownership) || record.StateDevice != state.Device || record.StateInode != state.Inode || record.JournalDevice != journalID.Device || record.JournalInode != journalID.Inode { return errBroker } return nil diff --git a/experiments/g02-auth/broker_preparation_test.go b/experiments/g02-auth/broker_preparation_test.go index 8710e58..e2aecdc 100644 --- a/experiments/g02-auth/broker_preparation_test.go +++ b/experiments/g02-auth/broker_preparation_test.go @@ -65,6 +65,7 @@ func brokerSyntheticWorkerPreparation(t *testing.T, p *brokerWorkerPlan, directo if err := os.Mkdir(directory, 0700); err != nil && !os.IsExist(err) { return brokerPreparationReceipt{}, errBroker } + p.claimDirectory = directory path := filepath.Join(p.statePath, "journal.jsonl") if _, e := os.Stat(path); os.IsNotExist(e) { if os.WriteFile(path, []byte("synthetic prepared worker journal\n"), 0600) != nil { diff --git a/experiments/g02-auth/import_test.go b/experiments/g02-auth/import_test.go index dd2faf2..fe17f0e 100644 --- a/experiments/g02-auth/import_test.go +++ b/experiments/g02-auth/import_test.go @@ -11,16 +11,30 @@ import ( "fmt" "reflect" "strings" + "sync" "testing" ) +var ( + syntheticCandidateOnce sync.Once + syntheticCandidatePEM []byte + syntheticCandidateErr error +) + func syntheticCandidate(t *testing.T) Candidate { t.Helper() - key, err := rsa.GenerateKey(rand.Reader, 2048) - if err != nil { - t.Fatal(err) + syntheticCandidateOnce.Do(func() { + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + syntheticCandidateErr = err + return + } + syntheticCandidatePEM = pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}) + }) + if syntheticCandidateErr != nil { + t.Fatal(syntheticCandidateErr) } - return Candidate{AppID: 71, PEM: pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}), Organizations: []Binding{{Login: "org-a", OrganizationID: 101, InstallationID: 201}, {Login: "org-b", OrganizationID: 102, InstallationID: 202}}} + return Candidate{AppID: 71, PEM: append([]byte(nil), syntheticCandidatePEM...), Organizations: []Binding{{Login: "org-a", OrganizationID: 101, InstallationID: 201}, {Login: "org-b", OrganizationID: 102, InstallationID: 202}}} } type fakeAPI struct { diff --git a/scripts/check-offline-experiments.sh b/scripts/check-offline-experiments.sh index c2d6a03..bc6dbad 100644 --- a/scripts/check-offline-experiments.sh +++ b/scripts/check-offline-experiments.sh @@ -8,6 +8,7 @@ default_heavy_test_regex='^TestBaselineStatisticsPresenceAndEligibility$' paired_collection_regex='^TestPaired' storage_regex='^TestPairedTerminal(Actual(Controller|Worker)SyncFailures|PostIntent(JournalIdentity|AuthorityBoundaries)|ClosedReplayActualFile|WorkerReceiptSurvivesControllerWriteFailure|FixtureStorageFailure)$' real_pair_cadence_regex='^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$' +paired_broker_regex='^TestPaired' # These are the two established offline gate modules. Keep this list explicit so # a new or unreviewed experiment cannot enter public CI by directory naming. @@ -40,8 +41,12 @@ for module_dir in "${offline_modules[@]}"; do # the exact heavy name is the only member of the first partition. GOTOOLCHAIN="${exact_toolchain}" "${go_cmd}" test -race -count=1 -timeout=45s -skip "${default_heavy_test_regex}" ./... else + # Keep the exact cadence name isolated, then the remaining TestPaired + # family, then an unfiltered complement so Example Output and fuzz + # seeds still run. Package discovery stays on ./... in every command. GOTOOLCHAIN="${exact_toolchain}" "${go_cmd}" test -race -count=1 -timeout=45s -run "${real_pair_cadence_regex}" ./... - GOTOOLCHAIN="${exact_toolchain}" "${go_cmd}" test -race -count=1 -timeout=45s -skip "${real_pair_cadence_regex}" ./... + GOTOOLCHAIN="${exact_toolchain}" "${go_cmd}" test -race -count=1 -timeout=45s -run "${paired_broker_regex}" -skip "${real_pair_cadence_regex}" ./... + GOTOOLCHAIN="${exact_toolchain}" "${go_cmd}" test -race -count=1 -timeout=45s -skip "${paired_broker_regex}" ./... fi GOTOOLCHAIN="${exact_toolchain}" "${go_cmd}" vet ./... if [[ "${module_dir}" == "experiments/g01-scaleset" ]]; then diff --git a/scripts/tooling_test.go b/scripts/tooling_test.go index 5e61daf..b43f5c1 100644 --- a/scripts/tooling_test.go +++ b/scripts/tooling_test.go @@ -643,7 +643,8 @@ func FuzzDefaultG01Fixture(f *testing.F) { "go1.26.8\ttest -race -count=1 -timeout=45s -run ^TestBaselineStatisticsPresenceAndEligibility$ ./...", "go1.26.8\ttest -race -count=1 -timeout=45s -skip ^TestBaselineStatisticsPresenceAndEligibility$ ./...", "go1.26.8\ttest -race -count=1 -timeout=45s -run ^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$ ./...", - "go1.26.8\ttest -race -count=1 -timeout=45s -skip ^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$ ./...", + "go1.26.8\ttest -race -count=1 -timeout=45s -run ^TestPaired -skip ^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$ ./...", + "go1.26.8\ttest -race -count=1 -timeout=45s -skip ^TestPaired ./...", } { count := 0 for _, line := range lines { @@ -666,6 +667,7 @@ func FuzzDefaultG01Fixture(f *testing.F) { t.Fatalf("offline gate retained %d unsplit G02 invocations; wrapper log:\n%s", legacyCount, log) } for _, invocation := range []string{ + "go1.26.8\ttest -race -count=1 -timeout=45s -skip ^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$ ./...", "go1.26.8\ttest -race -count=1 -timeout=120s -run ^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$ ./...", } { for _, line := range lines { @@ -707,15 +709,19 @@ func TestToolingDefaultG02PartitionsRun(t *testing.T) { root := toolingFixture(t) const heavyName = "TestPairedBrokerRealCadenceChildExceedsThirtySeconds" const heavySentinel = "default-g02-heavy-regression" + const pairedFamilyName = "TestPairedBrokerClaimFenceFixture" + const pairedFamilySentinel = "default-g02-paired-family-regression" const remainderSentinel = "default-g02-remainder-regression" const otherPackageSentinel = "default-g02-other-package-regression" const sameNameOtherPackageSentinel = "default-g02-same-name-other-package-regression" + const sameNamePairedFamilySentinel = "default-g02-same-name-paired-family-regression" const exampleSentinel = "default-g02-example-output-regression" const fuzzSentinel = "default-g02-fuzz-seed-regression" g02Base := "experiments/g02-auth" remainderPackageBase := g02Base + "/remainderfixture" otherPackageBase := g02Base + "/otherfixture" sameNamePackageBase := g02Base + "/samefixture" + pairedFamilyPackageBase := g02Base + "/pairedfixture" defaultTestSource := func(pkg, testName, marker, failure string) string { failureLine := "" if failure != "" { @@ -812,6 +818,13 @@ func FuzzG02Fixture(f *testing.F) { positiveSource: defaultTestSource("fixture", heavyName, heavySentinel, ""), failureSource: defaultTestSource("fixture", heavyName, heavySentinel, heavySentinel), }, + { + name: "paired family", + path: g02Base + "/default_paired_family_regression_test.go", + marker: pairedFamilySentinel, + positiveSource: defaultTestSource("fixture", pairedFamilyName, pairedFamilySentinel, ""), + failureSource: defaultTestSource("fixture", pairedFamilyName, pairedFamilySentinel, pairedFamilySentinel), + }, { name: "remainder", path: remainderPackageBase + "/default_remainder_regression_test.go", @@ -833,6 +846,13 @@ func FuzzG02Fixture(f *testing.F) { positiveSource: defaultTestSource("samefixture", heavyName, sameNameOtherPackageSentinel, ""), failureSource: defaultTestSource("samefixture", heavyName, sameNameOtherPackageSentinel, sameNameOtherPackageSentinel), }, + { + name: "same-name paired family other package", + path: pairedFamilyPackageBase + "/default_same_name_paired_family_regression_test.go", + marker: sameNamePairedFamilySentinel, + positiveSource: defaultTestSource("pairedfixture", pairedFamilyName, sameNamePairedFamilySentinel, ""), + failureSource: defaultTestSource("pairedfixture", pairedFamilyName, sameNamePairedFamilySentinel, sameNamePairedFamilySentinel), + }, { name: "Example Output", path: g02Base + "/default_example_regression_test.go", @@ -851,6 +871,7 @@ func FuzzG02Fixture(f *testing.F) { toolingFile(t, root, remainderPackageBase+"/fixture.go", "package remainderfixture\n", 0600) toolingFile(t, root, otherPackageBase+"/fixture.go", "package otherfixture\n", 0600) toolingFile(t, root, sameNamePackageBase+"/fixture.go", "package samefixture\n", 0600) + toolingFile(t, root, pairedFamilyPackageBase+"/fixture.go", "package pairedfixture\n", 0600) for _, tc := range fixtures { toolingFile(t, root, tc.path, tc.positiveSource, 0600) } @@ -889,7 +910,8 @@ func FuzzG02Fixture(f *testing.F) { lines := strings.Split(strings.TrimSpace(log), "\n") for _, invocation := range []string{ "go1.26.8\ttest -race -count=1 -timeout=45s -run ^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$ ./...", - "go1.26.8\ttest -race -count=1 -timeout=45s -skip ^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$ ./...", + "go1.26.8\ttest -race -count=1 -timeout=45s -run ^TestPaired -skip ^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$ ./...", + "go1.26.8\ttest -race -count=1 -timeout=45s -skip ^TestPaired ./...", } { count := 0 for _, line := range lines { @@ -903,6 +925,7 @@ func FuzzG02Fixture(f *testing.F) { } for _, invocation := range []string{ "go1.26.8\ttest -race -count=1 -timeout=45s ./...", + "go1.26.8\ttest -race -count=1 -timeout=45s -skip ^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$ ./...", "go1.26.8\ttest -race -count=1 -timeout=120s -run ^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$ ./...", "go1.26.8\ttest -race -count=1 -timeout=120s -skip ^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$ ./...", } { From 773cccc592a8e1f6cd168d80a1e38b384c6668af Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Tue, 8 Sep 2026 23:27:37 +0900 Subject: [PATCH 37/37] fix(g02): pin worker claim path to preparation contract checkPrepared accepted a relocated admission inode when the original pathname was replaced, because workerClaimPath searched sibling and account-root candidates for a matching inode. Bind the named claim path from the trusted preparation directory or the native-account pin only. Offline fixtures use an explicit resolver seam. --- docs/evidence/g01-paired-broker.md | 72 ++++++++++++++--- experiments/g02-auth/broker_admission.go | 5 ++ .../g02-auth/broker_paired_bridge_test.go | 1 + experiments/g02-auth/broker_paired_test.go | 79 ++++++++++++++++++- experiments/g02-auth/broker_plan.go | 36 +++------ .../g02-auth/broker_preparation_test.go | 10 +++ 6 files changed, 168 insertions(+), 35 deletions(-) diff --git a/docs/evidence/g01-paired-broker.md b/docs/evidence/g01-paired-broker.md index b0d67ab..2fd3675 100644 --- a/docs/evidence/g01-paired-broker.md +++ b/docs/evidence/g01-paired-broker.md @@ -116,6 +116,26 @@ canonical preparation reached mint and launch. That is Committed reviewer-local report/overlay paths are [discussion r3957639908](https://github.com/1XP-AI/gh-runnerd/pull/62#discussion_r3957639908). +A later coordinator reproduction at `92632dde67a98c31146386ce2ce144326b0bdff9` +showed the same-inode fence was incomplete: `workerClaimPath` searched a +sibling `worker-admission` directory and the account pin for any matching +inode. Moving the prepared claim inode into that sibling and writing `{}` at +the original pathname made `checkPrepared` return nil +([comment 5586268545](https://github.com/1XP-AI/gh-runnerd/pull/62#issuecomment-5586268545)): + +```text +GOTOOLCHAIN=go1.26.8 go test -count=1 -timeout=60s \ + -run '^TestPairedWorkerPreparationReceiptFencesClaimRelocation$' . +FAIL: relocated receipt inode bypassed changed canonical claim path + (relocated-inode and relocated-directory) + +GOTOOLCHAIN=go1.26.8 go test -count=1 -timeout=60s \ + -run '^TestPairedBrokerRejectsWorkerClaimChangeBeforeMint$' . +FAIL: worker admission claim relocated-inode crossed pre-mint fence: + err= mints=1 launches=1 + (same for relocated-directory) +``` + ## Implemented boundaries Production `validBrokerBuild` now requires the exact reviewed tag set @@ -144,11 +164,17 @@ worker authority/admission lease, rejects prior effects/uncertainty/reservation histories, and returns only a credential-free receipt. The broker binds the receipt's approval digest, state/journal/claim identities, and journal/claim digests. Every later `checkPrepared` reopens the worker journal and the -canonical admission claim (the receipt inode/digest, never a newly invented -root), compares both to the stored receipt, and checks the claim's version-1 -ownership/state/journal schema under a short exclusive file lease. Same-inode -mutation, replacement, missing, malformed, and locked claims fail at the -pre-auth, mint, and launch fences with zero remote/mint/launch as appropriate. +admission claim bound by the trusted preparation contract: the explicit +prepared directory in tests, otherwise the native-account worker pin. It +compares the named path's inode and digest to the stored receipt and checks +the claim's version-1 ownership/state/journal schema under a short exclusive +file lease. It does not search fixture siblings or other candidate paths for a +matching inode. Same-inode mutation, replacement, missing, malformed, locked, +relocated-inode, and relocated-directory claims fail at the pre-auth, mint, +and launch fences with zero remote/mint/launch as appropriate. Offline +executable fixtures bind the claim directory through the explicit +`resolveWorkerClaimDirectory` / `bindWorkerClaimDirectory` seam, the same +narrow pattern as `brokerBinaryOpener`. The child later reopens the worker journal and admission claim through the same canonical parser before worker effects, so those cases cannot mint/launch or authorize a retry. `TestPairedWorkerPreparationReceiptFencesJournalMutation` @@ -272,12 +298,39 @@ GOTOOLCHAIN=go1.26.8 go test -race -count=1 -timeout=45s \ PASS; g02-auth 24.639s; process wall time 25.48s ``` -Focused claim-fence and related tests: +Focused claim-fence and related tests after the pathname-binding fix: ```text GOTOOLCHAIN=go1.26.8 go test -count=1 -timeout=90s \ - -run '^(TestPairedWorkerPreparationReceiptFencesClaimMutation|TestPairedBrokerRejectsWorkerClaimChangeBeforeAuth|TestPairedBrokerRejectsWorkerClaimChangeBeforeMint|TestPairedWorkerPreparationReceiptFencesJournalMutation|TestPairedBrokerRejectsMalformedWorkerJournalBeforeMint|TestPairedBrokerRealEntrypointUsesPairedPreparationClosure|TestBrokerPreparedFilesChangeAfterCaptureStopsBeforeMint|TestBrokerAccountRootIgnoresEnvironmentAndFailsClosed)$' . -PASS; g02-auth 2.317s + -run '^(TestPairedWorkerPreparationReceiptFencesClaimRelocation|TestPairedWorkerPreparationReceiptFencesClaimMutation|TestPairedWorkerPreparationReceiptFencesJournalMutation|TestPairedBrokerRejectsWorkerClaimChangeBeforeAuth|TestPairedBrokerRejectsWorkerClaimChangeBeforeMint|TestPairedBrokerRealEntrypointUsesPairedPreparationClosure|TestPairedBrokerRejectsMalformedWorkerJournalBeforeMint|TestBrokerAccountRootIgnoresEnvironmentAndFailsClosed)$' . +PASS; g02-auth 2.514s +``` + +Current G02 45-second partitions after the pathname-binding fix: + +```text +GOTOOLCHAIN=go1.26.8 go test -race -count=1 -timeout=45s \ + -run '^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$' ./... +PASS; g02-auth 40.625s + +GOTOOLCHAIN=go1.26.8 go test -race -count=1 -timeout=45s \ + -run '^TestPaired' -skip '^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$' ./... +PASS; g02-auth 11.823s + +GOTOOLCHAIN=go1.26.8 go test -race -count=1 -timeout=45s \ + -skip '^TestPaired' ./... +PASS; g02-auth 25.740s +``` + +Declared offline gate after the pathname-binding fix: + +```text +/usr/bin/time -p env GOTOOLCHAIN=go1.26.8 bash scripts/check-offline-experiments.sh +G02 named cadence: g02-auth 40.781s +G02 remaining TestPaired family: g02-auth 11.511s +G02 unfiltered complement: g02-auth 26.138s +offline experiment checks passed: 2 module(s) +exit 0; process wall time 353.67s ``` The focused tooling matrix passed after the three-partition script correction: @@ -345,4 +398,5 @@ including stale/outdated ones. Hosted CI, independent review of the new head, and a clean exact-head Codex verdict remain required before any merge decision. Live recovery remains unauthorized and unproven. The named cadence 45-second partition still has only a few seconds of local headroom; that is a remaining -gap, not an approved timeout change. +gap, not an approved timeout change. Rollback is a source-only revert of this +pathname-binding follow-up. diff --git a/experiments/g02-auth/broker_admission.go b/experiments/g02-auth/broker_admission.go index 55cbd37..e583c39 100644 --- a/experiments/g02-auth/broker_admission.go +++ b/experiments/g02-auth/broker_admission.go @@ -30,6 +30,11 @@ func workerAdmissionDirectory() (string, error) { return workerDirectoryForAccount(user.LookupId) } +// resolveWorkerClaimDirectory is the trusted worker admission root. Production +// uses the native-account pin. Offline tests may replace it; production never +// searches fixture siblings or other candidate paths for a matching inode. +var resolveWorkerClaimDirectory = workerAdmissionDirectory + func brokerHomeDir(lookup func(string) (*user.User, error)) (string, error) { if lookup == nil { return "", errBroker diff --git a/experiments/g02-auth/broker_paired_bridge_test.go b/experiments/g02-auth/broker_paired_bridge_test.go index 5a4a411..54f6393 100644 --- a/experiments/g02-auth/broker_paired_bridge_test.go +++ b/experiments/g02-auth/broker_paired_bridge_test.go @@ -820,6 +820,7 @@ func runPairedBrokerBridge(t *testing.T, tags string) time.Duration { return &verifiedBrokerBinary{path: path, file: file, digest: a.ControllerBinarySHA256}, nil } defer func() { brokerBinaryOpener = oldOpener }() + bindWorkerClaimDirectory(t, filepath.Join(filepath.Dir(workerState), "worker-admission")) input, err := os.Open(inputPath) if err != nil { t.Fatal("broker input") diff --git a/experiments/g02-auth/broker_paired_test.go b/experiments/g02-auth/broker_paired_test.go index 1618d9a..706e076 100644 --- a/experiments/g02-auth/broker_paired_test.go +++ b/experiments/g02-auth/broker_paired_test.go @@ -165,6 +165,53 @@ func TestPairedWorkerPreparationReceiptFencesClaimMutation(t *testing.T) { } } +func TestPairedWorkerPreparationReceiptFencesClaimRelocation(t *testing.T) { + for _, kind := range []string{"relocated-inode", "relocated-directory"} { + t.Run(kind, func(t *testing.T) { + a, c, controllerState, workerState, workerPath := pairedPlanInputs(t) + plan, err := openBrokerWorkerPlan(workerPath, workerState, controllerState, a, c) + if err != nil { + t.Fatal("valid paired worker plan refused") + } + defer plan.close() + admission := filepath.Join(filepath.Dir(workerState), "canonical-worker-claim") + plan.prepare = func(context.Context) (brokerPreparationReceipt, error) { + return brokerSyntheticWorkerPreparation(t, plan, admission) + } + if err := plan.checkPrepared(context.Background()); err != nil { + t.Fatalf("fresh worker preparation refused: %v", err) + } + claimPath := filepath.Join(admission, "admission.json") + fallback := filepath.Join(filepath.Dir(workerState), "worker-admission") + switch kind { + case "relocated-inode": + if err := os.Mkdir(fallback, 0700); err != nil { + t.Fatal("fallback worker admission") + } + if err := os.Rename(claimPath, filepath.Join(fallback, "admission.json")); err != nil { + t.Fatal("relocate worker claim inode") + } + if err := os.WriteFile(claimPath, []byte("{}"), 0600); err != nil { + t.Fatal("replace canonical worker claim path") + } + case "relocated-directory": + if err := os.Rename(admission, fallback); err != nil { + t.Fatal("relocate worker claim directory") + } + if err := os.Mkdir(admission, 0700); err != nil { + t.Fatal("replacement worker claim directory") + } + if err := os.WriteFile(claimPath, []byte("{}"), 0600); err != nil { + t.Fatal("replace canonical worker claim path") + } + } + if err := plan.checkPrepared(context.Background()); err == nil { + t.Fatal("relocated receipt inode bypassed changed canonical claim path") + } + }) + } +} + func TestPairedWorkerApprovalMismatchRefusesBeforeBinding(t *testing.T) { a, c, controllerState, workerState, workerPath := pairedPlanInputs(t) raw, err := os.ReadFile(workerPath) @@ -232,6 +279,11 @@ func TestPairedApprovalRejectsInsufficientTerminalAuthority(t *testing.T) { } func pairedWorkerExecutePlan(t *testing.T, a *BrokerApproval, parent string, launch func(context.Context, []byte, string) error) (*brokerControllerPlan, string) { + t.Helper() + return pairedWorkerExecutePlanAt(t, a, parent, filepath.Join(parent, "canonical-worker-claim"), launch) +} + +func pairedWorkerExecutePlanAt(t *testing.T, a *BrokerApproval, parent, admission string, launch func(context.Context, []byte, string) error) (*brokerControllerPlan, string) { t.Helper() plan := brokerTestPlan(t, a, parent, launch) plan.controller.Phases = []string{"create", "before-ack", "inspect", "cleanup"} @@ -256,13 +308,12 @@ func pairedWorkerExecutePlan(t *testing.T, a *BrokerApproval, parent string, lau if err != nil { t.Fatal("valid paired worker plan refused") } - admission := filepath.Join(parent, "worker-admission") brokerAttachSyntheticWorkerPreparation(t, plan, admission) return plan, admission } func TestPairedBrokerRejectsWorkerClaimChangeBeforeAuth(t *testing.T) { - for _, kind := range []string{"hash", "replacement"} { + for _, kind := range []string{"hash", "replacement", "relocated-inode", "relocated-directory"} { t.Run(kind, func(t *testing.T) { a, candidate, api, fixture, attempt := newBrokerFixture(t) a.Mode, a.Phase, a.AllowVerificationAuthority = "paired-terminal", "paired-terminal", true @@ -292,6 +343,16 @@ func TestPairedBrokerRejectsWorkerClaimChangeBeforeAuth(t *testing.T) { if os.Rename(claimPath, claimPath+".retained") != nil || os.WriteFile(claimPath, data, 0600) != nil { t.Fatal("replace worker claim") } + case "relocated-inode": + fallback := filepath.Join(filepath.Dir(admission), "worker-admission") + if os.Mkdir(fallback, 0700) != nil || os.Rename(claimPath, filepath.Join(fallback, "admission.json")) != nil || os.WriteFile(claimPath, []byte("{}"), 0600) != nil { + t.Fatal("relocate worker claim inode") + } + case "relocated-directory": + fallback := filepath.Join(filepath.Dir(admission), "worker-admission") + if os.Rename(admission, fallback) != nil || os.Mkdir(admission, 0700) != nil || os.WriteFile(claimPath, []byte("{}"), 0600) != nil { + t.Fatal("relocate worker claim directory") + } } return receipt, nil } @@ -304,7 +365,7 @@ func TestPairedBrokerRejectsWorkerClaimChangeBeforeAuth(t *testing.T) { } func TestPairedBrokerRejectsWorkerClaimChangeBeforeMint(t *testing.T) { - for _, kind := range []string{"hash", "replacement", "missing", "malformed", "locked"} { + for _, kind := range []string{"hash", "replacement", "missing", "malformed", "locked", "relocated-inode", "relocated-directory"} { t.Run(kind, func(t *testing.T) { a, candidate, _, fixture, attempt := newBrokerFixture(t) a.Mode, a.Phase, a.AllowVerificationAuthority = "paired-terminal", "paired-terminal", true @@ -345,6 +406,16 @@ func TestPairedBrokerRejectsWorkerClaimChangeBeforeMint(t *testing.T) { if err != nil || syscall.Flock(int(held.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) != nil { t.Fatal("lock worker claim") } + case "relocated-inode": + fallback := filepath.Join(filepath.Dir(admission), "worker-admission") + if os.Mkdir(fallback, 0700) != nil || os.Rename(claimPath, filepath.Join(fallback, "admission.json")) != nil || os.WriteFile(claimPath, []byte("{}"), 0600) != nil { + t.Fatal("relocate worker claim inode") + } + case "relocated-directory": + fallback := filepath.Join(filepath.Dir(admission), "worker-admission") + if os.Rename(admission, fallback) != nil || os.Mkdir(admission, 0700) != nil || os.WriteFile(claimPath, []byte("{}"), 0600) != nil { + t.Fatal("relocate worker claim directory") + } } } return response, err @@ -613,6 +684,7 @@ func TestPairedBrokerRealEntrypointUsesPairedPreparationClosure(t *testing.T) { oldOpener := brokerBinaryOpener brokerBinaryOpener = func(string, BrokerApproval) (*verifiedBrokerBinary, error) { return binary, nil } defer func() { brokerBinaryOpener = oldOpener }() + bindWorkerClaimDirectory(t, filepath.Join(parent, "worker-admission")) result, err := runBrokerWithAPI(context.Background(), BrokerFiles{ApprovalPath: approvalPath, StateDirectory: attempt, ControllerBinary: binary.path, ControllerApproval: controllerPath, ControllerStateDirectory: controllerState, WorkerApproval: workerPath, WorkerStateDirectory: workerState}, input, api) if err != nil || result.Status != "paired_terminal_completed" || fixture.tokenCalls != 1 { t.Fatalf("real paired entrypoint did not complete one handoff: result=%+v err=%v mints=%d calls=%v", result, err, fixture.tokenCalls, fixture.calls) @@ -695,6 +767,7 @@ func TestPairedBrokerRejectsMalformedWorkerJournalBeforeMint(t *testing.T) { oldOpener := brokerBinaryOpener brokerBinaryOpener = func(string, BrokerApproval) (*verifiedBrokerBinary, error) { return binary, nil } defer func() { brokerBinaryOpener = oldOpener }() + bindWorkerClaimDirectory(t, filepath.Join(parent, "worker-admission")) _, err = runBrokerWithAPI(context.Background(), BrokerFiles{ApprovalPath: approvalPath, StateDirectory: attempt, ControllerBinary: binary.path, ControllerApproval: controllerPath, ControllerStateDirectory: controllerState, WorkerApproval: workerPath, WorkerStateDirectory: workerState}, input, api) if err == nil || fixture.tokenCalls != 0 { t.Fatalf("malformed worker journal crossed pre-mint boundary: err=%v mints=%d calls=%v", err, fixture.tokenCalls, fixture.calls) diff --git a/experiments/g02-auth/broker_plan.go b/experiments/g02-auth/broker_plan.go index 56e2794..dc7d4b8 100644 --- a/experiments/g02-auth/broker_plan.go +++ b/experiments/g02-auth/broker_plan.go @@ -140,33 +140,23 @@ func (p *brokerWorkerPlan) workerClaimPath() (string, error) { if p == nil || p.preparationReceipt.Claim == (brokerInode{}) { return "", errBroker } - match := func(path string) bool { - if path == "" { - return false + directory := p.claimDirectory + if directory == "" { + var err error + directory, err = resolveWorkerClaimDirectory() + if err != nil { + return "", errBroker } - info, err := os.Lstat(path) - return err == nil && info.Mode().IsRegular() && brokerFileIdentity(info) == p.preparationReceipt.Claim } - var candidates []string - if p.claimDirectory != "" { - candidates = append(candidates, filepath.Join(p.claimDirectory, "admission.json")) - } - candidates = append(candidates, filepath.Join(filepath.Dir(p.statePath), "worker-admission", "admission.json")) - seen := map[string]bool{} - for _, path := range candidates { - if seen[path] { - continue - } - seen[path] = true - if match(path) { - return path, nil - } + if !filepath.IsAbs(directory) || filepath.Clean(directory) != directory { + return "", errBroker } - directory, err := workerAdmissionDirectory() - if err == nil && match(filepath.Join(directory, "admission.json")) { - return filepath.Join(directory, "admission.json"), nil + path := filepath.Join(directory, "admission.json") + info, err := os.Lstat(path) + if err != nil || !info.Mode().IsRegular() || brokerFileIdentity(info) != p.preparationReceipt.Claim { + return "", errBroker } - return "", errBroker + return path, nil } func flockBrokerHandle(file *os.File) error { diff --git a/experiments/g02-auth/broker_preparation_test.go b/experiments/g02-auth/broker_preparation_test.go index e2aecdc..0e446b9 100644 --- a/experiments/g02-auth/broker_preparation_test.go +++ b/experiments/g02-auth/broker_preparation_test.go @@ -116,6 +116,16 @@ func brokerAttachSyntheticWorkerPreparation(t *testing.T, p *brokerControllerPla return brokerSyntheticWorkerPreparation(t, p.worker, directory) } } + +func bindWorkerClaimDirectory(t *testing.T, directory string) { + t.Helper() + if directory == "" || !filepath.IsAbs(directory) || filepath.Clean(directory) != directory { + t.Fatal("worker claim fixture directory") + } + previous := resolveWorkerClaimDirectory + resolveWorkerClaimDirectory = func() (string, error) { return directory, nil } + t.Cleanup(func() { resolveWorkerClaimDirectory = previous }) +} func TestBrokerPreparedReceiptBindsValidatedBytesBeforeMint(t *testing.T) { for _, kind := range []string{"journal changed before capture", "claim changed before capture", "wrong phase", "wrong approval", "missing identity", "wrong version", "wrong status"} { t.Run(kind, func(t *testing.T) {