diff --git a/docs/changes/unreleased/1815-security-affinity-accounting.md b/docs/changes/unreleased/1815-security-affinity-accounting.md new file mode 100644 index 0000000000..1e902b8d04 --- /dev/null +++ b/docs/changes/unreleased/1815-security-affinity-accounting.md @@ -0,0 +1,10 @@ +--- +kind: fixed +title: security audits distinguish incomplete analysis from a completed search +pr: 1815 +surface: [chat, engine] +invalidates: + - "Unicode session labels were also sent as machine affinity keys, which some model gateways rejected. Unsafe keys are now stable ASCII while the readable labels and selected models remain unchanged." + - "Failed hunt calls contributed empty batches without measured failure accounting, so every failed hunt could end as a clean pass. Failed attempts now survive checkpoints and reports, make the analysis incomplete and retain findings from completed hunts." + - "A partial audit result could conceal a later model refusal or run limit. Result-bearing stops now keep their failed or ceiling ending and mark Markdown, JSON and SARIF as incomplete without changing proof requirements." +--- diff --git a/internal/agentsession/affinity_test.go b/internal/agentsession/affinity_test.go new file mode 100644 index 0000000000..926ffbd80a --- /dev/null +++ b/internal/agentsession/affinity_test.go @@ -0,0 +1,46 @@ +package agentsession + +import ( + "strings" + "testing" +) + +// DISPLAY WORDS STAY WORDS, but a gateway may reuse the cache key as a request +// header. Exercise the actual wire body across turns, labels and sessions. +func TestUnicodeLabelsKeepTheirWordsAndUseStableASCIIWireAffinity(t *testing.T) { + root := repo(t) + label := "auth hunter · src/登录.py:12" + api := (&fakeAPI{}).then(callTool("read", `{"path":"app/views.py"}`)). + then(answer(`{"file":"app/views.py","line":4}`, 0)). + then(answer(`{"file":"app/views.py","line":4}`, 0)). + then(answer(`{"file":"app/views.py","line":4}`, 0)) + var heard []string + app := mustNew(t, api.serve(t), Config{Root: root, SessionModel: "vendor/work", + Watch: &Watch{Session: func(label string, _ SessionResult, _ error) { heard = append(heard, label) }}}) + for _, name := range []string{label, label, "auth hunter · src/登入.py:12"} { + opts := appxOptions("", root) + opts.Label = name + var dest map[string]any + result, err := app.Harness(t.Context(), "read and answer", findingSchema, &dest, opts) + if err != nil || result.IsError { + t.Fatalf("session %q: %+v, %v", name, result, err) + } + } + if strings.Join(heard, "|") != label+"|"+label+"|auth hunter · src/登入.py:12" { + t.Fatalf("display labels changed: %v", heard) + } + for _, body := range api.bodies { + if body.PromptCacheKey == "" || strings.IndexFunc(body.PromptCacheKey, func(r rune) bool { return r < 33 || r > 126 }) >= 0 { + t.Fatalf("unsafe wire affinity %q", body.PromptCacheKey) + } + if body.Model != "vendor/work" || len(body.Messages) == 0 || len(body.Tools) == 0 { + t.Fatalf("model, messages or tools lost: %+v", body) + } + } + if api.bodies[0].PromptCacheKey != api.bodies[1].PromptCacheKey { + t.Fatal("a tool turn changed the session's affinity") + } + if api.bodies[1].PromptCacheKey == api.bodies[2].PromptCacheKey || api.bodies[2].PromptCacheKey == api.bodies[3].PromptCacheKey { + t.Fatal("distinct sessions or Unicode labels share affinity") + } +} diff --git a/internal/agentsession/client.go b/internal/agentsession/client.go index d2ebad0410..16a155f4ea 100644 --- a/internal/agentsession/client.go +++ b/internal/agentsession/client.go @@ -23,6 +23,7 @@ package agentsession import ( "bytes" "context" + "crypto/sha256" "encoding/json" "errors" "fmt" @@ -141,7 +142,7 @@ func (c *Client) Complete(ctx context.Context, request Request) (*ai.Response, e } wire := wireRequest{ Model: request.Model, Messages: request.Messages, Tools: request.Tools, - ResponseFormat: request.ResponseFormat, PromptCacheKey: request.Thread, + ResponseFormat: request.ResponseFormat, PromptCacheKey: wireAffinity(request.Thread), } if len(wire.Tools) > 0 { wire.ToolChoice = "auto" @@ -189,6 +190,28 @@ func (c *Client) Complete(ctx context.Context, request Request) (*ai.Response, e return nil, last } +// wireAffinity keeps ordinary cache keys unchanged. A display label can contain +// any Unicode, but gateways also use affinity as a request header, which must +// be ASCII. Hash the entire unsafe key so its turns stay together without +// collapsing different labels or session numbers to the same stripped text. +func wireAffinity(thread string) string { + if strings.IndexFunc(thread, func(r rune) bool { return r < 33 || r > 126 }) < 0 { + return thread + } + sum := sha256.Sum256([]byte(thread)) + return fmt.Sprintf("agent-%x", sum) +} + +// Refusal is the first refusal that stopped all model calls. A program that +// retains a partial result still needs the typed ceiling or key refusal, even +// if its own phase calls wrapped the individual failures in opaque errors. +func (c *Client) Refusal() error { + if stopped := c.stopped.Load(); stopped != nil { + return *stopped + } + return nil +} + // errHeld is the model API asking a call to wait while calls in flight hold // what is left of the run's ceiling. var errHeld = errors.New("the run's ceiling is held by calls in flight") diff --git a/internal/manual/chat/sec.md b/internal/manual/chat/sec.md index 5575be211d..8d2471ab83 100644 --- a/internal/manual/chat/sec.md +++ b/internal/manual/chat/sec.md @@ -61,6 +61,25 @@ The full report is three files in the task's record folder, named at the end of `--compliance owasp,pci-dss` (at a shell) maps the findings to compliance frameworks — OWASP, PCI-DSS, SOC2, HIPAA, ISO27001 — and adds `sec-compliance.md`. A shell run's record folder is under `~/.codeaf/v3/carried/sec/`, one folder per run. +## A failed or incomplete audit — a model error, hunts that did not finish, no findings + +If a hunt cannot finish because a model call failed or its session stopped, sec +records that failure separately from a completed hunt with no findings. Its ending, +account and readable report say **analysis incomplete** and how many hunts failed. +Findings from completed hunts still go through the same checks and remain in the +report; the failure itself is never a security finding. If all hunts fail, sec +makes no claim that the code has no exploitable problems. + +The run ends as failed when any hunt fails, or on its ceiling if the run's dollar +or time limit caused the incomplete analysis. A run stopped or refused by the +model service later in the audit also says analysis incomplete, retaining any +findings already checked; a partial result never turns that stop into a pass. +The JSON report keeps the attempted +count and each failed strategy and reason in `metadata.hunt_coverage`; the SARIF +records unsuccessful execution and an error notification, alongside any findings. +Global stops are also recorded in `metadata.analysis_incomplete`. If the pipeline +cannot produce a result, the ending explains the stop without promising a report. + ## An earlier audit's report — find it from another conversation, run it again, the last audit, start fresh A sec run in the chat keeps its report in that task's record folder, `~/.codeaf/v3/projects///tasks//`. Every conversation in the project can see the run in `tasks`: its row's outcome is the first line of its account, which says what it found — `Security audit of the whole repository: 14 problems, 9 confirmed and 5 likely (1 critical, 1 high, 12 medium).` — and its `artifact` is that record folder, where `sec-report.md` is. diff --git a/internal/secaf/agents/hunt/coverage_test.go b/internal/secaf/agents/hunt/coverage_test.go new file mode 100644 index 0000000000..f53c247c6c --- /dev/null +++ b/internal/secaf/agents/hunt/coverage_test.go @@ -0,0 +1,29 @@ +package hunt + +import ( + "context" + "errors" + "testing" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/config" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +func TestResumedHuntsKeepPartialFindingsAndFailures(t *testing.T) { + withStrategyTable(t, + strategyEntry{schemas.HuntStrategyInjection, func(context.Context, appx.Harnesser, string, schemas.ReconResult, config.DepthProfile) (schemas.HuntResult, error) { + return schemas.HuntResult{}, errors.New("cannot search") + }}, + strategyEntry{schemas.HuntStrategyAuth, func(context.Context, appx.Harnesser, string, schemas.ReconResult, config.DepthProfile) (schemas.HuntResult, error) { + result := schemas.NewHuntResult() + result.Findings = []schemas.RawFinding{testFinding("app/a.py", "one")} + return result, nil + }}, + ) + withDedup(t, passthroughDedup) + result, err := RunHunt(t.Context(), &appx.Fake{}, ".", emptyRecon(), "standard", 2, 1, nil) + if err != nil || len(result.Findings) != 1 || result.Coverage == nil || result.Coverage.Attempted != 2 || len(result.Coverage.Failures) != 1 || result.Coverage.Failures[0].Strategy != "injection" { + t.Fatalf("partial hunt=%+v err=%v", result, err) + } +} diff --git a/internal/secaf/agents/hunt/hunt.go b/internal/secaf/agents/hunt/hunt.go index 1e603c873c..bc4bb37827 100644 --- a/internal/secaf/agents/hunt/hunt.go +++ b/internal/secaf/agents/hunt/hunt.go @@ -281,10 +281,8 @@ func strategyValues(strategies []schemas.HuntStrategy) []string { // // Concurrency parity: // -// - `asyncio.gather(..., return_exceptions=True)` means a hunter that raises -// is SILENTLY DROPPED — its findings are simply absent and nothing is -// logged. The Go port reproduces that: a per-index error slot that nothing -// reads. The only error RunHunt can return is dedup's. +// - A failed hunter contributes no findings, but codeaf records the failure +// separately so a resumed audit cannot mistake it for a completed search. // - Results are flattened in strategy order regardless of completion order, // so the port writes into a pre-sized slice rather than appending from the // goroutines. @@ -312,6 +310,7 @@ func RunHunt( sem := semaphore.NewWeighted(int64(concurrencyLimit(maxConcurrentHunters, len(strategies)))) perStrategy := make([][]schemas.RawFinding, len(strategies)) + failures := make([]error, len(strategies)) var wg sync.WaitGroup for i, strategy := range strategies { i, strategy := i, strategy @@ -319,7 +318,7 @@ func RunHunt( go func() { defer wg.Done() if err := sem.Acquire(ctx, 1); err != nil { - // Python: a cancelled task is an exception gather swallows. + failures[i] = err return } defer sem.Release(1) @@ -327,14 +326,16 @@ func RunHunt( ctx, strategyRunnerIndex[strategy], app, repoPath, recon, profile, ) if err != nil { - // Python: `return_exceptions=True` then `if isinstance(result, - // Exception): continue`. + failures[i] = err return } perStrategy[i] = findings }() } wg.Wait() + if err := ctx.Err(); err != nil { + return schemas.HuntResult{}, err + } allFindings := []schemas.RawFinding{} for _, findings := range perStrategy { @@ -352,6 +353,7 @@ func RunHunt( deduplicated.ChainCount = len(deduplicated.Chains) deduplicated.StrategiesRun = strategyValues(strategies) deduplicated.HuntDurationSeconds = time.Since(started).Seconds() + deduplicated.Coverage = coverageForErrors(strategies, failures) return deduplicated, nil } @@ -413,13 +415,15 @@ func RunHuntStreaming( ) var wg sync.WaitGroup - for _, strategy := range strategies { + failures := make([]error, len(strategies)) + for index, strategy := range strategies { strategy := strategy wg.Add(1) go func() { defer wg.Done() if err := sem.Acquire(ctx, 1); err != nil { + failures[index] = err return } hunterFindings, err := runSingleHunter( @@ -427,8 +431,7 @@ func RunHuntStreaming( ) sem.Release(1) // Python releases before taking the dedup lock. if err != nil { - // Python: gather(return_exceptions=True) swallows it, and the - // task never reaches its queue put. + failures[index] = err return } @@ -461,6 +464,9 @@ func RunHuntStreaming( } wg.Wait() close(findings) // Python: `await findings_queue.put(None)`. + if err := ctx.Err(); err != nil { + return schemas.HuntResult{}, err + } unique := make([]schemas.RawFinding, 0, len(fingerprintOrder)) for _, fingerprint := range fingerprintOrder { @@ -480,5 +486,21 @@ func RunHuntStreaming( deduplicated.ChainCount = len(deduplicated.Chains) deduplicated.StrategiesRun = strategyValues(strategies) deduplicated.HuntDurationSeconds = time.Since(started).Seconds() + deduplicated.Coverage = coverageForErrors(strategies, failures) return deduplicated, nil } + +// coverageForErrors retains selected invocations in order after all producers +// have joined. Complete runs keep the original report and checkpoint shape. +func coverageForErrors(strategies []schemas.HuntStrategy, failures []error) *schemas.HuntCoverage { + var failed []schemas.HuntFailure + for i, err := range failures { + if err != nil { + failed = append(failed, schemas.HuntFailure{Strategy: string(strategies[i]), Reason: err.Error()}) + } + } + if len(failed) == 0 { + return nil + } + return &schemas.HuntCoverage{Attempted: len(strategies), Failures: failed} +} diff --git a/internal/secaf/agents/hunt/hunt_test.go b/internal/secaf/agents/hunt/hunt_test.go index 391e127049..e2aa0efada 100644 --- a/internal/secaf/agents/hunt/hunt_test.go +++ b/internal/secaf/agents/hunt/hunt_test.go @@ -665,6 +665,9 @@ func TestRunHuntStreamingClosesOnHunterFailure(t *testing.T) { if count != 0 { t.Errorf("streamed %d batches, want 0", count) } + if result.Coverage == nil || !result.Coverage.AllFailed() || result.Coverage.Attempted != 1 { + t.Fatalf("failed streaming hunt lost accounting: %+v", result.Coverage) + } if result.TotalRaw != 0 || len(result.Findings) != 0 { t.Errorf("result = %+v, want an empty one", result) } diff --git a/internal/secaf/hunt_failure_test.go b/internal/secaf/hunt_failure_test.go new file mode 100644 index 0000000000..07c2e8c3fd --- /dev/null +++ b/internal/secaf/hunt_failure_test.go @@ -0,0 +1,194 @@ +package secaf + +import ( + "context" + "encoding/json" + "io" + "os" + "path/filepath" + "reflect" + "strings" + "sync/atomic" + "testing" + + "github.com/Agent-Field/codeaf/internal/delegate" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// RECON SUCCESS CANNOT MASK FAILED HUNTS. Fail the actual hunter schema calls +// after the real recon phase, and inspect the ending and all three reports. +func TestFailedHuntsCannotReportACleanAudit(t *testing.T) { + for _, tc := range []struct { + name string + partial bool + code int + status string + }{ + {"all", false, 400, delegate.StatusFail}, + {"partial", true, 400, delegate.StatusFail}, + {"dollar ceiling", false, 402, delegate.StatusBudget}, + {"refused key", false, 403, delegate.StatusFail}, + } { + t.Run(tc.name, func(t *testing.T) { + var scans atomic.Int32 + models := &stubModels{reject: func(title string) int { + if title != "ScanLocationsResult" { + return 0 + } + n := scans.Add(1) + if tc.partial && n > 1 { + return 0 + } + return tc.code + }} + repo := vulnerableRepo(t) + before := snapshot(t, repo) + host := &stubHost{workspace: repo, records: t.TempDir(), api: models.serve(t)} + run(context.Background(), host, options{brief: "whole repository quick", severity: "low", sessions: 4, maxTurns: 5}, io.Discard) + if host.ending == nil || host.ending.Status != tc.status { + t.Fatalf("failed analysis ended %+v, want %s", host.ending, tc.status) + } + ending := host.ending + if ending.Extra["status"] == "pass" || !strings.Contains(ending.Message, "incomplete") || !strings.Contains(firstLine(ending.Deliverable), "incomplete") { + t.Fatalf("failure hidden in ending %+v", ending) + } + data, err := os.ReadFile(filepath.Join(host.records, reportJSON)) + if err != nil { + t.Fatal(err) + } + var result schemas.SecurityAuditResult + if err := json.Unmarshal(data, &result); err != nil { + t.Fatal(err) + } + coverage, ok := result.Metadata["hunt_coverage"].(map[string]any) + if !ok || coverage["attempted"] != json.Number("6") { + t.Fatalf("missing hunt accounting: %#v", result.Metadata) + } + failed, _ := coverage["failures"].([]any) + want := 6 + if tc.partial { + want = 1 + } + if len(failed) != want { + t.Fatalf("failed attempts %d, want %d: %#v", len(failed), want, coverage) + } + if tc.partial && len(result.Findings) == 0 { + t.Fatal("findings from successful hunts were discarded") + } + readable, err := os.ReadFile(filepath.Join(host.records, reportMarkdown)) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(readable), "incomplete") { + t.Fatalf("readable report conceals failed hunts: %s", readable) + } + if !tc.partial && strings.Contains(strings.ToLower(string(readable)), "nothing it could show exploitable") { + t.Fatal("all-failed report makes a clean claim") + } + data, err = os.ReadFile(filepath.Join(host.records, reportSARIF)) + if err != nil { + t.Fatal(err) + } + var sarif struct { + Runs []struct { + Invocations []struct { + ExecutionSuccessful bool `json:"executionSuccessful"` + Notifications []any `json:"toolExecutionNotifications"` + } `json:"invocations"` + } `json:"runs"` + } + if err := json.Unmarshal(data, &sarif); err != nil { + t.Fatal(err) + } + if len(sarif.Runs) != 1 || len(sarif.Runs[0].Invocations) != 1 || sarif.Runs[0].Invocations[0].ExecutionSuccessful || len(sarif.Runs[0].Invocations[0].Notifications) == 0 { + t.Fatalf("SARIF conceals incomplete execution: %s", data) + } + if after := snapshot(t, repo); !reflect.DeepEqual(after, before) { + t.Fatal("audit changed the fixture") + } + }) + } +} + +// A refusal can arrive after every hunter answered. The proof chain may keep +// unclear findings, but that partial result must retain the global stop. +func TestAModelRefusalDuringProofCannotBecomeAPass(t *testing.T) { + for _, code := range []int{402, 403} { + t.Run(httpStatusWord(code), func(t *testing.T) { + var refused atomic.Int32 + models := &stubModels{reject: func(title string) int { + if title == "DataFlowTrace" { + refused.Add(1) + return code + } + return 0 + }} + host := &stubHost{workspace: vulnerableRepo(t), records: t.TempDir(), api: models.serve(t)} + run(t.Context(), host, options{brief: "whole repository quick", severity: "low", sessions: 4, maxTurns: 5}, io.Discard) + want := delegate.StatusFail + if code == 402 { + want = delegate.StatusBudget + } + if refused.Load() == 0 || host.ending == nil || host.ending.Status != want { + t.Fatalf("late refusal count=%d ending=%+v want=%s", refused.Load(), host.ending, want) + } + if !strings.Contains(firstLine(host.ending.Deliverable), "analysis incomplete") { + t.Fatalf("global stop concealed: %+v", host.ending) + } + data, err := os.ReadFile(filepath.Join(host.records, reportJSON)) + if err != nil { + t.Fatal(err) + } + var result schemas.SecurityAuditResult + if err := json.Unmarshal(data, &result); err != nil { + t.Fatal(err) + } + if len(result.Findings) == 0 || result.Metadata["hunt_coverage"] != nil || result.Metadata["analysis_incomplete"] == nil { + t.Fatalf("retained result/stop incorrect: %+v", result) + } + data, err = os.ReadFile(filepath.Join(host.records, reportSARIF)) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), `"executionSuccessful": false`) { + t.Fatalf("SARIF lost late refusal: %s", data) + } + }) + } +} + +func httpStatusWord(code int) string { + if code == 402 { + return "dollar ceiling" + } + return "refused key" +} + +func TestAStopDuringProofKeepsTheControlledFailureEnding(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + var stopped atomic.Bool + models := &stubModels{reject: func(title string) int { + if title == "DataFlowTrace" { + stopped.Store(true) + cancel() + return 400 + } + return 0 + }} + host := &stubHost{workspace: vulnerableRepo(t), records: t.TempDir(), api: models.serve(t)} + run(ctx, host, options{brief: "whole repository quick", severity: "low", sessions: 4, maxTurns: 5}, io.Discard) + if !stopped.Load() || host.ending == nil || host.ending.Status != delegate.StatusFail || !strings.Contains(host.ending.Message, "stopped") { + t.Fatalf("controlled stop became %+v", host.ending) + } +} + +func TestUnreadableAccountingMakesNoCleanClaim(t *testing.T) { + result := schemas.NewSecurityAuditResult() + result.Metadata["hunt_coverage"] = map[string]any{"attempted": 6} + for _, text := range []string{summary(Scope{}, nil, result, nil, "", ""), outcomeLine(result, ""), markdownReport(result, runFacts{})} { + if !strings.Contains(strings.ToLower(text), "incomplete") || strings.Contains(strings.ToLower(text), "nothing it could show exploitable") || strings.Contains(text, "all 1 hunts failed") || strings.Contains(text, "completed hunts") { + t.Fatalf("unknown accounting became a claim: %s", text) + } + } +} diff --git a/internal/secaf/markdown.go b/internal/secaf/markdown.go index 2b7ed8cdf9..bd31ddbcfc 100644 --- a/internal/secaf/markdown.go +++ b/internal/secaf/markdown.go @@ -64,9 +64,38 @@ func markdownReport(result schemas.SecurityAuditResult, run runFacts) string { if run.cutAt != "" { fmt.Fprintf(&b, "> **Cut short.** %s\n\n", cutSentence(run.cutAt, run.wall)) } + coverage := schemas.Coverage(result.Metadata) + interruption := schemas.Interruption(result.Metadata) + if interruption != "" { + fmt.Fprintf(&b, "> **Analysis incomplete.** %s.\n\n", interruption) + } + if coverage.Incomplete() { + fmt.Fprintf(&b, "> **Analysis incomplete.** %s.", coverage.Description()) + if !coverage.Unknown { + b.WriteString(" Findings from completed hunts remain in this report.") + } + b.WriteString("\n\n") + if len(coverage.Failures) > 0 { + b.WriteString("## Hunts that did not finish\n\n") + } + for _, failure := range coverage.Failures { + fmt.Fprintf(&b, "- %s: %s\n", oneLine(failure.Strategy), oneLine(failure.Reason)) + } + b.WriteString("\n") + } b.WriteString("## What it found\n\n") standing, unclear, ruledOut := sortFindings(result.Findings) - if len(standing) == 0 { + if coverage.Unknown || coverage.AllFailed() || (interruption != "" && len(standing) == 0) { + if coverage.AllFailed() { + b.WriteString("All hunts failed, so no conclusion about exploitable problems is available.") + } else if coverage.Unknown { + b.WriteString("Hunt accounting could not be read, so no conclusion about exploitable problems is available.") + } else { + b.WriteString("The analysis stopped, so no conclusion about exploitable problems is available.") + } + } else if len(standing) == 0 && coverage.Incomplete() { + b.WriteString("Completed hunts found nothing they could show exploitable; analysis is incomplete.") + } else if len(standing) == 0 { b.WriteString("Nothing it could show exploitable.") } else { fmt.Fprintf(&b, "%d problem%s: %d confirmed and %d likely", len(standing), plural(len(standing)), result.Confirmed, result.Likely) diff --git a/internal/secaf/orch/hunt_coverage_test.go b/internal/secaf/orch/hunt_coverage_test.go new file mode 100644 index 0000000000..f6ca640042 --- /dev/null +++ b/internal/secaf/orch/hunt_coverage_test.go @@ -0,0 +1,34 @@ +package orch + +import ( + "reflect" + "testing" + + "github.com/Agent-Field/codeaf/internal/secaf/phases" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +func TestIncompleteHuntCheckpointKeepsItsFailures(t *testing.T) { + o, _ := newTestOrchestrator(t) + hunt := schemas.NewHuntResult() + hunt.Coverage = &schemas.HuntCoverage{Attempted: 2, Failures: []schemas.HuntFailure{{Strategy: "auth", Reason: "no reply"}}} + if err := o.WriteCheckpoint(PhaseHunt, hunt); err != nil { + t.Fatal(err) + } + back, err := ReadCheckpoint(o, PhaseHunt, phases.BindHuntResult) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(back.Coverage, hunt.Coverage) { + t.Fatalf("checkpoint dropped failures: %+v", back) + } +} + +func TestReconFindingsDoNotEraseHuntFailures(t *testing.T) { + hunt := schemas.NewHuntResult() + hunt.Coverage = &schemas.HuntCoverage{Attempted: 2, Failures: []schemas.HuntFailure{{Strategy: "auth", Reason: "no reply"}}} + merged := MergeReconFindingsIntoHunt(hunt, []schemas.RawFinding{schemas.NewRawFinding()}) + if !reflect.DeepEqual(merged.Coverage, hunt.Coverage) || len(merged.Findings) != 1 { + t.Fatalf("merge lost findings or failures: %+v", merged) + } +} diff --git a/internal/secaf/orch/merge.go b/internal/secaf/orch/merge.go index 42dc1735cc..ab6c118d9f 100644 --- a/internal/secaf/orch/merge.go +++ b/internal/secaf/orch/merge.go @@ -122,6 +122,7 @@ func MergeReconFindingsIntoHunt(hunt schemas.HuntResult, reconFindings []schemas ChainCount: hunt.ChainCount, StrategiesRun: strategiesRun, HuntDurationSeconds: hunt.HuntDurationSeconds, + Coverage: hunt.Coverage, } } diff --git a/internal/secaf/orch/output.go b/internal/secaf/orch/output.go index 9cfc35365d..00808b8cc1 100644 --- a/internal/secaf/orch/output.go +++ b/internal/secaf/orch/output.go @@ -187,6 +187,9 @@ func (o *AuditOrchestrator) GenerateOutput( "findings_not_verified": o.FindingsNotVerified, "prove_drop_summary": o.ProveDropSummary, } + if hunt.Coverage != nil { + result.Metadata["hunt_coverage"] = *hunt.Coverage + } result.Sarif = "" result.Sarif = output.GenerateSarif(result) diff --git a/internal/secaf/orch/run.go b/internal/secaf/orch/run.go index 82706e5de4..358f241654 100644 --- a/internal/secaf/orch/run.go +++ b/internal/secaf/orch/run.go @@ -569,6 +569,7 @@ func (o *AuditOrchestrator) RunProve( ChainCount: hunt.ChainCount, StrategiesRun: hunt.StrategiesRun, HuntDurationSeconds: hunt.HuntDurationSeconds, + Coverage: hunt.Coverage, } o.FindingsNotVerified = len(hunt.Findings) - len(limitedHunt.Findings) diff --git a/internal/secaf/orch/run_test.go b/internal/secaf/orch/run_test.go index b393541231..c30fb62bd8 100644 --- a/internal/secaf/orch/run_test.go +++ b/internal/secaf/orch/run_test.go @@ -337,6 +337,7 @@ func TestRunFromCheckpoint_HuntBranch(t *testing.T) { hunt := schemas.NewHuntResult() hunt.TotalRaw = 9 hunt.StrategiesRun = []string{"injection"} + hunt.Coverage = &schemas.HuntCoverage{Attempted: 2, Failures: []schemas.HuntFailure{{Strategy: "auth", Reason: "no reply"}}} if err := o.WriteCheckpoint(PhaseRecon, reconFixture(t, "minimal")); err != nil { t.Fatalf("WriteCheckpoint: %v", err) } @@ -348,6 +349,9 @@ func TestRunFromCheckpoint_HuntBranch(t *testing.T) { if err != nil { t.Fatalf("RunFromCheckpoint: %v", err) } + if got := schemas.Coverage(result.Metadata); !got.Incomplete() || got.Attempted != 2 || len(got.Failures) != 1 { + t.Fatalf("resumed report dropped coverage: %+v", got) + } if result.TotalRawFindings != 9 { t.Errorf("total_raw_findings = %d, want the checkpointed 9", result.TotalRawFindings) } diff --git a/internal/secaf/output/sarif.go b/internal/secaf/output/sarif.go index b3175427bd..c0be6cd829 100644 --- a/internal/secaf/output/sarif.go +++ b/internal/secaf/output/sarif.go @@ -75,18 +75,33 @@ func GenerateSarifAs(result schemas.SecurityAuditResult, tool SarifTool) string results = append(results, buildResult(finding, tool.PropertyPrefix)) } + run := obj{ + {"tool", buildToolSection(included, tool)}, + {"results", results}, + {"automationDetails", obj{{"id", fmt.Sprintf("%s/audit/%s/%s", tool.PropertyPrefix, result.Repository, result.Timestamp.String())}}}, + } + coverage := schemas.Coverage(result.Metadata) + interruption := schemas.Interruption(result.Metadata) + if coverage.Incomplete() || interruption != "" { + message := "Security analysis incomplete: " + interruption + if coverage.Incomplete() { + message = "Security analysis incomplete: " + coverage.Description() + "." + if interruption != "" { + message += " " + interruption + } + } + run = append(run, kv{"invocations", []any{obj{ + {"executionSuccessful", false}, + {"toolExecutionNotifications", []any{obj{ + {"level", "error"}, + {"message", obj{{"text", message}}}, + }}}, + }}}) + } sarif := obj{ {"$schema", "https://json.schemastore.org/sarif-2.1.0.json"}, {"version", "2.1.0"}, - {"runs", []any{ - obj{ - {"tool", buildToolSection(included, tool)}, - {"results", results}, - {"automationDetails", obj{ - {"id", fmt.Sprintf("%s/audit/%s/%s", tool.PropertyPrefix, result.Repository, result.Timestamp.String())}, - }}, - }, - }}, + {"runs", []any{run}}, } return dumpsIndent(sarif, 2) } diff --git a/internal/secaf/phases/hunt.go b/internal/secaf/phases/hunt.go index d02dbef60f..d69898cf97 100644 --- a/internal/secaf/phases/hunt.go +++ b/internal/secaf/phases/hunt.go @@ -276,7 +276,7 @@ func HuntPhase( "hunt", "ai_gate", "cwe_expansion") } - dedupedFindings, totalRaw, err := runHunters( + dedupedFindings, totalRaw, failures, err := runHunters( ctx, app, nodeID, repoPath, recon, strategies, depth, maxConcurrentHunters, earlyStopFileThreshold, ) if err != nil { @@ -314,6 +314,10 @@ func HuntPhase( dedup.StrategiesRun = strategyValues(strategies) dedup.TotalRaw = totalRaw + dedup.Coverage = nil + if len(failures) > 0 { + dedup.Coverage = &schemas.HuntCoverage{Attempted: len(strategies), Failures: failures} + } app.Note(ctx, "HUNT phase complete", "phase", "hunt", "done") return afx.ToMap(dedup) @@ -371,7 +375,7 @@ func runHunters( depth string, maxConcurrentHunters int, earlyStopFileThreshold int, -) ([]schemas.RawFinding, int, error) { +) ([]schemas.RawFinding, int, []schemas.HuntFailure, error) { // Python: max(1, min(max_concurrent_hunters, len(strategies))) — the outer // max keeps an empty strategy list (or a non-positive limit) from asking // for a zero-capacity semaphore. @@ -447,7 +451,10 @@ func runHunters( fatalMu.Unlock() } - for _, strategy := range strategies { + // A failure belongs to an invocation, not a strategy name. Separate slots + // let concurrent producers retain selection order and duplicate attempts. + failures := make([]*schemas.HuntFailure, len(strategies)) + for index, strategy := range strategies { strategy := strategy wg.Add(1) go func() { @@ -487,8 +494,9 @@ func runHunters( "max_files_without_signal": earlyStopFileThreshold, }, BindHuntResult) if err != nil { - // Python: note(f"Hunt strategy failed: {strategy_name}: {exc}") - // then put([]). The exception TEXT is runtime-specific. + // Keep the other hunters and their findings, but remember that an + // empty failed batch is not a successful search with no findings. + failures[index] = &schemas.HuntFailure{Strategy: strategyName, Reason: err.Error()} app.Note(ctx, fmt.Sprintf("Hunt strategy failed: %s: %v", strategyName, err), "hunt", "error") findingsQueue <- []schemas.RawFinding{} return @@ -504,7 +512,13 @@ func runHunters( err := fatalErr fatalMu.Unlock() if err != nil { - return nil, 0, err + return nil, 0, nil, err + } + failed := make([]schemas.HuntFailure, 0) + for _, failure := range failures { + if failure != nil { + failed = append(failed, *failure) + } } - return result.findings, result.totalRaw, nil + return result.findings, result.totalRaw, failed, nil } diff --git a/internal/secaf/phases/hunt_coverage_test.go b/internal/secaf/phases/hunt_coverage_test.go new file mode 100644 index 0000000000..aa872143f8 --- /dev/null +++ b/internal/secaf/phases/hunt_coverage_test.go @@ -0,0 +1,37 @@ +package phases + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "testing" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// INVOCATIONS ARE COUNTED, including a repeated strategy, and an empty +// successful search must remain distinct from a failed search. +func TestHuntCoverageKeepsDuplicateAttemptsAndSelectionOrder(t *testing.T) { + spy := &callSpy{answers: map[string]json.RawMessage{"run_auth_hunter": json.RawMessage(`{"findings":[]}`)}, failures: map[string]error{"run_injection_hunter": errors.New("cannot search")}} + fake := &appx.Fake{CallFn: spy.fn()} + findings, raw, failed, err := runHunters(t.Context(), fake, testNodeID, "/repo", reconFixture(t, "minimal"), + []schemas.HuntStrategy{schemas.HuntStrategyInjection, schemas.HuntStrategyAuth, schemas.HuntStrategyInjection}, "quick", 3, 1) + if err != nil || len(findings) != 0 || raw != 0 { + t.Fatalf("findings=%v raw=%d err=%v", findings, raw, err) + } + want := []schemas.HuntFailure{{Strategy: "injection", Reason: "cannot search"}, {Strategy: "injection", Reason: "cannot search"}} + if !reflect.DeepEqual(failed, want) { + t.Fatalf("failures=%+v want=%+v", failed, want) + } +} + +func TestHuntCancellationStillStopsThePhase(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + _, _, _, err := runHunters(ctx, &appx.Fake{}, testNodeID, "/repo", reconFixture(t, "minimal"), []schemas.HuntStrategy{schemas.HuntStrategyAuth}, "quick", 1, 1) + if !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled hunt returned %v", err) + } +} diff --git a/internal/secaf/report.go b/internal/secaf/report.go index a0989c981e..b24ad797eb 100644 --- a/internal/secaf/report.go +++ b/internal/secaf/report.go @@ -145,14 +145,42 @@ func summary(scope Scope, changes *Changes, result schemas.SecurityAuditResult, if changes != nil { fmt.Fprintf(&b, " (%d changed files since %s, %d files near them)", len(changes.Changed), changes.BaseName, len(changes.Nearby)) } - if len(standing) == 0 { - b.WriteString(": nothing it could show exploitable") + coverage := schemas.Coverage(result.Metadata) + interruption := schemas.Interruption(result.Metadata) + if interruption != "" && !coverage.Incomplete() { + b.WriteString(": analysis incomplete; " + interruption) + if len(standing) > 0 { + b.WriteString("; completed work") + } + } + if coverage.Incomplete() { + b.WriteString(": analysis incomplete; " + coverage.Description()) + if !coverage.AllFailed() && !coverage.Unknown && (interruption == "" || len(standing) > 0) { + b.WriteString("; completed hunts") + } + } + if coverage.Unknown || coverage.AllFailed() || (interruption != "" && len(standing) == 0) { + if !coverage.AllFailed() { + b.WriteString("; no conclusion about exploitable problems is available") + } + b.WriteString(".\n") + } else if len(standing) == 0 { + if coverage.Incomplete() || interruption != "" { + b.WriteString(" found nothing they could show exploitable") + } else { + b.WriteString(": nothing it could show exploitable") + } if result.Inconclusive > 0 { fmt.Fprintf(&b, "; %d finding%s stayed unclear and %s in the report", result.Inconclusive, plural(result.Inconclusive), areIs(result.Inconclusive)) } b.WriteString(".\n") } else { - fmt.Fprintf(&b, ": %d problem%s, %d confirmed and %d likely", len(standing), plural(len(standing)), result.Confirmed, result.Likely) + if coverage.Incomplete() || interruption != "" { + b.WriteString(" found ") + } else { + b.WriteString(": ") + } + fmt.Fprintf(&b, "%d problem%s, %d confirmed and %d likely", len(standing), plural(len(standing)), result.Confirmed, result.Likely) if counts := severityCounts(standing); counts != "" { b.WriteString(" (" + counts + ")") } @@ -250,9 +278,25 @@ func plural(n int) string { // problems stand, or that none does. func outcomeLine(result schemas.SecurityAuditResult, cutAt string) string { standing := result.Confirmed + result.Likely - line := "it found nothing it could show exploitable" - if standing > 0 { - line = fmt.Sprintf("it found %d problem%s: %d confirmed, %d likely", standing, plural(standing), result.Confirmed, result.Likely) + line := outcomeLineWithoutCoverage(result) + if coverage := schemas.Coverage(result.Metadata); coverage.Incomplete() { + if coverage.AllFailed() { + line = fmt.Sprintf("analysis incomplete: all %d hunts failed", coverage.Attempted) + } else { + line = "analysis incomplete: " + coverage.Description() + if !coverage.Unknown { + line += "; completed hunts " + strings.TrimPrefix(outcomeLineWithoutCoverage(result), "it ") + } + } + } + if interruption := schemas.Interruption(result.Metadata); interruption != "" { + if !schemas.Coverage(result.Metadata).Incomplete() { + if standing == 0 { + line = "no conclusion about exploitable problems is available" + } + line = "analysis incomplete: " + line + } + line += "; " + interruption } if cutAt != "" { line += ", before its time ceiling cut it short during " + cutAt @@ -260,6 +304,15 @@ func outcomeLine(result schemas.SecurityAuditResult, cutAt string) string { return line } +// outcomeLineWithoutCoverage describes the retained findings alone, without +// treating an incomplete analysis as a conclusion about the whole repository. +func outcomeLineWithoutCoverage(result schemas.SecurityAuditResult) string { + if standing := result.Confirmed + result.Likely; standing > 0 { + return fmt.Sprintf("it found %d problem%s: %d confirmed, %d likely", standing, plural(standing), result.Confirmed, result.Likely) + } + return "it found nothing it could show exploitable" +} + // cutSentence says that the run's time ceiling stopped it, in which phase, and // what that left undone. func cutSentence(phase, wall string) string { diff --git a/internal/secaf/schemas/coverage.go b/internal/secaf/schemas/coverage.go new file mode 100644 index 0000000000..8517c7b5bf --- /dev/null +++ b/internal/secaf/schemas/coverage.go @@ -0,0 +1,70 @@ +package schemas + +import ( + "encoding/json" + "fmt" + "strings" +) + +// HuntCoverage is measured by the phase, never inferred from a model's answer +// or the number of findings. Each selected strategy invocation is one attempt, +// including repeated strategies; failures stay in selection order. +type HuntCoverage struct { + Attempted int `json:"attempted"` + Failures []HuntFailure `json:"failures"` + // Unknown marks unreadable persisted accounting, never a measured count. + Unknown bool `json:"unknown,omitempty"` +} + +type HuntFailure struct { + Strategy string `json:"strategy"` + Reason string `json:"reason"` +} + +func (c HuntCoverage) Incomplete() bool { return c.Unknown || len(c.Failures) > 0 } +func (c HuntCoverage) AllFailed() bool { return c.Attempted > 0 && len(c.Failures) == c.Attempted } + +func (c HuntCoverage) Description() string { + if c.Unknown { + return "hunt accounting could not be read" + } + return fmt.Sprintf("%d of %d hunts failed", len(c.Failures), c.Attempted) +} + +// Coverage reads the optional report metadata after either a local run or JSON +// round trip. Malformed accounting cannot become a claim of completed analysis. +func Coverage(metadata map[string]any) HuntCoverage { + raw, present := metadata["hunt_coverage"] + if !present { + return HuntCoverage{} + } + data, err := json.Marshal(raw) + var fields struct { + Attempted *int `json:"attempted"` + Failures *[]HuntFailure `json:"failures"` + Unknown bool `json:"unknown"` + } + if err == nil { + err = json.Unmarshal(data, &fields) + } + if err == nil && fields.Unknown { + return HuntCoverage{Unknown: true} + } + if err != nil || fields.Attempted == nil || fields.Failures == nil || *fields.Attempted < 1 || len(*fields.Failures) > *fields.Attempted { + return HuntCoverage{Unknown: true} + } + coverage := HuntCoverage{Attempted: *fields.Attempted, Failures: *fields.Failures} + for _, failure := range coverage.Failures { + if strings.TrimSpace(failure.Strategy) == "" || strings.TrimSpace(failure.Reason) == "" { + return HuntCoverage{Unknown: true} + } + } + return coverage +} + +// Interruption is a global stop after a partial result was produced. It is +// separate from hunt coverage because a ceiling or refusal can arrive later. +func Interruption(metadata map[string]any) string { + text, _ := metadata["analysis_incomplete"].(string) + return text +} diff --git a/internal/secaf/schemas/coverage_test.go b/internal/secaf/schemas/coverage_test.go new file mode 100644 index 0000000000..dab3e00609 --- /dev/null +++ b/internal/secaf/schemas/coverage_test.go @@ -0,0 +1,51 @@ +package schemas + +import ( + "encoding/json" + "reflect" + "testing" +) + +func TestIncompleteHuntCoverageSurvivesReportJSON(t *testing.T) { + want := HuntCoverage{Attempted: 3, Failures: []HuntFailure{{Strategy: "auth", Reason: "no reply"}}} + result := NewSecurityAuditResult() + result.Metadata["hunt_coverage"] = want + data, err := json.Marshal(result) + if err != nil { + t.Fatal(err) + } + var back SecurityAuditResult + if err := json.Unmarshal(data, &back); err != nil { + t.Fatal(err) + } + if got := Coverage(back.Metadata); !reflect.DeepEqual(got, want) { + t.Fatalf("coverage=%+v want=%+v", got, want) + } +} + +func TestUnreadableHuntCoverageCannotLookComplete(t *testing.T) { + for _, raw := range []any{nil, "bad", map[string]any{}, map[string]any{"attempted": 1}, map[string]any{"attempted": 1, "failures": nil}, map[string]any{"attempted": 1, "failures": []any{1, 2}}} { + if !Coverage(map[string]any{"hunt_coverage": raw}).Incomplete() { + t.Fatalf("unreadable coverage looks complete: %#v", raw) + } + } + if Coverage(nil).Incomplete() { + t.Fatal("a complete legacy report became incomplete") + } +} + +func TestUnknownAccountingMarkerSurvivesValidLookingCounts(t *testing.T) { + result := NewSecurityAuditResult() + result.Metadata["hunt_coverage"] = HuntCoverage{Attempted: 1, Failures: []HuntFailure{}, Unknown: true} + data, err := json.Marshal(result) + if err != nil { + t.Fatal(err) + } + var back SecurityAuditResult + if err := json.Unmarshal(data, &back); err != nil { + t.Fatal(err) + } + if got := Coverage(back.Metadata); !got.Unknown || !got.Incomplete() || got.Attempted != 0 { + t.Fatalf("unknown marker became measured counts: %+v", got) + } +} diff --git a/internal/secaf/schemas/hunt.go b/internal/secaf/schemas/hunt.go index fb763cb921..7b4f60b627 100644 --- a/internal/secaf/schemas/hunt.go +++ b/internal/secaf/schemas/hunt.go @@ -330,6 +330,9 @@ type HuntResult struct { ChainCount int `json:"chain_count"` StrategiesRun []string `json:"strategies_run"` HuntDurationSeconds float64 `json:"hunt_duration_seconds"` + // Coverage is measured by codeaf's phase. Complete hunts omit this optional + // extension, preserving the original model's wire shape and goldens. + Coverage *HuntCoverage `json:"hunt_coverage,omitempty"` } // DeduplicatedResult is the dedup lane output before PROVE prioritization diff --git a/internal/secaf/secaf.go b/internal/secaf/secaf.go index 5f66f4654c..6572a9e36b 100644 --- a/internal/secaf/secaf.go +++ b/internal/secaf/secaf.go @@ -36,6 +36,7 @@ import ( "github.com/Agent-Field/codeaf/internal/delegate" "github.com/Agent-Field/codeaf/internal/programguide" "github.com/Agent-Field/codeaf/internal/secaf/audit" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" ) // Name is the program's name: its chat command and its shell verb. @@ -277,15 +278,15 @@ func runAudit(ctx context.Context, host delegate.Host, o options, notes io.Write ending := delegate.Ending{CostUSD: spent, Reason: err.Error(), Extra: map[string]any{"sessions": sessions, "calls": calls}} switch { - case errors.Is(err, agentsession.ErrCeiling): + case ctx.Err() != nil: + ending.Status = delegate.StatusFail + ending.Message = "it was stopped before it finished" + case errors.Is(err, agentsession.ErrCeiling) || errors.Is(client.Refusal(), agentsession.ErrCeiling): ending.Status = delegate.StatusBudget ending.Message = "it reached the run's dollar ceiling before it finished, so it has no report" case errors.Is(auditCtx.Err(), context.DeadlineExceeded) && ctx.Err() == nil: ending.Status = delegate.StatusBudget ending.Message = "it reached the run's time ceiling before it finished, so it has no report" - case ctx.Err() != nil: - ending.Status = delegate.StatusFail - ending.Message = "it was stopped before it finished" default: ending.Status = delegate.StatusFail ending.Message = firstSentence(err.Error()) @@ -301,6 +302,28 @@ func runAudit(ctx context.Context, host delegate.Host, o options, notes io.Write if errors.Is(auditCtx.Err(), context.DeadlineExceeded) && ctx.Err() == nil { cut = stageWords[phases.current()] } + status := delegate.StatusPass + coverage := schemas.Coverage(result.Metadata) + if coverage.Incomplete() { + status = delegate.StatusFail + } + interruption := "" + switch { + case ctx.Err() != nil: + status, interruption = delegate.StatusFail, "the run was stopped before it finished" + case cut != "": + status, interruption = delegate.StatusBudget, "the run's time ceiling stopped the analysis" + case errors.Is(client.Refusal(), agentsession.ErrCeiling): + status, interruption = delegate.StatusBudget, "the run's dollar ceiling stopped the analysis" + case client.Refusal() != nil: + status, interruption = delegate.StatusFail, firstSentence(client.Refusal().Error()) + } + if interruption != "" { + if result.Metadata == nil { + result.Metadata = make(map[string]any) + } + result.Metadata["analysis_incomplete"] = interruption + } host.Stage(delegate.StageRecord{Stage: stageReport, Status: "running"}) files, err := reportFiles(records, result, len(request.ComplianceFrameworks) > 0, runFacts{ scope: scope, changes: changes, spent: spent, sessions: sessions, calls: calls, took: time.Since(started), @@ -309,16 +332,23 @@ func runAudit(ctx context.Context, host delegate.Host, o options, notes io.Write _, _ = fmt.Fprintf(notes, "[sec] could not write the full report: %v\n", err) } text := summary(scope, changes, result, files, cut, host.Ceilings().TimeWord()) + extra := map[string]any{ + "status": status, "reports": files, "sessions": sessions, "calls": calls, "cut_at": cut, + "confirmed": result.Confirmed, "likely": result.Likely, + "inconclusive": result.Inconclusive, "ruled_out": result.NotExploitable, + } + if coverage.Incomplete() { + extra["hunt_coverage"] = coverage + } + if interruption != "" { + extra["analysis_incomplete"] = interruption + } return delegate.Ending{ - Status: delegate.StatusPass, + Status: status, Message: outcomeLine(result, cut), CostUSD: spent, Deliverable: text, - Extra: map[string]any{ - "status": "pass", "reports": files, "sessions": sessions, "calls": calls, "cut_at": cut, - "confirmed": result.Confirmed, "likely": result.Likely, - "inconclusive": result.Inconclusive, "ruled_out": result.NotExploitable, - }, + Extra: extra, } } diff --git a/internal/secaf/secaf_test.go b/internal/secaf/secaf_test.go index 05c81e5b6e..28a6f156f2 100644 --- a/internal/secaf/secaf_test.go +++ b/internal/secaf/secaf_test.go @@ -60,6 +60,9 @@ type stubModels struct { calls int threads map[string]bool scanned []string + // reject injects a transport failure at a schema boundary while allowing + // the real pipeline's other agents to finish normally. + reject func(string) int } func (m *stubModels) serve(t *testing.T) delegate.ModelAPI { @@ -83,6 +86,14 @@ func (m *stubModels) serve(t *testing.T) delegate.ModelAPI { m.mu.Unlock() text := "no chains" if schema := schemaOf(body.ResponseFormat, body.Messages); schema != nil { + if m.reject != nil { + title, _ := schema["title"].(string) + if status := m.reject(title); status != 0 { + w.WriteHeader(status) + _, _ = io.WriteString(w, `{"error":{"message":"injected model refusal"}}`) + return + } + } value := minimal(schema, schema) if title, _ := schema["title"].(string); title == "ScanLocationsResult" { user := body.Messages[len(body.Messages)-1].Content[0].Text