diff --git a/README.md b/README.md index e65e6d51d..e5cd11712 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@
-Now includes Harness Orchestration — multi-turn coding agents with Claude Code, Codex, Gemini CLI, and OpenCode +Now includes Harness Orchestration — multi-turn coding agents with AForge, Claude Code, Codex, Gemini CLI, and OpenCode
@@ -251,7 +251,7 @@ Two examples already run at this load. The [deep-research engine](https://agentf - **[Reasoners & Skills](https://agentfield.ai/docs/build/building-blocks/reasoners?utm_source=github-readme&utm_campaign=github-readme&utm_id=github-readme-reasoners)** - `@app.reasoner()` for AI judgment, `@app.skill()` for deterministic code - **[Structured AI](https://agentfield.ai/docs/reference/sdks/python?utm_source=github-readme&utm_campaign=github-readme&utm_id=github-readme-structured-ai)** - `app.ai(schema=MyModel)` → typed Pydantic/Zod output from any LLM -- **[Harness](https://agentfield.ai/docs/build/intelligence/harness?utm_source=github-readme&utm_campaign=github-readme&utm_id=github-readme-harness)** - `app.harness("Fix the bug")` dispatches multi-turn tasks to Claude Code, Codex, Gemini CLI, or OpenCode +- **[Harness](https://agentfield.ai/docs/build/intelligence/harness?utm_source=github-readme&utm_campaign=github-readme&utm_id=github-readme-harness)** - `app.harness("Fix the bug")` dispatches multi-turn tasks to AForge, AgentField's own coding harness — no setup. Add `provider="claude-code"` (or `codex`, `gemini`, `opencode`) to orchestrate someone else's. - **[Cross-Agent Calls](https://agentfield.ai/docs/build/coordination/cross-agent-calls?utm_source=github-readme&utm_campaign=github-readme&utm_id=github-readme-cross-agent-calls)** - `app.call("other-agent.func")` routes through the control plane with full tracing - **[Discovery](https://agentfield.ai/docs/reference/sdks/python?utm_source=github-readme&utm_campaign=github-readme&utm_id=github-readme-discovery)** - `app.discover(tags=["ml*"])` finds agents and capabilities across the mesh. `tools="discover"` lets LLMs auto-invoke them. - **[Memory](https://agentfield.ai/docs/build/coordination/shared-memory?utm_source=github-readme&utm_campaign=github-readme&utm_id=github-readme-memory)** - `app.memory.set()` / `.get()` / `.similarity_search()` - KV + vector search, four scopes, no Redis needed @@ -283,7 +283,8 @@ Two examples already run at this load. The [deep-research engine](https://agentf | Feature | How | |---|---| | Structured output (Pydantic/Zod) | `app.ai(schema=MyModel)` | -| Multi-turn coding agents | `app.harness("task", provider="claude-code")` | +| Multi-turn coding agents | `app.harness("task")` — AForge by default | +| Orchestrate another harness | `app.harness("task", provider="claude-code")` | | LLM auto-discovers agents and tools | `app.ai(tools="discover")` | | Multimodal (text, image, audio) | `app.ai("Describe", image_url="...")` | | Streaming responses | `app.ai("...", stream=True)` | @@ -379,7 +380,9 @@ Two examples already run at this load. The [deep-research engine](https://agentf | Feature | How | |---|---| -| 4 providers | Claude Code, Codex, Gemini CLI, OpenCode | +| Zero-setup default harness | AForge (`aforge`), installed alongside `af` | +| Swap the worker, keep the loop | `provider="claude-code"` \| `"codex"` \| `"gemini"` \| `"opencode"` | +| Fleet-wide default override | `AGENTFIELD_HARNESS_PROVIDER=codex` | | Schema-constrained output | `schema=ResultModel` (Pydantic/Zod) | | Cost capping | `max_budget_usd=3.0` | | Turn limiting | `max_turns=100` | diff --git a/docs/design/harness-v2-design.md b/docs/design/harness-v2-design.md index 9ea47165a..dd2b998b3 100644 --- a/docs/design/harness-v2-design.md +++ b/docs/design/harness-v2-design.md @@ -26,8 +26,8 @@ app = Agent( node_id="my-agent", ai_config=AIConfig(model="openai/gpt-4o"), harness_config=HarnessConfig( - provider="claude-code", # Required — no implicit default - model="sonnet", + provider="claude-code", # Optional — defaults to "aforge" + model="sonnet", # Optional — defaults to the provider's own ), ) ``` @@ -38,8 +38,8 @@ import { Agent } from '@agentfield/sdk'; const agent = new Agent({ nodeId: 'my-agent', harnessConfig: { - provider: 'claude-code', // Required - model: 'sonnet', + provider: 'claude-code', // Optional — defaults to 'aforge' + model: 'sonnet', // Optional — defaults to the provider's own }, }); ``` @@ -112,7 +112,7 @@ app = Agent(node_id="minimal-agent") result = await app.harness( "Fix the bug", - provider="gemini", # Required when no harness_config + provider="gemini", # Optional — omit to use the default, "aforge" model="flash", cwd="/my/project", ) @@ -142,7 +142,7 @@ async def fix_issue(issue: dict) -> dict: ``` Agent ├── .ai() → AIConfig → LiteLLM → LLM APIs (100+ providers) -└── .harness() → HarnessConfig → HarnessRunner → Provider → {Claude Code, Codex, Gemini, OpenCode} +└── .harness() → HarnessConfig → HarnessRunner → Provider → {Aforge, Claude Code, Codex, Gemini, OpenCode} ``` ### 3.2 Component Stack @@ -350,12 +350,12 @@ Layer 4: Full retry (expensive, last resort class HarnessConfig(BaseModel): """Configuration for coding agent harness calls. - Provider is required — there is no implicit default. - All other fields have sensible defaults that can be overridden per-call. + Provider defaults to "aforge", AgentField's native harness. + All fields have sensible defaults that can be overridden per-call. """ - # Provider selection (required) - provider: str # "claude-code" | "codex" | "gemini" | "opencode" - model: str = "sonnet" + # Provider selection: explicit > AGENTFIELD_HARNESS_PROVIDER > "aforge" + provider: str = "aforge" # | "claude-code" | "codex" | "gemini" | "opencode" + model: Optional[str] = None # None → the provider's own default # Execution limits max_turns: int = 30 @@ -426,7 +426,8 @@ interface HarnessConfig { 1. HarnessConfig defaults (set at agent construction) 2. Per-call overrides (passed to .harness() method) → Per-call values win over HarnessConfig defaults - → If no HarnessConfig AND no per-call provider → raise error + → If no HarnessConfig AND no per-call provider → AGENTFIELD_HARNESS_PROVIDER, + then the default provider "aforge" ``` --- diff --git a/docs/harness-providers.md b/docs/harness-providers.md index 62995a11d..4598292b7 100644 --- a/docs/harness-providers.md +++ b/docs/harness-providers.md @@ -1,18 +1,83 @@ # Harness providers -AgentField harness providers run external coding agents. Install the provider -wrapper you need, install its CLI when required, and verify the runtime before -starting a workflow. +`app.harness()` hands a task to a coding agent — a multi-turn worker that reads, +writes, and edits files, then reports back through the same structured-output +contract as `app.ai()`. AgentField ships its own harness, **AForge**, and it is +the default: a call with no provider set runs `aforge`. Naming a different +provider swaps the worker without changing the surrounding loop, which is how +you orchestrate Claude Code, Codex, Gemini CLI, or OpenCode from a reasoner. + +## Default: AForge + +The `aforge` binary is provisioned alongside the `af` CLI — the curl installer, +the desktop app, and the published Docker images all ship it. To install or +repair it on demand: + +```bash +af aforge ensure +``` + +Set `OPENROUTER_API_KEY`, then call the harness with nothing else configured: + +```python +result = await app.harness("Fix the failing test in tests/test_auth.py", schema=Report) +``` + +```go +result, err := agent.Harness(ctx, task, schema, &dest, harness.Options{Cwd: repoRoot}) +``` + +```ts +const result = await app.harness(task, { schema }); +``` + +The model defaults to AForge's own default. Set `AFORGE_MODEL` to change it +process-wide, or pass `model=` per call. + +Verify the runtime before a paid run: + +```bash +af harness doctor --provider aforge +``` + +## Choosing a different provider + +Provider selection follows one precedence chain: + +| Order | Source | Example | +| --- | --- | --- | +| 1 | Explicit value on the call or in the agent's harness config | `app.harness(task, provider="codex")` | +| 2 | `AGENTFIELD_HARNESS_PROVIDER` environment variable | `AGENTFIELD_HARNESS_PROVIDER=claude-code` | +| 3 | Default | `aforge` | + +Same loop code, different worker: + +```python +# AForge — nothing to configure +report = await app.harness(task, schema=Report) + +# Orchestrate Claude Code instead +report = await app.harness(task, schema=Report, provider="claude-code") + +# ...or Codex, Gemini CLI, OpenCode +report = await app.harness(task, schema=Report, provider="codex") +``` + +The same override exists in every SDK — `harness.Options{Provider: harness.ProviderCodex}` +in Go, `{ provider: 'codex' }` in TypeScript — and an agent-wide default can be +set once on the agent's harness config (`HarnessConfig(provider="codex")` in +Python, `agent.HarnessConfig{Provider: "codex"}` in Go). ## Install -| Provider | Python extra | Required CLI | Authentication | -| --- | --- | --- | --- | -| `aforge` | None | `aforge` (`af aforge ensure`) | `OPENROUTER_API_KEY` | -| Claude Code | `agentfield[harness-claude]` | Bundled by `claude-agent-sdk` | Claude login or `ANTHROPIC_API_KEY` | -| Codex | `agentfield[harness-codex]` | `codex` | Codex login or `OPENAI_API_KEY` | -| Gemini | None | `gemini` | Gemini login, `GEMINI_API_KEY`, or `GOOGLE_API_KEY` | -| OpenCode | `agentfield[harness-opencode]` | `opencode` | Provider credentials configured in OpenCode | +| Provider | Install | Python extra | Required CLI | Authentication | +| --- | --- | --- | --- | --- | +| `aforge` (default) | `af aforge ensure` (shipped with `af`) | None | `aforge` | `OPENROUTER_API_KEY` | +| `claude-code` | `pip install 'agentfield[harness-claude]'` | `agentfield[harness-claude]` | Bundled by `claude-agent-sdk` | Claude login or `ANTHROPIC_API_KEY` | +| `codex` | `npm install -g @openai/codex` | `agentfield[harness-codex]` | `codex` | Codex login or `OPENAI_API_KEY` | +| `gemini` | `npm install -g @google/gemini-cli` | None | `gemini` | Gemini login, `GEMINI_API_KEY`, or `GOOGLE_API_KEY` | +| `opencode` | `curl -fsSL https://opencode.ai/install \| bash` | `agentfield[harness-opencode]` | `opencode` | Provider credentials configured in OpenCode | +| `grok` | Install the Grok Build CLI, then `grok login` | None | `grok` | `XAI_API_KEY` | Install every Python wrapper with: @@ -34,13 +99,31 @@ The pinned build, its download host and the opt-out are documented under [docs/ENVIRONMENT_VARIABLES.md](ENVIRONMENT_VARIABLES.md). The extras install Python wrappers. They do not replace the runtime preflight: -Aforge and Gemini are CLI-only, and Codex or OpenCode may still require a -separately available executable depending on the wrapper and platform. +AForge and Gemini are CLI-only, and Codex or OpenCode may still require a +separately available executable depending on the wrapper and platform. `grok` +is available in the Python SDK only. + +### AForge adapter contract + +AForge is registered as `aforge` in the Python, Go, and TypeScript SDKs. The +adapters default to the direct non-interactive contract, `aforge exec --json`, +send the task over stdin, and map AForge's usage ledger into AgentField turns, +token counts, and cost metrics. Set `AGENTFIELD_AFORGE_COMMAND=do` to opt into +the routed `aforge do --json --yes-spend` workflow instead. + +Set `AFORGE_MAX_CONCURRENT` to cap simultaneous AForge subprocesses. The +default is 8. `AGENTFIELD_HARNESS_TIMEOUT_SECONDS` is the outer watchdog; each +adapter gives AForge a five-second landing window to emit its exit-2 timeout +envelope. Schema runs use a unique output directory per invocation so parallel +jobs can safely share a checkout. Set `AFORGE_BIN` to an absolute path when the +binary is installed somewhere off `PATH`. ## Model selection and reasoning-effort variants -Every provider accepts a `model` option on `.harness()` calls. The model string -may carry a reasoning-effort variant after a `#` separator: +Every provider accepts a `model` option on `.harness()` calls. Leaving it unset +uses the provider's own default — AForge picks its own model, `claude-code` +keeps using `sonnet`. The model string may carry a reasoning-effort variant +after a `#` separator: ```python result = await app.harness( @@ -54,7 +137,7 @@ An explicit `variant="high"` keyword wins over the suffix. Per provider: | Provider | Model flag | Variant handling | | --- | --- | --- | -| `aforge` | `AFORGE_MODEL` env var with a bare OpenRouter slug (a leading `openrouter/` is stripped) | `AFORGE_EXEC_REASONING` (`off`, `low`, `medium`, or `high`) | +| `aforge` | `exec`: `--model` and `--plan-model`; `do`: `AFORGE_MODEL` (a leading `openrouter/` is stripped) | `AFORGE_EXEC_REASONING` (`off`, `low`, `medium`, or `high`) | | OpenCode | `-m ` | `--variant ` (provider-specific effort, e.g. `high`, `max`, `minimal`) | | Codex | `-m ` | `-c model_reasoning_effort=` | | Claude Code | SDK `model` option | No effort control — variant is dropped with a debug log | diff --git a/sdk/go/agent/harness.go b/sdk/go/agent/harness.go index 29462934c..f39d06970 100644 --- a/sdk/go/agent/harness.go +++ b/sdk/go/agent/harness.go @@ -10,10 +10,14 @@ import ( // providing lazy initialization and a convenience Harness() method. // HarnessConfig configures the default harness runner for the agent. type HarnessConfig struct { - // Provider is the default provider: "claude-code", "codex", "gemini", or "opencode". + // Provider is the default provider: "aforge", "claude-code", "codex", + // "gemini", or "opencode". When empty, AGENTFIELD_HARNESS_PROVIDER + // overrides the default, "aforge" (AgentField's native harness). An + // explicit value always wins. Provider string - // Model is the default model identifier. It may carry a + // Model is the default model identifier. Empty means the provider's own + // default. It may carry a // reasoning-effort variant after a "#" separator (e.g. // "openrouter/z-ai/glm-5.2#high"). Model string @@ -86,9 +90,7 @@ func (a *Agent) HarnessRunner() *harness.Runner { // } // var result ReviewResult // schema, _ := harness.StructToJSONSchema(result) -// hr, err := agent.Harness(ctx, "Review this code...", schema, &result, harness.Options{ -// Model: "sonnet", -// }) +// hr, err := agent.Harness(ctx, "Review this code...", schema, &result, harness.Options{}) func (a *Agent) Harness(ctx context.Context, prompt string, schema map[string]any, dest any, opts harness.Options) (*harness.Result, error) { result, err := a.HarnessRunner().Run(ctx, prompt, schema, dest, opts) if err == nil { diff --git a/sdk/go/agent/harness_test.go b/sdk/go/agent/harness_test.go index 78b51fc76..98bdb269d 100644 --- a/sdk/go/agent/harness_test.go +++ b/sdk/go/agent/harness_test.go @@ -4,6 +4,7 @@ import ( "context" "io" "log" + "path/filepath" "testing" "github.com/Agent-Field/agentfield/sdk/go/harness" @@ -112,28 +113,31 @@ func TestHarnessRunner_ConcurrentAccess(t *testing.T) { } func TestHarness_ErrorWithoutProvider(t *testing.T) { - // Harness() should fail when no provider is configured. - // The runner will return an error about a missing provider. + // With no provider configured, Harness() defaults to aforge and reaches + // provider execution. A missing binary may fail, but provider resolution does not. + t.Setenv(harness.ProviderEnvVar, "") a := newTestAgentForHarness(t) - _, err := a.Harness(context.Background(), "do something", nil, nil, harness.Options{}) - assert.Error(t, err) - assert.Contains(t, err.Error(), "provider") + missingAforge := filepath.Join(t.TempDir(), "missing-aforge") + result, err := a.Harness(context.Background(), "do something", nil, nil, harness.Options{BinPath: missingAforge}) + require.NoError(t, err) + require.NotNil(t, result) + assert.True(t, result.IsError) + assert.Contains(t, result.ErrorMessage, "missing-aforge") + assert.NotContains(t, result.ErrorMessage, "no harness provider specified") } func TestHarness_PassesOptsToRunner(t *testing.T) { // Verify that per-call Options are forwarded to the runner. // Using a non-existent provider triggers a provider-build error, - // which confirms the Options reached Run() (otherwise we'd get - // the "no harness provider specified" error instead). + // which confirms the Options reached Run(). a := newTestAgentForHarness(t) _, err := a.Harness(context.Background(), "test", nil, nil, harness.Options{ Provider: "nonexistent-provider", }) assert.Error(t, err) - // Should be a provider-build error, NOT the "no harness provider specified" error - assert.NotContains(t, err.Error(), "no harness provider specified") + assert.Contains(t, err.Error(), "unknown harness provider") } func TestHarnessConfig_PartialOverride(t *testing.T) { diff --git a/sdk/go/harness/aforge.go b/sdk/go/harness/aforge.go new file mode 100644 index 000000000..f23df3577 --- /dev/null +++ b/sdk/go/harness/aforge.go @@ -0,0 +1,340 @@ +package harness + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strconv" + "strings" + "sync" + "time" +) + +const ( + defaultAforgeMaxConcurrent = 8 + defaultAforgeTimeout = 1800 + aforgeLandingWindow = 5 +) + +var ( + aforgeSemaphore chan struct{} + aforgeSemOnce sync.Once +) + +// AforgeProvider invokes aforge's one-shot machine-readable harness mode. +type AforgeProvider struct { + BinPath string + runCLI func(ctx context.Context, cmd []string, env map[string]string, cwd string, timeout, idleSeconds int, stdin []byte) (*CLIResult, error) +} + +// NewAforgeProvider creates an Aforge provider. If binPath is empty, it +// defaults to "aforge". +func NewAforgeProvider(binPath string) *AforgeProvider { + if binPath == "" { + binPath = strings.TrimSpace(os.Getenv("AFORGE_BIN")) + if binPath == "" { + binPath = "aforge" + } + } + return &AforgeProvider{BinPath: binPath, runCLI: runCLIWithStdinIdle} +} + +func getAforgeSemaphore() chan struct{} { + aforgeSemOnce.Do(func() { + maxConcurrent := defaultAforgeMaxConcurrent + if raw := strings.TrimSpace(os.Getenv("AFORGE_MAX_CONCURRENT")); raw != "" { + if parsed, err := strconv.Atoi(raw); err == nil && parsed > 0 { + maxConcurrent = parsed + } + } + aforgeSemaphore = make(chan struct{}, maxConcurrent) + }) + return aforgeSemaphore +} + +func aforgeTimeout(options Options) int { + if options.Timeout > 0 { + return options.Timeout + } + if raw := strings.TrimSpace(os.Getenv("AGENTFIELD_HARNESS_TIMEOUT_SECONDS")); raw != "" { + if parsed, err := strconv.Atoi(raw); err == nil && parsed > 0 { + return parsed + } + } + return defaultAforgeTimeout +} + +func aforgeInnerTimeout(outer int) int { + if outer > aforgeLandingWindow { + return outer - aforgeLandingWindow + } + return 1 +} + +func stripOpenRouterPrefix(model string) string { + return strings.TrimPrefix(model, "openrouter/") +} + +func supportedAforgeVariant(variant string) (string, bool) { + normalized := strings.ToLower(strings.TrimSpace(variant)) + switch normalized { + case "off", "low", "medium", "high": + return normalized, true + default: + return "", false + } +} + +func aforgeTaskInput(prompt, systemPrompt string) string { + if systemPrompt = strings.TrimSpace(systemPrompt); systemPrompt != "" { + return systemPrompt + "\n\nTask:\n" + prompt + } + return prompt +} + +func aforgeCommand() (string, error) { + command := strings.ToLower(strings.TrimSpace(os.Getenv("AGENTFIELD_AFORGE_COMMAND"))) + if command == "" { + return "exec", nil + } + if command != "do" && command != "exec" { + return "", fmt.Errorf("AGENTFIELD_AFORGE_COMMAND must be do or exec, got %q", command) + } + return command, nil +} + +func parseAforgeEnvelope(stdout string) map[string]any { + // Both canonical `do` and `exec` print one JSON object. Parse that shape + // before the wrapper-compatible line scan. + var envelope map[string]any + if err := json.Unmarshal([]byte(strings.TrimSpace(stdout)), &envelope); err == nil { + _, hasDeliverable := envelope["deliverable"] + _, hasText := envelope["text"] + if hasDeliverable || hasText { + return envelope + } + } + + lines := strings.Split(stdout, "\n") + for i := len(lines) - 1; i >= 0; i-- { + line := strings.TrimSpace(lines[i]) + if line == "" { + continue + } + envelope = nil + if err := json.Unmarshal([]byte(line), &envelope); err != nil { + continue + } + _, hasDeliverable := envelope["deliverable"] + _, hasText := envelope["text"] + if hasDeliverable || hasText { + return envelope + } + } + return nil +} + +func aforgeNumber(value any) (float64, bool) { + switch number := value.(type) { + case float64: + return number, true + case float32: + return float64(number), true + case int: + return float64(number), true + case int64: + return float64(number), true + case json.Number: + parsed, err := number.Float64() + return parsed, err == nil + default: + return 0, false + } +} + +func aforgeCrashMessage(returnCode int, blockedOn, deliverable, stderr string) string { + cleanStderr := StripANSI(strings.TrimSpace(stderr)) + exitContext := fmt.Sprintf("aforge exit code %d", returnCode) + message := exitContext + if returnCode < 0 { + message = fmt.Sprintf("Process killed by signal %d. %s", -returnCode, exitContext) + } + switch { + case cleanStderr != "": + message += ". stderr: " + truncate(cleanStderr, 1000) + case blockedOn != "": + message += ". blocked_on: " + truncate(blockedOn, 1000) + case deliverable != "": + message += ". partial: " + truncate(deliverable, 1000) + } + return message +} + +func (p *AforgeProvider) Execute(ctx context.Context, prompt string, options Options) (*RawResult, error) { + sem := getAforgeSemaphore() + select { + case sem <- struct{}{}: + defer func() { <-sem }() + case <-ctx.Done(): + return nil, ctx.Err() + } + + root := options.ProjectDir + if root == "" { + root = options.Cwd + } + if root == "" { + root = "." + } + outerTimeout := aforgeTimeout(options) + command, commandErr := aforgeCommand() + if commandErr != nil { + return &RawResult{ + IsError: true, ErrorMessage: commandErr.Error(), FailureType: FailureCrash, + Metrics: Metrics{}, + }, nil + } + var cmd []string + input := []byte(prompt) + if command == "exec" { + cmd = []string{ + p.BinPath, "exec", "--json", "-w", root, + "--timeout", strconv.Itoa(aforgeInnerTimeout(outerTimeout)), + "--context-fill", "60", "--completion-reserve", "65536", + } + if systemPrompt := strings.TrimSpace(options.SystemPrompt); systemPrompt != "" { + cmd = append(cmd, "--system", systemPrompt) + } + } else { + cmd = []string{ + p.BinPath, "do", "--json", "--yes-spend", "-w", root, + "--timeout", strconv.Itoa(aforgeInnerTimeout(outerTimeout)), + } + input = []byte(aforgeTaskInput(prompt, options.SystemPrompt)) + } + + model, variant := options.resolveModelAndVariant() + env := make(map[string]string) + if command == "exec" { + env["AFORGE_MODELS"] = "" + } + if model != "" { + modelSlug := stripOpenRouterPrefix(model) + env["AFORGE_MODEL"] = modelSlug + if command == "exec" { + cmd = append(cmd, "--model", modelSlug, "--plan-model", modelSlug) + } + } + if normalized, ok := supportedAforgeVariant(variant); ok { + env["AFORGE_EXEC_REASONING"] = normalized + } + // Caller-supplied environment wins over values derived from model/variant. + for key, value := range options.Env { + env[key] = value + } + + started := time.Now() + cliResult, err := p.runCLI(ctx, cmd, env, "", outerTimeout, 0, input) + apiMS := int(time.Since(started).Milliseconds()) + if err != nil { + if isExecNotFound(err) { + return &RawResult{ + IsError: true, + ErrorMessage: fmt.Sprintf("AForge binary not found at '%s'. Install it with `af aforge ensure`, or set AFORGE_BIN to its path.", p.BinPath), + FailureType: FailureCrash, + Metrics: Metrics{DurationAPIMS: apiMS}, + }, nil + } + if strings.Contains(strings.ToLower(err.Error()), "timed out") || strings.Contains(strings.ToLower(err.Error()), "deadline exceeded") { + return &RawResult{ + IsError: true, + ErrorMessage: err.Error(), + FailureType: FailureTimeout, + Metrics: Metrics{DurationAPIMS: apiMS}, + }, nil + } + return nil, err + } + + envelope := parseAforgeEnvelope(cliResult.Stdout) + resultText := "" + blockedOn := "" + stop := "" + usage := map[string]any{} + if envelope != nil { + textKey := "deliverable" + if command == "exec" { + textKey = "text" + } + if text, ok := envelope[textKey].(string); ok { + resultText = strings.TrimSpace(text) + } + if value, ok := envelope["blocked_on"].(string); ok { + blockedOn = strings.TrimSpace(value) + } + if value, ok := envelope["usage"].(map[string]any); ok { + usage = value + } + if command == "exec" { + if value, ok := envelope["stop"].(string); ok { + stop = strings.TrimSpace(value) + } + } + } + + isError := cliResult.ReturnCode != 0 || resultText == "" || blockedOn != "" + if command == "exec" { + isError = cliResult.ReturnCode < 0 || resultText == "" || + (cliResult.ReturnCode != 0 && cliResult.ReturnCode != 2 && cliResult.ReturnCode != 3) + } + metrics := Metrics{DurationAPIMS: apiMS} + if value, ok := aforgeNumber(usage["calls"]); ok { + metrics.NumTurns = int(value) + } + if command == "exec" { + if value, ok := aforgeNumber(envelope["turns"]); ok { + metrics.NumTurns = int(value) + } + } + if value, ok := aforgeNumber(usage["prompt_tokens"]); ok { + metrics.InputTokens = int(value) + } + if value, ok := aforgeNumber(usage["completion_tokens"]); ok { + metrics.OutputTokens = int(value) + } + if value, ok := aforgeNumber(usage["cached_tokens"]); ok { + metrics.CacheReadTokens = int(value) + } + spend, hasSpend := aforgeNumber(envelope["spend"]) + legacyCost, hasLegacyCost := aforgeNumber(usage["cost"]) + if hasSpend && spend > 0 { + cost := spend + metrics.CostUSD = &cost + } else if hasLegacyCost && legacyCost > 0 { + cost := legacyCost + metrics.CostUSD = &cost + } + + messages := []map[string]any(nil) + if envelope != nil { + messages = []map[string]any{envelope} + } + raw := &RawResult{ + Result: resultText, + Messages: messages, + Metrics: metrics, + IsError: isError, + FailureType: FailureNone, + ReturnCode: cliResult.ReturnCode, + } + if isError { + if (command == "do" && cliResult.ReturnCode == 2) || (command == "exec" && cliResult.ReturnCode == 4) { + raw.FailureType = FailureTimeout + } else { + raw.FailureType = FailureCrash + } + raw.ErrorMessage = aforgeCrashMessage(cliResult.ReturnCode, firstNonEmpty(blockedOn, stop), resultText, cliResult.Stderr) + } + return raw, nil +} diff --git a/sdk/go/harness/aforge_integration_test.go b/sdk/go/harness/aforge_integration_test.go new file mode 100644 index 000000000..d86138a09 --- /dev/null +++ b/sdk/go/harness/aforge_integration_test.go @@ -0,0 +1,88 @@ +//go:build integration + +package harness + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestAforgeProvider_Integration drives the real aforge binary and its default +// `exec` JSON envelope. Set AFORGE_INTEGRATION=1 so TestMain does not shadow it. +func TestAforgeProvider_Integration(t *testing.T) { + binPath, err := exec.LookPath("aforge") + if err != nil { + t.Skip("aforge binary not installed") + } + if os.Getenv("OPENROUTER_API_KEY") == "" && os.Getenv("OPENAI_API_KEY") == "" { + t.Skip("OPENROUTER_API_KEY or OPENAI_API_KEY is required") + } + + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second) + defer cancel() + raw, err := NewAforgeProvider(binPath).Execute(ctx, "Reply with exactly: HELLO_AGENTFIELD", Options{Timeout: 300}) + require.NoError(t, err) + require.NotNil(t, raw) + assert.False(t, raw.IsError, raw.ErrorMessage) + assert.Contains(t, raw.Result, "HELLO_AGENTFIELD") + assert.NotEmpty(t, raw.Messages) + assert.Greater(t, raw.Metrics.NumTurns, 0) + assert.Greater(t, raw.Metrics.InputTokens, 0) + assert.Greater(t, raw.Metrics.OutputTokens, 0) + require.NotNil(t, raw.Metrics.CostUSD) + assert.Greater(t, *raw.Metrics.CostUSD, 0.0) + t.Logf( + "aforge metrics: duration_ms=%d calls=%d input_tokens=%d output_tokens=%d cache_read_tokens=%d cost_usd=%.10f", + raw.Metrics.DurationAPIMS, raw.Metrics.NumTurns, raw.Metrics.InputTokens, + raw.Metrics.OutputTokens, raw.Metrics.CacheReadTokens, *raw.Metrics.CostUSD, + ) +} + +func TestAforgeRunner_Integration_Schema(t *testing.T) { + if _, err := exec.LookPath("aforge"); err != nil { + t.Skip("aforge binary not installed") + } + if os.Getenv("OPENROUTER_API_KEY") == "" && os.Getenv("OPENAI_API_KEY") == "" { + t.Skip("OPENROUTER_API_KEY or OPENAI_API_KEY is required") + } + + workDir := t.TempDir() + schema := map[string]any{ + "type": "object", + "properties": map[string]any{ + "greeting": map[string]any{"type": "string"}, + "number": map[string]any{"type": "integer"}, + }, + "required": []string{"greeting", "number"}, + } + + var parsed map[string]any + result, err := NewRunner(Options{Provider: ProviderAforge}).Run( + context.Background(), + `Return greeting="Hello from Aforge" and number=42. Follow the OUTPUT REQUIREMENTS precisely.`, + schema, + &parsed, + Options{Cwd: workDir, MaxRetries: 1, Timeout: 300}, + ) + require.NoError(t, err) + require.NotNil(t, result) + assert.False(t, result.IsError, result.ErrorMessage) + assert.Equal(t, "Hello from Aforge", parsed["greeting"]) + assert.EqualValues(t, 42, parsed["number"]) + require.NotNil(t, result.CostUSD) + t.Logf( + "aforge schema metrics: duration_ms=%d calls=%d input_tokens=%d output_tokens=%d cache_read_tokens=%d cost_usd=%.10f", + result.DurationMS, result.NumTurns, result.InputTokens, result.OutputTokens, + result.CacheReadTokens, *result.CostUSD, + ) + matches, err := filepath.Glob(filepath.Join(workDir, ".agentfield-out-*")) + require.NoError(t, err) + assert.Empty(t, matches) +} diff --git a/sdk/go/harness/aforge_test.go b/sdk/go/harness/aforge_test.go new file mode 100644 index 000000000..434dbfa0c --- /dev/null +++ b/sdk/go/harness/aforge_test.go @@ -0,0 +1,406 @@ +package harness + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func aforgeEnvelope(deliverable string, settled bool, blockedOn, usage string) string { + if usage == "" { + usage = `{"calls":1,"prompt_tokens":0,"completion_tokens":0,"cached_tokens":0,"cost":0}` + } + return fmt.Sprintf(`{"settled":%t,"deliverable":%q,"blocked_on":%q,"spend_usd":0.0123,"elapsed_ms":12,"usage":%s}`, + settled, deliverable, blockedOn, usage) +} + +func aforgeExecEnvelope(text, stop, usage string, turns int) string { + if usage == "" { + usage = `{"calls":1,"prompt_tokens":0,"completion_tokens":0,"cached_tokens":0,"cost":0}` + } + return fmt.Sprintf(`{"text":%q,"stop":%q,"usage":%s,"artifacts":[],"turns":%d,"elapsed_ms":12}`, + text, stop, usage, turns) +} + +func useAforgeDo(t *testing.T) { + t.Helper() + t.Setenv("AGENTFIELD_AFORGE_COMMAND", "do") +} + +func TestAforgeProviderMapsDoCommandEnvelopeAndMetrics(t *testing.T) { + useAforgeDo(t) + var capturedCmd []string + var capturedEnv map[string]string + var capturedCwd string + var capturedTimeout, capturedIdle int + var capturedStdin []byte + + p := NewAforgeProvider("/opt/aforge") + p.runCLI = func(_ context.Context, cmd []string, env map[string]string, cwd string, timeout, idleSeconds int, stdin []byte) (*CLIResult, error) { + capturedCmd = append([]string(nil), cmd...) + capturedEnv = env + capturedCwd = cwd + capturedTimeout = timeout + capturedIdle = idleSeconds + capturedStdin = append([]byte(nil), stdin...) + return &CLIResult{ + Stdout: aforgeEnvelope(" final answer ", true, "", + `{"calls":3,"prompt_tokens":100,"completion_tokens":50,"cached_tokens":20,"cost":0.0123}`), + ReturnCode: 0, + }, nil + } + + raw, err := p.Execute(context.Background(), "prompt that stays off argv", Options{ + ProjectDir: "/project", + Cwd: "/project/nested", + SystemPrompt: " be precise ", + Model: "openrouter/z-ai/glm-5.2#high", + }) + require.NoError(t, err) + require.NotNil(t, raw) + assert.Equal(t, []string{ + "/opt/aforge", "do", "--json", "--yes-spend", "-w", "/project", + "--timeout", "1795", + }, capturedCmd) + assert.Equal(t, "z-ai/glm-5.2", capturedEnv["AFORGE_MODEL"]) + assert.Equal(t, "high", capturedEnv["AFORGE_EXEC_REASONING"]) + assert.Empty(t, capturedCwd) + assert.Equal(t, defaultAforgeTimeout, capturedTimeout) + assert.Zero(t, capturedIdle) + assert.Equal(t, "be precise\n\nTask:\nprompt that stays off argv", string(capturedStdin)) + for _, arg := range capturedCmd { + assert.NotContains(t, arg, "prompt that stays off argv") + } + + assert.Equal(t, "final answer", raw.Result) + assert.False(t, raw.IsError) + assert.Equal(t, FailureNone, raw.FailureType) + assert.Equal(t, 0, raw.ReturnCode) + assert.Equal(t, 3, raw.Metrics.NumTurns) + assert.Equal(t, 100, raw.Metrics.InputTokens) + assert.Equal(t, 50, raw.Metrics.OutputTokens) + assert.Equal(t, 20, raw.Metrics.CacheReadTokens) + require.NotNil(t, raw.Metrics.CostUSD) + assert.InDelta(t, 0.0123, *raw.Metrics.CostUSD, 1e-9) + require.Len(t, raw.Messages, 1) + assert.Equal(t, " final answer ", raw.Messages[0]["deliverable"]) +} + +func TestAforgeProviderMapsExecCommandEnvelopeAndMetrics(t *testing.T) { + t.Setenv("AGENTFIELD_AFORGE_COMMAND", "") + var capturedCmd []string + var capturedEnv map[string]string + var capturedStdin []byte + p := NewAforgeProvider("/opt/aforge") + p.runCLI = func(_ context.Context, cmd []string, env map[string]string, _ string, _, _ int, stdin []byte) (*CLIResult, error) { + capturedCmd = append([]string(nil), cmd...) + capturedEnv = env + capturedStdin = append([]byte(nil), stdin...) + return &CLIResult{Stdout: aforgeExecEnvelope(" linear answer ", "done", + `{"calls":3,"prompt_tokens":100,"completion_tokens":50,"cached_tokens":20,"cost":0.0123}`, 4)}, nil + } + + raw, err := p.Execute(context.Background(), "prompt that stays off argv", Options{ + ProjectDir: "/project", + SystemPrompt: " be precise ", + Model: "openrouter/deepseek/deepseek-v4-flash-0731", + }) + require.NoError(t, err) + assert.Equal(t, []string{ + "/opt/aforge", "exec", "--json", "-w", "/project", + "--timeout", "1795", "--context-fill", "60", "--completion-reserve", "65536", + "--system", "be precise", + "--model", "deepseek/deepseek-v4-flash-0731", + "--plan-model", "deepseek/deepseek-v4-flash-0731", + }, capturedCmd) + assert.Equal(t, "", capturedEnv["AFORGE_MODELS"]) + assert.Equal(t, "deepseek/deepseek-v4-flash-0731", capturedEnv["AFORGE_MODEL"]) + assert.Equal(t, "prompt that stays off argv", string(capturedStdin)) + assert.Equal(t, "linear answer", raw.Result) + assert.False(t, raw.IsError) + assert.Equal(t, 4, raw.Metrics.NumTurns) + assert.Equal(t, 100, raw.Metrics.InputTokens) + require.NotNil(t, raw.Metrics.CostUSD) + assert.InDelta(t, 0.0123, *raw.Metrics.CostUSD, 1e-9) +} + +func TestAforgeProviderExecBudgetPartialIsUsable(t *testing.T) { + t.Setenv("AGENTFIELD_AFORGE_COMMAND", "exec") + p := NewAforgeProvider("aforge") + p.runCLI = func(context.Context, []string, map[string]string, string, int, int, []byte) (*CLIResult, error) { + return &CLIResult{Stdout: aforgeExecEnvelope("usable", "budget", "", 2), ReturnCode: 2}, nil + } + + raw, err := p.Execute(context.Background(), "hello", Options{}) + require.NoError(t, err) + assert.False(t, raw.IsError) + assert.Equal(t, FailureNone, raw.FailureType) + assert.Equal(t, "usable", raw.Result) +} + +func TestAforgeProviderBinaryEnvironmentOverride(t *testing.T) { + t.Setenv("AFORGE_BIN", "/opt/aforge-env") + assert.Equal(t, "/opt/aforge-env", NewAforgeProvider("").BinPath) + assert.Equal(t, "/explicit/aforge", NewAforgeProvider("/explicit/aforge").BinPath) +} + +func TestAforgeProviderModelVariantAndEnvironmentPrecedence(t *testing.T) { + useAforgeDo(t) + var captured []map[string]string + p := NewAforgeProvider("aforge") + p.runCLI = func(_ context.Context, _ []string, env map[string]string, _ string, _, _ int, _ []byte) (*CLIResult, error) { + copyEnv := make(map[string]string, len(env)) + for key, value := range env { + copyEnv[key] = value + } + captured = append(captured, copyEnv) + return &CLIResult{Stdout: aforgeEnvelope("done", true, "", ""), ReturnCode: 0}, nil + } + + _, err := p.Execute(context.Background(), "hello", Options{Model: "openrouter/x/y#turbo"}) + require.NoError(t, err) + _, err = p.Execute(context.Background(), "hello", Options{ + Model: "openrouter/x/y#low", + Variant: " HIGH ", + Env: map[string]string{ + "AFORGE_MODEL": "override/model", + "AFORGE_EXEC_REASONING": "off", + "EXTRA": "1", + }, + }) + require.NoError(t, err) + + assert.Equal(t, "x/y", captured[0]["AFORGE_MODEL"]) + assert.NotContains(t, captured[0], "AFORGE_EXEC_REASONING") + assert.Equal(t, "override/model", captured[1]["AFORGE_MODEL"]) + assert.Equal(t, "off", captured[1]["AFORGE_EXEC_REASONING"]) + assert.Equal(t, "1", captured[1]["EXTRA"]) +} + +func TestAforgeProviderRootAndTimeoutResolution(t *testing.T) { + useAforgeDo(t) + t.Setenv("AGENTFIELD_HARNESS_TIMEOUT_SECONDS", "2400") + var commands [][]string + var timeouts []int + p := NewAforgeProvider("aforge") + p.runCLI = func(_ context.Context, cmd []string, _ map[string]string, _ string, timeout, idleSeconds int, _ []byte) (*CLIResult, error) { + commands = append(commands, append([]string(nil), cmd...)) + timeouts = append(timeouts, timeout) + assert.Zero(t, idleSeconds) + return &CLIResult{Stdout: aforgeEnvelope("done", true, "", ""), ReturnCode: 0}, nil + } + + _, err := p.Execute(context.Background(), "hello", Options{Cwd: "/cwd-only"}) + require.NoError(t, err) + _, err = p.Execute(context.Background(), "hello", Options{Timeout: 7}) + require.NoError(t, err) + + assert.Equal(t, []string{"aforge", "do", "--json", "--yes-spend", "-w", "/cwd-only", "--timeout", "2395"}, commands[0]) + assert.Equal(t, []string{"aforge", "do", "--json", "--yes-spend", "-w", ".", "--timeout", "2"}, commands[1]) + assert.Equal(t, []int{2400, 7}, timeouts) +} + +func TestAforgeProviderExitSemantics(t *testing.T) { + useAforgeDo(t) + tests := []struct { + name string + code int + deliverable string + blockedOn string + stderr string + wantFailure FailureType + wantMessage string + }{ + {name: "success", code: 0, deliverable: "done", wantFailure: FailureNone}, + {name: "timeout with partial", code: 2, deliverable: "usable", wantFailure: FailureTimeout, wantMessage: "partial: usable"}, + {name: "blocked", code: 1, blockedOn: "Which repository?", wantFailure: FailureCrash, wantMessage: "blocked_on: Which repository?"}, + {name: "error", code: 1, stderr: "\x1b[31mauthentication exploded\x1b[0m", wantFailure: FailureCrash, wantMessage: "authentication exploded"}, + {name: "zero without deliverable", code: 0, wantFailure: FailureCrash, wantMessage: "aforge exit code 0"}, + {name: "signal", code: -9, wantFailure: FailureCrash, wantMessage: "Process killed by signal 9"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + p := NewAforgeProvider("aforge") + p.runCLI = func(context.Context, []string, map[string]string, string, int, int, []byte) (*CLIResult, error) { + return &CLIResult{ + Stdout: aforgeEnvelope(test.deliverable, test.code == 0, test.blockedOn, ""), + Stderr: test.stderr, + ReturnCode: test.code, + }, nil + } + + raw, err := p.Execute(context.Background(), "hello", Options{}) + require.NoError(t, err) + assert.Equal(t, test.wantFailure != FailureNone, raw.IsError) + assert.Equal(t, test.wantFailure, raw.FailureType) + if test.wantMessage != "" { + assert.Contains(t, raw.ErrorMessage, test.wantMessage) + } + }) + } +} + +func TestAforgeProviderParsesLastEnvelopeAndLeavesZeroCostUnknown(t *testing.T) { + useAforgeDo(t) + p := NewAforgeProvider("aforge") + p.runCLI = func(context.Context, []string, map[string]string, string, int, int, []byte) (*CLIResult, error) { + return &CLIResult{Stdout: "stray diagnostic\n{\"type\":\"event\"}\n" + + aforgeEnvelope("real result", true, "", `{"calls":1,"cost":0}`), ReturnCode: 0}, nil + } + + raw, err := p.Execute(context.Background(), "hello", Options{}) + require.NoError(t, err) + assert.Equal(t, "real result", raw.Result) + assert.Nil(t, raw.Metrics.CostUSD, "zero provider cost remains unknown") +} + +func TestAforgeProviderParsesPrettyPrintedEnvelope(t *testing.T) { + useAforgeDo(t) + p := NewAforgeProvider("aforge") + p.runCLI = func(context.Context, []string, map[string]string, string, int, int, []byte) (*CLIResult, error) { + var value map[string]any + require.NoError(t, json.Unmarshal([]byte(aforgeEnvelope("pretty result", true, "", "")), &value)) + pretty, err := json.MarshalIndent(value, "", " ") + require.NoError(t, err) + return &CLIResult{Stdout: string(pretty), ReturnCode: 0}, nil + } + + raw, err := p.Execute(context.Background(), "hello", Options{}) + require.NoError(t, err) + assert.Equal(t, "pretty result", raw.Result) + assert.False(t, raw.IsError) +} + +func TestAforgeProviderMissingBinaryAndTimeout(t *testing.T) { + useAforgeDo(t) + t.Run("missing binary", func(t *testing.T) { + p := NewAforgeProvider("aforge-missing") + p.runCLI = func(context.Context, []string, map[string]string, string, int, int, []byte) (*CLIResult, error) { + return nil, fmt.Errorf("exec: executable file not found in $PATH") + } + raw, err := p.Execute(context.Background(), "hello", Options{}) + require.NoError(t, err) + assert.True(t, raw.IsError) + assert.Equal(t, FailureCrash, raw.FailureType) + assert.Contains(t, raw.ErrorMessage, "aforge-missing") + }) + + t.Run("timeout", func(t *testing.T) { + p := NewAforgeProvider("aforge") + p.runCLI = func(context.Context, []string, map[string]string, string, int, int, []byte) (*CLIResult, error) { + return nil, fmt.Errorf("CLI command timed out after 1s: aforge do") + } + raw, err := p.Execute(context.Background(), "hello", Options{}) + require.NoError(t, err) + assert.True(t, raw.IsError) + assert.Equal(t, FailureTimeout, raw.FailureType) + }) +} + +func TestAforgeProviderConcurrencyLimit(t *testing.T) { + useAforgeDo(t) + t.Setenv("AFORGE_MAX_CONCURRENT", "2") + aforgeSemaphore = nil + aforgeSemOnce = sync.Once{} + t.Cleanup(func() { + aforgeSemaphore = nil + aforgeSemOnce = sync.Once{} + }) + + var current int64 + var maxSeen int64 + p := NewAforgeProvider("aforge") + p.runCLI = func(context.Context, []string, map[string]string, string, int, int, []byte) (*CLIResult, error) { + active := atomic.AddInt64(¤t, 1) + for { + previous := atomic.LoadInt64(&maxSeen) + if active <= previous || atomic.CompareAndSwapInt64(&maxSeen, previous, active) { + break + } + } + time.Sleep(50 * time.Millisecond) + atomic.AddInt64(¤t, -1) + return &CLIResult{Stdout: aforgeEnvelope("done", true, "", ""), ReturnCode: 0}, nil + } + + var wg sync.WaitGroup + for i := 0; i < 6; i++ { + wg.Add(1) + go func() { + defer wg.Done() + _, _ = p.Execute(context.Background(), "hello", Options{}) + }() + } + wg.Wait() + assert.LessOrEqual(t, maxSeen, int64(2)) +} + +func TestAforgeRunnerConcurrentSameCwdUsesIsolatedSchemaFiles(t *testing.T) { + useAforgeDo(t) + cwd := t.TempDir() + script := writeTestScript(t, cwd, "aforge-test", `#!/bin/sh +prompt=$(cat) +output_path=$(printf '%s' "$prompt" | tr '\n' ' ' | sed -n 's/.*create this file: \([^ ]*\.agentfield_output\.json\).*/\1/p') +case "$prompt" in + *first*) payload='{"name":"first","count":1}' ;; + *) payload='{"name":"second","count":2}' ;; +esac +mkdir -p "$(dirname "$output_path")" +printf '%s' "$payload" > "$output_path" +printf '%s\n' '{"settled":true,"deliverable":"done","blocked_on":"","spend_usd":0,"elapsed_ms":1,"usage":{"calls":1,"prompt_tokens":0,"completion_tokens":0,"cached_tokens":0,"cost":0}}' +`) + + type output struct { + Name string `json:"name"` + Count int `json:"count"` + } + schema := map[string]any{ + "type": "object", + "properties": map[string]any{ + "name": map[string]any{"type": "string"}, + "count": map[string]any{"type": "integer"}, + }, + "required": []any{"name", "count"}, + } + runner := NewRunner(Options{Provider: ProviderAforge, BinPath: script}) + + type runResult struct { + result *Result + dest output + err error + } + results := make(chan runResult, 2) + for _, prompt := range []string{"first", "second"} { + prompt := prompt + go func() { + var dest output + result, err := runner.Run(context.Background(), prompt, schema, &dest, Options{Cwd: cwd}) + results <- runResult{result: result, dest: dest, err: err} + }() + } + + seen := map[string]int{} + for i := 0; i < 2; i++ { + got := <-results + require.NoError(t, got.err) + require.NotNil(t, got.result) + assert.False(t, got.result.IsError, got.result.ErrorMessage) + seen[got.dest.Name] = got.dest.Count + } + assert.Equal(t, map[string]int{"first": 1, "second": 2}, seen) + matches, err := filepath.Glob(filepath.Join(cwd, ".agentfield-out-*")) + require.NoError(t, err) + assert.Empty(t, matches) + _, err = os.Stat(filepath.Join(cwd, outputFilename)) + assert.True(t, os.IsNotExist(err)) +} diff --git a/sdk/go/harness/cli.go b/sdk/go/harness/cli.go index 24d090a1a..c2ce6936f 100644 --- a/sdk/go/harness/cli.go +++ b/sdk/go/harness/cli.go @@ -74,6 +74,17 @@ func RunCLI(ctx context.Context, cmd []string, env map[string]string, cwd string // string value ("") causes that variable to be removed from the environment // rather than set to empty — use this to unset inherited variables. func RunCLIWithStdin(ctx context.Context, cmd []string, env map[string]string, cwd string, timeout int, stdin []byte) (*CLIResult, error) { + return runCLIWithStdin(ctx, cmd, env, cwd, timeout, stdin, nil) +} + +// runCLIWithStdinIdle is the provider-internal variant with an explicit idle +// watchdog. A value <= 0 disables the watchdog. Aforge uses this because its +// machine-readable mode is intentionally silent until the final JSON envelope. +func runCLIWithStdinIdle(ctx context.Context, cmd []string, env map[string]string, cwd string, timeout, idleSeconds int, stdin []byte) (*CLIResult, error) { + return runCLIWithStdin(ctx, cmd, env, cwd, timeout, stdin, &idleSeconds) +} + +func runCLIWithStdin(ctx context.Context, cmd []string, env map[string]string, cwd string, timeout int, stdin []byte, idleOverride *int) (*CLIResult, error) { if timeout > 0 { var cancel context.CancelFunc ctx, cancel = context.WithTimeout(ctx, time.Duration(timeout)*time.Second) @@ -191,6 +202,9 @@ func RunCLIWithStdin(ctx context.Context, cmd []string, env map[string]string, c }() idleSeconds := resolveIdleSeconds() + if idleOverride != nil { + idleSeconds = *idleOverride + } idleTimedOut := false var waitErr error diff --git a/sdk/go/harness/factory.go b/sdk/go/harness/factory.go index c58420207..9b8a0d3d0 100644 --- a/sdk/go/harness/factory.go +++ b/sdk/go/harness/factory.go @@ -1,11 +1,31 @@ package harness -import "fmt" +import ( + "fmt" + "os" + "strings" +) + +// ResolveProviderName applies harness provider precedence: an explicit name +// wins, then AGENTFIELD_HARNESS_PROVIDER, then DefaultProvider ("aforge"). +// Blank / whitespace-only values are treated as unset. +func ResolveProviderName(name string) string { + if trimmed := strings.TrimSpace(name); trimmed != "" { + return trimmed + } + if envName := strings.TrimSpace(os.Getenv(ProviderEnvVar)); envName != "" { + return envName + } + return DefaultProvider +} // BuildProvider creates a Provider instance for the given provider name. -// Supported providers: "claude-code", "codex", "gemini", "opencode". +// Supported providers: "aforge", "claude-code", "codex", "gemini", "opencode". func BuildProvider(name string, binPath string) (Provider, error) { + name = ResolveProviderName(name) switch name { + case ProviderAforge: + return NewAforgeProvider(binPath), nil case ProviderClaudeCode: return NewClaudeCodeProvider(binPath), nil case ProviderCodex: @@ -16,8 +36,8 @@ func BuildProvider(name string, binPath string) (Provider, error) { return NewOpenCodeProvider(binPath, ""), nil default: return nil, fmt.Errorf( - "unknown harness provider: %q (supported: %s, %s, %s, %s)", - name, ProviderClaudeCode, ProviderCodex, ProviderGemini, ProviderOpenCode, + "unknown harness provider: %q (supported: %s, %s, %s, %s, %s)", + name, ProviderAforge, ProviderClaudeCode, ProviderCodex, ProviderGemini, ProviderOpenCode, ) } } diff --git a/sdk/go/harness/factory_test.go b/sdk/go/harness/factory_test.go new file mode 100644 index 000000000..be480ed4b --- /dev/null +++ b/sdk/go/harness/factory_test.go @@ -0,0 +1,48 @@ +package harness + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestResolveProviderName(t *testing.T) { + tests := []struct { + name string + explicit string + env string + want string + }{ + {name: "default", want: ProviderAforge}, + {name: "environment", env: ProviderCodex, want: ProviderCodex}, + {name: "explicit wins", explicit: ProviderGemini, env: ProviderCodex, want: ProviderGemini}, + {name: "blank explicit", explicit: " ", want: ProviderAforge}, + {name: "blank environment", env: " ", want: ProviderAforge}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv(ProviderEnvVar, tt.env) + assert.Equal(t, tt.want, ResolveProviderName(tt.explicit)) + }) + } +} + +func TestBuildProvider_DefaultsToAforge(t *testing.T) { + t.Setenv(ProviderEnvVar, "") + provider, err := BuildProvider("", "") + require.NoError(t, err) + require.NotNil(t, provider) + _, ok := provider.(*AforgeProvider) + assert.True(t, ok) +} + +func TestBuildProvider_RejectsUnknownName(t *testing.T) { + provider, err := BuildProvider("nope", "") + assert.Nil(t, provider) + require.Error(t, err) + assert.Contains(t, err.Error(), "nope") + assert.True(t, strings.Contains(err.Error(), "supported: aforge, claude-code, codex, gemini, opencode")) +} diff --git a/sdk/go/harness/provider.go b/sdk/go/harness/provider.go index 2f6ab37fd..f986f2240 100644 --- a/sdk/go/harness/provider.go +++ b/sdk/go/harness/provider.go @@ -3,6 +3,8 @@ package harness import "context" const ( + // ProviderAforge is the provider name for the Aforge CLI. + ProviderAforge = "aforge" // ProviderOpenCode is the provider name for OpenCode CLI. ProviderOpenCode = "opencode" // ProviderClaudeCode is the provider name for Claude Code CLI. @@ -13,6 +15,17 @@ const ( ProviderGemini = "gemini" ) +const ( + // DefaultProvider is the harness provider used when no provider is + // configured explicitly and AGENTFIELD_HARNESS_PROVIDER is unset. + // AForge is AgentField's native coding harness. + DefaultProvider = ProviderAforge + + // ProviderEnvVar overrides the default provider process-wide. An + // explicitly configured provider still wins over it. + ProviderEnvVar = "AGENTFIELD_HARNESS_PROVIDER" +) + // Provider is the interface that CLI-based harness providers implement. // Each provider knows how to invoke a specific coding agent (opencode, // claude-code, etc.) and return a RawResult. @@ -23,7 +36,9 @@ type Provider interface { // Options control a single harness invocation. Fields are optional; // zero values mean "use default". type Options struct { - // Provider name: "opencode", "claude-code". + // Provider name: "aforge", "opencode", "claude-code", "codex", or + // "gemini". An explicit value wins over AGENTFIELD_HARNESS_PROVIDER; + // when both are empty, the provider defaults to "aforge". Provider string // Model identifier passed to the coding agent. It may carry a diff --git a/sdk/go/harness/runner.go b/sdk/go/harness/runner.go index 45f811d0d..23a71f01b 100644 --- a/sdk/go/harness/runner.go +++ b/sdk/go/harness/runner.go @@ -52,26 +52,29 @@ type schemaAware interface { // is non-nil, the validated output is unmarshalled into it. func (r *Runner) Run(ctx context.Context, prompt string, schema map[string]any, dest any, overrides Options) (*Result, error) { opts := r.mergeOptions(overrides) - - if opts.Provider == "" { - return nil, fmt.Errorf( - "no harness provider specified: set Provider in runner defaults or pass it to Run()", - ) - } + opts.Provider = ResolveProviderName(opts.Provider) provider, err := r.buildProvider(opts) if err != nil { return nil, err } - // Determine output directory for schema files. - outputDir := opts.Cwd + // Always isolate schema output in a per-run directory under the effective + // project root. Concurrent calls sharing one checkout must not overwrite or + // clean up each other's fixed .agentfield_output.json file. + outputDir := opts.ProjectDir + if outputDir == "" { + outputDir = opts.Cwd + } if outputDir == "" { outputDir = "." } var tempOutputDir string - if opts.ProjectDir != "" { - tempOutputDir, err = os.MkdirTemp(opts.ProjectDir, ".agentfield-out-") + if schema != nil { + if err = os.MkdirAll(outputDir, 0o700); err != nil { + return nil, fmt.Errorf("creating schema output root: %w", err) + } + tempOutputDir, err = os.MkdirTemp(outputDir, ".agentfield-out-") if err != nil { return nil, fmt.Errorf("creating temp output dir: %w", err) } diff --git a/sdk/go/harness/runner_invariant_test.go b/sdk/go/harness/runner_invariant_test.go index 20157e06b..ca138c1d7 100644 --- a/sdk/go/harness/runner_invariant_test.go +++ b/sdk/go/harness/runner_invariant_test.go @@ -178,6 +178,7 @@ func TestInvariant_Runner_RepairJSON_ProjectionExtended(t *testing.T) { // registered provider name returns a non-nil provider from BuildProvider. func TestInvariant_Runner_ProviderFactoryExhaustiveness(t *testing.T) { knownProviders := []string{ + ProviderAforge, ProviderClaudeCode, ProviderCodex, ProviderGemini, @@ -197,7 +198,6 @@ func TestInvariant_Runner_ProviderFactoryExhaustiveness(t *testing.T) { // unknown provider names return an error with a non-nil error value. func TestInvariant_Runner_ProviderFactoryUnknownReturnsError(t *testing.T) { unknownNames := []string{ - "", "nonexistent", "gpt-4", "anthropic", @@ -310,6 +310,13 @@ func TestInvariant_Runner_BuildProviderWithBinPath(t *testing.T) { provider string checkPath func(t *testing.T, prov Provider) }{ + { + provider: ProviderAforge, + checkPath: func(t *testing.T, prov Provider) { + p := prov.(*AforgeProvider) + assert.Equal(t, customBinPath, p.BinPath) + }, + }, { provider: ProviderClaudeCode, checkPath: func(t *testing.T, prov Provider) { diff --git a/sdk/go/harness/runner_test.go b/sdk/go/harness/runner_test.go index 4d744ec3d..f027bd4bb 100644 --- a/sdk/go/harness/runner_test.go +++ b/sdk/go/harness/runner_test.go @@ -37,11 +37,15 @@ func TestRunner_Run_NoSchema(t *testing.T) { runner := NewRunner(Options{Provider: "opencode"}) // We can't easily test with real providers, so test the merge/validation logic - t.Run("missing provider", func(t *testing.T) { - r := NewRunner(Options{}) - _, err := r.Run(context.Background(), "test", nil, nil, Options{}) - assert.Error(t, err) - assert.Contains(t, err.Error(), "no harness provider") + t.Run("empty provider defaults to aforge", func(t *testing.T) { + t.Setenv(ProviderEnvVar, "") + r := NewRunner(Options{BinPath: filepath.Join(t.TempDir(), "missing-aforge")}) + result, err := r.Run(context.Background(), "test", nil, nil, Options{}) + require.NoError(t, err) + require.NotNil(t, result) + assert.True(t, result.IsError) + assert.Contains(t, result.ErrorMessage, "missing-aforge") + assert.NotContains(t, result.ErrorMessage, "no harness provider specified") }) t.Run("unknown provider", func(t *testing.T) { @@ -567,6 +571,7 @@ func TestBuildProvider(t *testing.T) { wantType string wantErr bool }{ + {"aforge", "aforge", "*harness.AforgeProvider", false}, {"claude-code", "claude-code", "*harness.ClaudeCodeProvider", false}, {"codex", "codex", "*harness.CodexProvider", false}, {"gemini", "gemini", "*harness.GeminiProvider", false}, @@ -589,8 +594,8 @@ func TestBuildProvider(t *testing.T) { } func TestRunner_BuildProvider_UsesFactory(t *testing.T) { - // Verify the runner can now build all 4 providers - for _, name := range []string{"claude-code", "codex", "gemini", "opencode"} { + // Verify the runner can build every registered provider. + for _, name := range []string{"aforge", "claude-code", "codex", "gemini", "opencode"} { t.Run(name, func(t *testing.T) { runner := NewRunner(Options{Provider: name}) _, err := runner.Run(context.Background(), "test", nil, nil, Options{}) diff --git a/sdk/go/harness/testmain_test.go b/sdk/go/harness/testmain_test.go index f574d80d5..de877dd55 100644 --- a/sdk/go/harness/testmain_test.go +++ b/sdk/go/harness/testmain_test.go @@ -27,6 +27,11 @@ printf '%s\n' 'stub gemini result' printf '%s\n' 'stub opencode result' `, } + if os.Getenv("AFORGE_INTEGRATION") != "1" { + stubs["aforge"] = `#!/bin/sh +printf '%s\n' '{"settled":true,"deliverable":"stub aforge result","blocked_on":"","spend_usd":0,"elapsed_ms":1,"usage":{"calls":1,"prompt_tokens":0,"completion_tokens":0,"cached_tokens":0,"cost":0}}' +` + } for name, content := range stubs { if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o755); err != nil { diff --git a/sdk/python/agentfield/agent.py b/sdk/python/agentfield/agent.py index 004164e57..cf92f9d3a 100644 --- a/sdk/python/agentfield/agent.py +++ b/sdk/python/agentfield/agent.py @@ -3735,8 +3735,9 @@ async def harness( prompt: Task description for the coding agent. schema: Pydantic BaseModel class for structured output validation. provider: Override provider ("aforge", "claude-code", "codex", "gemini", - "opencode"). - model: Override model identifier. + "opencode", "grok"). Omit to use ``AGENTFIELD_HARNESS_PROVIDER`` + when set, otherwise ``aforge``. + model: Override model identifier. Empty uses the provider's own default. max_turns: Maximum agent iterations. max_budget_usd: Cost cap in USD. tools: Allowed tools list. diff --git a/sdk/python/agentfield/harness/_defaults.py b/sdk/python/agentfield/harness/_defaults.py new file mode 100644 index 000000000..b9b731718 --- /dev/null +++ b/sdk/python/agentfield/harness/_defaults.py @@ -0,0 +1,24 @@ +"""Default harness provider selection. + +`aforge` is AgentField's native harness. Provider precedence is +explicit value > AGENTFIELD_HARNESS_PROVIDER env var > DEFAULT_HARNESS_PROVIDER. +""" + +from __future__ import annotations + +import os +from typing import Optional + +DEFAULT_HARNESS_PROVIDER = "aforge" +HARNESS_PROVIDER_ENV_VAR = "AGENTFIELD_HARNESS_PROVIDER" + + +def resolve_harness_provider(explicit: Optional[str] = None) -> str: + if isinstance(explicit, str) and explicit.strip(): + return explicit.strip() + + env_provider = os.environ.get(HARNESS_PROVIDER_ENV_VAR) + if isinstance(env_provider, str) and env_provider.strip(): + return env_provider.strip() + + return DEFAULT_HARNESS_PROVIDER diff --git a/sdk/python/agentfield/harness/_runner.py b/sdk/python/agentfield/harness/_runner.py index 46fe61ecc..216a58b22 100644 --- a/sdk/python/agentfield/harness/_runner.py +++ b/sdk/python/agentfield/harness/_runner.py @@ -10,6 +10,7 @@ import time from typing import Any, Dict, List, Optional +from agentfield.harness._defaults import resolve_harness_provider from agentfield.harness._result import FailureType, HarnessResult, RawResult from agentfield.harness._schema import ( build_followup_prompt, @@ -275,12 +276,8 @@ async def run( } options = _resolve_options(self._config, overrides) - resolved_provider = options.get("provider") - if not resolved_provider: - raise ValueError( - "No harness provider specified. Set 'provider' in HarnessConfig " - "or pass it to .harness() call." - ) + resolved_provider = resolve_harness_provider(options.get("provider")) + options["provider"] = resolved_provider resolved_cwd = str(options.get("cwd") or ".") provider_instance = self._build_provider(str(resolved_provider), options) diff --git a/sdk/python/agentfield/harness/providers/aforge.py b/sdk/python/agentfield/harness/providers/aforge.py index 1b36d4539..f6e32f7e0 100644 --- a/sdk/python/agentfield/harness/providers/aforge.py +++ b/sdk/python/agentfield/harness/providers/aforge.py @@ -7,7 +7,7 @@ import logging import os import time -from typing import ClassVar, Dict, Optional +from typing import ClassVar from agentfield.harness._availability import ensure_cli_available, provider_unavailable from agentfield.harness._cli import ( @@ -25,12 +25,21 @@ def _strip_openrouter_prefix(model: str) -> str: """Strip one leading ``openrouter/`` prefix from a model slug.""" - prefix = "openrouter/" - return model[len(prefix) :] if model.startswith(prefix) else model + return model.removeprefix("openrouter/") def _parse_envelope(stdout: str) -> dict[str, object] | None: - """Return the last JSON object containing an aforge ``text`` field.""" + """Return the last canonical ``do`` or ``exec`` envelope.""" + # Both surfaces print one JSON object. Parse that shape before falling back + # to the line-oriented form tolerated for wrappers that prepend diagnostics. + try: + value = json.loads(stdout.strip()) + except ValueError: + pass + else: + if isinstance(value, dict) and ("deliverable" in value or "text" in value): + return value + for line in reversed( [line.strip() for line in stdout.splitlines() if line.strip()] ): @@ -38,7 +47,7 @@ def _parse_envelope(stdout: str) -> dict[str, object] | None: value = json.loads(line) except ValueError: continue - if isinstance(value, dict) and "text" in value: + if isinstance(value, dict) and ("deliverable" in value or "text" in value): return value return None @@ -50,27 +59,48 @@ def _numeric(value: object) -> int | float | None: return None -def _crash_message(returncode: int, stop: str, stderr: str) -> str: +def _task_input(prompt: str, system_prompt: object) -> str: + if isinstance(system_prompt, str) and system_prompt.strip(): + return f"{system_prompt.strip()}\n\nTask:\n{prompt}" + return prompt + + +def _crash_message( + returncode: int, + blocked_on: str, + deliverable: str | None, + stderr: str, +) -> str: """Build a consistent, bounded aforge crash message.""" clean_stderr = strip_ansi(stderr.strip()) - exit_context = f"aforge exit code {returncode}, stop={stop!s}" + exit_context = f"aforge exit code {returncode}" if returncode < 0: message = f"Process killed by signal {-returncode}. {exit_context}" else: message = exit_context if clean_stderr: message += f". stderr: {clean_stderr[:1000]}" + elif blocked_on: + message += f". blocked_on: {blocked_on[:1000]}" + elif deliverable: + message += f". partial: {deliverable[:1000]}" return message class AforgeProvider: - """Aforge CLI provider. Invokes ``aforge exec --json`` subprocess.""" + """Aforge CLI provider. + + ``exec`` is the default direct one-shot entry point. Set + ``AGENTFIELD_AFORGE_COMMAND=do`` to opt into Aforge's routed workflow. + """ _MAX_CONCURRENT: ClassVar[int] = int(os.environ.get("AFORGE_MAX_CONCURRENT", "8")) - _concurrency_sem: ClassVar[Optional[asyncio.Semaphore]] = None + _concurrency_sem: ClassVar[asyncio.Semaphore | None] = None def __init__(self, bin_path: str = "aforge"): - self._bin = bin_path + self._bin = ( + os.environ.get("AFORGE_BIN", bin_path) if bin_path == "aforge" else bin_path + ) @classmethod def _get_semaphore(cls) -> asyncio.Semaphore: @@ -93,16 +123,61 @@ async def _execute_impl(self, prompt: str, options: dict[str, object]) -> RawRes # project_dir is the canonical agent root; a nested task cwd must not # restrict access to sibling paths under the shared project root. root = str(options.get("project_dir") or options.get("cwd") or ".") - cmd = [self._bin, "exec", "--json", "-w", root] - - system_prompt = options.get("system_prompt") - if isinstance(system_prompt, str) and system_prompt.strip(): - cmd.extend(["--system", system_prompt.strip()]) - + timeout_seconds = int( + os.environ.get("AGENTFIELD_HARNESS_TIMEOUT_SECONDS", "1800") + ) model_value, variant_value = resolve_model_and_variant(options) - env: Dict[str, str] = {} + # Leave a small landing window so aforge can emit its honest timeout + # envelope before the outer subprocess watchdog has to kill it. + aforge_timeout = max(1, timeout_seconds - 5) + command = os.environ.get("AGENTFIELD_AFORGE_COMMAND", "exec").strip().lower() + if command not in {"do", "exec"}: + return RawResult( + is_error=True, + error_message=( + "AGENTFIELD_AFORGE_COMMAND must be 'do' or 'exec', " + f"got {command!r}" + ), + failure_type=FailureType.CRASH, + metrics=Metrics(), + ) + if command == "exec": + cmd = [ + self._bin, + "exec", + "--json", + "-w", + root, + "--timeout", + str(aforge_timeout), + "--context-fill", + "60", + "--completion-reserve", + "65536", + ] + system_prompt = options.get("system_prompt") + if isinstance(system_prompt, str) and system_prompt.strip(): + cmd.extend(["--system", system_prompt.strip()]) + input_text = prompt + else: + cmd = [ + self._bin, + "do", + "--json", + "--yes-spend", + "-w", + root, + "--timeout", + str(aforge_timeout), + ] + input_text = _task_input(prompt, options.get("system_prompt")) + + env: dict[str, str] = {"AFORGE_MODELS": ""} if command == "exec" else {} if model_value: - env["AFORGE_MODEL"] = _strip_openrouter_prefix(model_value) + model_slug = _strip_openrouter_prefix(model_value) + env["AFORGE_MODEL"] = model_slug + if command == "exec": + cmd.extend(["--model", model_slug, "--plan-model", model_slug]) if variant_value: normalized_variant = variant_value.strip().lower() @@ -121,9 +196,6 @@ async def _execute_impl(self, prompt: str, options: dict[str, object]) -> RawRes } ) - timeout_seconds = int( - os.environ.get("AGENTFIELD_HARNESS_TIMEOUT_SECONDS", "1800") - ) start_api = time.monotonic() try: @@ -135,7 +207,7 @@ async def _execute_impl(self, prompt: str, options: dict[str, object]) -> RawRes # Aforge is stdout-silent until its final envelope; disable the # no-progress watchdog so legitimate long runs are not killed. idle_seconds=0, - input_text=prompt, + input_text=input_text, ) except FileNotFoundError as exc: raise provider_unavailable("aforge", self._bin) from exc @@ -151,22 +223,30 @@ async def _execute_impl(self, prompt: str, options: dict[str, object]) -> RawRes envelope = _parse_envelope(stdout) result_text: str | None = None - stop = "" usage: dict[object, object] = {} - turns = 0 + calls = 0 + blocked_on = "" + stop = "" if envelope is not None: - text_value = envelope.get("text") + text_value = envelope.get("text" if command == "exec" else "deliverable") if isinstance(text_value, str) and text_value.strip(): result_text = text_value.strip() - stop_value = envelope.get("stop") - if isinstance(stop_value, str): - stop = stop_value + blocked_value = envelope.get("blocked_on") + if isinstance(blocked_value, str): + blocked_on = blocked_value.strip() usage_value = envelope.get("usage") if isinstance(usage_value, dict): usage = usage_value - turns_value = _numeric(envelope.get("turns")) - if turns_value is not None: - turns = int(turns_value) + calls_value = _numeric(usage.get("calls")) + if calls_value is not None: + calls = int(calls_value) + if command == "exec": + stop_value = envelope.get("stop") + if isinstance(stop_value, str): + stop = stop_value.strip() + turns_value = _numeric(envelope.get("turns")) + if turns_value is not None: + calls = int(turns_value) clean_stderr = strip_ansi(stderr.strip()) if stderr else "" logger.info( @@ -178,23 +258,38 @@ async def _execute_impl(self, prompt: str, options: dict[str, object]) -> RawRes if not result_text and clean_stderr: logger.warning("aforge no text. stderr: %s", clean_stderr[:800]) - if returncode < 0: - is_error = True - elif returncode in (2, 3) and result_text: - is_error = False - elif returncode != 0: - is_error = True + if command == "exec": + # Budget and turn-cap exits with a usable landing are partial + # successes under the original exec adapter contract. + is_error = ( + returncode < 0 + or result_text is None + or returncode not in {0, 2, 3} + ) else: - is_error = result_text is None - - failure_type = FailureType.CRASH if is_error else FailureType.NONE - error_message = _crash_message(returncode, stop, stderr) if is_error else None + is_error = returncode != 0 or result_text is None or bool(blocked_on) + if not is_error: + failure_type = FailureType.NONE + elif (command == "do" and returncode == 2) or ( + command == "exec" and returncode == 4 + ): + failure_type = FailureType.TIMEOUT + else: + failure_type = FailureType.CRASH + error_message = ( + _crash_message(returncode, blocked_on or stop, result_text, stderr) + if is_error + else None + ) input_tokens_value = _numeric(usage.get("prompt_tokens")) output_tokens_value = _numeric(usage.get("completion_tokens")) cached_tokens_value = _numeric(usage.get("cached_tokens")) + spend_value = _numeric(envelope.get("spend")) if envelope else None cost_value = _numeric(usage.get("cost")) - if cost_value is not None and cost_value > 0: + if spend_value is not None and spend_value > 0: + total_cost = float(spend_value) + elif cost_value is not None and cost_value > 0: total_cost = float(cost_value) else: total_cost = estimate_cli_cost( @@ -208,7 +303,7 @@ async def _execute_impl(self, prompt: str, options: dict[str, object]) -> RawRes messages=[envelope] if envelope is not None else [], metrics=Metrics( duration_api_ms=api_ms, - num_turns=turns, + num_turns=calls, total_cost_usd=total_cost, session_id="", input_tokens=int(input_tokens_value or 0), diff --git a/sdk/python/agentfield/harness/providers/claude.py b/sdk/python/agentfield/harness/providers/claude.py index 39bdb8059..5c5cbeb1d 100644 --- a/sdk/python/agentfield/harness/providers/claude.py +++ b/sdk/python/agentfield/harness/providers/claude.py @@ -16,6 +16,11 @@ logger = logging.getLogger("agentfield.harness.claude") +# Claude Code's historical default in AgentField. HarnessConfig.model used to +# default to "sonnet"; now the default is empty and each provider owns its own +# default, so claude-code keeps sonnet here. +DEFAULT_CLAUDE_MODEL = "sonnet" + def _get_claude_sdk() -> Any: """Lazy import of claude_agent_sdk.""" @@ -75,8 +80,7 @@ async def execute(self, prompt: str, options: dict[str, object]) -> RawResult: agent_options: dict[str, object] = {} model_value, variant_value = resolve_model_and_variant(options) - if model_value is not None: - agent_options["model"] = model_value + agent_options["model"] = model_value or DEFAULT_CLAUDE_MODEL if variant_value: # claude_agent_sdk has no reasoning-effort knob; drop the variant # rather than passing an invalid "model#variant" model id. diff --git a/sdk/python/agentfield/types.py b/sdk/python/agentfield/types.py index f9091d3ad..abe3860c7 100644 --- a/sdk/python/agentfield/types.py +++ b/sdk/python/agentfield/types.py @@ -273,15 +273,29 @@ class DiscoveryResult: xml: Optional[str] = None +def _default_harness_provider() -> str: + # Imported lazily: agentfield.harness imports agentfield.types. + from agentfield.harness._defaults import resolve_harness_provider + + return resolve_harness_provider() + + class HarnessConfig(BaseModel): provider: str = Field( - ..., + default_factory=_default_harness_provider, + description=( + 'Coding agent provider: "aforge" (default) | "claude-code" | "codex" | ' + '"gemini" | "opencode" | "grok". Unset resolves to the ' + 'AGENTFIELD_HARNESS_PROVIDER env var when present, else "aforge".' + ), + ) + model: Optional[str] = Field( + default=None, description=( - 'Coding agent provider: "aforge" | "claude-code" | "codex" | ' - '"gemini" | "opencode" | "grok"' + "Model identifier. None/empty means the provider's own default " + "(aforge picks its own; claude-code uses sonnet)." ), ) - model: str = Field(default="sonnet", description="Default model identifier.") max_turns: int = Field(default=30, description="Maximum agent iterations.") max_budget_usd: Optional[float] = Field( default=None, description="Cost cap in USD." diff --git a/sdk/python/tests/test_harness_defaults.py b/sdk/python/tests/test_harness_defaults.py new file mode 100644 index 000000000..58ffcf39a --- /dev/null +++ b/sdk/python/tests/test_harness_defaults.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +from typing import Any +from unittest.mock import patch + +import pytest + +from agentfield.harness._result import RawResult +from agentfield.harness._runner import HarnessRunner +from agentfield.types import HarnessConfig + + +class _Provider: + async def execute(self, prompt: str, options: dict[str, Any]) -> RawResult: + return RawResult(result="ok") + + +def test_default_provider_is_aforge(monkeypatch): + monkeypatch.delenv("AGENTFIELD_HARNESS_PROVIDER", raising=False) + assert HarnessConfig().provider == "aforge" + + +def test_env_provider_is_honoured(monkeypatch): + monkeypatch.setenv("AGENTFIELD_HARNESS_PROVIDER", "codex") + assert HarnessConfig().provider == "codex" + + +def test_explicit_provider_beats_env(monkeypatch): + monkeypatch.setenv("AGENTFIELD_HARNESS_PROVIDER", "codex") + assert HarnessConfig(provider="gemini").provider == "gemini" + + +def test_blank_env_is_ignored(monkeypatch): + monkeypatch.setenv("AGENTFIELD_HARNESS_PROVIDER", " ") + assert HarnessConfig().provider == "aforge" + + +async def _run_and_capture_provider( + tmp_path, config: HarnessConfig | None = None, **overrides: Any +) -> str: + captured: dict[str, str] = {} + + def fake_build_provider(factory_config): + captured["provider"] = factory_config.provider + return _Provider() + + with patch("agentfield.harness._runner.build_provider", fake_build_provider): + await HarnessRunner(config=config).run( + "hello", cwd=str(tmp_path), **overrides + ) + return captured["provider"] + + +@pytest.mark.asyncio +async def test_zero_setup_runner_selects_aforge(tmp_path, monkeypatch): + monkeypatch.delenv("AGENTFIELD_HARNESS_PROVIDER", raising=False) + assert await _run_and_capture_provider(tmp_path) == "aforge" + + +@pytest.mark.asyncio +async def test_runner_honours_env_provider(tmp_path, monkeypatch): + monkeypatch.setenv("AGENTFIELD_HARNESS_PROVIDER", "opencode") + assert await _run_and_capture_provider(tmp_path) == "opencode" + + +@pytest.mark.asyncio +async def test_runner_explicit_provider_beats_env_and_config(tmp_path, monkeypatch): + monkeypatch.setenv("AGENTFIELD_HARNESS_PROVIDER", "gemini") + config = HarnessConfig(provider="codex") + assert ( + await _run_and_capture_provider(tmp_path, config, provider="opencode") + == "opencode" + ) + + +def test_model_default_is_empty(monkeypatch): + monkeypatch.delenv("AGENTFIELD_HARNESS_PROVIDER", raising=False) + assert HarnessConfig().model is None diff --git a/sdk/python/tests/test_harness_functional.py b/sdk/python/tests/test_harness_functional.py index 000ada1c8..43c1c994b 100644 --- a/sdk/python/tests/test_harness_functional.py +++ b/sdk/python/tests/test_harness_functional.py @@ -5,7 +5,7 @@ Prerequisites ~~~~~~~~~~~~~ -- Coding agent CLIs installed (claude, codex, opencode) +- Coding agent CLIs installed (aforge, claude, codex, opencode) - Valid API keys / auth configured for each provider - Internet access for API calls @@ -18,6 +18,7 @@ pytest tests/test_harness_functional.py -m harness_live -v -k codex --timeout=300 pytest tests/test_harness_functional.py -m harness_live -v -k claude --timeout=300 pytest tests/test_harness_functional.py -m harness_live -v -k opencode --timeout=300 + pytest tests/test_harness_functional.py -m harness_live -v -k aforge --timeout=300 """ from __future__ import annotations @@ -26,13 +27,12 @@ import shutil import subprocess import tempfile -from typing import List import pytest from pydantic import BaseModel -from agentfield.harness._runner import HarnessRunner from agentfield.harness._result import HarnessResult # noqa: F401 +from agentfield.harness._runner import HarnessRunner from agentfield.types import HarnessConfig # ──────────────────────────────────────────────────────────────────────── @@ -52,7 +52,7 @@ class CodeReviewResponse(BaseModel): summary: str score: int - suggestions: List[str] + suggestions: list[str] # ──────────────────────────────────────────────────────────────────────── @@ -61,6 +61,7 @@ class CodeReviewResponse(BaseModel): HAS_CODEX = shutil.which("codex") is not None HAS_OPENCODE = shutil.which("opencode") is not None +HAS_AFORGE = shutil.which("aforge") is not None try: import claude_agent_sdk # noqa: F401 @@ -106,6 +107,78 @@ def work_dir(): pytestmark = [pytest.mark.harness_live, pytest.mark.asyncio] +# ════════════════════════════════════════════════════════════════════════ +# AFORGE +# ════════════════════════════════════════════════════════════════════════ + + +@pytest.mark.skipif(not HAS_AFORGE, reason="aforge CLI not installed") +class TestAforgeLive: + """Live tests against Aforge's canonical ``aforge do --json`` mode.""" + + async def test_basic_prompt(self, work_dir: str) -> None: + """Provider returns text plus measured benchmark usage.""" + from agentfield.harness.providers.aforge import AforgeProvider + + result = await AforgeProvider().execute( + "Reply with exactly: HELLO_AGENTFIELD", + {"cwd": work_dir}, + ) + + assert not result.is_error, f"Aforge returned error: {result.error_message}" + assert result.result is not None + assert "HELLO_AGENTFIELD" in result.result + assert result.metrics.num_turns > 0 + assert result.metrics.input_tokens > 0 + assert result.metrics.output_tokens > 0 + assert result.metrics.total_cost_usd is not None + assert result.metrics.total_cost_usd > 0 + print( + "aforge metrics", + { + "duration_ms": result.metrics.duration_api_ms, + "calls": result.metrics.num_turns, + "input_tokens": result.metrics.input_tokens, + "output_tokens": result.metrics.output_tokens, + "cache_read_tokens": result.metrics.cache_read_tokens, + "cost_usd": result.metrics.total_cost_usd, + }, + ) + + async def test_schema_pipeline(self, work_dir: str) -> None: + """Full schema pipeline writes, parses, and cleans isolated output.""" + result = await HarnessRunner().run( + 'Return exactly: greeting="Hello from Aforge" and number=42. ' + "Follow the OUTPUT REQUIREMENTS below precisely.", + provider="aforge", + schema=SimpleResponse, + cwd=work_dir, + max_retries=1, + ) + + assert not result.is_error, ( + f"Schema pipeline failed: {result.error_message}\n" + f"Raw result: {result.result!r}" + ) + assert isinstance(result.parsed, SimpleResponse) + assert result.parsed.greeting == "Hello from Aforge" + assert result.parsed.number == 42 + print( + "aforge schema metrics", + { + "duration_ms": result.duration_ms, + "calls": result.num_turns, + "input_tokens": result.input_tokens, + "output_tokens": result.output_tokens, + "cache_read_tokens": result.cache_read_tokens, + "cost_usd": result.cost_usd, + }, + ) + assert not any( + name.startswith(".agentfield-out-") for name in os.listdir(work_dir) + ) + + # ════════════════════════════════════════════════════════════════════════ # CODEX # ════════════════════════════════════════════════════════════════════════ diff --git a/sdk/python/tests/test_harness_provider_aforge.py b/sdk/python/tests/test_harness_provider_aforge.py index 1ae720b0f..6f38498ea 100644 --- a/sdk/python/tests/test_harness_provider_aforge.py +++ b/sdk/python/tests/test_harness_provider_aforge.py @@ -16,9 +16,31 @@ def mock_aforge_available(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr( "agentfield.harness._availability.shutil.which", lambda path: path ) + monkeypatch.setenv("AGENTFIELD_AFORGE_COMMAND", "do") def _envelope( + text: str = "done", + *, + usage: dict[str, object] | None = None, + settled: bool = True, + blocked_on: str | None = None, +) -> str: + envelope = { + "deliverable": text, + "usage": usage or {}, + "artifacts": [], + "nodes": 2, + "seconds": 0.012, + "settled": settled, + "spend": 0.0, + } + if blocked_on is not None: + envelope["blocked_on"] = blocked_on + return json.dumps(envelope) + + +def _exec_envelope( text: str = "done", *, stop: str = "done", @@ -55,7 +77,6 @@ async def fake_run_cli( "cached_tokens": 20, "cost": 0.0123, }, - turns=4, ), "", 0, @@ -72,12 +93,103 @@ async def fake_run_cli( assert raw.metrics.output_tokens == 50 assert raw.metrics.cache_read_tokens == 20 assert raw.metrics.cache_creation_tokens == 0 - assert raw.metrics.num_turns == 4 + assert raw.metrics.num_turns == 3 assert raw.metrics.total_cost_usd == 0.0123 assert raw.metrics.model == "openrouter/z-ai/glm-5.2" assert raw.metrics.duration_api_ms >= 0 assert raw.returncode == 0 - assert raw.messages[0]["text"] == " final answer " + assert raw.messages[0]["deliverable"] == " final answer " + + +@pytest.mark.asyncio +async def test_aforge_exec_mode_maps_original_contract_and_pins_model( + monkeypatch: pytest.MonkeyPatch, +): + captured: dict[str, object] = {} + + async def fake_run_cli( + cmd, *, env=None, cwd=None, timeout=None, idle_seconds=None, input_text=None + ): + captured.update(cmd=cmd, env=env, input_text=input_text) + return ( + _exec_envelope( + " linear answer ", + usage={ + "calls": 3, + "prompt_tokens": 100, + "completion_tokens": 50, + "cached_tokens": 20, + "cost": 0.0123, + }, + turns=4, + ), + "", + 0, + ) + + monkeypatch.delenv("AGENTFIELD_AFORGE_COMMAND") + monkeypatch.setattr("agentfield.harness.providers.aforge.run_cli", fake_run_cli) + raw = await AforgeProvider("/opt/aforge").execute( + "prompt that stays off argv", + { + "project_dir": "/project", + "system_prompt": " be precise ", + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + }, + ) + + assert captured["cmd"] == [ + "/opt/aforge", + "exec", + "--json", + "-w", + "/project", + "--timeout", + "1795", + "--context-fill", + "60", + "--completion-reserve", + "65536", + "--system", + "be precise", + "--model", + "deepseek/deepseek-v4-flash-0731", + "--plan-model", + "deepseek/deepseek-v4-flash-0731", + ] + assert captured["env"] == { + "AFORGE_MODELS": "", + "AFORGE_MODEL": "deepseek/deepseek-v4-flash-0731", + } + assert captured["input_text"] == "prompt that stays off argv" + assert raw.result == "linear answer" + assert raw.is_error is False + assert raw.metrics.num_turns == 4 + assert raw.metrics.input_tokens == 100 + assert raw.metrics.total_cost_usd == 0.0123 + + +@pytest.mark.asyncio +async def test_aforge_exec_mode_accepts_budget_partial( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setenv("AGENTFIELD_AFORGE_COMMAND", "exec") + monkeypatch.setattr( + "agentfield.harness.providers.aforge.run_cli", + AsyncMock(return_value=(_exec_envelope("usable", stop="budget"), "", 2)), + ) + + raw = await AforgeProvider().execute("hello", {}) + + assert raw.result == "usable" + assert raw.is_error is False + assert raw.failure_type is FailureType.NONE + + +def test_aforge_binary_environment_override(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("AFORGE_BIN", "/opt/aforge-env") + assert AforgeProvider()._bin == "/opt/aforge-env" + assert AforgeProvider("/explicit/aforge")._bin == "/explicit/aforge" @pytest.mark.asyncio @@ -200,7 +312,7 @@ async def fake_run_cli( @pytest.mark.asyncio -async def test_aforge_passes_stripped_system_prompt_flag( +async def test_aforge_prepends_stripped_system_prompt_to_stdin( monkeypatch: pytest.MonkeyPatch, ): captured: dict[str, Any] = {} @@ -208,15 +320,17 @@ async def test_aforge_passes_stripped_system_prompt_flag( async def fake_run_cli( cmd, *, env=None, cwd=None, timeout=None, idle_seconds=None, input_text=None ): - _ = env, cwd, timeout, idle_seconds, input_text + _ = env, cwd, timeout, idle_seconds captured["cmd"] = cmd + captured["input_text"] = input_text return _envelope(), "", 0 monkeypatch.setattr("agentfield.harness.providers.aforge.run_cli", fake_run_cli) await AforgeProvider().execute("hello", {"system_prompt": " be precise "}) - assert captured["cmd"][-2:] == ["--system", "be precise"] + assert "--system" not in captured["cmd"] + assert captured["input_text"] == "be precise\n\nTask:\nhello" @pytest.mark.asyncio @@ -240,19 +354,25 @@ async def fake_run_cli( ) await provider.execute("hello", {"cwd": "/cwd-only"}) - assert captured_cmds[0][:6] == [ + assert captured_cmds[0] == [ "aforge", - "exec", + "do", "--json", + "--yes-spend", "-w", "/project", + "--timeout", + "1795", ] - assert captured_cmds[1][:6] == [ + assert captured_cmds[1] == [ "aforge", - "exec", + "do", "--json", + "--yes-spend", "-w", "/cwd-only", + "--timeout", + "1795", ] @@ -281,7 +401,7 @@ async def fake_run_cli( cmd, *, env=None, cwd=None, timeout=None, idle_seconds=None, input_text=None ): _ = cmd, env, cwd, timeout, idle_seconds, input_text - return _envelope("", stop="error"), "authentication exploded", 5 + return _envelope(""), "authentication exploded", 1 monkeypatch.setattr("agentfield.harness.providers.aforge.run_cli", fake_run_cli) @@ -289,38 +409,38 @@ async def fake_run_cli( assert raw.is_error is True assert raw.failure_type is FailureType.CRASH - assert "aforge exit code 5" in (raw.error_message or "") + assert "aforge exit code 1" in (raw.error_message or "") assert "authentication exploded" in (raw.error_message or "") @pytest.mark.asyncio -async def test_aforge_budget_exit_with_text_is_success( +async def test_aforge_timeout_exit_with_partial_is_timeout( monkeypatch: pytest.MonkeyPatch, ): async def fake_run_cli( cmd, *, env=None, cwd=None, timeout=None, idle_seconds=None, input_text=None ): _ = cmd, env, cwd, timeout, idle_seconds, input_text - return _envelope("usable landing", stop="budget"), "", 2 + return _envelope("usable partial", settled=False), "", 2 monkeypatch.setattr("agentfield.harness.providers.aforge.run_cli", fake_run_cli) raw = await AforgeProvider().execute("hello", {}) - assert raw.result == "usable landing" - assert raw.is_error is False - assert raw.failure_type is FailureType.NONE + assert raw.result == "usable partial" + assert raw.is_error is True + assert raw.failure_type is FailureType.TIMEOUT @pytest.mark.asyncio -async def test_aforge_turn_cap_without_text_is_crash( +async def test_aforge_blocked_question_is_crash( monkeypatch: pytest.MonkeyPatch, ): async def fake_run_cli( cmd, *, env=None, cwd=None, timeout=None, idle_seconds=None, input_text=None ): _ = cmd, env, cwd, timeout, idle_seconds, input_text - return _envelope("", stop="turn-cap"), "turn cap reached", 3 + return _envelope("", blocked_on="Which repository?"), "", 1 monkeypatch.setattr("agentfield.harness.providers.aforge.run_cli", fake_run_cli) @@ -328,7 +448,8 @@ async def fake_run_cli( assert raw.is_error is True assert raw.failure_type is FailureType.CRASH - assert "aforge exit code 3" in (raw.error_message or "") + assert "aforge exit code 1" in (raw.error_message or "") + assert "Which repository?" in (raw.error_message or "") @pytest.mark.asyncio @@ -389,6 +510,24 @@ async def fake_run_cli( assert raw.is_error is False +@pytest.mark.asyncio +async def test_aforge_parses_pretty_printed_envelope( + monkeypatch: pytest.MonkeyPatch, +): + async def fake_run_cli( + cmd, *, env=None, cwd=None, timeout=None, idle_seconds=None, input_text=None + ): + _ = cmd, env, cwd, timeout, idle_seconds, input_text + return json.dumps(json.loads(_envelope("pretty result")), indent=2), "", 0 + + monkeypatch.setattr("agentfield.harness.providers.aforge.run_cli", fake_run_cli) + + raw = await AforgeProvider().execute("hello", {}) + + assert raw.result == "pretty result" + assert raw.is_error is False + + @pytest.mark.asyncio async def test_aforge_missing_or_zero_cost_falls_back_to_estimate( monkeypatch: pytest.MonkeyPatch, diff --git a/sdk/python/tests/test_harness_provider_claude.py b/sdk/python/tests/test_harness_provider_claude.py index 1fa3b0b86..0997b2116 100644 --- a/sdk/python/tests/test_harness_provider_claude.py +++ b/sdk/python/tests/test_harness_provider_claude.py @@ -87,6 +87,30 @@ def fake_query(*, prompt: str, options: FakeClaudeAgentOptions): assert len(raw.messages) == 2 +@pytest.mark.asyncio +async def test_execute_defaults_model_to_sonnet(monkeypatch): + from agentfield.harness.providers.claude import ClaudeCodeProvider + + captured: dict[str, Any] = {} + + class FakeClaudeAgentOptions: + def __init__(self, **kwargs: Any) -> None: + self.kwargs = kwargs + + def fake_query(*, prompt: str, options: FakeClaudeAgentOptions): + captured["options"] = options + return _AsyncStream([{"type": "result", "result": "ok"}]) + + fake_sdk = ModuleType("claude_agent_sdk") + setattr(fake_sdk, "ClaudeAgentOptions", FakeClaudeAgentOptions) + setattr(fake_sdk, "query", fake_query) + monkeypatch.setitem(__import__("sys").modules, "claude_agent_sdk", fake_sdk) + + await ClaudeCodeProvider().execute("hello", {}) + + assert captured["options"].kwargs["model"] == "sonnet" + + @pytest.mark.asyncio async def test_execute_extracts_result_from_subtype_success(monkeypatch): """Claude Agent SDK sends subtype='success' instead of type='result'.""" diff --git a/sdk/python/tests/test_harness_runner.py b/sdk/python/tests/test_harness_runner.py index cbb689ff0..033f428e6 100644 --- a/sdk/python/tests/test_harness_runner.py +++ b/sdk/python/tests/test_harness_runner.py @@ -265,10 +265,21 @@ async def test_run_with_schema_injects_prompt_suffix_and_parses_output(tmp_path) @pytest.mark.asyncio -async def test_run_raises_when_no_provider_set(tmp_path): +async def test_run_defaults_to_aforge_when_no_provider_set(tmp_path, monkeypatch): + monkeypatch.delenv("AGENTFIELD_HARNESS_PROVIDER", raising=False) runner = HarnessRunner() - with pytest.raises(ValueError, match="No harness provider specified"): - await runner.run("hello", cwd=str(tmp_path)) + provider = MockProvider([RawResult(result="ok")]) + captured: dict[str, str] = {} + + def fake_build_provider(config): + captured["provider"] = config.provider + return provider + + with patch("agentfield.harness._runner.build_provider", fake_build_provider): + result = await runner.run("hello", cwd=str(tmp_path)) + + assert captured["provider"] == "aforge" + assert result.result == "ok" @pytest.mark.asyncio diff --git a/sdk/python/tests/test_harness_types.py b/sdk/python/tests/test_harness_types.py index 26f206c45..1a95db42a 100644 --- a/sdk/python/tests/test_harness_types.py +++ b/sdk/python/tests/test_harness_types.py @@ -1,23 +1,21 @@ # pyright: reportMissingImports=false import pytest -from pydantic import ValidationError - from agentfield.harness._result import HarnessResult, Metrics, RawResult from agentfield.harness.providers._factory import build_provider from agentfield.types import HarnessConfig -def test_harness_config_provider_required(): - with pytest.raises(ValidationError): - HarnessConfig() +def test_harness_config_provider_defaults_to_aforge(monkeypatch): + monkeypatch.delenv("AGENTFIELD_HARNESS_PROVIDER", raising=False) + assert HarnessConfig().provider == "aforge" def test_harness_config_defaults(): cfg = HarnessConfig(provider="codex") assert cfg.provider == "codex" - assert cfg.model == "sonnet" + assert cfg.model is None assert cfg.max_turns == 30 assert cfg.max_budget_usd is None assert cfg.max_retries == 3 diff --git a/sdk/python/tests/test_types.py b/sdk/python/tests/test_types.py index 9ee9805ef..03bfeb696 100644 --- a/sdk/python/tests/test_types.py +++ b/sdk/python/tests/test_types.py @@ -470,7 +470,7 @@ class TestHarnessConfig: def test_defaults(self): hc = HarnessConfig(provider="claude-code") assert hc.provider == "claude-code" - assert hc.model == "sonnet" + assert hc.model is None assert hc.max_turns == 30 assert hc.max_budget_usd is None assert hc.max_retries == 3 @@ -493,9 +493,9 @@ def test_custom_values(self): assert hc.tools == ["Bash"] assert hc.permission_mode == "auto" - def test_provider_required(self): - with pytest.raises(Exception): - HarnessConfig() # type: ignore[call-arg] + def test_provider_defaults_to_aforge(self, monkeypatch): + monkeypatch.delenv("AGENTFIELD_HARNESS_PROVIDER", raising=False) + assert HarnessConfig().provider == "aforge" def test_json_roundtrip(self): hc = HarnessConfig(provider="gemini", model="gemini-2.5-flash") diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json index 49696665b..1c597ad8f 100644 --- a/sdk/typescript/package.json +++ b/sdk/typescript/package.json @@ -14,7 +14,7 @@ "dev": "tsup --watch", "lint": "tsc --noEmit", "test": "vitest run --config vitest.config.ts", - "test:functional": "vitest run tests/harness_functional.test.ts --timeout=300000", + "test:functional": "vitest run --config vitest.functional.config.ts", "test:core": "vitest run --config vitest.config.ts", "test:coverage": "vitest run --config vitest.config.ts --coverage", "test:coverage:core": "vitest run --config vitest.config.ts --coverage", diff --git a/sdk/typescript/src/harness/cli.ts b/sdk/typescript/src/harness/cli.ts index f279460fc..2ed9e82f8 100644 --- a/sdk/typescript/src/harness/cli.ts +++ b/sdk/typescript/src/harness/cli.ts @@ -33,20 +33,27 @@ export function runCli( cwd?: string; timeout?: number; idleSeconds?: number; + inputText?: string; } ): Promise { return new Promise((resolve, reject) => { const [bin, ...args] = cmd; const env = { ...process.env, ...options?.env }; applyOpenRouterAttributionEnv(env); + const hasInput = options?.inputText !== undefined; // 'ignore' on stdin gives the child an immediate EOF instead of an open - // pipe that never closes (a hang risk if the child probes stdin). + // pipe that never closes (a hang risk if the child probes stdin). Providers + // with large or sensitive prompts can explicitly pipe text over stdin. const proc = spawn(bin, args, { env, cwd: options?.cwd, - stdio: ['ignore', 'pipe', 'pipe'], + stdio: [hasInput ? 'pipe' : 'ignore', 'pipe', 'pipe'], }); + if (hasInput) { + proc.stdin?.end(options.inputText); + } + let stdout = ''; let stderr = ''; let settled = false; @@ -54,11 +61,11 @@ export function runCli( // Both stdout and stderr are drained concurrently via their own 'data' // listeners, so a full stderr pipe cannot deadlock the read of stdout. - proc.stdout.on('data', (data: Uint8Array | string) => { + proc.stdout!.on('data', (data: Uint8Array | string) => { stdout += data.toString(); lastActivity = Date.now(); }); - proc.stderr.on('data', (data: Uint8Array | string) => { + proc.stderr!.on('data', (data: Uint8Array | string) => { stderr += data.toString(); lastActivity = Date.now(); }); diff --git a/sdk/typescript/src/harness/providers/aforge.ts b/sdk/typescript/src/harness/providers/aforge.ts new file mode 100644 index 000000000..5aa8667b5 --- /dev/null +++ b/sdk/typescript/src/harness/providers/aforge.ts @@ -0,0 +1,286 @@ +import type { HarnessProvider } from './base.js'; +import type { RawResult } from '../types.js'; +import { createMetrics, createRawResult } from '../types.js'; +import { runCli } from '../cli.js'; +import { resolveModelAndVariant } from '../modelVariant.js'; + +const REASONING_VARIANTS = new Set(['off', 'low', 'medium', 'high']); +const DEFAULT_TIMEOUT_SECONDS = 1800; +const LANDING_WINDOW_SECONDS = 5; +const DEFAULT_MAX_CONCURRENT = 8; +const ANSI_PATTERN = /\x1B\[[0-?]*[ -/]*[@-~]/g; + +class Semaphore { + private active = 0; + private readonly waiters: Array<() => void> = []; + + public constructor(private readonly limit: number) {} + + public async use(operation: () => Promise): Promise { + await this.acquire(); + try { + return await operation(); + } finally { + this.release(); + } + } + + private acquire(): Promise { + if (this.active < this.limit) { + this.active += 1; + return Promise.resolve(); + } + return new Promise((resolve) => { + this.waiters.push(() => { + this.active += 1; + resolve(); + }); + }); + } + + private release(): void { + this.active -= 1; + this.waiters.shift()?.(); + } +} + +function resolveMaxConcurrent(): number { + const parsed = Number.parseInt(process.env.AFORGE_MAX_CONCURRENT ?? '', 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_MAX_CONCURRENT; +} + +const aforgeSemaphore = new Semaphore(resolveMaxConcurrent()); + +function stripOpenRouterPrefix(model: string): string { + return model.startsWith('openrouter/') ? model.slice('openrouter/'.length) : model; +} + +function parseEnvelope(stdout: string): Record | undefined { + // Both canonical `do` and `exec` print one JSON object. Parse that shape + // before the wrapper-compatible line scan. + try { + const value: unknown = JSON.parse(stdout.trim()); + if (typeof value === 'object' && value !== null && !Array.isArray(value) + && ('deliverable' in value || 'text' in value)) { + return value as Record; + } + } catch { + // Fall through to the wrapper-compatible line scan. + } + + const lines = stdout.split('\n').map((line) => line.trim()).filter(Boolean); + for (let index = lines.length - 1; index >= 0; index -= 1) { + try { + const value: unknown = JSON.parse(lines[index]); + if (typeof value === 'object' && value !== null && !Array.isArray(value) + && ('deliverable' in value || 'text' in value)) { + return value as Record; + } + } catch { + // Tolerate stray stdout from a wrapper and keep looking for the envelope. + } + } + return undefined; +} + +function numeric(value: unknown): number | undefined { + return typeof value === 'number' && Number.isFinite(value) ? value : undefined; +} + +function timeoutSeconds(): number { + const parsed = Number.parseInt(process.env.AGENTFIELD_HARNESS_TIMEOUT_SECONDS ?? '', 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_TIMEOUT_SECONDS; +} + +function innerTimeout(outer: number): number { + return outer > LANDING_WINDOW_SECONDS ? outer - LANDING_WINDOW_SECONDS : 1; +} + +function taskInput(prompt: string, systemPrompt: unknown): string { + return typeof systemPrompt === 'string' && systemPrompt.trim() + ? `${systemPrompt.trim()}\n\nTask:\n${prompt}` + : prompt; +} + +function crashMessage(exitCode: number, blockedOn: string, deliverable: string | undefined, stderr: string): string { + const cleanStderr = stderr.trim().replace(ANSI_PATTERN, ''); + const exitContext = `aforge exit code ${exitCode}`; + let message = exitCode < 0 ? `Process killed by signal ${-exitCode}. ${exitContext}` : exitContext; + if (cleanStderr) { + message += `. stderr: ${cleanStderr.slice(0, 1000)}`; + } else if (blockedOn) { + message += `. blocked_on: ${blockedOn.slice(0, 1000)}`; + } else if (deliverable) { + message += `. partial: ${deliverable.slice(0, 1000)}`; + } + return message; +} + +function stringOptions(value: unknown): Record { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + return {}; + } + const result: Record = {}; + for (const [key, item] of Object.entries(value)) { + if (typeof item === 'string') { + result[key] = item; + } + } + return result; +} + +/** + * Aforge CLI provider. `exec` is the default direct one-shot entry point; set + * `AGENTFIELD_AFORGE_COMMAND=do` to opt into Aforge's routed workflow. + */ +export class AforgeProvider implements HarnessProvider { + private readonly bin: string; + + public constructor(bin = 'aforge') { + this.bin = bin === 'aforge' ? (process.env.AFORGE_BIN?.trim() || bin) : bin; + } + + public async execute(prompt: string, options: Record): Promise { + return aforgeSemaphore.use(() => this.executeImpl(prompt, options)); + } + + private async executeImpl(prompt: string, options: Record): Promise { + const projectDir = typeof options.projectDir === 'string' + ? options.projectDir + : typeof options.project_dir === 'string' + ? options.project_dir + : undefined; + const cwd = typeof options.cwd === 'string' ? options.cwd : undefined; + const root = projectDir ?? cwd ?? '.'; + const outerTimeout = timeoutSeconds(); + const command = (process.env.AGENTFIELD_AFORGE_COMMAND ?? 'exec').trim().toLowerCase(); + if (command !== 'do' && command !== 'exec') { + return createRawResult({ + isError: true, + errorMessage: `AGENTFIELD_AFORGE_COMMAND must be 'do' or 'exec', got ${JSON.stringify(command)}`, + failureType: 'crash', + metrics: createMetrics(), + }); + } + const systemPrompt = options.systemPrompt ?? options.system_prompt; + const cmd = command === 'exec' + ? [ + this.bin, + 'exec', + '--json', + '-w', + root, + '--timeout', + String(innerTimeout(outerTimeout)), + '--context-fill', + '60', + '--completion-reserve', + '65536', + ] + : [ + this.bin, + 'do', + '--json', + '--yes-spend', + '-w', + root, + '--timeout', + String(innerTimeout(outerTimeout)), + ]; + if (command === 'exec' && typeof systemPrompt === 'string' && systemPrompt.trim()) { + cmd.push('--system', systemPrompt.trim()); + } + + const { model, variant } = resolveModelAndVariant(options); + const env: Record = command === 'exec' ? { AFORGE_MODELS: '' } : {}; + if (model) { + const slug = stripOpenRouterPrefix(model); + env.AFORGE_MODEL = slug; + if (command === 'exec') { + cmd.push('--model', slug, '--plan-model', slug); + } + } + if (variant) { + const normalized = variant.trim().toLowerCase(); + if (REASONING_VARIANTS.has(normalized)) { + env.AFORGE_EXEC_REASONING = normalized; + } + } + Object.assign(env, stringOptions(options.env)); + + const startApi = Date.now(); + try { + const { stdout, stderr, exitCode } = await runCli(cmd, { + env, + cwd: undefined, + timeout: outerTimeout * 1000, + idleSeconds: 0, + inputText: command === 'exec' ? prompt : taskInput(prompt, systemPrompt), + }); + const envelope = parseEnvelope(stdout); + const outputValue = command === 'exec' ? envelope?.text : envelope?.deliverable; + const resultText = typeof outputValue === 'string' && outputValue.trim() + ? outputValue.trim() + : undefined; + const blockedOn = typeof envelope?.blocked_on === 'string' ? envelope.blocked_on.trim() : ''; + const stop = typeof envelope?.stop === 'string' ? envelope.stop.trim() : ''; + const usage = typeof envelope?.usage === 'object' && envelope.usage !== null && !Array.isArray(envelope.usage) + ? envelope.usage as Record + : {}; + + const isError = command === 'exec' + ? exitCode < 0 || resultText === undefined || ![0, 2, 3].includes(exitCode) + : exitCode !== 0 || resultText === undefined || blockedOn !== ''; + const inputTokens = Math.trunc(numeric(usage.prompt_tokens) ?? 0); + const outputTokens = Math.trunc(numeric(usage.completion_tokens) ?? 0); + const cacheReadTokens = Math.trunc(numeric(usage.cached_tokens) ?? 0); + const calls = Math.trunc(numeric(command === 'exec' ? envelope?.turns : usage.calls) ?? 0); + const nativeSpend = numeric(envelope?.spend); + const legacyCost = numeric(usage.cost); + const providerCost = nativeSpend !== undefined && nativeSpend > 0 ? nativeSpend : legacyCost; + + return createRawResult({ + result: resultText, + messages: envelope ? [envelope] : [], + metrics: createMetrics({ + durationApiMs: Date.now() - startApi, + numTurns: calls, + totalCostUsd: providerCost !== undefined && providerCost > 0 ? providerCost : undefined, + usage, + sessionId: '', + inputTokens, + outputTokens, + cacheReadTokens, + cacheCreationTokens: 0, + totalTokens: inputTokens + outputTokens, + model, + }), + isError, + errorMessage: isError ? crashMessage(exitCode, blockedOn || stop, resultText, stderr) : undefined, + failureType: isError + ? ((command === 'do' && exitCode === 2) || (command === 'exec' && exitCode === 4) + ? 'timeout' + : 'crash') + : 'none', + returnCode: exitCode, + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (message.includes('ENOENT')) { + return createRawResult({ + isError: true, + errorMessage: `AForge binary not found at '${this.bin}'. Install it with \`af aforge ensure\`, or set AFORGE_BIN to its path.`, + failureType: 'crash', + metrics: createMetrics({ durationApiMs: Date.now() - startApi }), + }); + } + const timedOut = /timed out|deadline exceeded|no progress/i.test(message); + return createRawResult({ + isError: true, + errorMessage: message, + failureType: timedOut ? 'timeout' : 'crash', + metrics: createMetrics({ durationApiMs: Date.now() - startApi }), + }); + } + } +} diff --git a/sdk/typescript/src/harness/providers/factory.ts b/sdk/typescript/src/harness/providers/factory.ts index 432d904a7..ebf9d97c9 100644 --- a/sdk/typescript/src/harness/providers/factory.ts +++ b/sdk/typescript/src/harness/providers/factory.ts @@ -1,29 +1,51 @@ import type { HarnessProvider } from './base.js'; import type { HarnessConfig } from '../types.js'; -export const SUPPORTED_PROVIDERS = new Set(['claude-code', 'codex', 'gemini', 'opencode']); +export const SUPPORTED_PROVIDERS = new Set(['aforge', 'claude-code', 'codex', 'gemini', 'opencode']); +export const DEFAULT_HARNESS_PROVIDER = 'aforge'; +export const HARNESS_PROVIDER_ENV_VAR = 'AGENTFIELD_HARNESS_PROVIDER'; + +/** + * Applies harness provider precedence: an explicit name wins, then + * AGENTFIELD_HARNESS_PROVIDER, then DEFAULT_HARNESS_PROVIDER ("aforge"). + * Blank / whitespace-only values are treated as unset. + */ +export function resolveProviderName(explicit?: string): string { + const explicitName = explicit?.trim(); + if (explicitName) { + return explicitName; + } + + const envName = process.env[HARNESS_PROVIDER_ENV_VAR]?.trim(); + return envName || DEFAULT_HARNESS_PROVIDER; +} export async function buildProvider(config: HarnessConfig): Promise { - if (!SUPPORTED_PROVIDERS.has(config.provider)) { + const provider = resolveProviderName(config.provider); + if (!SUPPORTED_PROVIDERS.has(provider)) { throw new Error( - `Unknown harness provider: "${config.provider}". Supported: ${[...SUPPORTED_PROVIDERS].sort().join(', ')}` + `Unknown harness provider: "${provider}". Supported: ${[...SUPPORTED_PROVIDERS].sort().join(', ')}` ); } - if (config.provider === 'claude-code') { + if (provider === 'aforge') { + const { AforgeProvider } = await import('./aforge.js'); + return new AforgeProvider(config.aforgeBin ?? 'aforge'); + } + if (provider === 'claude-code') { const { ClaudeCodeProvider } = await import('./claude.js'); return new ClaudeCodeProvider(); } - if (config.provider === 'codex') { + if (provider === 'codex') { const { CodexProvider } = await import('./codex.js'); return new CodexProvider(config.codexBin ?? 'codex'); } - if (config.provider === 'gemini') { + if (provider === 'gemini') { const { GeminiProvider } = await import('./gemini.js'); return new GeminiProvider(config.geminiBin ?? 'gemini'); } - if (config.provider === 'opencode') { + if (provider === 'opencode') { const { OpenCodeProvider } = await import('./opencode.js'); return new OpenCodeProvider(config.opencodeBin ?? 'opencode'); } - throw new Error(`Provider "${config.provider}" is not yet implemented.`); + throw new Error(`Provider "${provider}" is not yet implemented.`); } diff --git a/sdk/typescript/src/harness/providers/index.ts b/sdk/typescript/src/harness/providers/index.ts index c754fcf25..ae479f4e2 100644 --- a/sdk/typescript/src/harness/providers/index.ts +++ b/sdk/typescript/src/harness/providers/index.ts @@ -1,5 +1,6 @@ export type { HarnessProvider } from './base.js'; export { buildProvider, SUPPORTED_PROVIDERS } from './factory.js'; +export { AforgeProvider } from './aforge.js'; export { ClaudeCodeProvider } from './claude.js'; export { CodexProvider } from './codex.js'; export { GeminiProvider } from './gemini.js'; diff --git a/sdk/typescript/src/harness/runner.ts b/sdk/typescript/src/harness/runner.ts index 55d8dfe7b..722ed80b1 100644 --- a/sdk/typescript/src/harness/runner.ts +++ b/sdk/typescript/src/harness/runner.ts @@ -1,5 +1,7 @@ +import fs from 'node:fs'; +import path from 'node:path'; import { buildPromptSuffix, cleanupTempFiles, getOutputPath, parseAndValidate } from './schema.js'; -import { buildProvider } from './providers/factory.js'; +import { buildProvider, resolveProviderName } from './providers/factory.js'; import type { HarnessProvider } from './providers/base.js'; import { createHarnessResult, @@ -41,6 +43,8 @@ type RunnerOptions = Omit & { initialDelay?: number; maxDelay?: number; backoffFactor?: number; + projectDir?: string; + aforgeBin?: string; codexBin?: string; geminiBin?: string; opencodeBin?: string; @@ -52,27 +56,32 @@ export class HarnessRunner { public async run(prompt: string, options: HarnessOptions = {}) { const { schema, ...rest } = options; const resolved = this.resolveOptions(this.config, rest); + resolved.provider = resolveProviderName(resolved.provider); - if (!resolved.provider) { - throw new Error("No harness provider specified. Set 'provider' in HarnessConfig or pass it to .harness() call."); - } - - const cwd = resolved.cwd ?? '.'; const provider = await this.buildProvider(resolved.provider, resolved); - const effectivePrompt = schema === undefined ? prompt : `${prompt}${buildPromptSuffix(schema, cwd)}`; + const cwd = resolved.cwd ?? '.'; + const outputRoot = resolved.projectDir ?? cwd; + let outputDir: string | undefined; + if (schema !== undefined) { + fs.mkdirSync(outputRoot, { recursive: true }); + outputDir = fs.mkdtempSync(path.join(outputRoot, '.agentfield-out-')); + } + const effectivePrompt = schema === undefined ? prompt : `${prompt}${buildPromptSuffix(schema, outputDir!)}`; const startTime = Date.now(); try { const raw = await this.executeWithRetry(provider, effectivePrompt, resolved); if (schema !== undefined) { - return this.handleSchemaOutput(raw, schema, cwd, startTime); + return this.handleSchemaOutput(raw, schema, outputDir!, startTime); } return createHarnessResult({ result: raw.result, isError: raw.isError, errorMessage: raw.errorMessage, + failureType: raw.failureType, + returnCode: raw.returnCode, costUsd: raw.metrics.totalCostUsd, numTurns: raw.metrics.numTurns, durationMs: Date.now() - startTime, @@ -82,7 +91,8 @@ export class HarnessRunner { }); } finally { if (schema !== undefined) { - cleanupTempFiles(cwd); + cleanupTempFiles(outputDir!); + fs.rmSync(outputDir!, { recursive: true, force: true }); } } } @@ -105,6 +115,8 @@ export class HarnessRunner { 'systemPrompt', 'env', 'cwd', + 'projectDir', + 'aforgeBin', 'codexBin', 'geminiBin', 'opencodeBin', @@ -179,6 +191,8 @@ export class HarnessRunner { result: raw.result, parsed, isError: false, + failureType: raw.failureType, + returnCode: raw.returnCode, costUsd: raw.metrics.totalCostUsd, numTurns: raw.metrics.numTurns, durationMs: Date.now() - startTime, @@ -192,6 +206,8 @@ export class HarnessRunner { result: raw.result, isError: true, errorMessage: 'Schema validation failed after parse and cosmetic repair attempts.', + failureType: 'schema', + returnCode: raw.returnCode, costUsd: raw.metrics.totalCostUsd, numTurns: raw.metrics.numTurns, durationMs: Date.now() - startTime, @@ -203,7 +219,7 @@ export class HarnessRunner { private async buildProvider(providerName: string, options: RunnerOptions): Promise { const { provider: _, ...rest } = options; - return buildProvider({ provider: providerName as HarnessConfig['provider'], ...rest }); + return buildProvider({ provider: providerName as NonNullable, ...rest }); } private computeBackoffDelay( diff --git a/sdk/typescript/src/harness/types.ts b/sdk/typescript/src/harness/types.ts index 397fc320f..e1385e8fa 100644 --- a/sdk/typescript/src/harness/types.ts +++ b/sdk/typescript/src/harness/types.ts @@ -1,5 +1,11 @@ export interface HarnessConfig { - provider: 'claude-code' | 'codex' | 'gemini' | 'opencode'; + /** + * Coding agent provider. Defaults to `aforge`, AgentField's native harness. + * When unset, `AGENTFIELD_HARNESS_PROVIDER` is consulted before the default. + * An explicit value always wins. + */ + provider?: 'aforge' | 'claude-code' | 'codex' | 'gemini' | 'opencode'; + /** Model identifier. Empty means the provider's own default. */ model?: string; /** * Provider-specific reasoning-effort variant (e.g. `high`, `minimal`). @@ -17,13 +23,21 @@ export interface HarnessConfig { systemPrompt?: string; env?: Record; cwd?: string; + projectDir?: string; + aforgeBin?: string; codexBin?: string; geminiBin?: string; opencodeBin?: string; } export interface HarnessOptions { + /** + * Coding agent provider. Defaults to `aforge`, AgentField's native harness. + * When unset, `AGENTFIELD_HARNESS_PROVIDER` is consulted before the default. + * An explicit value always wins. + */ provider?: string; + /** Model identifier. Empty means the provider's own default. */ model?: string; /** * Provider-specific reasoning-effort variant (e.g. `high`, `minimal`). @@ -41,6 +55,8 @@ export interface HarnessOptions { systemPrompt?: string; env?: Record; cwd?: string; + projectDir?: string; + aforgeBin?: string; codexBin?: string; geminiBin?: string; opencodeBin?: string; @@ -64,12 +80,16 @@ export interface Metrics { model?: string; } +export type FailureType = 'none' | 'crash' | 'timeout' | 'api_error' | 'no_output' | 'schema'; + export interface RawResult { result?: string; messages: Array>; metrics: Metrics; isError: boolean; errorMessage?: string; + failureType?: FailureType; + returnCode?: number; } export interface HarnessResult { @@ -77,6 +97,8 @@ export interface HarnessResult { parsed?: unknown; isError: boolean; errorMessage?: string; + failureType?: FailureType; + returnCode?: number; costUsd?: number; numTurns: number; durationMs: number; diff --git a/sdk/typescript/tests/harness_cli.test.ts b/sdk/typescript/tests/harness_cli.test.ts index 367308806..896c9a01b 100644 --- a/sdk/typescript/tests/harness_cli.test.ts +++ b/sdk/typescript/tests/harness_cli.test.ts @@ -20,10 +20,11 @@ class MockStream extends EventEmitter { } type MockChild = EventEmitter & - Pick; + Pick; const createProcess = (): MockChild => { const proc = new EventEmitter() as MockChild; + proc.stdin = { end: vi.fn() } as unknown as ChildProcessWithoutNullStreams['stdin']; proc.stdout = new MockStream() as ChildProcessWithoutNullStreams['stdout']; proc.stderr = new MockStream() as ChildProcessWithoutNullStreams['stderr']; proc.kill = vi.fn(); @@ -90,6 +91,23 @@ describe('harness cli utilities', () => { await expect(pending).resolves.toMatchObject({ exitCode: 0 }); }); + it('pipes an explicit prompt over stdin and closes the stream', async () => { + const proc = createProcess(); + spawnMock.mockReturnValueOnce(proc as unknown as ReturnType); + + const pending = runCli(['aforge', 'do', '--json'], { inputText: 'keep me off argv' }); + + expect(spawnMock).toHaveBeenCalledWith('aforge', ['do', '--json'], { + env: expect.any(Object), + cwd: undefined, + stdio: ['pipe', 'pipe', 'pipe'] + }); + expect(proc.stdin.end).toHaveBeenCalledWith('keep me off argv'); + + proc.emit('close', 0); + await expect(pending).resolves.toMatchObject({ exitCode: 0 }); + }); + it('rejects on child process errors and on timeouts', async () => { const errorProc = createProcess(); spawnMock.mockReturnValueOnce(errorProc as unknown as ReturnType); diff --git a/sdk/typescript/tests/harness_functional.test.ts b/sdk/typescript/tests/harness_functional.test.ts index 3f0993562..a0214696f 100644 --- a/sdk/typescript/tests/harness_functional.test.ts +++ b/sdk/typescript/tests/harness_functional.test.ts @@ -4,11 +4,12 @@ * These tests invoke REAL coding agents and make real API calls. * They are NOT included in the default `vitest run` — run explicitly: * - * npx vitest run tests/harness_functional.test.ts --timeout=300000 + * npx vitest run --config vitest.functional.config.ts * * Run a single provider: - * npx vitest run tests/harness_functional.test.ts -t "Codex" --timeout=300000 - * npx vitest run tests/harness_functional.test.ts -t "OpenCode" --timeout=300000 + * npx vitest run --config vitest.functional.config.ts -t "Aforge" + * npx vitest run --config vitest.functional.config.ts -t "Codex" + * npx vitest run --config vitest.functional.config.ts -t "OpenCode" */ import { describe, it, expect, beforeEach, afterEach } from 'vitest'; @@ -33,6 +34,8 @@ function hasBinary(name: string): boolean { const HAS_CODEX = hasBinary('codex'); const HAS_OPENCODE = hasBinary('opencode'); +const HAS_AFORGE = hasBinary('aforge') + && Boolean(process.env.OPENROUTER_API_KEY || process.env.OPENAI_API_KEY); /** Plain JSON Schema — no Zod dependency needed. */ const simpleSchema = { @@ -77,6 +80,60 @@ function cleanupWorkDir(dir: string): void { } } +// ════════════════════════════════════════════════════════════════════════ +// AFORGE +// ════════════════════════════════════════════════════════════════════════ + +describe.skipIf(!HAS_AFORGE)('Aforge Functional', () => { + beforeEach(() => { + workDir = createWorkDir(); + }); + afterEach(() => { + cleanupWorkDir(workDir); + }); + + it('basic prompt returns the exec JSON deliverable and provider metrics', async () => { + const { AforgeProvider } = await import('../src/harness/providers/aforge.js'); + const result = await new AforgeProvider().execute( + 'Reply with exactly: HELLO_AGENTFIELD', + { cwd: workDir }, + ); + + expect(result.isError).toBe(false); + expect(result.result).toContain('HELLO_AGENTFIELD'); + expect(result.messages).toHaveLength(1); + expect(result.metrics.numTurns).toBeGreaterThan(0); + expect(result.metrics.inputTokens).toBeGreaterThan(0); + expect(result.metrics.outputTokens).toBeGreaterThan(0); + expect(result.metrics.totalCostUsd).toBeGreaterThan(0); + console.info('aforge metrics', JSON.stringify(result.metrics)); + }, 300_000); + + it('schema pipeline writes, parses, and cleans isolated output', async () => { + const result = await new HarnessRunner().run( + 'Return exactly: greeting="Hello from Aforge" and number=42. Follow the OUTPUT REQUIREMENTS below precisely.', + { + provider: 'aforge', + schema: simpleSchema, + cwd: workDir, + maxRetries: 1, + }, + ); + + expect(result.isError).toBe(false); + expect(result.parsed).toMatchObject({ greeting: 'Hello from Aforge', number: 42 }); + console.info('aforge schema metrics', JSON.stringify({ + durationMs: result.durationMs, + calls: result.numTurns, + inputTokens: result.inputTokens, + outputTokens: result.outputTokens, + cacheReadTokens: result.cacheReadTokens, + costUsd: result.costUsd, + })); + expect(fs.readdirSync(workDir).filter((name) => name.startsWith('.agentfield-out-'))).toEqual([]); + }, 300_000); +}); + // ════════════════════════════════════════════════════════════════════════ // CODEX // ════════════════════════════════════════════════════════════════════════ diff --git a/sdk/typescript/tests/harness_provider_aforge.test.ts b/sdk/typescript/tests/harness_provider_aforge.test.ts new file mode 100644 index 000000000..267d51ca9 --- /dev/null +++ b/sdk/typescript/tests/harness_provider_aforge.test.ts @@ -0,0 +1,265 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { AforgeProvider } from '../src/harness/providers/aforge.js'; +import { buildProvider, SUPPORTED_PROVIDERS } from '../src/harness/providers/factory.js'; +import * as cli from '../src/harness/cli.js'; + +function envelope( + deliverable = 'done', + options: { settled?: boolean; blockedOn?: string; usage?: Record } = {} +): string { + return JSON.stringify({ + settled: options.settled ?? true, + deliverable, + blocked_on: options.blockedOn ?? '', + spend_usd: 0.0123, + elapsed_ms: 12, + usage: options.usage ?? {}, + }); +} + +function execEnvelope( + text = 'done', + options: { stop?: string; usage?: Record; turns?: number } = {} +): string { + return JSON.stringify({ + text, + stop: options.stop ?? 'done', + usage: options.usage ?? {}, + artifacts: [], + turns: options.turns ?? 1, + elapsed_ms: 12, + }); +} + +afterEach(() => { + vi.restoreAllMocks(); + delete process.env.AGENTFIELD_HARNESS_TIMEOUT_SECONDS; + delete process.env.AFORGE_BIN; + delete process.env.AGENTFIELD_AFORGE_COMMAND; +}); + +beforeEach(() => { + process.env.AGENTFIELD_AFORGE_COMMAND = 'do'; +}); + +describe('aforge provider', () => { + it('honors AFORGE_BIN unless a binary is explicit', async () => { + process.env.AFORGE_BIN = '/opt/aforge-env'; + vi.spyOn(cli, 'runCli').mockResolvedValue({ stdout: envelope(), stderr: '', exitCode: 0 }); + + await new AforgeProvider().execute('hello', {}); + await new AforgeProvider('/explicit/aforge').execute('hello', {}); + + expect(vi.mocked(cli.runCli).mock.calls[0][0][0]).toBe('/opt/aforge-env'); + expect(vi.mocked(cli.runCli).mock.calls[1][0][0]).toBe('/explicit/aforge'); + }); + + it('maps the do command, stdin prompt, JSON envelope, and metrics', async () => { + vi.spyOn(cli, 'runCli').mockResolvedValue({ + stdout: envelope(' final answer ', { + usage: { + calls: 3, + prompt_tokens: 100, + completion_tokens: 50, + cached_tokens: 20, + cost: 0.0123, + }, + }), + stderr: '', + exitCode: 0, + }); + + const result = await new AforgeProvider('/opt/aforge').execute('prompt that stays off argv', { + projectDir: '/project', + cwd: '/project/nested', + systemPrompt: ' be precise ', + model: 'openrouter/z-ai/glm-5.2#high', + }); + + expect(cli.runCli).toHaveBeenCalledWith( + ['/opt/aforge', 'do', '--json', '--yes-spend', '-w', '/project', '--timeout', '1795'], + { + env: { AFORGE_MODEL: 'z-ai/glm-5.2', AFORGE_EXEC_REASONING: 'high' }, + cwd: undefined, + timeout: 1_800_000, + idleSeconds: 0, + inputText: 'be precise\n\nTask:\nprompt that stays off argv', + } + ); + expect(result.result).toBe('final answer'); + expect(result.isError).toBe(false); + expect(result.failureType).toBe('none'); + expect(result.returnCode).toBe(0); + expect(result.metrics).toMatchObject({ + numTurns: 3, + totalCostUsd: 0.0123, + inputTokens: 100, + outputTokens: 50, + cacheReadTokens: 20, + cacheCreationTokens: 0, + totalTokens: 150, + model: 'openrouter/z-ai/glm-5.2', + }); + expect(result.messages[0].deliverable).toBe(' final answer '); + }); + + it('maps the opt-in exec command and original envelope contract', async () => { + delete process.env.AGENTFIELD_AFORGE_COMMAND; + vi.spyOn(cli, 'runCli').mockResolvedValue({ + stdout: execEnvelope(' linear answer ', { + turns: 4, + usage: { + calls: 3, + prompt_tokens: 100, + completion_tokens: 50, + cached_tokens: 20, + cost: 0.0123, + }, + }), + stderr: '', + exitCode: 0, + }); + + const result = await new AforgeProvider('/opt/aforge').execute('prompt that stays off argv', { + projectDir: '/project', + systemPrompt: ' be precise ', + model: 'openrouter/deepseek/deepseek-v4-flash-0731#high', + }); + + expect(cli.runCli).toHaveBeenCalledWith( + [ + '/opt/aforge', 'exec', '--json', '-w', '/project', '--timeout', '1795', + '--context-fill', '60', '--completion-reserve', '65536', + '--system', 'be precise', + '--model', 'deepseek/deepseek-v4-flash-0731', + '--plan-model', 'deepseek/deepseek-v4-flash-0731', + ], + { + env: { + AFORGE_MODELS: '', + AFORGE_MODEL: 'deepseek/deepseek-v4-flash-0731', + AFORGE_EXEC_REASONING: 'high', + }, + cwd: undefined, + timeout: 1_800_000, + idleSeconds: 0, + inputText: 'prompt that stays off argv', + } + ); + expect(result.result).toBe('linear answer'); + expect(result.isError).toBe(false); + expect(result.metrics.numTurns).toBe(4); + expect(result.metrics.inputTokens).toBe(100); + expect(result.metrics.totalCostUsd).toBe(0.0123); + }); + + it('accepts an exec budget partial with usable text', async () => { + process.env.AGENTFIELD_AFORGE_COMMAND = 'exec'; + vi.spyOn(cli, 'runCli').mockResolvedValue({ + stdout: execEnvelope('usable', { stop: 'budget', turns: 2 }), + stderr: '', + exitCode: 2, + }); + + const result = await new AforgeProvider().execute('hello', {}); + + expect(result.result).toBe('usable'); + expect(result.isError).toBe(false); + expect(result.failureType).toBe('none'); + }); + + it('uses cwd as the root and gives aforge a timeout landing window', async () => { + process.env.AGENTFIELD_HARNESS_TIMEOUT_SECONDS = '2400'; + vi.spyOn(cli, 'runCli').mockResolvedValue({ stdout: envelope(), stderr: '', exitCode: 0 }); + + await new AforgeProvider().execute('hello', { cwd: '/cwd-only' }); + + expect(cli.runCli).toHaveBeenCalledWith( + ['aforge', 'do', '--json', '--yes-spend', '-w', '/cwd-only', '--timeout', '2395'], + expect.objectContaining({ timeout: 2_400_000, idleSeconds: 0, inputText: 'hello' }) + ); + }); + + it('ignores unknown variants and lets caller env override derived env', async () => { + vi.spyOn(cli, 'runCli').mockResolvedValue({ stdout: envelope(), stderr: '', exitCode: 0 }); + const provider = new AforgeProvider(); + + await provider.execute('hello', { model: 'openrouter/x/y#turbo' }); + await provider.execute('hello', { + model: 'openrouter/x/y#low', + variant: 'HIGH', + env: { AFORGE_MODEL: 'override/model', AFORGE_EXEC_REASONING: 'off', EXTRA: '1' }, + }); + + expect(vi.mocked(cli.runCli).mock.calls[0][1]?.env).toEqual({ AFORGE_MODEL: 'x/y' }); + expect(vi.mocked(cli.runCli).mock.calls[1][1]?.env).toEqual({ + AFORGE_MODEL: 'override/model', + AFORGE_EXEC_REASONING: 'off', + EXTRA: '1', + }); + }); + + it.each([ + ['success', 0, 'done', '', 'none', false], + ['timeout with partial', 2, 'usable', '', 'timeout', true], + ['blocked', 1, '', 'Which repository?', 'crash', true], + ['zero without deliverable', 0, '', '', 'crash', true], + ['signal', -9, '', '', 'crash', true], + ])('applies %s exit semantics', async (_name, exitCode, deliverable, blockedOn, failureType, wantError) => { + vi.spyOn(cli, 'runCli').mockResolvedValue({ + stdout: envelope(deliverable as string, { settled: exitCode === 0, blockedOn: blockedOn as string }), + stderr: '', + exitCode: exitCode as number, + }); + + const result = await new AforgeProvider().execute('hello', {}); + + expect(result.isError).toBe(wantError); + expect(result.failureType).toBe(failureType); + }); + + it('parses the last envelope and leaves zero cost unknown', async () => { + vi.spyOn(cli, 'runCli').mockResolvedValue({ + stdout: `stray diagnostic\n{"type":"event"}\n${envelope('real result', { usage: { calls: 1, cost: 0 } })}`, + stderr: '', + exitCode: 0, + }); + + const result = await new AforgeProvider().execute('hello', {}); + + expect(result.result).toBe('real result'); + expect(result.metrics.totalCostUsd).toBeUndefined(); + }); + + it('parses the canonical pretty-printed envelope', async () => { + const pretty = JSON.stringify(JSON.parse(envelope('pretty result')), null, 2); + vi.spyOn(cli, 'runCli').mockResolvedValue({ stdout: pretty, stderr: '', exitCode: 0 }); + + const result = await new AforgeProvider().execute('hello', {}); + + expect(result.result).toBe('pretty result'); + expect(result.isError).toBe(false); + }); + + it('classifies missing binaries and timeouts', async () => { + vi.spyOn(cli, 'runCli').mockRejectedValueOnce(new Error('spawn aforge ENOENT')); + const missing = await new AforgeProvider('aforge-missing').execute('hello', {}); + expect(missing.isError).toBe(true); + expect(missing.failureType).toBe('crash'); + expect(missing.errorMessage).toContain('aforge-missing'); + + vi.spyOn(cli, 'runCli').mockRejectedValueOnce(new Error('CLI timed out after 1ms')); + const timeout = await new AforgeProvider().execute('hello', {}); + expect(timeout.isError).toBe(true); + expect(timeout.failureType).toBe('timeout'); + }); +}); + +describe('aforge provider factory', () => { + it('registers aforge and threads aforgeBin', async () => { + expect(SUPPORTED_PROVIDERS.has('aforge')).toBe(true); + const provider = await buildProvider({ provider: 'aforge', aforgeBin: '/opt/aforge' }); + expect(provider).toBeInstanceOf(AforgeProvider); + }); +}); diff --git a/sdk/typescript/tests/harness_provider_factory.test.ts b/sdk/typescript/tests/harness_provider_factory.test.ts new file mode 100644 index 000000000..236af0d35 --- /dev/null +++ b/sdk/typescript/tests/harness_provider_factory.test.ts @@ -0,0 +1,58 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { AforgeProvider } from '../src/harness/providers/aforge.js'; +import { + buildProvider, + HARNESS_PROVIDER_ENV_VAR, + resolveProviderName, +} from '../src/harness/providers/factory.js'; +import type { HarnessConfig } from '../src/harness/types.js'; + +let originalProviderEnv: string | undefined; + +beforeEach(() => { + originalProviderEnv = process.env[HARNESS_PROVIDER_ENV_VAR]; + delete process.env[HARNESS_PROVIDER_ENV_VAR]; +}); + +afterEach(() => { + if (originalProviderEnv === undefined) { + delete process.env[HARNESS_PROVIDER_ENV_VAR]; + } else { + process.env[HARNESS_PROVIDER_ENV_VAR] = originalProviderEnv; + } +}); + +describe('harness provider factory', () => { + it('defaults to aforge', () => { + expect(resolveProviderName(undefined)).toBe('aforge'); + }); + + it('honours the environment fallback', () => { + process.env[HARNESS_PROVIDER_ENV_VAR] = 'codex'; + expect(resolveProviderName(undefined)).toBe('codex'); + }); + + it('prefers an explicit provider over the environment', () => { + process.env[HARNESS_PROVIDER_ENV_VAR] = 'codex'; + expect(resolveProviderName('gemini')).toBe('gemini'); + }); + + it('treats blank explicit and environment values as unset', () => { + expect(resolveProviderName(' ')).toBe('aforge'); + + process.env[HARNESS_PROVIDER_ENV_VAR] = ' '; + expect(resolveProviderName(undefined)).toBe('aforge'); + }); + + it('builds the aforge provider when no provider is configured', async () => { + await expect(buildProvider({})).resolves.toBeInstanceOf(AforgeProvider); + }); + + it('rejects genuinely unknown providers with the supported list', async () => { + const config = { provider: 'nope' } as unknown as HarnessConfig; + await expect(buildProvider(config)).rejects.toThrow( + 'Unknown harness provider: "nope". Supported: aforge, claude-code, codex, gemini, opencode' + ); + }); +}); diff --git a/sdk/typescript/tests/harness_runner.test.ts b/sdk/typescript/tests/harness_runner.test.ts index 1fe393bc3..304ce35b4 100644 --- a/sdk/typescript/tests/harness_runner.test.ts +++ b/sdk/typescript/tests/harness_runner.test.ts @@ -21,6 +21,7 @@ function makeTempDir(): string { afterEach(() => { vi.restoreAllMocks(); + vi.unstubAllEnvs(); for (const dir of tempDirs.splice(0, tempDirs.length)) { fs.rmSync(dir, { recursive: true, force: true }); } @@ -45,13 +46,19 @@ class MockProvider implements HarnessProvider { } class FileWritingProvider extends MockProvider { + public readonly outputPaths: string[] = []; + public constructor(private readonly payload: string, result?: RawResult) { super([result ?? createRawResult({ result: 'ok' })]); } public override async execute(prompt: string, options: Record): Promise { - const cwd = typeof options.cwd === 'string' ? options.cwd : '.'; - fs.writeFileSync(getOutputPath(cwd), this.payload, 'utf8'); + const match = prompt.match(/(\S*\.agentfield_output\.json)/); + if (!match) { + throw new Error('schema prompt did not contain an output path'); + } + this.outputPaths.push(match[1]); + fs.writeFileSync(match[1], this.payload, 'utf8'); return super.execute(prompt, options); } } @@ -68,6 +75,8 @@ describe('harness runner', () => { systemPrompt: 'base', env: { A: '1' }, cwd: '/tmp/base', + projectDir: '/tmp/project', + aforgeBin: 'aforge', codexBin: 'codex', geminiBin: 'gemini', opencodeBin: 'opencode', @@ -88,6 +97,8 @@ describe('harness runner', () => { expect(options.maxBudgetUsd).toBe(2); expect(options.env).toEqual({ B: '2' }); expect(options.cwd).toBe('/tmp/override'); + expect(options.projectDir).toBe('/tmp/project'); + expect(options.aforgeBin).toBe('aforge'); }); it('isTransient matches transient errors and rejects non-transient', () => { @@ -128,6 +139,8 @@ describe('harness runner', () => { const provider = new MockProvider([ createRawResult({ result: 'done', + failureType: 'none', + returnCode: 0, metrics: createMetrics({ numTurns: 2, totalCostUsd: 0.42, sessionId: 'sess-1' }), }), ]); @@ -142,6 +155,8 @@ describe('harness runner', () => { expect(result.costUsd).toBe(0.42); expect(result.numTurns).toBe(2); expect(result.sessionId).toBe('sess-1'); + expect(result.failureType).toBe('none'); + expect(result.returnCode).toBe(0); }); it('run with schema injects suffix and parses output', async () => { @@ -154,14 +169,67 @@ describe('harness runner', () => { const result = await runner.run('produce json', { provider: 'codex', schema, cwd }); expect(provider.lastPrompt).toContain('OUTPUT REQUIREMENTS'); - expect(provider.lastPrompt).toContain(getOutputPath(cwd)); + expect(provider.outputPaths).toHaveLength(1); + expect(provider.lastPrompt).toContain(provider.outputPaths[0]); + expect(provider.outputPaths[0]).toMatch(/\.agentfield-out-[^/]+\/\.agentfield_output\.json$/); expect(result.isError).toBe(false); expect(result.parsed).toEqual({ name: 'ok', count: 1 }); + expect(fs.existsSync(provider.outputPaths[0])).toBe(false); }); - it('run throws when no provider is configured', async () => { + it('isolates concurrent schema runs that share one cwd', async () => { + const cwd = makeTempDir(); + const schema = z.object({ name: z.string(), count: z.number() }); + const outputPaths: string[] = []; + const provider: HarnessProvider = { + async execute(prompt: string): Promise { + const match = prompt.match(/(\S*\.agentfield_output\.json)/); + if (!match) { + throw new Error('schema prompt did not contain an output path'); + } + outputPaths.push(match[1]); + const first = prompt.startsWith('first'); + await new Promise((resolve) => setTimeout(resolve, 10)); + fs.writeFileSync(match[1], JSON.stringify({ name: first ? 'first' : 'second', count: first ? 1 : 2 })); + return createRawResult({ result: 'done' }); + }, + }; + vi.spyOn(factory, 'buildProvider').mockResolvedValue(provider); + const runner = new HarnessRunner(); - await expect(runner.run('hello', {})).rejects.toThrow(/No harness provider specified/); + const [first, second] = await Promise.all([ + runner.run('first', { provider: 'aforge', schema, cwd }), + runner.run('second', { provider: 'aforge', schema, cwd }), + ]); + + expect(first.parsed).toEqual({ name: 'first', count: 1 }); + expect(second.parsed).toEqual({ name: 'second', count: 2 }); + expect(new Set(outputPaths).size).toBe(2); + expect(fs.readdirSync(cwd).filter((name) => name.startsWith('.agentfield-out-'))).toEqual([]); + expect(fs.existsSync(getOutputPath(cwd))).toBe(false); + }); + + it('run defaults to aforge when no provider is configured', async () => { + vi.stubEnv('AGENTFIELD_HARNESS_PROVIDER', ''); + const provider = new MockProvider(); + const buildProviderSpy = vi.spyOn(factory, 'buildProvider').mockResolvedValue(provider); + const runner = new HarnessRunner(); + await runner.run('hello', {}); + + expect(buildProviderSpy).toHaveBeenCalledWith(expect.objectContaining({ provider: 'aforge' })); + expect(provider.lastOptions?.provider).toBe('aforge'); + }); + + it('per-call provider wins over the runner default and environment', async () => { + vi.stubEnv('AGENTFIELD_HARNESS_PROVIDER', 'gemini'); + const provider = new MockProvider(); + const buildProviderSpy = vi.spyOn(factory, 'buildProvider').mockResolvedValue(provider); + const runner = new HarnessRunner({ provider: 'codex' }); + + await runner.run('hello', { provider: 'opencode' }); + + expect(buildProviderSpy).toHaveBeenCalledWith(expect.objectContaining({ provider: 'opencode' })); + expect(provider.lastOptions?.provider).toBe('opencode'); }); it('retries on transient error then succeeds', async () => { diff --git a/sdk/typescript/vitest.functional.config.ts b/sdk/typescript/vitest.functional.config.ts new file mode 100644 index 000000000..32814c370 --- /dev/null +++ b/sdk/typescript/vitest.functional.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + include: ['tests/harness_functional.test.ts'], + testTimeout: 300_000, + hookTimeout: 30_000, + }, +});