Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 40 additions & 8 deletions packages/agent-harness/src/detect-max-turns.ts
Original file line number Diff line number Diff line change
@@ -1,23 +1,55 @@
/**
* Detect max-turns exhaustion from agent CLI stdout/stderr.
*
* Claude Code (`-p` / stdin) emits `Error: Reached max turns (N)` on stdout with exit code 1.
* Match known phrases across both streams since agents differ.
* Claude Code (`-p` / stdin) emits `Error: Reached max turns (N)` on stdout
* with exit code 1. Match that as a complete diagnostic line — a substring
* scan of the whole transcript also sees this repository's own source, lint
* dumps, and diffs (DEV-70 Codex cron false positive).
*
* Do not treat stderr as trusted: Codex writes its tool transcript there.
* Callers should also skip detection when the harness cannot impose a CLI
* turn limit (`supportsMaxTurns`).
*/

import { isSourceOrDiffLine, outputLines } from "./output-lines.js";

const MAX_TURNS_PATTERNS = [
/Reached max turns/i,
/max turns reached/i,
/maximum turns reached/i,
/^(?:error:\s*)?reached max turns(?:\s*\(\d+\))?[.!]?$/i,
/^(?:error:\s*)?max(?:imum)? turns reached[.!]?$/i,
] as const;

/**
* Return the diagnostic line that indicates a max-turns limit, if any.
*
* @param stdout - Captured standard output
* @param stderr - Captured standard error
*/
export function findMaxTurnsReachedLine(stdout: string, stderr: string): string | undefined {
const lines = [...outputLines(stderr), ...outputLines(stdout)];
const matched = lines.find((line) => {
const normalized = line.normalized.trim();
if (!normalized || isSourceOrDiffLine(normalized)) {
return false;
}
return MAX_TURNS_PATTERNS.some((pattern) => pattern.test(normalized));
});
return matched?.raw.trim();
}

/**
* Return whether agent output indicates the conversation hit a max-turns limit.
*
* @param stdout - Captured standard output
* @param stderr - Captured standard error
* @param supportsMaxTurns - When false, skip scanning (harness cannot hit a CLI turn limit)
*/
export function detectMaxTurnsReached(stdout: string, stderr: string): boolean {
const combined = `${stdout}\n${stderr}`;
return MAX_TURNS_PATTERNS.some((pattern) => pattern.test(combined));
export function detectMaxTurnsReached(
stdout: string,
stderr: string,
supportsMaxTurns: boolean = true,
): boolean {
if (!supportsMaxTurns) {
return false;
}
return findMaxTurnsReachedLine(stdout, stderr) !== undefined;
}
50 changes: 3 additions & 47 deletions packages/agent-harness/src/detect-usage-limit.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@
* attempting text-based detection of opencode plan limits.
*/

import type { OutputLine } from "./output-lines.js";
import { isSourceOrDiffLine, outputLines } from "./output-lines.js";

const USAGE_LIMIT_PATTERNS = [
// Keep subscription messages anchored to the whole line. Codex writes its
// tool transcript to stderr, so a substring match also sees source such as
Expand All @@ -59,13 +62,6 @@ const PROVIDER_LIMIT_PATTERNS = [
/\b429\b/,
] as const;

const ANSI_ESCAPE_PATTERN = new RegExp(`${String.fromCodePoint(0x1b)}\\[[0-?]*[ -/]*[@-~]`, "g");

interface OutputLine {
raw: string;
normalized: string;
}

const RESET_PATTERNS = [
// "resets 7:20pm (Asia/Ho_Chi_Minh)", "resets at 9am", "resets in 2 hours"
/resets?\s+(?:at\s+|in\s+)?([0-9][^\n.]*?)(?:\.|\n|$)/i,
Expand Down Expand Up @@ -101,46 +97,6 @@ function extractResetHint(text: string): string | undefined {
return undefined;
}

/**
* Strip terminal styling before matching, while retaining the original line
* for diagnostics.
*/
function stripAnsi(text: string): string {
return text.replace(ANSI_ESCAPE_PATTERN, "");
}

/**
* Split one captured stream into matchable lines.
*/
function outputLines(text: string): OutputLine[] {
return text.split(/\r?\n/).map((raw) => ({
raw,
normalized: stripAnsi(raw),
}));
}

/**
* Avoid treating source code, comments, quoted Markdown, search results, and
* diff hunks as provider errors. Agent transcripts can include arbitrary file
* content from tools such as `sed`, `rg`, and `git diff`, including literal
* provider diagnostics such as "429 Too Many Requests".
*/
function isSourceOrDiffLine(line: string): boolean {
const trimmed = line.trim();
return (
/^(?:diff --git|index\b|---\s|\+\+\+\s|@@\s)/i.test(trimmed) ||
/^[+-](?![+-])/.test(trimmed) ||
/^(?:\/\/|\/\*|\*|#|>|```|~~~)/.test(trimmed) ||
// `rg -n`, grep, and compiler-style locations: path:line[:column]:content.
/^(?:(?:\.?\.?\/|\/)?(?:[^:\s]+\/)+[^:\s]+|[^:\s]+\.[a-z\d]+):\d+(?::\d+)?:/i.test(trimmed) ||
/^(?:const|let|var|function|class|import|export|return)\b/.test(trimmed) ||
/^(?:super|throw\s+new\s+Error|[\w$.]+\.(?:error|warn|log))\s*\(/.test(trimmed) ||
/^(?:["'`]).*(?:["'`])[,;)]?$/.test(trimmed) ||
/\b(?:includes|startsWith|endsWith|\.match|\.test)\s*\(/.test(trimmed) ||
/=>/.test(trimmed)
);
}

/**
* Provider rate-limit phrases need stronger evidence than a subscription
* limit phrase. In particular, a bare `429` or `Too Many Requests` in agent
Expand Down
1 change: 1 addition & 0 deletions packages/agent-harness/src/harnesses/claude-code.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ export class ClaudeCodeHarness implements AgentHarness {
readonly defaultPath = "claude";
readonly promptFlag = "-p";
readonly supportedModes = ["plan", "readonly"] as const;
readonly supportsMaxTurns = true;

/**
* Build `claude` CLI flags for non-interactive (`-p`) execution.
Expand Down
2 changes: 1 addition & 1 deletion packages/agent-harness/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ export {
} from "./spawn-agent.js";

// Max-turns detection
export { detectMaxTurnsReached } from "./detect-max-turns.js";
export { detectMaxTurnsReached, findMaxTurnsReachedLine } from "./detect-max-turns.js";

// Usage/rate-limit detection
export { detectUsageLimit, resetHintToMs, type UsageLimitResult } from "./detect-usage-limit.js";
Expand Down
53 changes: 53 additions & 0 deletions packages/agent-harness/src/output-lines.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
/**
* Line-level helpers for scanning agent CLI transcripts.
*
* Codex writes its full tool transcript to stderr, so callers must not treat
* either stream as trusted diagnostics. Split into lines, strip ANSI, and skip
* source / diff / search output before matching error phrases.
*/

export interface OutputLine {
raw: string;
normalized: string;
}

const ANSI_ESCAPE_PATTERN = new RegExp(`${String.fromCodePoint(0x1b)}\\[[0-?]*[ -/]*[@-~]`, "g");

/**
* Strip terminal styling before matching, while retaining the original line
* for diagnostics.
*/
export function stripAnsi(text: string): string {
return text.replace(ANSI_ESCAPE_PATTERN, "");
}

/**
* Split one captured stream into matchable lines.
*/
export function outputLines(text: string): OutputLine[] {
return text.split(/\r?\n/).map((raw) => ({
raw,
normalized: stripAnsi(raw),
}));
}

/**
* Avoid treating source code, comments, quoted Markdown, search results, and
* diff hunks as provider errors. Agent transcripts can include arbitrary file
* content from tools such as `sed`, `rg`, and `git diff`.
*/
export function isSourceOrDiffLine(line: string): boolean {
const trimmed = line.trim();
return (
/^(?:diff --git|index\b|---\s|\+\+\+\s|@@\s)/i.test(trimmed) ||
/^[+-](?![+-])/.test(trimmed) ||
/^(?:\/\/|\/\*|\*|#|>|```|~~~)/.test(trimmed) ||
// `rg -n`, grep, and compiler-style locations: path:line[:column]:content.
/^(?:(?:\.?\.?\/|\/)?(?:[^:\s]+\/)+[^:\s]+|[^:\s]+\.[a-z\d]+):\d+(?::\d+)?:/i.test(trimmed) ||
/^(?:const|let|var|function|class|import|export|return)\b/.test(trimmed) ||
/^(?:super|throw\s+new\s+Error|[\w$.]+\.(?:error|warn|log))\s*\(/.test(trimmed) ||
/^(?:["'`]).*(?:["'`])[,;)]?$/.test(trimmed) ||
/\b(?:includes|startsWith|endsWith|\.match|\.test)\s*\(/.test(trimmed) ||
/=>/.test(trimmed)
);
}
2 changes: 1 addition & 1 deletion packages/agent-harness/src/runners/bun.ts
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,6 @@ export async function runAgentBun(
stdout,
stderr,
exitCode,
maxTurnsReached: detectMaxTurnsReached(stdout, stderr),
maxTurnsReached: detectMaxTurnsReached(stdout, stderr, harness.supportsMaxTurns === true),
};
}
2 changes: 1 addition & 1 deletion packages/agent-harness/src/runners/node.ts
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ export async function runAgentNode(
stdout,
stderr,
exitCode: code ?? 1,
maxTurnsReached: detectMaxTurnsReached(stdout, stderr),
maxTurnsReached: detectMaxTurnsReached(stdout, stderr, harness.supportsMaxTurns === true),
});
}
});
Expand Down
7 changes: 7 additions & 0 deletions packages/agent-harness/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,13 @@ export interface AgentHarness {
* Empty / omitted means only `"default"` is supported.
*/
readonly supportedModes?: readonly Exclude<AgentRunMode, "default">[];
/**
* Whether this harness accepts a CLI turn limit (`--max-turns` or
* equivalent) and can emit a max-turns diagnostic. Omitted / false means
* callers skip transcript scanning so tool output cannot be mistaken for a
* turn-limit error.
*/
readonly supportsMaxTurns?: boolean;
/**
* Whether this harness's constrained modes still allow unrestricted
* network and MCP tool use (web search, web fetch, MCP servers).
Expand Down
40 changes: 39 additions & 1 deletion packages/agent-harness/tests/detect-max-turns.test.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,16 @@
import { describe, expect, test } from "bun:test";

import { detectMaxTurnsReached } from "../src/detect-max-turns.js";
import {
detectMaxTurnsReached,
findMaxTurnsReachedLine,
} from "../src/detect-max-turns.js";

describe("detectMaxTurnsReached", () => {
test("detects Claude Code stdout message", () => {
expect(detectMaxTurnsReached("Error: Reached max turns (1)\n", "")).toBe(true);
expect(findMaxTurnsReachedLine("Error: Reached max turns (1)\n", "")).toBe(
"Error: Reached max turns (1)",
);
});

test("detects message on stderr", () => {
Expand All @@ -22,4 +28,36 @@ describe("detectMaxTurnsReached", () => {
test("returns false for normal output", () => {
expect(detectMaxTurnsReached("Hello! How can I help you today?\n", "")).toBe(false);
});

test("skips scanning when the harness cannot impose a turn limit", () => {
expect(detectMaxTurnsReached("Error: Reached max turns (1)\n", "", false)).toBe(false);
});

test("ignores the phrase inside this repo's source and lint dumps", () => {
const transcript = [
`@getdevintern/code lint: 3445 │ │ \`\\n🔄 Moving \${taskKey} back to '\${todoStatus}' due to max turns reached...\`,`,
" /max turns reached/i,",
' console.log("⚠️ Agent reached maximum turns limit without completing the task");',
" // Check if Agent reached max turns or had other issues",
].join("\n");

expect(detectMaxTurnsReached(transcript, "")).toBe(false);
expect(detectMaxTurnsReached("", transcript)).toBe(false);
});

test("ignores compact diff additions", () => {
const diff = [
"diff --git a/packages/code/src/index.ts b/packages/code/src/index.ts",
`+ \`\\n🔄 Moving \${taskKey} back to '\${todoStatus}' due to max turns reached...\`,`,
].join("\n");

expect(detectMaxTurnsReached("", diff)).toBe(false);
});

test("ignores file-location search output", () => {
const source = "packages/code/src/index.ts:3445:due to max turns reached...";

expect(detectMaxTurnsReached(source, "")).toBe(false);
expect(detectMaxTurnsReached("", source)).toBe(false);
});
});
2 changes: 2 additions & 0 deletions packages/agent-harness/tests/harnesses.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ describe("ClaudeCodeHarness", () => {
expect(h.displayName).toBe("Claude Code");
expect(h.defaultPath).toBe("claude");
expect(h.promptFlag).toBe("-p");
expect(h.supportsMaxTurns).toBe(true);
});

test("buildArgs empty", () => {
Expand Down Expand Up @@ -76,6 +77,7 @@ describe("CodexHarness", () => {
expect(h.displayName).toBe("Codex");
expect(h.defaultPath).toBe("codex");
expect(h.promptFlag).toBeUndefined();
expect(h.supportsMaxTurns).toBeUndefined();
});

test("buildArgs empty", () => {
Expand Down
19 changes: 17 additions & 2 deletions packages/code/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import {
buildPromptArgs,
detectIncompleteImplementation,
detectMaxTurnsReached,
findMaxTurnsReachedLine,
detectOpenQuestions,
detectSandboxProviders,
detectUsageLimit,
Expand Down Expand Up @@ -2678,9 +2679,15 @@ async function runClarityCheck(
}

// Check if Agent reached max turns or had other issues
if (detectMaxTurnsReached(stdoutOutput, stderrOutput)) {
if (
detectMaxTurnsReached(stdoutOutput, stderrOutput, harness.supportsMaxTurns === true)
) {
console.log("\n⚠️ Clarity assessment reached maximum conversation turns");
console.log(" This may indicate task complexity or insufficient details");
const matchedLine = findMaxTurnsReachedLine(stdoutOutput, stderrOutput);
if (matchedLine) {
console.log(` Matched output: ${matchedLine}`);
}
if (!skipComments) {
console.log(
" Will attempt to proceed with implementation but posting failure to task tracker...\n",
Expand Down Expand Up @@ -3395,14 +3402,22 @@ async function runAgentHarness(
return;
}

const maxTurnsReached = detectMaxTurnsReached(stdoutOutput, stderrOutput);
const maxTurnsReached = detectMaxTurnsReached(
stdoutOutput,
stderrOutput,
harness.supportsMaxTurns === true,
);

if (maxTurnsReached) {
console.log("⚠️ Agent reached maximum turns limit without completing the task");
console.log(" The task may be too complex or require more turns to complete");
console.log(
" Consider breaking it into smaller tasks or increasing the max-turns limit",
);
const matchedLine = findMaxTurnsReachedLine(stdoutOutput, stderrOutput);
if (matchedLine) {
console.log(` Matched output: ${matchedLine}`);
}

// Save incomplete implementation for analysis
if (taskKey && stdoutOutput.trim()) {
Expand Down
6 changes: 5 additions & 1 deletion packages/code/src/lib/address-review.ts
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,11 @@ export async function runAgent(
agent.on("close", (code: number | null) => {
clearTimeout(timeout);
sandboxCleanup().catch(() => {});
const maxTurnsReached = detectMaxTurnsReached(stdoutOutput, stderrOutput);
const maxTurnsReached = detectMaxTurnsReached(
stdoutOutput,
stderrOutput,
harness.supportsMaxTurns === true,
);
const output = stdoutOutput + stderrOutput;

resolve({
Expand Down
Loading
Loading