qa.ts

/**
 * Headless QA (Phase 4) — run the target repo's `qa-agent` mode headlessly.
 *
 * Decision documented in docs/phase4-qa.md: the `qa-agent` mode's
 * core loop (start the app → run `node qa/runner.js` → read `qa/last-report.json`
 * → correlate with the server log → diagnose → fix → verify) is driven entirely
 * by read + execute_command (+ edit for remediation) tools. The Playwright MCP
 * server referenced in the mode's customInstructions is OPTIONAL enrichment
 * ("you can also use Playwright MCP tools directly") — the harness cannot serve
 * MCP (the vendored McpHub is a stub and MCP tools are excluded from
 * `selectToolsForMode`), so browser interaction is done via the repo's own
 * Playwright CLI (`qa/runner.js`, `npx playwright ...`) through
 * `execute_command`. That is choice (b)-via-CLI from the Phase 4 spec.
 *
 * `runQa` therefore runs a second harness session against a worktree in
 * `--mode qa-agent` (spliced from the target repo's `.roomodes` +
 * `.roo/rules-qa-agent/` automatically by the vendored prompt builder — same
 * mechanism as the reviewer). When the target repo has no `qa-agent` mode, a
 * generic QA checklist (`GENERIC_QA_CHECKLIST`) is used as a
 * `systemPromptOverride` so the session still gets a real QA operating
 * procedure.
 *
 * Non-edit discipline: QA may RUN the app + tests (execute_command) but must
 * NOT be able to modify source files unexpectedly. `write_to_file` is absent
 * from BOTH the advertised tool list (`qaTools()`) and the executor
 * (`createQaHeadlessExecutor`) — a model that calls it gets a clear
 * "not implemented" error and the loop's consecutive-mistake bound will
 * eventually trip.
 *
 * Verdict parsing mirrors the reviewer (`src/orchestrator/reviewer.ts`) but is
 * FAIL-CLOSED: a QA report that neither explicitly passes nor explicitly fails
 * is treated as `fail`, because QA gates a deploy — an inconclusive QA run must
 * never let a deployment through.
 */

import { execFileSync } from "node:child_process"
import * as fs from "node:fs"
import * as path from "node:path"

import { HeadlessSession } from "../engine/loop.js"
import { Logger } from "../engine/logger.js"
import { OllamaClient } from "../llm/ollama.js"
import { OpenRouterClient } from "../llm/openrouter.js"
import { envBoolean, resolvePerModeEnv } from "../cli.js"
import { appendBrowserActionTool, appendCodeIntelTools, loadCustomModes } from "../engine/prompt.js"
import { createQaHeadlessExecutor } from "../tools/executor.js"
import { getNativeTools } from "../vendor/zoo-code/src/core/prompts/tools/native-tools/index.js"
import { addCustomInstructions } from "../vendor/zoo-code/src/core/prompts/sections/custom-instructions.js"
import type { ChatTool, LlmClient, SessionResult } from "../engine/types.js"
import type { MemoryStore } from "../memory/types.js"
import type { SessionBudget } from "../budget/budget.js"

/** Default mode slug used for QA sessions (the qa-agent mode). */
export const DEFAULT_QA_MODE = "qa-agent"

/** Tools the QA agent may call: read + list + command + reporting. NO write. */
const QA_TOOL_NAMES = new Set([
	"read_file",
	"list_files",
	"execute_command",
	"attempt_completion",
	"ask_followup_question",
])

/**
 * The QA-mode tool schemas (read-only + command, no write tools), plus
 * browser_action and the four code-intelligence tools — QA can boot the app,
 * visually verify it in a headless browser (the phase4 gap "live
 * Playwright-MCP interactivity is not available headlessly" is partially
 * closed for agent-driven checks; see src/tools/browser/tool.ts), and
 * navigate source to substantiate findings (see src/codeintel/).
 */
export function qaTools(): ChatTool[] {
	const tools = getNativeTools().filter(
		(t) => t.type === "function" && QA_TOOL_NAMES.has(t.function.name),
	) as unknown as ChatTool[]
	return appendCodeIntelTools(appendBrowserActionTool(tools))
}

/**
 * Generic QA checklist used when the target repo defines no `qa-agent` mode.
 * The text deliberately opens with "You are a QA agent" so e2e mock
 * scenarios can classify QA sessions deterministically (see
 * scripts/e2e/mock-openrouter.mjs `qa-orchestrate`).
 */
export const GENERIC_QA_CHECKLIST = [
	"You are a QA agent for this project. Your job is to verify the work done in this workspace",
	"and report evidence with REAL command output. You operate in a loop: boot → test → exercise →",
	"report. You do not write or edit source files — QA is verification only; report problems,",
	"do not fix them.",
	"",
	"## How to perform QA",
	"",
	"0. FIRST, before anything else: run `git status --porcelain` and",
	"   `git diff <upstream>...HEAD` to see exactly what changed in this worktree — including",
	"   UNTRACKED files (this pipeline's workers frequently do not commit; an untracked new",
	"   file IS the change under review, same as a committed diff would be). Verify against",
	"   THAT real change, not against whatever existing file happens to be topically similar",
	"   or easiest to find — reviewing/exercising the wrong file produces a verdict about code",
	"   that was never actually touched.",
	"1. This checklist is generic (`npm test`/`pytest`/\"boot the application\") because it is a",
	"   fallback for when the repo defines no project-specific QA mode — it will not literally",
	"   apply to every repo (e.g. a kernel/systems project has no `npm start` to boot). Discover",
	"   the REAL build/verify/boot commands from the repo itself first — check the Makefile",
	"   (`make help`, or just read it), README, CI config, and `scripts/` — and use those,",
	"   not whatever this checklist's generic examples happen to name.",
	"2. Boot the application or relevant service using the repo's own start script / command.",
	"   Run it as a background process if it would block; set a timeout on every wait loop and",
	"   every command. Any scratch you need (probe scripts, temp output captures) goes in",
	"   `<workspace>/.headlesscode/scratch/` — NEVER write to `/tmp` or any other path outside",
	"   the workspace.",
	"3. Run the project's real test/verification suite and capture the REAL output — counts of",
	"   passed/failed tests (or the equivalent pass/fail signal for this repo's toolchain) are",
	"   the primary evidence.",
	"4. Exercise the changed behavior identified in step 0: run the relevant command, script,",
	"   or browser check (e.g. a Playwright runner script the repo already has) and capture output.",
	"5. If anything fails, report it — do not modify source files to make it pass.",
	"",
	"## Definition of done",
	"",
	"A task is complete when every check ran clean and the changed behavior works as expected.",
	"Report a final summary with evidence: the real commands you ran and their output, test",
	"counts, and any errors found.",
	"",
	"## How to finish",
	"",
	"Call attempt_completion with a structured summary:",
	"- A verdict line: `QA PASS` (all checks clean) or `QA FAIL` (any error, failed test, or",
	"  behavior that does not work).",
	"- An `## Evidence` section listing each command you ran and its real output.",
	"- The final baseline numbers you personally confirmed.",
].join("\n")

export interface RunQaOptions {
	/** The worktree to run QA against (checked out on the worker's branch). */
	workspaceRoot: string
	/** The QA checklist task text (default: built from the mode/workspace). */
	taskText?: string
	/** Mode slug (default: `qa-agent` — the QA mode, auto-spliced). */
	mode?: string
	/** Model id (default: env OPENROUTER_MODEL / client default). */
	model?: string
	/** LLM client; inject a fake in tests (default: OpenRouterClient). */
	llmClient?: LlmClient
	/** OpenRouter base URL override (e2e points this at the mock server). */
	baseUrl?: string
	/**
	 * Loop iteration cap — a backstop, not the primary guard (see `budget`
	 * below). Default is deliberately generous (200): QA that boots the
	 * app, runs a real test suite, and exercises changed behavior can
	 * legitimately need many tool calls, and cutting it off mid-work
	 * silently loses the real evidence rather than ending cleanly (see
	 * `runQa`'s session-failure fallback below) — a review session hitting
	 * this exact problem at the old default of 40 was a real incident.
	 */
	maxIterations?: number
	/**
	 * Cost/duration backstop (SessionBudget's maxCostUsd/maxDurationMs) —
	 * the preferred way to bound a QA session: a true runaway gets caught
	 * by spend or wall-clock time, not by an arbitrary tool-call count that
	 * penalizes legitimate thorough work.
	 */
	budget?: Pick<SessionBudget, "maxCostUsd" | "maxDurationMs">
	/** Phase 3 memory store (optional; QA can reuse the project's memory). */
	memory?: MemoryStore | null
	/** Project scope for memory (default: basename of workspaceRoot). */
	project?: string
}

export type QaVerdict = "pass" | "fail" | "error"

export interface QaResult {
	/** pass = all QA checks clean; fail = problems found; error = session error. */
	verdict: QaVerdict
	/** The evidence section extracted from the report (real command output). */
	evidence: string
	/** The full final summary from the QA session. */
	summary: string
	/**
	 * Issue #34: absolute path to the QA session's complete final report
	 * (`<workspaceRoot>/.headlesscode/reports/<sessionId>.md`), when the
	 * session succeeded and the report write succeeded. The orchestrator
	 * persists this so the full reasoning behind a QA verdict is one
	 * file-read away, not a re-run away. Absent for a session error (no
	 * report was ever produced).
	 */
	reportPath?: string
}

function currentBranch(workspaceRoot: string): string {
	try {
		return execFileSync("git", ["-C", workspaceRoot, "branch", "--show-current"], {
			encoding: "utf-8",
			timeout: 5000,
		}).trim()
	} catch {
		return "current branch"
	}
}

function defaultTaskText(workspaceRoot: string, mode: string): string {
	return (
		`Run the QA checklist defined in your operating instructions against this workspace ` +
		`(branch: ${currentBranch(workspaceRoot)}, mode: ${mode}).\n\n` +
		`FIRST: run \`git diff origin/master...HEAD --stat\` (or the equivalent against this repo's ` +
		`actual base branch) to see EXACTLY which files changed. That diff is your scope — it tells ` +
		`you which behavior to exercise. A real incident: without this instruction, a QA session spent ` +
		`over a dozen iterations exploring an unrelated chat-UI input component and an unrelated ` +
		`dependency-pinning question before ever looking at the actual diff, on a change that only ` +
		`touched two shell scripts and a docs file.\n\n` +
		`Boot the app and confirm it starts cleanly (a QUICK smoke test — no new JS/console errors on ` +
		`load — not a deep exploration of unrelated areas), run the relevant test suite, then exercise ` +
		`SPECIFICALLY the behavior the diff touched. If the diff doesn't touch the browser-facing app at ` +
		`all (e.g. only scripts/docs/backend), the boot-and-smoke-test step still applies but do not go ` +
		`looking for unrelated things to click through — there's nothing in scope for deeper UI ` +
		`interaction. Report EVIDENCE with real command output. ` +
		`When you are done, call attempt_completion with a structured summary: an Evidence section ` +
		`with the real commands you ran and their output, and the final baseline numbers you ` +
		`personally confirmed.\n\n` +
		`The VERY LAST LINE of your attempt_completion result must be exactly one of:\n` +
		`QA_VERDICT: PASS\n` +
		`QA_VERDICT: FAIL\n` +
		`Nothing else on that line — no prose, no punctuation, no markdown formatting. This is the ONLY ` +
		`line the orchestrator parses to decide pass/fail; everything else in your report is for a ` +
		`human reader.`
	)
}

/**
 * Run one headless QA session against `workspaceRoot` using the target repo's
 * `qa-agent` mode (auto-spliced from .roomodes + .roo/rules-qa-agent/) or a
 * generic checklist when the mode is absent. Returns the parsed verdict /
 * evidence / summary. A failed session (LLM error, max iterations) is reported
 * as `verdict: "error"` — the caller must never treat an inconclusive QA run
 * as a pass.
 */
export async function runQa(options: RunQaOptions): Promise<QaResult> {
	const {
		workspaceRoot,
		mode = DEFAULT_QA_MODE,
		model,
		llmClient,
		baseUrl,
		maxIterations = 200,
		budget,
		memory = null,
		project,
	} = options

	// Detect whether the target repo actually defines the requested mode. If
	// not, fall back to the generic checklist as a system prompt override.
	const customModes = await loadCustomModes(workspaceRoot)
	const modeExists = customModes.some((m) => m.slug === mode)
	// 2026-08-27 (mirrors reviewer.ts's runReview — same incident): when
	// modeExists is true, systemPromptOverride stays undefined and loop.ts's
	// own buildPrompt() path already splices in `.roo/rules/` via
	// addCustomInstructions. But the GENERIC_QA_CHECKLIST fallback is used
	// VERBATIM by loop.ts, bypassing buildPrompt (and addCustomInstructions
	// with it) entirely — a QA session taking this fallback path would be
	// just as blind to project-specific guidance (e.g. joeos's documented
	// curlee compiler location) as the review session was found to be.
	const workspaceCustomInstructions = modeExists ? "" : await addCustomInstructions("", "", workspaceRoot, mode, {})
	const systemPromptOverride = modeExists
		? undefined
		: workspaceCustomInstructions
			? `${GENERIC_QA_CHECKLIST}\n\n${workspaceCustomInstructions}`
			: GENERIC_QA_CHECKLIST

	// Mirror reviewer.ts's runReview gate (issue #142 follow-up): unlike
	// reviewer.ts, this constructed an OpenRouterClient unconditionally, so
	// a local-backend setup (HEADLESSCODE_CODE_MODE_BACKEND=ollama +
	// HEADLESSCODE_LOCAL_BACKEND_MODES including this QA mode) had no effect
	// on QA sessions at all, even though the worker and reviewer both
	// respected it. Same gate, same precedence (an explicit llmClient/model/
	// baseUrl injected by a caller still wins).
	const useLocalBackend =
		process.env.HEADLESSCODE_CODE_MODE_BACKEND === "ollama" &&
		(process.env.HEADLESSCODE_LOCAL_BACKEND_MODES ?? "code")
			.split(",")
			.map((s) => s.trim())
			.filter(Boolean)
			.includes(mode)
	// Same fix as reviewer.ts's effectiveModel (2026-08-27): downstream
	// consumers of `model` (session-start logs, cost records, the
	// `request.model` OllamaClient sends) must see the LOCAL model id when
	// local backend is active, or a local QA session logs itself as the
	// cloud model throughout even though it never touches OpenRouter.
	const effectiveModel = useLocalBackend ? (resolvePerModeEnv("HEADLESSCODE_CODE_MODE_MODEL", mode) ?? model) : model
	const client =
		llmClient ??
		(useLocalBackend
			? new OllamaClient({
					baseUrl: baseUrl ?? resolvePerModeEnv("HEADLESSCODE_OLLAMA_URL", mode),
					defaultModel: effectiveModel,
					// OllamaClient has its own independent abort timer
					// (ollama.ts's DEFAULT_OLLAMA_TIMEOUT_MS, 300s), separate
					// from the llmTimeoutMs passed to HeadlessSession below —
					// verified live 2026-08-28 (cli.ts's LOCAL_LLM_TIMEOUT_MS
					// doc comment has the full story): raising the session-
					// level timeout alone still left QA sessions dying at
					// exactly 300000ms because this constructor never heard
					// about it.
					timeoutMs: useLocalBackend ? 630_000 : undefined,
				})
			: new OpenRouterClient({ apiKey: process.env.HEADLESSCODE_OPENROUTER_API_KEY, defaultModel: model, baseUrl }))

	// Mirror every log line to <worktree>/qa.log — see reviewer.ts's runReview
	// for the full rationale (same fix, same incident: review/QA run
	// in-process rather than as a spawned subprocess with redirected stdout
	// like a worker gets via harness.log, so there was no `tail -f`able file).
	const logFilePath = path.join(workspaceRoot, "qa.log")
	fs.appendFileSync(logFilePath, `\n===== headlesscode qa start: ${new Date().toISOString()} =====\n`, "utf-8")
	const logger = new Logger({ level: "info", filePath: logFilePath })

	const session = new HeadlessSession({
		workspaceRoot,
		mode,
		model: effectiveModel,
		taskText: options.taskText ?? defaultTaskText(workspaceRoot, mode),
		maxIterations,
		budget,
		systemPromptOverride,
		tools: qaTools(),
		executor: createQaHeadlessExecutor(workspaceRoot),
		llmClient: client,
		// Issue #144 (mirrors reviewer.ts): local inference is free — a
		// fabricated dollar figure in the QA log is noise at best.
		trackCost: !useLocalBackend,
		memory,
		project,
		logger,
		// 2026-08-27 (mirrors reviewer.ts's runReview — same incident, same
		// fix): without this, a local QA session's bare/fabricated text reply
		// is accepted as an ordinary success, so a QA verdict parser has no
		// way to tell "genuinely verified" from "session produced garbage" —
		// exactly the gap that let a local reviewer session silently rubber-
		// stamp a worker's false completion. Forcing explicit completion here
		// makes a garbage QA reply a retried mistake / genuine session error
		// instead of a silent false pass.
		requireExplicitCompletion: useLocalBackend && !envBoolean("HEADLESSCODE_ALLOW_TEXT_ONLY_COMPLETION"),
		// Same gap as cli.ts's LOCAL_LLM_TIMEOUT_MS (see its doc comment for
		// the full story): this session construction never set llmTimeoutMs
		// at all, so a QA session on the local daemon always used the
		// generic DEFAULT_LLM_TIMEOUT_MS (300s) — shorter than the shim's
		// own deliberately-raised 600s upstream patience, so the harness
		// gives up first on a genuinely slow (not hung) local call.
		llmTimeoutMs: useLocalBackend ? 630_000 : undefined,
		// Same gap as cli.ts's LOCAL_MAX_TOKENS (see its doc comment for the
		// full incident): unset here, a QA session on the local daemon falls
		// back to loop.ts's DEFAULT_MAX_TOKENS (32768) — sized for a cloud
		// reasoning model's 128K+ window, not this daemon's real
		// 65,536-token total context, where one runaway generation can crash
		// the whole session outright.
		maxTokens: useLocalBackend ? 8192 : undefined,
	})

	const result: SessionResult = await session.run()
	if (result.status !== "success" || result.result === undefined) {
		const error = result.error ?? "unknown QA session error"
		return {
			verdict: "error",
			evidence: "",
			summary: `QA session failed: ${error}`,
		}
	}
	return { ...parseQaResult(result.result), reportPath: result.reportPath }
}

/**
 * Run QA, retrying ONLY when the session itself failed (verdict "error" —
 * a crash, budget stop, bounded-failure mistake limit, or an
 * attempt_completion whose result never actually populated), up to
 * `maxRetries` additional attempts. A real "pass" or "fail" is returned
 * immediately, first try.
 *
 * Mirrors reviewer.ts's runReviewWithRetries — same class of bug, same fix.
 * Caught live 2026-08-05 running issue #18's own round: a QA session ended
 * with no real result (`result.result === undefined` despite the
 * HeadlessSession itself reporting "success"), correctly classified as
 * verdict "error" by runQa — but `cli.ts` had nowhere to route that
 * distinctly from a real "fail": the group's top-level status stayed "done",
 * cost got recorded, and the round was silently considered fully settled
 * with an empty-evidence "failed" QA that nobody would ever look at again.
 */
export async function runQaWithRetries(options: RunQaOptions, maxRetries = 2): Promise<QaResult> {
	let last: QaResult = { verdict: "error", evidence: "", summary: "no attempt made" }
	for (let attempt = 0; attempt <= maxRetries; attempt++) {
		last = await runQa(options)
		if (last.verdict !== "error") {
			return last
		}
	}
	return last
}

/**
 * Parse a QA session's final report into { verdict, evidence, summary }.
 *
 * Deterministic + tested:
 *   - "pass"  — explicit pass markers (QA PASS, all tests pass, no errors
 *               found, definition of done satisfied, `errors: []`, …).
 *   - "fail"  — explicit fail markers (QA FAIL, does not work, definition of
 *               done NOT met) — deliberately narrow, see below;
 *   - default "fail" when neither is stated (FAIL-CLOSED: an inconclusive QA
 *     report never gates a deploy through).
 *
 * The fail-marker list is deliberately narrow — the SAME class of bug as
 * `reviewer.ts`'s parseReviewResult, found in the same incident: it used to
 * include bare "failed"/"failure"/"failures found"/"errors found"/"bug"/
 * "regression"/"broken"/"failing" ANYWHERE in the report. A QA report citing
 * real baseline test counts ("12 failed, 1187 passed" — pre-existing on
 * master, unrelated to the change) contains the bare word "failed", which
 * alone tripped the fail branch — and fail always wins over pass in this
 * function's logic, so ANY QA report mentioning ordinary baseline numbers
 * would be force-classified as "fail" even when QA genuinely passed. The
 * negative lookbehind `(?<!no\s)failed` only protects the exact phrase "no
 * failed" — it does nothing for "12 failed", which is the actual shape
 * baseline reporting takes. Narrowed to `qa fail` / `definition of done
 * not` / `does not work` — explicit, structured verdict language, not
 * incidental words that show up constantly in normal test-output prose.
 * Do not add generic words back without a specific report shape that needs
 * them AND a test proving no false-positive on baseline/prose language.
 *
 * Evidence: an `## Evidence` / `Evidence:` section when present, else the full
 * summary text (which still contains the real command output).
 */
export function parseQaResult(text: string): QaResult {
	const summary = text.trim()
	const lower = summary.toLowerCase()

	// Extract an "## Evidence" / "Evidence:" section if present.
	let evidence = ""
	const section = summary.match(
		/(?:^|\n)(?:#{1,6}\s*)?evidence\s*:?\s*\n([\s\S]*?)(?=\n#{1,6}\s|\n\s*(?:verdict|summary)\b|\s*$)/i,
	)
	if (section?.[1]) {
		evidence = section[1].trim()
	}

	// Primary path: an explicit, structured "QA_VERDICT: PASS"/"QA_VERDICT: FAIL"
	// line (required by defaultTaskText) is authoritative — exact match, no
	// heuristics. Same rationale as reviewer.ts's parseReviewResult: free-form
	// regex heuristics on prose have no ceiling on false positives (a real
	// incident here — baseline "N failed" test counts forced a fail verdict on
	// a QA session that explicitly said "QA PASS").
	const structuredVerdict = summary.match(/^QA_VERDICT:\s*(PASS|FAIL)\s*$/im)
	if (structuredVerdict) {
		const verdict: QaVerdict = structuredVerdict[1]!.toUpperCase() === "PASS" ? "pass" : "fail"
		return { verdict, evidence: evidence || summary, summary }
	}

	// Fallback for a session that didn't emit the required line.
	//
	// "QA PASS"/"QA FAIL" as a real verdict declaration must LEAD a line
	// (like the structured QA_VERDICT: line above) -- not match anywhere in
	// flowing prose. Verified live 2026-08-28: a QA report that correctly
	// reported failure ("a QA PASS requires all checks to have run clean...
	// I must fail rather than guess") got silently flipped to "pass"
	// because the old bare `\bqa\s*pass\b` match fired on the substring
	// "QA PASS" inside that EXPLANATORY sentence, which was never a real
	// verdict declaration — and the actual conclusion ("I must fail")
	// never matched `\bqa\s*fail\b` since it lacks the literal word "qa".
	// Same bug class this file's own history already documents once (an
	// over-eager FAIL match on "12 failed" baseline counts, see
	// testBaselineFailedCountsDoNotFalselyFailAPassingQaReport) — this
	// time in the opposite direction, an over-eager PASS match.
	const qaPassLeadsLine = /^\s*(?:[#*-]\s*)*qa\s*pass\b/im.test(summary)
	const qaFailLeadsLine = /^\s*(?:[#*-]\s*)*qa\s*fail\b/im.test(summary)
	const hasPassMarker =
		qaPassLeadsLine ||
		/\b(all\s*tests?\s*pass|no\s+errors?\s+found|no\s+failures|definition\s+of\s+done\s+(is\s+|was\s+)?(met|satisfied)|errors?\s*:\s*\[\s*\]|passed\s+(\d+)\/\d+|verified\s+ok)\b/i.test(
			lower,
		)
	const hasFailMarker = qaFailLeadsLine || /\b(definition\s+of\s+done\s+not|does\s+not\s+work)\b/i.test(lower)

	let verdict: QaVerdict
	if (hasPassMarker && !hasFailMarker) {
		verdict = "pass"
	} else if (hasFailMarker) {
		verdict = "fail"
	} else {
		// Fail-closed: no explicit verdict → do not pass.
		verdict = "fail"
	}

	return { verdict, evidence: evidence || summary, summary }
}