From 45f32c298dec94a89ae3055be42a2383a70e61b3 Mon Sep 17 00:00:00 2001 From: plind-junior <59729252+plind-junior@users.noreply.github.com> Date: Fri, 26 Jun 2026 19:02:47 +0900 Subject: [PATCH] feat(dual-solve): surface engine logs --- CHANGELOG.md | 3 ++ src/vouch/cli.py | 14 +++++++- src/vouch/dual_solve.py | 51 ++++++++++++++++++++++++++++- src/vouch/web/dual_solve_api.py | 4 ++- src/vouch/web/static/dual_solve.css | 3 ++ src/vouch/web/static/dual_solve.js | 13 +++++++- tests/test_dual_solve.py | 45 +++++++++++++++++++++++-- tests/test_web_dual_solve.py | 10 ++++-- 8 files changed, 134 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75723dbd..5ba7890e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,9 @@ All notable changes to vouch are documented here. Format follows - dual-solve JSON, review-ui job, and choose responses now include `changed_files` for each candidate and the kept branch, so desktop and browser clients can show the resulting files without parsing unified diffs. +- dual-solve JSON and review-ui job responses now include each engine's returned + output log plus a deterministic recommendation hint based on success and diff + scope, so clients can compare Claude and Codex results before choosing. - `vouch review-ui --allow-dual-solve` — a browser SPA that runs `dual-solve` on a github issue link, streams progress over the review-ui's websocket, shows both engines' diffs side by side, and lets you pick the winner. Off by default; diff --git a/src/vouch/cli.py b/src/vouch/cli.py index ca465cae..9cf69649 100644 --- a/src/vouch/cli.py +++ b/src/vouch/cli.py @@ -2029,21 +2029,33 @@ def dual_solve_cmd(issue_url: str, claude_effort: str, codex_effort: str, _emit_json({ "issue": {"number": issue.number, "title": issue.title, "url": issue.url}, + "recommendation": ds_mod.recommendation(candidates), "candidates": [ {"engine": c.engine, "branch": c.branch, "ok": c.ok, "error": c.error, "changed_files": ds_mod.changed_files(c.diff), - "diff": c.diff} for c in candidates + "log": c.log, "diff": c.diff} for c in candidates ], }) return for c in candidates: click.echo(f"\n=== {c.engine} ({c.branch}) ===", err=True) + if c.log.strip(): + click.echo("--- engine log ---", err=True) + click.echo(c.log) if c.ok: + if c.log.strip(): + click.echo("--- diff ---", err=True) click.echo(c.diff) else: click.echo(f"(failed: {c.error})", err=True) + rec = ds_mod.recommendation(candidates) + if rec.get("reason"): + label = f"recommendation: {rec['engine']}" if rec.get("engine") \ + else "recommendation: no automatic pick" + click.echo(f"{label} -- {rec['reason']}", err=True) + ok = [c for c in candidates if c.ok] if not ok: raise click.ClickException("both engines failed; nothing to choose") diff --git a/src/vouch/dual_solve.py b/src/vouch/dual_solve.py index d7734af5..dedbb82d 100644 --- a/src/vouch/dual_solve.py +++ b/src/vouch/dual_solve.py @@ -37,6 +37,7 @@ "parse_issue_ref", "parse_summary", "prepare", + "recommendation", "record_to_kb", "repo_root", "run_candidate", @@ -73,6 +74,7 @@ class Candidate: worktree: Path diff: str = "" sha: str = "" + log: str = "" ok: bool = False error: str | None = None @@ -97,6 +99,53 @@ def changed_files(diff: str) -> list[str]: return files +def _diff_stats(candidate: Candidate) -> tuple[int, int]: + files = changed_files(candidate.diff) + return len(files), len(candidate.diff.splitlines()) + + +def recommendation(candidates: list[Candidate]) -> dict[str, str | None]: + """Return a deterministic reviewer hint for the two dual-solve candidates. + + This is deliberately a scope heuristic, not an automated quality judgment: + successful candidates beat failed ones; then the smaller changed-file count + and smaller diff win. Ties stay unresolved for the human reviewer. + """ + ok = [c for c in candidates if c.ok] + if not ok: + return { + "engine": None, + "reason": "neither engine produced a usable diff.", + } + if len(ok) == 1: + return { + "engine": ok[0].engine, + "reason": f"only {ok[0].engine} produced a usable diff.", + } + + ranked = sorted(ok, key=_diff_stats) + best = ranked[0] + other = ranked[1] + best_files, best_lines = _diff_stats(best) + other_files, other_lines = _diff_stats(other) + if (best_files, best_lines) == (other_files, other_lines): + return { + "engine": None, + "reason": ( + "both engines produced equally scoped diffs; " + "review the logs and tests before choosing." + ), + } + return { + "engine": best.engine, + "reason": ( + f"{best.engine} has the smaller scoped diff " + f"({best_files} files, {best_lines} lines vs " + f"{other_files} files, {other_lines} lines)." + ), + } + + def parse_issue_ref(ref: str) -> tuple[str | None, str]: """Normalize an issue reference for ``gh issue view``. @@ -209,7 +258,7 @@ def run_candidate(engine: Engine, issue: Issue, prompt: str, root: Path, return cand try: - engine.fix(cwd=str(worktree), prompt=prompt) + cand.log = engine.fix(cwd=str(worktree), prompt=prompt) except Exception as exc: cand.error = f"engine failed: {exc}" return cand diff --git a/src/vouch/web/dual_solve_api.py b/src/vouch/web/dual_solve_api.py index 5b0e08df..c696cb94 100644 --- a/src/vouch/web/dual_solve_api.py +++ b/src/vouch/web/dual_solve_api.py @@ -68,9 +68,10 @@ def _serialize(job: DualSolveJob) -> dict[str, Any]: "candidates": [ {"engine": c.engine, "branch": c.branch, "ok": c.ok, "error": c.error, "changed_files": ds.changed_files(c.diff), - "diff": c.diff} + "log": c.log, "diff": c.diff} for c in job.candidates ], + "recommendation": ds.recommendation(job.candidates), "proposed_ids": list(job.proposed_ids), "kept_branch": job.kept_branch, "changed_files": ds.changed_files(kept.diff) if kept is not None else [], @@ -200,4 +201,5 @@ async def dual_solve_choose(req: _ChooseReq) -> dict[str, Any]: "kept_branch": job.kept_branch, "proposed_ids": ids, "changed_files": ds.changed_files(chosen.diff) if chosen is not None else [], + "recommendation": ds.recommendation(job.candidates), } diff --git a/src/vouch/web/static/dual_solve.css b/src/vouch/web/static/dual_solve.css index e94148a8..e77e4bf8 100644 --- a/src/vouch/web/static/dual_solve.css +++ b/src/vouch/web/static/dual_solve.css @@ -3,9 +3,12 @@ .ds-run input { flex: 1; } .ds-progress { background:#111; color:#ddd; padding:.5rem; white-space:pre-wrap; } .ds-error { color:#b00; } +.ds-recommendation { border:1px solid #ddd; padding:.5rem; background:#f8f8f8; } .ds-panes { display:grid; grid-template-columns:1fr 1fr; gap:1rem; } .ds-pane { border:1px solid #ccc; padding:.5rem; overflow:auto; } .ds-changed-files { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size:12px; margin:.25rem 0 .5rem; } +.ds-log { margin:.25rem 0 .75rem; } +.ds-log pre { max-height:220px; overflow:auto; white-space:pre-wrap; background:#111; color:#ddd; padding:.5rem; } .ds-file-head { font-weight:600; margin-top:.5rem; } .ds-pane pre { margin:0; font-size:12px; overflow-x:auto; } .ln-add { background:#e6ffed; display:block; } diff --git a/src/vouch/web/static/dual_solve.js b/src/vouch/web/static/dual_solve.js index fc0e9b85..a7772013 100644 --- a/src/vouch/web/static/dual_solve.js +++ b/src/vouch/web/static/dual_solve.js @@ -36,7 +36,7 @@ export default { const job = reactive({ id: null, status: "idle", progress: [], candidates: [], issue: null, error: null, kept_branch: null, proposed_ids: [], - changed_files: [], + changed_files: [], recommendation: null, }); function applyState(s) { @@ -64,6 +64,7 @@ export default { async function run() { job.progress = []; job.error = null; job.candidates = []; job.kept_branch = null; job.proposed_ids = []; job.changed_files = []; + job.recommendation = null; const r = await fetch("/dual-solve/run", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ @@ -105,6 +106,12 @@ export default {

#{{job.issue.number}} {{job.issue.title}}

{{ job.progress.join('\\n') }}

{{ job.error }}

+

+ recommendation: + {{job.recommendation.engine}} + no automatic pick + -- {{job.recommendation.reason}} +

@@ -113,6 +120,10 @@ export default { +
+ {{c.engine}} log + 
{{c.log}}
+
{{f.path}}
{{l.text}}\\n
diff --git a/tests/test_dual_solve.py b/tests/test_dual_solve.py index 12d7584d..0f0979fe 100644 --- a/tests/test_dual_solve.py +++ b/tests/test_dual_solve.py @@ -5,6 +5,7 @@ """ from __future__ import annotations +import json from pathlib import Path import pytest @@ -56,6 +57,32 @@ def test_changed_files_extracts_paths_from_git_diff(): assert ds.changed_files(diff) == ["README.md", "src/new.py"] +def test_recommendation_prefers_smaller_successful_diff(): + claude = ds.Candidate( + "claude", "b-claude", Path("/w/claude"), + diff="diff --git a/a.txt b/a.txt\n+one\n", ok=True, + ) + codex = ds.Candidate( + "codex", "b-codex", Path("/w/codex"), + diff="diff --git a/a.txt b/a.txt\n+one\n+two\n", ok=True, + ) + + rec = ds.recommendation([codex, claude]) + + assert rec["engine"] == "claude" + assert "smaller scoped diff" in (rec["reason"] or "") + + +def test_recommendation_avoids_tiebreaking_equal_scope(): + claude = ds.Candidate("claude", "b1", Path("/a"), diff="d", ok=True) + codex = ds.Candidate("codex", "b2", Path("/b"), diff="x", ok=True) + + rec = ds.recommendation([claude, codex]) + + assert rec["engine"] is None + assert "equally scoped" in (rec["reason"] or "") + + def test_require_engines_raises_when_missing(monkeypatch): monkeypatch.setattr(ds.shutil, "which", lambda b: None) with pytest.raises(RuntimeError, match="not on PATH"): @@ -140,6 +167,7 @@ def test_run_candidate_success_commits_and_captures_sha(tmp_path): assert cand.engine == "claude" assert cand.branch == "vouch-dual/3-fix-bug-claude" assert cand.diff == "patch text" and cand.sha == "abc123" + assert cand.log == "done" assert any(c[:5] == ["git", "-C", str(root), "worktree", "add"] for c in fr.calls) assert any(c[:4] == ["git", "-C", str(wt), "commit"] for c in fr.calls) assert any(c and c[0] == "claude" for c in fr.calls) @@ -390,8 +418,16 @@ def test_cli_dual_solve_json_is_noninteractive(monkeypatch, tmp_path): from vouch.cli import cli issue = ds.Issue("t", "b", number=1) - cands = [ds.Candidate("claude", "b1", tmp_path / "a", diff="DA", ok=True), - ds.Candidate("codex", "b2", tmp_path / "b", diff="DB", ok=True)] + cands = [ + ds.Candidate( + "claude", "b1", tmp_path / "a", + diff="diff --git a/a b/a\n+1\n", log="claude log", ok=True, + ), + ds.Candidate( + "codex", "b2", tmp_path / "b", + diff="diff --git a/b b/b\n+1\n+2\n", log="codex log", ok=True, + ), + ] monkeypatch.setattr("vouch.dual_solve._require_engines", lambda: None) monkeypatch.setattr("vouch.dual_solve.repo_root", lambda r, c: tmp_path) monkeypatch.setattr("vouch.dual_solve.prepare", @@ -403,7 +439,10 @@ def test_cli_dual_solve_json_is_noninteractive(monkeypatch, tmp_path): r = CliRunner().invoke(cli, ["dual-solve", "o/n#1", "--json"]) assert r.exit_code == 0, r.output - assert '"engine"' in r.output and "DA" in r.output and "DB" in r.output + body = json.loads(r.output) + assert body["recommendation"]["engine"] == "claude" + assert body["candidates"][0]["log"] == "claude log" + assert body["candidates"][1]["log"] == "codex log" # --json must not prompt and must not finalize/record. assert finalize_called["n"] == 0 diff --git a/tests/test_web_dual_solve.py b/tests/test_web_dual_solve.py index 00a4bcd5..4ea528df 100644 --- a/tests/test_web_dual_solve.py +++ b/tests/test_web_dual_solve.py @@ -80,9 +80,11 @@ def __init__(self, *, repo_root, runner, image): def _fake_prepare(monkeypatch, *, calls): issue = ds.Issue("Fix bug", "body", number=4, url="u") cA = ds.Candidate("claude", "vouch-dual/4-fix-bug-claude", Path("/w/claude"), - diff="diff --git a/x b/x\n+1\n", sha="s1", ok=True) + diff="diff --git a/x b/x\n+1\n", sha="s1", + log="claude fixed it", ok=True) cX = ds.Candidate("codex", "vouch-dual/4-fix-bug-codex", Path("/w/codex"), - diff="diff --git a/y b/y\n+2\n", sha="s2", ok=True) + diff="diff --git a/y b/y\n+2\n+3\n", sha="s2", + log="codex fixed it too", ok=True) def fake(store, issue_ref, root, runner, *, claude_effort="high", codex_effort="high", autonomy="edit", dry_run=False, @@ -117,6 +119,9 @@ def test_run_starts_job_and_reaches_ready(git_kb, monkeypatch): assert [x["engine"] for x in state["candidates"]] == ["claude", "codex"] assert state["candidates"][0]["changed_files"] == ["x"] assert state["candidates"][1]["changed_files"] == ["y"] + assert state["candidates"][0]["log"] == "claude fixed it" + assert state["candidates"][1]["log"] == "codex fixed it too" + assert state["recommendation"]["engine"] == "claude" # autonomy is forced to edit regardless of input assert calls[0]["autonomy"] == "edit" @@ -200,6 +205,7 @@ def test_choose_winner_finalizes_and_returns_ids(git_kb, monkeypatch): assert r.json()["proposed_ids"] == ["prop-1", "prop-2"] assert r.json()["kept_branch"] == "vouch-dual/4-fix-bug-codex" assert r.json()["changed_files"] == ["y"] + assert r.json()["recommendation"]["engine"] == "claude" assert captured["winner"] == "codex" assert captured["record"] is True and captured["reason"] == "cleaner"