diff --git a/src/vouch/capture.py b/src/vouch/capture.py index 06b56e38..55ec1ee9 100644 --- a/src/vouch/capture.py +++ b/src/vouch/capture.py @@ -11,6 +11,7 @@ from __future__ import annotations import json +import os import subprocess import time from dataclasses import dataclass @@ -38,7 +39,15 @@ class CaptureConfig: def load_config(store: KBStore) -> CaptureConfig: - """Read ``capture:`` from config.yaml; fall back to defaults.""" + """Read ``capture:`` from config.yaml; fall back to defaults. + + ``VOUCH_CAPTURE_DISABLE=1`` overrides to disabled: vouch's own LLM + subprocesses (compile, session-split, transcript grading) set it so the + agent session they spawn does not capture itself back into the KB as a + fresh pending summary on every run. + """ + if os.environ.get("VOUCH_CAPTURE_DISABLE") == "1": + return CaptureConfig(enabled=False) try: loaded = yaml.safe_load(store.config_path.read_text(encoding="utf-8")) except (OSError, UnicodeDecodeError, yaml.YAMLError): diff --git a/src/vouch/cli.py b/src/vouch/cli.py index b38824bc..974572f4 100644 --- a/src/vouch/cli.py +++ b/src/vouch/cli.py @@ -2228,13 +2228,26 @@ def session_list_cmd(as_json: bool) -> None: default=None, help="Pin the transcript source instead of auto-detecting.", ) -def session_transcript_cmd(session_id: str, agent: str | None) -> None: +@click.option( + "--grade", is_flag=True, + help="Annotate dialog with cached LLM review-relevance grades.", +) +@click.option( + "--regrade", is_flag=True, + help="Re-run the LLM grading even when a cached grade exists.", +) +def session_transcript_cmd( + session_id: str, agent: str | None, grade: bool, regrade: bool +) -> None: """Emit the normalized transcript of a captured session as JSON.""" from . import transcript store = _load_store() with _cli_errors(): - _emit_json(transcript.load_transcript(store, session_id, agent=agent)) + _emit_json(transcript.load_transcript( + store, session_id, agent=agent, + grade=grade or regrade, regrade=regrade, + )) @session.command("summarize") diff --git a/src/vouch/index_db.py b/src/vouch/index_db.py index 38d904c0..afacb755 100644 --- a/src/vouch/index_db.py +++ b/src/vouch/index_db.py @@ -14,7 +14,7 @@ import datetime as _dt import json import sqlite3 -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from contextlib import contextmanager, suppress from pathlib import Path @@ -75,6 +75,13 @@ detected_at TEXT NOT NULL ); +CREATE TABLE IF NOT EXISTS transcript_grades ( + session_id TEXT PRIMARY KEY, + content_hash TEXT NOT NULL, + grades TEXT NOT NULL, + graded_at TEXT NOT NULL +); + CREATE TABLE IF NOT EXISTS prov_edges ( src_id TEXT NOT NULL, dst_id TEXT NOT NULL, @@ -329,6 +336,51 @@ def stats(kb_dir: Path) -> dict[str, int]: } +# --- transcript grade cache ------------------------------------------------- +# +# LLM review-relevance grades for a session transcript. Derived cache like +# everything else in state.db: keyed by a hash of the raw session file so a +# session that grew since grading re-grades, and losing the DB just means +# paying for the grading again. + + +def get_transcript_grades( + kb_dir: Path, session_id: str, content_hash: str +) -> dict[str, object] | None: + """The cached grades payload for (session, exact file content), or None.""" + with open_db(kb_dir) as conn: + row = conn.execute( + "SELECT content_hash, grades, graded_at FROM transcript_grades " + "WHERE session_id = ?", + (session_id,), + ).fetchone() + if row is None or row[0] != content_hash: + return None + try: + grades = json.loads(row[1]) + except (TypeError, ValueError): + return None + if not isinstance(grades, dict): + return None + return {"grades": grades, "graded_at": row[2]} + + +def put_transcript_grades( + kb_dir: Path, + session_id: str, + *, + content_hash: str, + grades: Mapping[str, object], + graded_at: str, +) -> None: + with open_db(kb_dir) as conn: + conn.execute( + "INSERT OR REPLACE INTO transcript_grades " + "(session_id, content_hash, grades, graded_at) VALUES (?, ?, ?, ?)", + (session_id, content_hash, json.dumps(grades), graded_at), + ) + + # --- embeddings storage -------------------------------------------------- diff --git a/src/vouch/jsonl_server.py b/src/vouch/jsonl_server.py index d6a12525..fcc70cca 100644 --- a/src/vouch/jsonl_server.py +++ b/src/vouch/jsonl_server.py @@ -401,7 +401,11 @@ def _h_session_transcript(p: dict) -> dict: agent = p.get("agent") if agent is not None and agent not in ("claude", "codex"): raise ValueError(f"unknown agent: {agent!r} (expected 'claude' or 'codex')") - return transcript.load_transcript(_store(), session_id, agent=agent) + return transcript.load_transcript( + _store(), session_id, agent=agent, + grade=bool(p.get("grade", False)), + regrade=bool(p.get("regrade", False)), + ) def _h_propose_entity(p: dict) -> dict: diff --git a/src/vouch/llm_draft.py b/src/vouch/llm_draft.py index 3c3f1943..6368f38a 100644 --- a/src/vouch/llm_draft.py +++ b/src/vouch/llm_draft.py @@ -11,6 +11,7 @@ from __future__ import annotations import json +import os import re import subprocess import tempfile @@ -47,6 +48,12 @@ def run_llm( input=prompt, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=timeout_seconds, + # An agent CLI used as llm_cmd fires its own vouch hooks; the + # temp cwd shields cwd-discovery, but hooks that resolve the + # project dir from their own env still capture the drafting + # session back into the KB as a new pending summary each run. + # capture.load_config honors this kill-switch. + env={**os.environ, "VOUCH_CAPTURE_DISABLE": "1"}, ) except subprocess.TimeoutExpired as e: raise LLMDraftError(f"{label} timed out after {timeout_seconds:.0f}s") from e diff --git a/src/vouch/server.py b/src/vouch/server.py index fbfe3bac..533c9db4 100644 --- a/src/vouch/server.py +++ b/src/vouch/server.py @@ -595,18 +595,29 @@ def kb_list_sessions() -> dict[str, Any]: @mcp.tool() -def kb_session_transcript(session_id: str, agent: str | None = None) -> dict[str, Any]: +def kb_session_transcript( + session_id: str, + agent: str | None = None, + grade: bool = False, + regrade: bool = False, +) -> dict[str, Any]: """Render a captured session's full transcript from its raw agent JSONL. Read-only. Locates the raw Claude Code / Codex file on disk and normalizes it into message blocks (text, thinking, tool_use with paired results). + Injected scaffolding (system reminders, hook payloads, command output) + arrives tagged with a ``noise`` kind so clients can collapse it. ``agent`` restricts the search ("claude" | "codex"); omit to try both. + ``grade=True`` adds cached LLM review-relevance annotations via + ``compile.llm_cmd`` (``regrade=True`` bypasses the cache). Degrades to compact capture observations when the raw file is unavailable. """ from . import transcript if agent is not None and agent not in ("claude", "codex"): raise ValueError(f"unknown agent: {agent!r} (expected 'claude' or 'codex')") - return transcript.load_transcript(_store(), session_id, agent=agent) + return transcript.load_transcript( + _store(), session_id, agent=agent, grade=grade, regrade=regrade, + ) @mcp.tool() diff --git a/src/vouch/transcript.py b/src/vouch/transcript.py index 5ca10d34..7f731bca 100644 --- a/src/vouch/transcript.py +++ b/src/vouch/transcript.py @@ -72,6 +72,58 @@ def i(key: str) -> int: } +# Injected/scaffolding user-role content the reviewer never typed. Tagged, not +# dropped: the raw file stays the evidence of record and the console renders +# noise behind a stub. Kinds: +# meta — the JSONL entry itself carries isMeta: true +# system-reminder — blocks (memory recalls, hook nags) +# hook-context — SessionStart / UserPromptSubmit / notification payloads, +# including vouch's own per-prompt kb injection +# command — slash-command and IDE scaffolding +_NOISE_PREFIXES: tuple[tuple[str, str], ...] = ( + ("", "system-reminder"), + ("", "hook-context"), + ("", "hook-context"), + ("", "hook-context"), + ("[SYSTEM NOTIFICATION", "hook-context"), + ("SessionStart hook", "hook-context"), + ("UserPromptSubmit hook", "hook-context"), + ("", "command"), + ("", "command"), + ("", "command"), + ("", "command"), + ("", "command"), + ("", "command"), + ("", "command"), + ("", "command"), + ("", "command"), + ("Caveat: The messages below were generated by the user while running", "command"), +) + + +def classify_noise(text: str) -> str | None: + """The noise kind for an injected user-role text block, or None. + + Prefix-matched on the stripped text: injected payloads arrive as whole + content parts, so a genuine prompt that merely *mentions* one of these + tags mid-sentence is never tagged. + """ + stripped = text.lstrip() + for prefix, kind in _NOISE_PREFIXES: + if stripped.startswith(prefix): + return kind + return None + + +def _rollup_noise(message: dict[str, Any]) -> None: + """Stamp message["noise"] when every block is noise (one-line UI stub).""" + blocks = message["blocks"] + kinds = {b.get("noise") for b in blocks} + if blocks and None not in kinds: + only = kinds.pop() if len(kinds) == 1 else "mixed" + message["noise"] = only + + def _result_text(content: Any) -> str: """tool_result.content is a string, or a list of {type:text,text} parts.""" if isinstance(content, str): @@ -194,14 +246,25 @@ def flush() -> None: # user entry flush() + is_meta = bool(obj.get("isMeta")) + + def _text_block(text: str, *, _meta: bool = is_meta) -> dict[str, Any]: + block: dict[str, Any] = {"type": "text", "text": text} + kind = "meta" if _meta else classify_noise(text) + if kind is not None: + block["noise"] = kind + return block + if isinstance(content, str): text = content.strip() if text: - messages.append({ + message = { "role": "user", "id": None, "model": None, "timestamp": ts if isinstance(ts, str) else None, - "tokens": None, "blocks": [{"type": "text", "text": text}], - }) + "tokens": None, "blocks": [_text_block(text)], + } + _rollup_noise(message) + messages.append(message) continue parts = content if isinstance(content, list) else [] user_blocks: list[dict[str, Any]] = [] @@ -224,13 +287,15 @@ def flush() -> None: elif part.get("type") == "text": text = str(part.get("text", "")).strip() if text: - user_blocks.append({"type": "text", "text": text}) + user_blocks.append(_text_block(text)) if user_blocks: - messages.append({ + message = { "role": "user", "id": None, "model": None, "timestamp": ts if isinstance(ts, str) else None, "tokens": None, "blocks": user_blocks, - }) + } + _rollup_noise(message) + messages.append(message) flush() return {"session": session, "messages": messages, "truncated": truncated} @@ -325,11 +390,17 @@ def flush() -> None: if role == "user": flush() if text: - messages.append({ + text_block: dict[str, Any] = {"type": "text", "text": text} + kind = classify_noise(text) + if kind is not None: + text_block["noise"] = kind + message = { "role": "user", "id": None, "model": None, "timestamp": None, "tokens": None, - "blocks": [{"type": "text", "text": text}], - }) + "blocks": [text_block], + } + _rollup_noise(message) + messages.append(message) elif role == "assistant": if current is None: current = new_assistant() @@ -370,13 +441,22 @@ def _degraded(store: KBStore, session_id: str, reason: str) -> dict[str, Any]: def load_transcript( - store: KBStore, session_id: str, *, agent: str | None = None + store: KBStore, + session_id: str, + *, + agent: str | None = None, + grade: bool = False, + regrade: bool = False, ) -> dict[str, Any]: """Locate + parse the raw transcript for ``session_id``. ``agent`` restricts the search ("claude" | "codex"); when None both are tried. Returns the normalized schema on success, or a degraded result (compact capture observations) when the raw file is missing/too large. + + ``grade=True`` additionally annotates dialog messages with LLM + review-relevance grades (cached; ``regrade=True`` bypasses the cache). + Grades are presentation metadata — they never reorder or drop messages. """ path: Path | None = None source_agent = "" @@ -403,4 +483,17 @@ def load_transcript( parsed = parse_claude_transcript(path, max_messages=MAX_MESSAGES) else: parsed = parse_codex_transcript(path, max_messages=MAX_MESSAGES) - return {"available": True, "source": {"agent": source_agent, "path": str(path)}, **parsed} + result = { + "available": True, + "source": {"agent": source_agent, "path": str(path)}, + **parsed, + } + + from . import transcript_grading + + result["grading_available"] = transcript_grading.grading_available(store) + if grade or regrade: + transcript_grading.apply_grades( + store, session_id, result, raw_path=path, regrade=regrade, + ) + return result diff --git a/src/vouch/transcript_grading.py b/src/vouch/transcript_grading.py new file mode 100644 index 00000000..d9ef6721 --- /dev/null +++ b/src/vouch/transcript_grading.py @@ -0,0 +1,190 @@ +"""LLM review-relevance grading for session transcripts. + +The Review console shows a captured session's full dialog; even after the +deterministic noise tagging in ``transcript.py`` hides injected scaffolding, +a long session buries the few exchanges a reviewer actually needs. This +module asks the deployment-configured LLM (``compile.llm_cmd`` — the same +command the wiki compiler and synthesize use) to grade the surviving dialog +messages as ``key`` or ``low``; ungraded messages are implicitly normal. + +Grades are presentation metadata only: they attach to messages, never +reorder or remove them, and the raw session file stays the evidence of +record. Every LLM reply is mechanically validated — out-of-range indices, +unknown grades, and non-dialog targets are dropped; an unusable reply +degrades to an error note on the transcript, never an exception to the +caller. Results cache in state.db keyed by the raw file's content hash. +""" + +from __future__ import annotations + +import hashlib +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from . import index_db +from .compile import load_config +from .llm_draft import LLMDraftError, parse_object, run_llm +from .storage import KBStore + +# Cap what one grading call sends: enough dialog for judgment, bounded cost. +MAX_EXCERPT_CHARS = 400 +MAX_PROMPT_MESSAGES = 400 +MAX_NOTE_CHARS = 200 + +_VALID_GRADES = ("key", "low") + +_PROMPT_HEADER = """\ +You grade a coding-agent session dialog for a human reviewer who must decide +whether the session's knowledge-base proposals are trustworthy. The reviewer +skims: mark only the messages that change what they should look at. + +Grade "key": a message that states the user's actual goal, a decision, a +constraint, a discovered fact, or an outcome the reviewer must see. +Grade "low": a message that carries no review value (acknowledgements, +pleasantries, progress chatter, repetition). +Everything you do not list is treated as normal. + +Reply with ONLY a JSON object, no prose: +{"grades": [{"i": , "grade": "key"|"low", "note": "<=120 chars, only for key"}]} + +Dialog (index. role: text): +""" + + +def grading_available(store: KBStore) -> bool: + """True when the deployment configured an LLM command to grade with.""" + return load_config(store).llm_cmd is not None + + +def _dialog_lines(transcript: dict[str, Any]) -> tuple[list[str], list[int]]: + """Numbered prompt lines for gradable messages + their message indices. + + Gradable = a dialog message with visible (non-noise) text. Tool payloads + and thinking are excluded — the reviewer's transcript collapses those + already, and they would dwarf the dialog in the prompt. + """ + lines: list[str] = [] + indices: list[int] = [] + for i, message in enumerate(transcript.get("messages", [])): + if message.get("noise"): + continue + texts = [ + b.get("text", "") + for b in message.get("blocks", []) + if b.get("type") == "text" and not b.get("noise") + ] + text = "\n".join(t for t in texts if t).strip() + if not text: + continue + excerpt = text[:MAX_EXCERPT_CHARS] + lines.append(f"{i}. {message.get('role', '?')}: {excerpt}") + indices.append(i) + if len(lines) >= MAX_PROMPT_MESSAGES: + break + return lines, indices + + +def _validate_grades(raw: dict[str, Any], gradable: set[int]) -> dict[str, dict[str, Any]]: + """Keep only well-formed grades for messages that were actually offered.""" + out: dict[str, dict[str, Any]] = {} + entries = raw.get("grades") + if not isinstance(entries, list): + return out + for entry in entries: + if not isinstance(entry, dict): + continue + idx = entry.get("i") + grade = entry.get("grade") + if not isinstance(idx, int) or idx not in gradable: + continue + if grade not in _VALID_GRADES: + continue + note = entry.get("note") + note = note.strip()[:MAX_NOTE_CHARS] if isinstance(note, str) else None + out[str(idx)] = {"grade": grade, "note": note if grade == "key" else None} + return out + + +def _attach(transcript: dict[str, Any], grades: dict[str, Any], *, graded_at: str, + cached: bool) -> None: + messages = transcript.get("messages", []) + for key, value in grades.items(): + try: + idx = int(key) + except (TypeError, ValueError): + continue + if not isinstance(value, dict) or value.get("grade") not in _VALID_GRADES: + continue + if 0 <= idx < len(messages): + note = value.get("note") + messages[idx]["relevance"] = { + "grade": value["grade"], + "note": note if isinstance(note, str) else None, + } + transcript["grading"] = { + "graded_at": graded_at, + "cached": cached, + "graded_messages": len(grades), + } + + +def apply_grades( + store: KBStore, + session_id: str, + transcript: dict[str, Any], + *, + raw_path: Path, + regrade: bool = False, +) -> None: + """Annotate ``transcript`` in place with cached-or-fresh LLM grades. + + Failure shape: ``transcript["grading"] = {"error": …}`` — the transcript + itself always renders. + """ + config = load_config(store) + if config.llm_cmd is None: + transcript["grading"] = {"error": "compile.llm_cmd is not configured"} + return + + try: + content_hash = hashlib.sha256(raw_path.read_bytes()).hexdigest() + except OSError as e: + transcript["grading"] = {"error": f"cannot hash transcript: {e}"} + return + + if not regrade: + hit = index_db.get_transcript_grades(store.kb_dir, session_id, content_hash) + if hit is not None: + grades = hit["grades"] + assert isinstance(grades, dict) + _attach( + transcript, grades, + graded_at=str(hit["graded_at"]), cached=True, + ) + return + + lines, indices = _dialog_lines(transcript) + if not lines: + transcript["grading"] = {"error": "no dialog messages to grade"} + return + + prompt = _PROMPT_HEADER + "\n".join(lines) + "\n" + try: + raw = run_llm( + config.llm_cmd, prompt, + timeout_seconds=config.timeout_seconds, + label="compile.llm_cmd", + ) + parsed = parse_object(raw, noun="transcript grades") + except LLMDraftError as e: + transcript["grading"] = {"error": str(e)} + return + + grades = _validate_grades(parsed, set(indices)) + graded_at = datetime.now(UTC).isoformat() + index_db.put_transcript_grades( + store.kb_dir, session_id, + content_hash=content_hash, grades=grades, graded_at=graded_at, + ) + _attach(transcript, grades, graded_at=graded_at, cached=False) diff --git a/tests/test_capture.py b/tests/test_capture.py index 17961882..b087374a 100644 --- a/tests/test_capture.py +++ b/tests/test_capture.py @@ -723,3 +723,14 @@ def test_capture_e2e_sessionstart_cleanup_then_finalize(tmp_path): # Total proposals: old + new assert len(pending_after) >= 2 + + +def test_observe_noop_under_capture_disable_env( + store: KBStore, monkeypatch: pytest.MonkeyPatch, +) -> None: + """VOUCH_CAPTURE_DISABLE=1 (set by vouch-spawned llm subprocesses) wins + over an enabled capture config — the drafting session must not capture + itself back into the KB.""" + monkeypatch.setenv("VOUCH_CAPTURE_DISABLE", "1") + assert cap.observe(store, "s1", tool="Bash", summary="Ran: x") is False + assert not cap.buffer_path(store, "s1").exists() diff --git a/tests/test_session_transcript.py b/tests/test_session_transcript.py index 19c0a6d9..98c602c5 100644 --- a/tests/test_session_transcript.py +++ b/tests/test_session_transcript.py @@ -275,3 +275,87 @@ def test_load_transcript_codex_source( out = transcript.load_transcript(store, sid) assert out["available"] is True assert out["source"]["agent"] == "codex" + + +# --- noise tagging ---------------------------------------------------------- + + +def _parse(tmp_path: Path, lines: list[dict]) -> dict: + f = tmp_path / "s.jsonl" + _write_jsonl(f, lines) + return transcript.parse_claude_transcript(f) + + +def _user_line(content, **extra) -> dict: + return {"type": "user", "message": {"role": "user", "content": content}, **extra} + + +def test_system_reminder_block_tagged_and_rolled_up(tmp_path: Path) -> None: + parsed = _parse(tmp_path, [_user_line( + [{"type": "text", "text": "nag nag"}], + )]) + (m,) = parsed["messages"] + assert m["blocks"][0]["noise"] == "system-reminder" + assert m["noise"] == "system-reminder" + + +def test_is_meta_entry_tagged_meta(tmp_path: Path) -> None: + parsed = _parse(tmp_path, [_user_line("Caveat: local commands…", isMeta=True)]) + (m,) = parsed["messages"] + assert m["blocks"][0]["noise"] == "meta" + assert m["noise"] == "meta" + + +def test_mixed_message_keeps_real_text_unrolled(tmp_path: Path) -> None: + parsed = _parse(tmp_path, [_user_line([ + {"type": "text", "text": "recall"}, + {"type": "text", "text": "fix the login bug"}, + ])]) + (m,) = parsed["messages"] + assert m["blocks"][0]["noise"] == "system-reminder" + assert "noise" not in m["blocks"][1] + assert "noise" not in m + + +def test_prompt_mentioning_tag_mid_sentence_not_tagged(tmp_path: Path) -> None: + parsed = _parse(tmp_path, [_user_line( + [{"type": "text", "text": "why does appear in my logs?"}], + )]) + (m,) = parsed["messages"] + assert "noise" not in m["blocks"][0] + assert "noise" not in m + + +def test_command_scaffolding_tagged(tmp_path: Path) -> None: + parsed = _parse(tmp_path, [ + _user_line([{"type": "text", "text": "/clear"}]), + _user_line([{"type": "text", "text": "ok"}]), + _user_line([{"type": "text", "text": "x.py"}]), + ]) + kinds = [m["noise"] for m in parsed["messages"]] + assert kinds == ["command", "command", "command"] + + +def test_hook_payloads_tagged(tmp_path: Path) -> None: + parsed = _parse(tmp_path, [ + _user_line([{ + "type": "text", + "text": "kb ctx", + }]), + _user_line([{"type": "text", "text": "[SYSTEM NOTIFICATION - NOT USER INPUT] task done"}]), + ]) + kinds = [m["noise"] for m in parsed["messages"]] + assert kinds == ["hook-context", "hook-context"] + + +def test_codex_user_noise_tagged(tmp_path: Path) -> None: + f = tmp_path / "rollout.jsonl" + _write_jsonl(f, [ + {"type": "response_item", "payload": {"type": "message", "role": "user", + "content": [{"type": "input_text", "text": "x"}]}}, + {"type": "response_item", "payload": {"type": "message", "role": "user", + "content": [{"type": "input_text", "text": "real prompt"}]}}, + ]) + parsed = transcript.parse_codex_transcript(f) + assert parsed["messages"][0]["noise"] == "system-reminder" + assert "noise" not in parsed["messages"][1] diff --git a/tests/test_transcript_grading.py b/tests/test_transcript_grading.py new file mode 100644 index 00000000..d152a696 --- /dev/null +++ b/tests/test_transcript_grading.py @@ -0,0 +1,183 @@ +"""LLM review-relevance grading — validation, caching, degradation.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest +import yaml + +from vouch import transcript +from vouch.storage import KBStore + +SID = "aaaaaaaa-1111-2222-3333-bbbbbbbbbbbb" + + +def _write_session(root: Path) -> Path: + f = root / "-proj" / f"{SID}.jsonl" + f.parent.mkdir(parents=True, exist_ok=True) + lines = [ + {"type": "user", "message": {"role": "user", "content": [ + {"type": "text", "text": "secret recall"}, + ]}}, + {"type": "user", "message": {"role": "user", "content": [ + {"type": "text", "text": "fix the login redirect"}, + ]}}, + {"type": "assistant", "message": {"id": "m1", "role": "assistant", "content": [ + {"type": "text", "text": "The redirect drops the port."}, + ], "usage": {}}}, + {"type": "user", "message": {"role": "user", "content": [ + {"type": "text", "text": "thanks"}, + ]}}, + ] + f.write_text("\n".join(json.dumps(x) for x in lines) + "\n", encoding="utf-8") + return f + + +def _fake_llm(tmp_path: Path, response: dict | str) -> str: + """A stdin→stdout llm_cmd that logs each call's prompt for assertions.""" + script = tmp_path / "fake_llm.py" + out = json.dumps(response) if isinstance(response, dict) else response + script.write_text( + "import sys, pathlib\n" + f"log = pathlib.Path({str(tmp_path / 'calls.log')!r})\n" + "prompt = sys.stdin.read()\n" + "with log.open('a') as fh:\n" + " fh.write(prompt.replace(chr(10), ' ') + chr(10))\n" + f"sys.stdout.write({out!r})\n", + encoding="utf-8", + ) + return f"{sys.executable} {script}" + + +def _calls(tmp_path: Path) -> list[str]: + log = tmp_path / "calls.log" + return log.read_text(encoding="utf-8").splitlines() if log.exists() else [] + + +@pytest.fixture +def store(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> KBStore: + s = KBStore.init(tmp_path / "repo") + monkeypatch.chdir(s.root) + monkeypatch.setenv( + "VOUCH_CLAUDE_PROJECTS_DIR", str(tmp_path / "projects"), + ) + _write_session(tmp_path / "projects") + return s + + +def _set_llm(store: KBStore, cmd: str | None) -> None: + cfg = yaml.safe_load(store.config_path.read_text()) + cfg["compile"] = {"llm_cmd": cmd} if cmd else {} + store.config_path.write_text(yaml.safe_dump(cfg)) + + +def test_grades_attach_and_invalid_entries_dropped( + store: KBStore, tmp_path: Path, +) -> None: + _set_llm(store, _fake_llm(tmp_path, {"grades": [ + {"i": 1, "grade": "key", "note": "the actual goal"}, + {"i": 3, "grade": "low"}, + {"i": 99, "grade": "key"}, # out of range — dropped + {"i": 2, "grade": "meh"}, # unknown grade — dropped + {"i": 0, "grade": "key"}, # noise message, not offered — dropped + ]})) + + t = transcript.load_transcript(store, SID, grade=True) + + assert t["grading_available"] is True + assert t["messages"][1]["relevance"] == {"grade": "key", "note": "the actual goal"} + assert t["messages"][3]["relevance"] == {"grade": "low", "note": None} + assert "relevance" not in t["messages"][0] + assert "relevance" not in t["messages"][2] + assert t["grading"]["cached"] is False + assert t["grading"]["graded_messages"] == 2 + + +def test_noise_never_reaches_the_llm(store: KBStore, tmp_path: Path) -> None: + _set_llm(store, _fake_llm(tmp_path, {"grades": []})) + + transcript.load_transcript(store, SID, grade=True) + + (prompt,) = _calls(tmp_path) + assert "secret recall" not in prompt + assert "fix the login redirect" in prompt + + +def test_second_grade_serves_from_cache(store: KBStore, tmp_path: Path) -> None: + _set_llm(store, _fake_llm(tmp_path, {"grades": [{"i": 1, "grade": "key"}]})) + + first = transcript.load_transcript(store, SID, grade=True) + second = transcript.load_transcript(store, SID, grade=True) + + assert len(_calls(tmp_path)) == 1 + assert second["grading"]["cached"] is True + assert second["messages"][1]["relevance"] == first["messages"][1]["relevance"] + + +def test_regrade_bypasses_cache(store: KBStore, tmp_path: Path) -> None: + _set_llm(store, _fake_llm(tmp_path, {"grades": []})) + + transcript.load_transcript(store, SID, grade=True) + t = transcript.load_transcript(store, SID, regrade=True) + + assert len(_calls(tmp_path)) == 2 + assert t["grading"]["cached"] is False + + +def test_no_llm_cmd_reports_unavailable(store: KBStore) -> None: + _set_llm(store, None) + + t = transcript.load_transcript(store, SID, grade=True) + + assert t["grading_available"] is False + assert "not configured" in t["grading"]["error"] + assert t["messages"], "transcript must still render" + + +def test_garbage_llm_output_degrades_cleanly( + store: KBStore, tmp_path: Path, +) -> None: + _set_llm(store, _fake_llm(tmp_path, "definitely not json")) + + t = transcript.load_transcript(store, SID, grade=True) + + assert "error" in t["grading"] + assert all("relevance" not in m for m in t["messages"]) + # a failed grading must not poison the cache + _set_llm(store, _fake_llm(tmp_path, {"grades": [{"i": 1, "grade": "key"}]})) + t2 = transcript.load_transcript(store, SID, grade=True) + assert t2["messages"][1].get("relevance") == {"grade": "key", "note": None} + + +def test_plain_load_never_grades(store: KBStore, tmp_path: Path) -> None: + _set_llm(store, _fake_llm(tmp_path, {"grades": []})) + + t = transcript.load_transcript(store, SID) + + assert _calls(tmp_path) == [] + assert "grading" not in t + assert t["grading_available"] is True + + +def test_grading_subprocess_gets_capture_kill_switch( + store: KBStore, tmp_path: Path, +) -> None: + """The llm_cmd child env carries VOUCH_CAPTURE_DISABLE=1 so the agent + session it spawns cannot file a fresh pending summary on every grade.""" + script = tmp_path / "env_probe.py" + script.write_text( + "import os, sys, pathlib\n" + f"out = pathlib.Path({str(tmp_path / 'env.log')!r})\n" + "sys.stdin.read()\n" + "out.write_text(os.environ.get('VOUCH_CAPTURE_DISABLE', 'unset'))\n" + 'sys.stdout.write(\'{"grades": []}\')\n', + encoding="utf-8", + ) + _set_llm(store, f"{sys.executable} {script}") + + transcript.load_transcript(store, SID, grade=True) + + assert (tmp_path / "env.log").read_text() == "1" diff --git a/webapp/src/components/transcript/MessageBlock.tsx b/webapp/src/components/transcript/MessageBlock.tsx index 689260d2..60de95b1 100644 --- a/webapp/src/components/transcript/MessageBlock.tsx +++ b/webapp/src/components/transcript/MessageBlock.tsx @@ -1,4 +1,5 @@ -import { Bot, User } from 'lucide-react' +import { Bot, ChevronDown, Star, User } from 'lucide-react' +import { useState } from 'react' import type { TranscriptMessage } from '../../lib/transcript' import { Markdown } from '../Markdown' import { ThinkingBlock } from './ThinkingBlock' @@ -6,30 +7,85 @@ import { ToolBlock } from './ToolBlock' export function MessageBlock({ message, + dialog = false, + showHidden = false, onOpenSubagent, }: { message: TranscriptMessage + /** Dialog view: render only the visible conversation text — no tools, + * thinking, or injected blocks (the working steps live behind stubs). */ + dialog?: boolean + showHidden?: boolean onOpenSubagent?: (sessionId: string) => void }) { const isUser = message.role === 'user' + const relevance = message.relevance + // Low-relevance dialog collapses to its header until the reviewer asks. + const [lowOpen, setLowOpen] = useState(false) + const [revealed, setRevealed] = useState>(new Set()) + const collapsedLow = relevance?.grade === 'low' && !lowOpen + return ( -
-
+
+
{isUser ? : } {isUser ? 'user' : 'assistant'} {message.model && {message.model}} + {relevance?.grade === 'key' && ( + + key moment + {relevance.note && — {relevance.note}} + + )} + {relevance?.grade === 'low' && ( + + )}
-
- {message.blocks.map((b, i) => { - if (b.type === 'thinking') return - if (b.type === 'tool_use') return - return ( -
- {b.text} -
- ) - })} -
+ {!collapsedLow && ( +
+ {message.blocks.map((b, i) => { + if (b.type === 'thinking') + return dialog ? null : + if (b.type === 'tool_use') + return dialog ? null : ( + + ) + if (b.noise && dialog) return null + if (b.noise && !showHidden && !revealed.has(i)) { + return ( + + ) + } + return ( +
+ {b.text} +
+ ) + })} +
+ )}
) } diff --git a/webapp/src/lib/transcript.ts b/webapp/src/lib/transcript.ts index 2b98057f..969a9ba6 100644 --- a/webapp/src/lib/transcript.ts +++ b/webapp/src/lib/transcript.ts @@ -14,8 +14,16 @@ export interface ToolResult { subagent_session_id: string | null } +/** Injected/scaffolding content the reviewer never typed — collapsed by default. */ +export type NoiseKind = 'meta' | 'system-reminder' | 'hook-context' | 'command' | 'mixed' + +export interface Relevance { + grade: 'key' | 'low' + note: string | null +} + export type TranscriptBlock = - | { type: 'text'; text: string } + | { type: 'text'; text: string; noise?: NoiseKind } | { type: 'thinking'; text: string } | { type: 'tool_use' @@ -32,6 +40,17 @@ export interface TranscriptMessage { timestamp: string | null tokens: Tokens | null blocks: TranscriptBlock[] + /** Set when every block is noise — the whole message renders as a stub. */ + noise?: NoiseKind + /** LLM review-relevance annotation; absent = normal. */ + relevance?: Relevance +} + +export interface GradingStatus { + graded_at?: string + cached?: boolean + graded_messages?: number + error?: string } export interface SessionMeta { @@ -61,6 +80,8 @@ export type Transcript = session: SessionMeta messages: TranscriptMessage[] truncated: boolean + grading_available?: boolean + grading?: GradingStatus } | { available: false; reason: string; observations: Observation[] } @@ -68,8 +89,11 @@ export function fetchTranscript( conn: VouchConnectionInfo, sessionId: string, agent?: string, + opts?: { grade?: boolean; regrade?: boolean }, ): Promise { const params: Record = { session_id: sessionId } if (agent) params.agent = agent + if (opts?.grade) params.grade = true + if (opts?.regrade) params.regrade = true return rpc(conn, 'kb.session_transcript', params) } diff --git a/webapp/src/views/TranscriptView.test.tsx b/webapp/src/views/TranscriptView.test.tsx index 6558ee06..421d9f2d 100644 --- a/webapp/src/views/TranscriptView.test.tsx +++ b/webapp/src/views/TranscriptView.test.tsx @@ -64,6 +64,7 @@ describe('TranscriptView', () => { } }) renderWithProviders() + await userEvent.click(await screen.findByRole('button', { name: /full transcript/i })) await waitFor(() => expect(screen.getByText('Task')).toBeInTheDocument()) await userEvent.click(screen.getByRole('button', { name: /Task/i })) await userEvent.click(screen.getByRole('button', { name: /view subagent/i })) @@ -72,3 +73,160 @@ describe('TranscriptView', () => { await waitFor(() => expect(screen.getByText('Task')).toBeInTheDocument()) }) }) + +const noisyTranscript = { + available: true, + source: { agent: 'claude', path: '/x' }, + session: { + id: 's1', + agent: 'claude', + cwd: null, + git_branch: null, + title: null, + started_at: null, + ended_at: null, + model: null, + tokens: { input: 0, output: 0, cache_read: 0, cache_creation: 0 }, + }, + grading_available: true, + messages: [ + { + role: 'user', + id: null, + model: null, + timestamp: null, + tokens: null, + noise: 'system-reminder', + blocks: [{ type: 'text', text: 'recall blob', noise: 'system-reminder' }], + }, + { + role: 'user', + id: null, + model: null, + timestamp: null, + tokens: null, + noise: 'hook-context', + blocks: [{ type: 'text', text: 'injected kb context', noise: 'hook-context' }], + }, + { + role: 'user', + id: null, + model: null, + timestamp: null, + tokens: null, + blocks: [ + { type: 'text', text: 'inline reminder', noise: 'system-reminder' }, + { type: 'text', text: 'fix the login bug' }, + ], + }, + { + role: 'assistant', + id: 'm0', + model: null, + timestamp: null, + tokens: null, + blocks: [ + { + type: 'tool_use', + id: 't1', + name: 'Bash', + input: { command: 'pytest -q' }, + result: { content: 'ok', is_error: false, subagent_session_id: null }, + }, + ], + }, + { + role: 'assistant', + id: 'm1', + model: null, + timestamp: null, + tokens: null, + blocks: [{ type: 'text', text: 'on it' }], + }, + ], + truncated: false, +} + +describe('TranscriptView noise filtering', () => { + it('dialog view shows only prompts and replies, folding work into stubs', async () => { + vi.mocked(rpc).mockResolvedValue(noisyTranscript) + renderWithProviders() + await waitFor(() => expect(screen.getByText('fix the login bug')).toBeInTheDocument()) + // injected content is silently absent from the dialog + expect(screen.queryByText('recall blob')).not.toBeInTheDocument() + expect(screen.queryByText('injected kb context')).not.toBeInTheDocument() + expect(screen.queryByText('inline reminder')).not.toBeInTheDocument() + // the tool-only assistant message folds into a working-steps stub + const stub = screen.getByRole('button', { name: /1 working step hidden \(1 tool run\)/i }) + expect(screen.queryByText('Bash')).not.toBeInTheDocument() + await userEvent.click(stub) + await waitFor(() => expect(screen.getByText('Bash')).toBeInTheDocument()) + }) + + it('full transcript brings back tools and stubs injected messages', async () => { + vi.mocked(rpc).mockResolvedValue(noisyTranscript) + renderWithProviders() + await waitFor(() => expect(screen.getByText('fix the login bug')).toBeInTheDocument()) + await userEvent.click(screen.getByRole('button', { name: /full transcript/i })) + await waitFor(() => expect(screen.getByText('Bash')).toBeInTheDocument()) + // injected messages stay behind a stub until expanded + expect(screen.queryByText('recall blob')).not.toBeInTheDocument() + await userEvent.click( + screen.getByRole('button', { name: /▸ 2 injected\/system messages hidden/i }), + ) + await waitFor(() => expect(screen.getByText('recall blob')).toBeInTheDocument()) + expect(screen.getByText('injected kb context')).toBeInTheDocument() + }) + + it('full transcript stubs a noise block inside a substantive message', async () => { + vi.mocked(rpc).mockResolvedValue(noisyTranscript) + renderWithProviders() + await waitFor(() => expect(screen.getByText('fix the login bug')).toBeInTheDocument()) + await userEvent.click(screen.getByRole('button', { name: /full transcript/i })) + expect(screen.queryByText('inline reminder')).not.toBeInTheDocument() + await userEvent.click(screen.getByRole('button', { name: /system-reminder hidden/i })) + await waitFor(() => expect(screen.getByText('inline reminder')).toBeInTheDocument()) + }) + + it('the global toggle reveals and re-hides injected messages', async () => { + vi.mocked(rpc).mockResolvedValue(noisyTranscript) + renderWithProviders() + await waitFor(() => expect(screen.getByText('fix the login bug')).toBeInTheDocument()) + await userEvent.click(screen.getByRole('button', { name: /full transcript/i })) + await userEvent.click( + screen.getByRole('button', { name: /2 injected\/system messages hidden — show/i }), + ) + await waitFor(() => expect(screen.getByText('recall blob')).toBeInTheDocument()) + await userEvent.click( + screen.getByRole('button', { name: /hide injected\/system messages/i }), + ) + await waitFor(() => expect(screen.queryByText('recall blob')).not.toBeInTheDocument()) + }) + + it('grades on demand and renders key/low annotations', async () => { + const graded = { + ...noisyTranscript, + grading: { graded_at: '2026-07-14T00:00:00Z', cached: false, graded_messages: 2 }, + messages: noisyTranscript.messages.map((m, i) => + i === 2 + ? { ...m, relevance: { grade: 'key', note: 'the actual ask' } } + : i === 4 + ? { ...m, relevance: { grade: 'low', note: null } } + : m, + ), + } + vi.mocked(rpc).mockImplementation(async (_c, _m, params) => + (params as Record).grade ? graded : noisyTranscript, + ) + renderWithProviders() + await waitFor(() => expect(screen.getByText('fix the login bug')).toBeInTheDocument()) + await userEvent.click(screen.getByRole('button', { name: /grade relevance with llm/i })) + await waitFor(() => expect(screen.getByText(/key moment/i)).toBeInTheDocument()) + expect(screen.getByText(/the actual ask/)).toBeInTheDocument() + expect(screen.getByText(/graded/)).toBeInTheDocument() + // low-relevance dialog collapses its body behind the header chip + expect(screen.queryByText('on it')).not.toBeInTheDocument() + await userEvent.click(screen.getByRole('button', { name: /low relevance/i })) + await waitFor(() => expect(screen.getByText('on it')).toBeInTheDocument()) + }) +}) diff --git a/webapp/src/views/TranscriptView.tsx b/webapp/src/views/TranscriptView.tsx index 0ca86be1..94a0513c 100644 --- a/webapp/src/views/TranscriptView.tsx +++ b/webapp/src/views/TranscriptView.tsx @@ -1,11 +1,12 @@ import { useQuery } from '@tanstack/react-query' -import { useState } from 'react' +import { EyeOff, MessagesSquare, ScrollText, Sparkles } from 'lucide-react' +import { useMemo, useState } from 'react' import { EmptyState } from '../components/EmptyState' import { ErrorCard } from '../components/ErrorCard' import { MessageBlock } from '../components/transcript/MessageBlock' import { VouchRpcError } from '../lib/rpc' import { fetchTranscript } from '../lib/transcript' -import type { Observation } from '../lib/transcript' +import type { Observation, TranscriptMessage } from '../lib/transcript' import type { VouchConnectionInfo } from '../lib/types' function Degraded({ reason, observations }: { reason: string; observations: Observation[] }) { @@ -33,6 +34,53 @@ function Degraded({ reason, observations }: { reason: string; observations: Obse ) } +/** A message the dialog view shows: it has at least one visible text block. */ +function hasDialogText(m: TranscriptMessage): boolean { + if (m.noise) return false + return m.blocks.some((b) => b.type === 'text' && !b.noise && b.text.trim() !== '') +} + +type RenderItem = + | { kind: 'message'; index: number } + /** Consecutive messages the current view hides, folded into one stub row. */ + | { kind: 'stub'; start: number; indices: number[]; tools: number; thinking: number; injected: number } + +function groupMessages(messages: TranscriptMessage[], dialog: boolean): RenderItem[] { + const items: RenderItem[] = [] + for (let i = 0; i < messages.length; i++) { + const m = messages[i] + const hidden = dialog ? !hasDialogText(m) : !!m.noise + if (!hidden) { + items.push({ kind: 'message', index: i }) + continue + } + let stub = items[items.length - 1] + if (!stub || stub.kind !== 'stub') { + stub = { kind: 'stub', start: i, indices: [], tools: 0, thinking: 0, injected: 0 } + items.push(stub) + } + stub.indices.push(i) + if (m.noise) stub.injected += 1 + for (const b of m.blocks) { + if (b.type === 'tool_use') stub.tools += 1 + else if (b.type === 'thinking') stub.thinking += 1 + } + } + return items +} + +function stubLabel(s: Extract, dialog: boolean): string { + if (!dialog) { + return `${s.indices.length} injected/system ${s.indices.length === 1 ? 'message' : 'messages'} hidden` + } + const parts: string[] = [] + if (s.tools > 0) parts.push(`${s.tools} tool ${s.tools === 1 ? 'run' : 'runs'}`) + if (s.thinking > 0) parts.push(`${s.thinking} thinking`) + if (s.injected > 0) parts.push(`${s.injected} injected`) + const detail = parts.length > 0 ? ` (${parts.join(', ')})` : '' + return `${s.indices.length} working ${s.indices.length === 1 ? 'step' : 'steps'} hidden${detail}` +} + export function TranscriptView({ conn, sessionId, @@ -45,12 +93,35 @@ export function TranscriptView({ // Subagent drill-down replaces the shown transcript with the child's, // keeping a back stack to the parent session. const [stack, setStack] = useState<{ id: string; agent?: string }[]>([{ id: sessionId, agent }]) + // The reviewer's default is the conversation itself — prompts and replies. + // "full transcript" brings back the working steps (tools, thinking), with + // injected/system content still behind stubs. + const [dialog, setDialog] = useState(true) + const [showHidden, setShowHidden] = useState(false) + const [expanded, setExpanded] = useState>(new Set()) + // grade.n keys the query so "grade" and "re-grade" each fetch exactly once. + const [grade, setGrade] = useState<{ on: boolean; fresh: boolean; n: number }>({ + on: false, + fresh: false, + n: 0, + }) const top = stack[stack.length - 1] const q = useQuery({ - queryKey: ['transcript', conn.endpoint, top.id], - queryFn: () => fetchTranscript(conn, top.id, top.agent), + queryKey: ['transcript', conn.endpoint, top.id, grade.n], + queryFn: () => + fetchTranscript(conn, top.id, top.agent, { + grade: grade.on, + regrade: grade.fresh, + }), + placeholderData: (prev) => prev, }) + const t = q.data + const items = useMemo( + () => (t?.available ? groupMessages(t.messages, dialog) : []), + [t, dialog], + ) + if (q.isPending) return
Loading transcript…
if (q.isError) { const e = q.error @@ -63,7 +134,10 @@ export function TranscriptView({
) } - const t = q.data + if (!t) return null + + const noiseCount = t.available ? t.messages.filter((m) => m.noise).length : 0 + return (
{stack.length > 1 && ( @@ -85,18 +159,105 @@ export function TranscriptView({ {t.session.tokens.input + t.session.tokens.output} tokens {t.source.agent}
+
+
+ + +
+ {!dialog && noiseCount > 0 && ( + + )} + {t.grading_available && !t.grading && ( + + )} + {t.grading && !t.grading.error && ( + + + ✓ graded{t.grading.cached ? ' (cached)' : ''} — {t.grading.graded_messages}{' '} + marked + + + + )} + {t.grading?.error && ( + grading failed: {t.grading.error} + )} +
{t.truncated && (
transcript truncated at {t.messages.length} messages
)} - {t.messages.map((m, i) => ( - setStack((s) => [...s, { id, agent: t.source.agent }])} - /> - ))} + {items.map((it) => { + if (it.kind === 'message') { + return ( + setStack((s) => [...s, { id, agent: t.source.agent }])} + /> + ) + } + const stubKey = `${dialog ? 'd' : 'f'}-${it.start}` + const isOpen = (!dialog && showHidden) || expanded.has(stubKey) + if (!isOpen) { + return ( + + ) + } + return it.indices.map((mi) => ( +
+ setStack((s) => [...s, { id, agent: t.source.agent }])} + /> +
+ )) + })} )}