diff --git a/CHANGELOG.md b/CHANGELOG.md index 96385951..35f3adf2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ All notable changes to vouch are documented here. Format follows ## [Unreleased] ### Added +- the per-prompt recall hook (`vouch context-hook`) is now instructional + and always visible: with relevant approved items it tells the model to + open its reply with **"From vouch memory:"** and ground in the cited + items; with no relevant items it says so explicitly ("Nothing in vouch + on this.") instead of injecting nothing — so recall can never be + silently mistaken for "vouch did nothing". an opt-in confidence + short-circuit (`retrieval.short_circuit.{enabled,min_confidence}`) + lets a high-confidence non-action lookup collapse to a verbatim + vouched answer; "do work" prompts never short-circuit. - `vouch install-mcp claude-code` now registers vouch as a **local-scope** MCP server in `~/.claude.json` (`projects[].mcpServers`), the same thing `claude mcp add` does. a committed `.mcp.json` is a diff --git a/README.md b/README.md index e4aaaad3..658946ce 100644 --- a/README.md +++ b/README.md @@ -127,7 +127,7 @@ vouch install-mcp claude-code # creates .vouch/ (if missing) + wires Claud > **Why the extra registration?** A committed `.mcp.json` is a *project*-scope server, and Claude Code only loads one after a per-user approval — which the **VS Code extension never prompts for**, so `.mcp.json` alone leaves the `kb_*` tools invisible in the extension (they sit at "pending approval", while the hooks quietly work — easy to misread as "connected"). The local-scope entry `install-mcp` writes is trusted on sight, so a fresh install just connects. Verify with `claude mcp list` (`vouch … ✔ Connected`). Pass `--no-approve` to skip it and approve `.mcp.json` yourself. -> **One-time approval.** Claude Code only loads a project's `.mcp.json` servers after a per-user approval. The terminal CLI prompts for it on the next `claude` launch — but the **VS Code extension never shows this prompt**, so the `kb.*` tools stay unavailable there while the hooks keep working (which makes it easy to think vouch is connected when it isn't). If `claude mcp list` run inside the project shows `vouch … ⏸ Pending approval`, either run `claude` in the project folder from any terminal and accept the prompt, or create `.claude/settings.local.json` (user-local, not committed) with `{"enabledMcpjsonServers": ["vouch"]}`. The same key in the committed `.claude/settings.json` is deliberately ignored — a repo can't approve its own servers — which is why `install-mcp` can't do this step for you. Verify with `claude mcp list` (`✔ Connected`), then reload the VS Code window. +> **What you'll see.** Every prompt is checked against the KB first. When vouch knows something relevant, the answer opens with **"From vouch memory:"**, grounded in the cited items; when it doesn't, it opens with *"Nothing in vouch on this."* — recall is visible on every turn, never silent. (A fresh KB knows almost nothing yet: work a session or two so capture fills it, then ask about the project again.) **2. Point `compile` at an LLM** — the only step that needs a model. In `.vouch/config.yaml`: diff --git a/src/vouch/hooks.py b/src/vouch/hooks.py index 6fef47fd..6b1276b4 100644 --- a/src/vouch/hooks.py +++ b/src/vouch/hooks.py @@ -6,12 +6,19 @@ context block drawn from *approved* KB knowledge — so recall costs the agent zero tool calls. It never raises: on any problem it returns "" (inject nothing), so a hook failure can never block the user's turn. + +The injected block is instructional, not just informational: when the KB has +relevant approved items the model is told to open its reply with +"From vouch memory:" and ground the answer in the cited items, and when the +KB has nothing relevant it is told to say so — so the user always sees that +vouch was consulted instead of silence that looks like vouch did nothing. """ from __future__ import annotations import json import logging +import math from typing import Any import yaml @@ -38,6 +45,94 @@ def _render(pack: dict[str, Any]) -> str: return "\n".join(lines) +# --- confidence short-circuit (opt-in) ------------------------------------- +# +# Retrieval scores are unbounded (fts/semantic hybrid), so squash them to a +# 0-1 "confidence" so the config knob (retrieval.short_circuit.min_confidence) +# reads the way a user expects: conf = 1 - exp(-score / K). This is a +# heuristic, NOT a calibrated probability — the action gate below, not this +# number, is what keeps the short-circuit safe. +# +# "Do work" prompts must never collapse to a memory pass-through: a high +# retrieval score cannot tell these apart from a lookup — an action prompt +# about a topic the KB knows still scores high — so an action is never +# short-circuited, however confident the match. + +_SHORT_CIRCUIT_K = 5.0 +_DEFAULT_MIN_CONFIDENCE = 0.8 + +_ACTION_VERBS = frozenset({ + "implement", "fix", "refactor", "add", "create", "build", "run", "change", + "update", "edit", "write", "delete", "remove", "make", "install", "deploy", + "commit", "merge", "rename", "move", "generate", "wire", "patch", "migrate", + "rewrite", "optimize", "optimise", "debug", "configure", "rebase", "push", + "revert", "bump", "replace", "integrate", "scaffold", "draft", "compile", + "set", "apply", "convert", "extract", "split", "rework", "port", "cut", +}) + + +def short_circuit_cfg(cfg: dict) -> tuple[bool, float]: + """Read ``retrieval.short_circuit`` defensively. Returns (enabled, min_confidence). + + Opt-in (default off). ``min_confidence`` is a 0-1 threshold on the squashed + top-item confidence; malformed values fall back to the default. + """ + retrieval = cfg.get("retrieval") if isinstance(cfg, dict) else None + sc = retrieval.get("short_circuit") if isinstance(retrieval, dict) else None + if not isinstance(sc, dict): + return (False, _DEFAULT_MIN_CONFIDENCE) + enabled = bool(sc.get("enabled", False)) + try: + threshold = float(sc.get("min_confidence", _DEFAULT_MIN_CONFIDENCE)) + except (TypeError, ValueError): + threshold = _DEFAULT_MIN_CONFIDENCE + if not (0.0 <= threshold <= 1.0): + threshold = _DEFAULT_MIN_CONFIDENCE + return (enabled, threshold) + + +def _confidence(score: float) -> float: + """Squash an unbounded retrieval score into a 0-1 heuristic confidence.""" + if score <= 0: + return 0.0 + return 1.0 - math.exp(-score / _SHORT_CIRCUIT_K) + + +def _top_confidence(pack: Any) -> float: + """Highest item confidence in a context pack (0.0 when empty/malformed).""" + if not isinstance(pack, dict): + return 0.0 + best = 0.0 + for item in pack.get("items", []): + try: + best = max(best, _confidence(float(item.get("score") or 0))) + except (TypeError, ValueError): + continue + return best + + +def _looks_like_action(prompt: str) -> bool: + """True when the prompt's first real word is an imperative 'do work' verb.""" + for tok in prompt.strip().lower().split(): + word = "".join(ch for ch in tok if ch.isalpha()) + if not word: + continue # skip leading punctuation / emoji tokens + return word in _ACTION_VERBS + return False + + +def _envelope(block: str) -> str: + """Wrap an injected context block in the UserPromptSubmit hook envelope.""" + return json.dumps( + { + "hookSpecificOutput": { + "hookEventName": "UserPromptSubmit", + "additionalContext": block, + } + } + ) + + def build_claude_prompt_hook(store: KBStore, stdin_text: str) -> str: """Return the stdout a host should inject for this prompt, or "" for none.""" try: @@ -51,15 +146,23 @@ def build_claude_prompt_hook(store: KBStore, stdin_text: str) -> str: return "" session_id = payload.get("session_id") session_id = str(session_id) if session_id else None + + # Load config once (defensively): both the salience reflex and the + # confidence short-circuit read it, and a config failure must never break + # the hook (module contract — see docstring). + cfg: dict[str, Any] = {} + try: + loaded = yaml.safe_load(store.config_path.read_text(encoding="utf-8")) + cfg = loaded if isinstance(loaded, dict) else {} + except Exception: + cfg = {} + # Feed the entity-salience reflex (#223) so repeated mentions of an # entity within a session sharpen ranking on subsequent turns -- this # was previously computed but never actually recorded from the hook # path, so the reflex sat dormant for claude-code sessions (#425). if session_id: try: - cfg = yaml.safe_load(store.config_path.read_text(encoding="utf-8")) or {} - if not isinstance(cfg, dict): - cfg = {} _enabled, window, _top_k = salience_mod.reflex_cfg(cfg) salience_mod.record_query(session_id, prompt, window=window) except Exception: @@ -78,16 +181,36 @@ def build_claude_prompt_hook(store: KBStore, stdin_text: str) -> str: return "" body = _render(pack) if isinstance(pack, dict) else "" if not body: - return "" - block = ( - "Relevant knowledge from the project's vouch KB " - "(approved & cited — consider it before answering):\n" + body - ) - return json.dumps( - { - "hookSpecificOutput": { - "hookEventName": "UserPromptSubmit", - "additionalContext": block, - } - } - ) + # Say so explicitly even when empty — the user wants to see that vouch + # was consulted, not silence that looks like vouch did nothing. + return _envelope( + "[vouch memory] I searched the project's vouch knowledge base for this " + 'prompt and found nothing relevant. Open your reply with "Nothing in ' + 'vouch on this." then answer from your own knowledge.' + ) + + sc_enabled, min_conf = short_circuit_cfg(cfg) + if sc_enabled and not _looks_like_action(prompt) and _top_confidence(pack) >= min_conf: + # High-confidence lookup: let the model collapse to a near-instant + # pass-through. The escape hatch ("if they do NOT fully answer") covers + # the score-is-not-answer gap — a high score means "strongly related", + # not "definitely the answer". An action prompt never reaches here. + block = ( + "[vouch memory] A high-confidence match was found in the project's " + "vouch knowledge base. If the cited item(s) below fully answer the " + 'prompt, reply with ONLY "From vouch memory:" then the relevant ' + "item(s) verbatim and their [ev-...] id(s), and STOP -- no extra " + "reasoning, caveats, or tool calls. If they do NOT fully answer it, " + 'continue normally: open with "From vouch memory:" and ground in them.' + "\n\n" + body + ) + else: + block = ( + "[vouch memory] I searched the project's vouch knowledge base for this " + "prompt. Approved, cited items are below. Before you reason on your own: " + 'open your reply with "From vouch memory:" and ground your answer in the ' + "relevant item(s), citing each id in [brackets]. If none are actually " + 'relevant, say "Nothing relevant in vouch on this" and then answer from ' + "your own knowledge.\n\n" + body + ) + return _envelope(block) diff --git a/tests/test_hooks.py b/tests/test_hooks.py index 5ba4a32d..12d69b0a 100644 --- a/tests/test_hooks.py +++ b/tests/test_hooks.py @@ -45,16 +45,110 @@ def test_relevant_prompt_yields_additional_context( assert "tuesdays" in env["hookSpecificOutput"]["additionalContext"] +def test_relevant_prompt_banner_instructs_from_vouch_memory( + store: KBStore, monkeypatch: pytest.MonkeyPatch +) -> None: + """The block is instructional: the model is told to open its reply with + "From vouch memory:" and ground in the cited items — that visible opener + is the user-facing proof recall ran.""" + _force_hit(monkeypatch) + out = hooks.build_claude_prompt_hook(store, json.dumps({"prompt": "when do deploys run"})) + ctx = json.loads(out)["hookSpecificOutput"]["additionalContext"] + assert ctx.startswith("[vouch memory]") + assert 'open your reply with "From vouch memory:"' in ctx + + def test_raw_non_json_stdin_is_tolerated(store: KBStore, monkeypatch: pytest.MonkeyPatch) -> None: _force_hit(monkeypatch) out = hooks.build_claude_prompt_hook(store, "when do deploys run") assert "tuesdays" in json.loads(out)["hookSpecificOutput"]["additionalContext"] -def test_no_hits_injects_nothing(store: KBStore, monkeypatch: pytest.MonkeyPatch) -> None: +def test_no_hits_injects_explicit_nothing_banner( + store: KBStore, monkeypatch: pytest.MonkeyPatch +) -> None: + """An empty result still injects — the user must see vouch was consulted, + not silence that looks like vouch did nothing.""" monkeypatch.setattr(context.index_db, "search_semantic", lambda *a, **k: []) monkeypatch.setattr(context.index_db, "search", lambda *a, **k: []) - assert hooks.build_claude_prompt_hook(store, json.dumps({"prompt": "zzznomatch"})) == "" + out = hooks.build_claude_prompt_hook(store, json.dumps({"prompt": "zzznomatch"})) + ctx = json.loads(out)["hookSpecificOutput"]["additionalContext"] + assert "found nothing relevant" in ctx + assert '"Nothing in vouch on this."' in ctx + + +def _stub_pack(monkeypatch: pytest.MonkeyPatch, score: float) -> None: + # stub the pack builder itself (module attr on hooks), not index_db: the + # short-circuit reads the fused item score, which hybrid retrieval rescales. + pack = { + "items": [ + {"summary": "deploys run on tuesdays", "citations": ["ev-1"], "score": score} + ] + } + monkeypatch.setattr(hooks, "build_context_pack", lambda *a, **k: pack) + + +def _enable_short_circuit(store: KBStore, threshold: float = 0.8) -> None: + store.config_path.write_text( + f"retrieval:\n short_circuit:\n enabled: true\n min_confidence: {threshold}\n", + encoding="utf-8", + ) + + +def test_short_circuit_banner_on_high_confidence_lookup( + store: KBStore, monkeypatch: pytest.MonkeyPatch +) -> None: + _enable_short_circuit(store) + _stub_pack(monkeypatch, score=50.0) # conf = 1-exp(-10) ≈ 1.0 + out = hooks.build_claude_prompt_hook(store, json.dumps({"prompt": "when do deploys run"})) + ctx = json.loads(out)["hookSpecificOutput"]["additionalContext"] + assert "high-confidence match" in ctx + assert 'reply with ONLY "From vouch memory:"' in ctx + + +def test_short_circuit_never_fires_for_action_prompts( + store: KBStore, monkeypatch: pytest.MonkeyPatch +) -> None: + """A 'do work' prompt must never collapse to a memory pass-through, + however confident the match — the action gate, not the score, keeps + the short-circuit safe.""" + _enable_short_circuit(store) + _stub_pack(monkeypatch, score=50.0) + out = hooks.build_claude_prompt_hook(store, json.dumps({"prompt": "fix the deploy pipeline"})) + ctx = json.loads(out)["hookSpecificOutput"]["additionalContext"] + assert "high-confidence match" not in ctx + assert 'open your reply with "From vouch memory:"' in ctx + + +def test_short_circuit_off_by_default( + store: KBStore, monkeypatch: pytest.MonkeyPatch +) -> None: + _stub_pack(monkeypatch, score=50.0) # confident, but the knob is opt-in + out = hooks.build_claude_prompt_hook(store, json.dumps({"prompt": "when do deploys run"})) + ctx = json.loads(out)["hookSpecificOutput"]["additionalContext"] + assert "high-confidence match" not in ctx + + +def test_short_circuit_respects_threshold( + store: KBStore, monkeypatch: pytest.MonkeyPatch +) -> None: + _enable_short_circuit(store, threshold=0.99) + _stub_pack(monkeypatch, score=8.5) # conf ≈ 0.82 < 0.99 + out = hooks.build_claude_prompt_hook(store, json.dumps({"prompt": "when do deploys run"})) + assert "high-confidence match" not in json.loads(out)["hookSpecificOutput"]["additionalContext"] + + +def test_short_circuit_cfg_is_defensive() -> None: + default = (False, 0.8) + assert hooks.short_circuit_cfg({}) == default + assert hooks.short_circuit_cfg({"retrieval": None}) == default + assert hooks.short_circuit_cfg({"retrieval": {"short_circuit": "yes"}}) == default + assert hooks.short_circuit_cfg( + {"retrieval": {"short_circuit": {"enabled": True, "min_confidence": "high"}}} + ) == (True, 0.8) + assert hooks.short_circuit_cfg( + {"retrieval": {"short_circuit": {"enabled": True, "min_confidence": 7}}} + ) == (True, 0.8) def test_non_dict_json_payload_is_safe(store: KBStore) -> None: