diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 29e7c00..2a30b5b 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -3,7 +3,7 @@ "name": "codebase-index", "displayName": "Codebase Index", "description": "Give Claude a precise local map of your codebase: find implementations, trace behavior, and predict change impact with file-line evidence.", - "version": "1.9.0", + "version": "2.0.0", "author": { "name": "codebase-index contributors" }, diff --git a/.claude/skills/codebase-index/.skill_version b/.claude/skills/codebase-index/.skill_version index f8e233b..227cea2 100644 --- a/.claude/skills/codebase-index/.skill_version +++ b/.claude/skills/codebase-index/.skill_version @@ -1 +1 @@ -1.9.0 +2.0.0 diff --git a/.claude/skills/codebase-index/SKILL.md b/.claude/skills/codebase-index/SKILL.md index 9404686..432005b 100644 --- a/.claude/skills/codebase-index/SKILL.md +++ b/.claude/skills/codebase-index/SKILL.md @@ -1,25 +1,26 @@ --- name: codebase-index -description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository. -allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob +description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository, and verify evidence gathered earlier before relying on it. +allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index verify *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob --- # Codebase Index Use the local index before reading repository files. -The operating principle is **Find → Trace → Predict**: +The operating principle is **Find → Trace → Verify → Predict**: - **Find** the implementation with ranked retrieval. - **Trace** behavior through definitions, callers, dependencies, and paths. +- **Verify** that evidence you already hold is still true before relying on it. - **Predict** change impact while preserving an explicit evidence trail. ## Route the question | Intent | Command | |---|---| -| Where is X implemented? | `codebase-index search "X" --json` | -| How does X work? | `codebase-index explain "X" --json` | +| Where is X implemented? | `codebase-index search "X" --session --json` | +| How does X work? | `codebase-index explain "X" --session --json` | | What is this codebase? | `codebase-index architecture --json` | | Find a named symbol | `codebase-index symbol "X" --json` | | Who calls or references X? | `codebase-index refs "X" --json` | @@ -27,6 +28,7 @@ The operating principle is **Find → Trace → Predict**: | What does my current diff affect? | `codebase-index diff-impact --json` | | How are X and Y connected? | `codebase-index path "X" "Y" --json` | | Describe X and its neighborhood | `codebase-index describe "X" --json` | +| Is what I read earlier still true? | `codebase-index verify --session --json` | | Produce a human graph | `codebase-index graph "X" --output ` | Use `search --mode symbol` for exact symbol work, `--mode fts` for text and @@ -38,21 +40,41 @@ or routing remain unclear. ## Evidence protocol -1. Run the best-matching command with `--json`. -2. Check `index` before trusting the payload: +1. Pick one session tag for this conversation (for example `auth-fix-1`) and + pass `--session ` to every `search` and `explain`. +2. Run the best-matching command with `--json`. +3. Check `index` before trusting the payload: - missing → run `codebase-index index`, then repeat; - stale with fewer than 20 changed files → run `codebase-index update`; - stale with 20 or more changed files → run `codebase-index index`; - fresh → continue. -3. Start with ranks 1–3. Read only `recommended_reads` line ranges. -4. Trace one additional hop only when the question requires behavior, +4. Start with ranks 1–3. Read only `recommended_reads` line ranges. +5. Trace one additional hop only when the question requires behavior, ownership, or impact. -5. Answer with `file:line` evidence and state uncertainty explicitly. +6. Before answering or editing from evidence gathered earlier in the task, run + `codebase-index verify --session --json` and reread anything whose + state is not `valid` or `relocated`. +7. Answer with `file:line` evidence and state uncertainty explicitly. Do not open whole files when a line range is available. A snippet may already be sufficient. `skeletonized: true` means the response intentionally folded unrelated body lines; read the supplied range when the missing body matters. +## Evidence memory + +- `reused: true` with `snippet: null` — this session already received that + exact text and its source is unchanged. Use your earlier copy; if you can no + longer see it, Read the range. +- `memory.invalidated` — evidence this session received has changed since. + Treat your earlier copy as wrong and reread before relying on it. +- `stale: true` — the index is older than the file. Run `codebase-index update` + or Read the range. +- A tag belongs to one context. Never give it to a subagent or another + conversation. Start a new tag after the context is cleared or compacted, or + whenever earlier snippets are no longer visible to you. + +Verdict states and citing evidence in notes: [references/memory.md](references/memory.md). + ## Confidence contract - **high** — answer from the indexed evidence. diff --git a/.claude/skills/codebase-index/references/commands.md b/.claude/skills/codebase-index/references/commands.md index 0efb464..79c91a6 100644 --- a/.claude/skills/codebase-index/references/commands.md +++ b/.claude/skills/codebase-index/references/commands.md @@ -17,6 +17,9 @@ Useful search options: - `--offset ` - `--raw` to disable snippet skeletonization - `--no-fallback` to suppress fallback suggestions +- `--session ` to name this conversation's context: unchanged evidence it + already received comes back as `reused: true` without the snippet, and + evidence that changed is listed under `memory.invalidated` `explain` uses the HOW_IT_WORKS intent and a larger default token budget. Prefer it over repeatedly rewording a broad search. @@ -49,6 +52,18 @@ codebase-index graph "" --direction both --depth 2 --output graph.html For headless work, use `--output`; do not use `--open`. Exports also support `--format graphml|dot|neo4j`. +## Evidence + +```bash +codebase-index verify --session --json +codebase-index verify "" ... --json +``` + +- `verify` is read-only and needs no index: it checks evidence against the + working tree. `all_valid` is true only when every checked span still holds. +- `--strict` exits 1 when anything is invalid (useful in scripts). +- See [memory.md](memory.md) for verdict states and when to reread. + ## Index health ```bash diff --git a/.claude/skills/codebase-index/references/memory.md b/.claude/skills/codebase-index/references/memory.md new file mode 100644 index 0000000..fa533ad --- /dev/null +++ b/.claude/skills/codebase-index/references/memory.md @@ -0,0 +1,49 @@ +# Evidence Memory + +Load this when you use `--session`, when a packet contains `reused`, `stale` or +`memory`, or when you want to cite evidence for later. + +## What is guaranteed + +Evidence is identified by the exact bytes of a file span. `reused: true` is set +only when this session already received the same text — or the whole span — +and those bytes are unchanged now. Nothing is decided by query similarity, and +memory stores no source text. + +## Verdict states + +| state | still true? | do this | +|---|---|---| +| `valid` | yes | use it | +| `relocated` | yes — identical content moved inside the file | use it; cite the new lines | +| `changed` | no | reread the range before relying on it | +| `ambiguous` | no — identical content now occurs more than once | reread | +| `deleted` | no — file gone; a rename or move counts as deleted | search again | +| `excluded` | no — now ignored or secret-like | do not read it | +| `unreadable` | no | search again | + +## Reread even when evidence is valid + +- You are about to edit and need the exact current text. +- The earlier snippet was a skeleton or a signature and you need the body. +- Your context was cleared or compacted, or you cannot see the earlier snippet. +- A tool output was truncated. + +## Citing evidence across conversations + +`codebase-index verify --session --json` lists everything the session +received as `path:start-end@hash` references. Keep the references next to the +conclusions they support in notes or handoffs: + +```text +Refunds are capped at the invoice total [billing/refund.py:3-4@3f9a2c1b7d4e8a90] +``` + +Later, from any agent, check them before trusting the note: + +```bash +codebase-index verify "billing/refund.py:3-4@3f9a2c1b7d4e8a90" --json +``` + +`all_valid: true` means every cited span still holds exactly. Otherwise reread +the invalid spans and re-derive the conclusion; do not patch the old one. diff --git a/.claude/skills/codebase-index/references/response-contract.md b/.claude/skills/codebase-index/references/response-contract.md index 798f152..98af67f 100644 --- a/.claude/skills/codebase-index/references/response-contract.md +++ b/.claude/skills/codebase-index/references/response-contract.md @@ -16,6 +16,9 @@ Each result can contain: - `snippet` - `skeletonized` - `elided_lines` +- `stale` — only when the index text no longer matches the file +- `reused` — only with `--session`, when `snippet` was withheld because this + session already received that exact text from unchanged source `recommended_reads` is the read plan. Start with its first one to three entries and use exact line ranges. An entry with `truncated: true` was capped at the @@ -23,6 +26,11 @@ definition head (`max_read_lines`, default 120); `line_end_full` gives the real extent. Read the capped range first and continue only when the head is not enough. +With `--session`, the packet also carries `memory`: `session`, `reused`, +`tokens_saved`, and `invalidated` — references to evidence the session received +that has since changed. `available: false` means nothing was withheld. See +[memory.md](memory.md). + `pagination.has_more` and `pagination.next_offset` indicate additional results. Prefer a more specific command or a larger token budget before paging. diff --git a/.claude/skills/codebase-index/scripts/cbx b/.claude/skills/codebase-index/scripts/cbx index 94a9114..18c3ed0 100644 --- a/.claude/skills/codebase-index/scripts/cbx +++ b/.claude/skills/codebase-index/scripts/cbx @@ -4,7 +4,7 @@ # - Whitelists subcommands so the skill can never invoke destructive ones (clean/init/watch). set -euo pipefail -ALLOWED="search explain architecture symbol refs impact diff-impact path describe graph stats doctor update index" +ALLOWED="search explain architecture symbol refs impact diff-impact path describe verify graph stats doctor update index" sub="${1:-}" case " $ALLOWED " in diff --git a/.claude/skills/codebase-index/scripts/cbx.ps1 b/.claude/skills/codebase-index/scripts/cbx.ps1 index eee56ae..6419e37 100644 --- a/.claude/skills/codebase-index/scripts/cbx.ps1 +++ b/.claude/skills/codebase-index/scripts/cbx.ps1 @@ -10,7 +10,7 @@ param( $ErrorActionPreference = "Stop" $allowed = @( "search", "explain", "architecture", "symbol", "refs", "impact", "diff-impact", - "path", "describe", "graph", "stats", "doctor", "update", "index" + "path", "describe", "verify", "graph", "stats", "doctor", "update", "index" ) if ($allowed -notcontains $Subcommand) { diff --git a/.codex/skills/codebase-index/.skill_version b/.codex/skills/codebase-index/.skill_version index f8e233b..227cea2 100644 --- a/.codex/skills/codebase-index/.skill_version +++ b/.codex/skills/codebase-index/.skill_version @@ -1 +1 @@ -1.9.0 +2.0.0 diff --git a/.codex/skills/codebase-index/SKILL.md b/.codex/skills/codebase-index/SKILL.md index 9404686..432005b 100644 --- a/.codex/skills/codebase-index/SKILL.md +++ b/.codex/skills/codebase-index/SKILL.md @@ -1,25 +1,26 @@ --- name: codebase-index -description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository. -allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob +description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository, and verify evidence gathered earlier before relying on it. +allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index verify *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob --- # Codebase Index Use the local index before reading repository files. -The operating principle is **Find → Trace → Predict**: +The operating principle is **Find → Trace → Verify → Predict**: - **Find** the implementation with ranked retrieval. - **Trace** behavior through definitions, callers, dependencies, and paths. +- **Verify** that evidence you already hold is still true before relying on it. - **Predict** change impact while preserving an explicit evidence trail. ## Route the question | Intent | Command | |---|---| -| Where is X implemented? | `codebase-index search "X" --json` | -| How does X work? | `codebase-index explain "X" --json` | +| Where is X implemented? | `codebase-index search "X" --session --json` | +| How does X work? | `codebase-index explain "X" --session --json` | | What is this codebase? | `codebase-index architecture --json` | | Find a named symbol | `codebase-index symbol "X" --json` | | Who calls or references X? | `codebase-index refs "X" --json` | @@ -27,6 +28,7 @@ The operating principle is **Find → Trace → Predict**: | What does my current diff affect? | `codebase-index diff-impact --json` | | How are X and Y connected? | `codebase-index path "X" "Y" --json` | | Describe X and its neighborhood | `codebase-index describe "X" --json` | +| Is what I read earlier still true? | `codebase-index verify --session --json` | | Produce a human graph | `codebase-index graph "X" --output ` | Use `search --mode symbol` for exact symbol work, `--mode fts` for text and @@ -38,21 +40,41 @@ or routing remain unclear. ## Evidence protocol -1. Run the best-matching command with `--json`. -2. Check `index` before trusting the payload: +1. Pick one session tag for this conversation (for example `auth-fix-1`) and + pass `--session ` to every `search` and `explain`. +2. Run the best-matching command with `--json`. +3. Check `index` before trusting the payload: - missing → run `codebase-index index`, then repeat; - stale with fewer than 20 changed files → run `codebase-index update`; - stale with 20 or more changed files → run `codebase-index index`; - fresh → continue. -3. Start with ranks 1–3. Read only `recommended_reads` line ranges. -4. Trace one additional hop only when the question requires behavior, +4. Start with ranks 1–3. Read only `recommended_reads` line ranges. +5. Trace one additional hop only when the question requires behavior, ownership, or impact. -5. Answer with `file:line` evidence and state uncertainty explicitly. +6. Before answering or editing from evidence gathered earlier in the task, run + `codebase-index verify --session --json` and reread anything whose + state is not `valid` or `relocated`. +7. Answer with `file:line` evidence and state uncertainty explicitly. Do not open whole files when a line range is available. A snippet may already be sufficient. `skeletonized: true` means the response intentionally folded unrelated body lines; read the supplied range when the missing body matters. +## Evidence memory + +- `reused: true` with `snippet: null` — this session already received that + exact text and its source is unchanged. Use your earlier copy; if you can no + longer see it, Read the range. +- `memory.invalidated` — evidence this session received has changed since. + Treat your earlier copy as wrong and reread before relying on it. +- `stale: true` — the index is older than the file. Run `codebase-index update` + or Read the range. +- A tag belongs to one context. Never give it to a subagent or another + conversation. Start a new tag after the context is cleared or compacted, or + whenever earlier snippets are no longer visible to you. + +Verdict states and citing evidence in notes: [references/memory.md](references/memory.md). + ## Confidence contract - **high** — answer from the indexed evidence. diff --git a/.codex/skills/codebase-index/references/commands.md b/.codex/skills/codebase-index/references/commands.md index 0efb464..79c91a6 100644 --- a/.codex/skills/codebase-index/references/commands.md +++ b/.codex/skills/codebase-index/references/commands.md @@ -17,6 +17,9 @@ Useful search options: - `--offset ` - `--raw` to disable snippet skeletonization - `--no-fallback` to suppress fallback suggestions +- `--session ` to name this conversation's context: unchanged evidence it + already received comes back as `reused: true` without the snippet, and + evidence that changed is listed under `memory.invalidated` `explain` uses the HOW_IT_WORKS intent and a larger default token budget. Prefer it over repeatedly rewording a broad search. @@ -49,6 +52,18 @@ codebase-index graph "" --direction both --depth 2 --output graph.html For headless work, use `--output`; do not use `--open`. Exports also support `--format graphml|dot|neo4j`. +## Evidence + +```bash +codebase-index verify --session --json +codebase-index verify "" ... --json +``` + +- `verify` is read-only and needs no index: it checks evidence against the + working tree. `all_valid` is true only when every checked span still holds. +- `--strict` exits 1 when anything is invalid (useful in scripts). +- See [memory.md](memory.md) for verdict states and when to reread. + ## Index health ```bash diff --git a/.codex/skills/codebase-index/references/memory.md b/.codex/skills/codebase-index/references/memory.md new file mode 100644 index 0000000..fa533ad --- /dev/null +++ b/.codex/skills/codebase-index/references/memory.md @@ -0,0 +1,49 @@ +# Evidence Memory + +Load this when you use `--session`, when a packet contains `reused`, `stale` or +`memory`, or when you want to cite evidence for later. + +## What is guaranteed + +Evidence is identified by the exact bytes of a file span. `reused: true` is set +only when this session already received the same text — or the whole span — +and those bytes are unchanged now. Nothing is decided by query similarity, and +memory stores no source text. + +## Verdict states + +| state | still true? | do this | +|---|---|---| +| `valid` | yes | use it | +| `relocated` | yes — identical content moved inside the file | use it; cite the new lines | +| `changed` | no | reread the range before relying on it | +| `ambiguous` | no — identical content now occurs more than once | reread | +| `deleted` | no — file gone; a rename or move counts as deleted | search again | +| `excluded` | no — now ignored or secret-like | do not read it | +| `unreadable` | no | search again | + +## Reread even when evidence is valid + +- You are about to edit and need the exact current text. +- The earlier snippet was a skeleton or a signature and you need the body. +- Your context was cleared or compacted, or you cannot see the earlier snippet. +- A tool output was truncated. + +## Citing evidence across conversations + +`codebase-index verify --session --json` lists everything the session +received as `path:start-end@hash` references. Keep the references next to the +conclusions they support in notes or handoffs: + +```text +Refunds are capped at the invoice total [billing/refund.py:3-4@3f9a2c1b7d4e8a90] +``` + +Later, from any agent, check them before trusting the note: + +```bash +codebase-index verify "billing/refund.py:3-4@3f9a2c1b7d4e8a90" --json +``` + +`all_valid: true` means every cited span still holds exactly. Otherwise reread +the invalid spans and re-derive the conclusion; do not patch the old one. diff --git a/.codex/skills/codebase-index/references/response-contract.md b/.codex/skills/codebase-index/references/response-contract.md index 798f152..98af67f 100644 --- a/.codex/skills/codebase-index/references/response-contract.md +++ b/.codex/skills/codebase-index/references/response-contract.md @@ -16,6 +16,9 @@ Each result can contain: - `snippet` - `skeletonized` - `elided_lines` +- `stale` — only when the index text no longer matches the file +- `reused` — only with `--session`, when `snippet` was withheld because this + session already received that exact text from unchanged source `recommended_reads` is the read plan. Start with its first one to three entries and use exact line ranges. An entry with `truncated: true` was capped at the @@ -23,6 +26,11 @@ definition head (`max_read_lines`, default 120); `line_end_full` gives the real extent. Read the capped range first and continue only when the head is not enough. +With `--session`, the packet also carries `memory`: `session`, `reused`, +`tokens_saved`, and `invalidated` — references to evidence the session received +that has since changed. `available: false` means nothing was withheld. See +[memory.md](memory.md). + `pagination.has_more` and `pagination.next_offset` indicate additional results. Prefer a more specific command or a larger token budget before paging. diff --git a/.codex/skills/codebase-index/scripts/cbx b/.codex/skills/codebase-index/scripts/cbx index 94a9114..18c3ed0 100644 --- a/.codex/skills/codebase-index/scripts/cbx +++ b/.codex/skills/codebase-index/scripts/cbx @@ -4,7 +4,7 @@ # - Whitelists subcommands so the skill can never invoke destructive ones (clean/init/watch). set -euo pipefail -ALLOWED="search explain architecture symbol refs impact diff-impact path describe graph stats doctor update index" +ALLOWED="search explain architecture symbol refs impact diff-impact path describe verify graph stats doctor update index" sub="${1:-}" case " $ALLOWED " in diff --git a/.codex/skills/codebase-index/scripts/cbx.ps1 b/.codex/skills/codebase-index/scripts/cbx.ps1 index eee56ae..6419e37 100644 --- a/.codex/skills/codebase-index/scripts/cbx.ps1 +++ b/.codex/skills/codebase-index/scripts/cbx.ps1 @@ -10,7 +10,7 @@ param( $ErrorActionPreference = "Stop" $allowed = @( "search", "explain", "architecture", "symbol", "refs", "impact", "diff-impact", - "path", "describe", "graph", "stats", "doctor", "update", "index" + "path", "describe", "verify", "graph", "stats", "doctor", "update", "index" ) if ($allowed -notcontains $Subcommand) { diff --git a/.gitattributes b/.gitattributes index 5a02ab9..0885947 100644 --- a/.gitattributes +++ b/.gitattributes @@ -5,3 +5,12 @@ *.py text eol=lf # cbx — POSIX-скрипт без расширения; CRLF в worktree ломает shebang при копировании cbx text eol=lf +# .skill_version хранит ровно "\n"; scripts/sync_skill_copies.py пишет и +# сравнивает байты, поэтому при core.autocrlf=true git подставлял CRLF в worktree и +# test_real_repo_is_in_sync падал на любом свежем клоне под Windows. +.skill_version text eol=lf +# sample_repo сравнивается побайтно: tests/fixtures/index-1.10.0/index.sqlite хранит sha256 +# её файлов. Файлы без правила (web/app.ts) git под Windows с core.autocrlf=true отдавал +# с CRLF, под Linux/macOS с LF, и update на CI переиндексировал файл. -text отключает +# любую конверсию: в рабочем дереве те же байты, что в репозитории, на любой ОС. +tests/fixtures/sample_repo/** -text diff --git a/.gitignore b/.gitignore index 1c5d496..30036c7 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ *.sqlite *.sqlite-wal *.sqlite-shm +# Exception: a real 1.10.0 index, kept to test the 1.x -> 2.0 upgrade path. +!tests/fixtures/index-1.10.0/index.sqlite # The skill itself (.claude/skills/codebase-index/) MAY be committed so the team shares it. # The cache above must not be. diff --git a/.opencode/skills/codebase-index/.skill_version b/.opencode/skills/codebase-index/.skill_version index f8e233b..227cea2 100644 --- a/.opencode/skills/codebase-index/.skill_version +++ b/.opencode/skills/codebase-index/.skill_version @@ -1 +1 @@ -1.9.0 +2.0.0 diff --git a/.opencode/skills/codebase-index/SKILL.md b/.opencode/skills/codebase-index/SKILL.md index 9404686..432005b 100644 --- a/.opencode/skills/codebase-index/SKILL.md +++ b/.opencode/skills/codebase-index/SKILL.md @@ -1,25 +1,26 @@ --- name: codebase-index -description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository. -allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob +description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository, and verify evidence gathered earlier before relying on it. +allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index verify *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob --- # Codebase Index Use the local index before reading repository files. -The operating principle is **Find → Trace → Predict**: +The operating principle is **Find → Trace → Verify → Predict**: - **Find** the implementation with ranked retrieval. - **Trace** behavior through definitions, callers, dependencies, and paths. +- **Verify** that evidence you already hold is still true before relying on it. - **Predict** change impact while preserving an explicit evidence trail. ## Route the question | Intent | Command | |---|---| -| Where is X implemented? | `codebase-index search "X" --json` | -| How does X work? | `codebase-index explain "X" --json` | +| Where is X implemented? | `codebase-index search "X" --session --json` | +| How does X work? | `codebase-index explain "X" --session --json` | | What is this codebase? | `codebase-index architecture --json` | | Find a named symbol | `codebase-index symbol "X" --json` | | Who calls or references X? | `codebase-index refs "X" --json` | @@ -27,6 +28,7 @@ The operating principle is **Find → Trace → Predict**: | What does my current diff affect? | `codebase-index diff-impact --json` | | How are X and Y connected? | `codebase-index path "X" "Y" --json` | | Describe X and its neighborhood | `codebase-index describe "X" --json` | +| Is what I read earlier still true? | `codebase-index verify --session --json` | | Produce a human graph | `codebase-index graph "X" --output ` | Use `search --mode symbol` for exact symbol work, `--mode fts` for text and @@ -38,21 +40,41 @@ or routing remain unclear. ## Evidence protocol -1. Run the best-matching command with `--json`. -2. Check `index` before trusting the payload: +1. Pick one session tag for this conversation (for example `auth-fix-1`) and + pass `--session ` to every `search` and `explain`. +2. Run the best-matching command with `--json`. +3. Check `index` before trusting the payload: - missing → run `codebase-index index`, then repeat; - stale with fewer than 20 changed files → run `codebase-index update`; - stale with 20 or more changed files → run `codebase-index index`; - fresh → continue. -3. Start with ranks 1–3. Read only `recommended_reads` line ranges. -4. Trace one additional hop only when the question requires behavior, +4. Start with ranks 1–3. Read only `recommended_reads` line ranges. +5. Trace one additional hop only when the question requires behavior, ownership, or impact. -5. Answer with `file:line` evidence and state uncertainty explicitly. +6. Before answering or editing from evidence gathered earlier in the task, run + `codebase-index verify --session --json` and reread anything whose + state is not `valid` or `relocated`. +7. Answer with `file:line` evidence and state uncertainty explicitly. Do not open whole files when a line range is available. A snippet may already be sufficient. `skeletonized: true` means the response intentionally folded unrelated body lines; read the supplied range when the missing body matters. +## Evidence memory + +- `reused: true` with `snippet: null` — this session already received that + exact text and its source is unchanged. Use your earlier copy; if you can no + longer see it, Read the range. +- `memory.invalidated` — evidence this session received has changed since. + Treat your earlier copy as wrong and reread before relying on it. +- `stale: true` — the index is older than the file. Run `codebase-index update` + or Read the range. +- A tag belongs to one context. Never give it to a subagent or another + conversation. Start a new tag after the context is cleared or compacted, or + whenever earlier snippets are no longer visible to you. + +Verdict states and citing evidence in notes: [references/memory.md](references/memory.md). + ## Confidence contract - **high** — answer from the indexed evidence. diff --git a/.opencode/skills/codebase-index/references/commands.md b/.opencode/skills/codebase-index/references/commands.md index 0efb464..79c91a6 100644 --- a/.opencode/skills/codebase-index/references/commands.md +++ b/.opencode/skills/codebase-index/references/commands.md @@ -17,6 +17,9 @@ Useful search options: - `--offset ` - `--raw` to disable snippet skeletonization - `--no-fallback` to suppress fallback suggestions +- `--session ` to name this conversation's context: unchanged evidence it + already received comes back as `reused: true` without the snippet, and + evidence that changed is listed under `memory.invalidated` `explain` uses the HOW_IT_WORKS intent and a larger default token budget. Prefer it over repeatedly rewording a broad search. @@ -49,6 +52,18 @@ codebase-index graph "" --direction both --depth 2 --output graph.html For headless work, use `--output`; do not use `--open`. Exports also support `--format graphml|dot|neo4j`. +## Evidence + +```bash +codebase-index verify --session --json +codebase-index verify "" ... --json +``` + +- `verify` is read-only and needs no index: it checks evidence against the + working tree. `all_valid` is true only when every checked span still holds. +- `--strict` exits 1 when anything is invalid (useful in scripts). +- See [memory.md](memory.md) for verdict states and when to reread. + ## Index health ```bash diff --git a/.opencode/skills/codebase-index/references/memory.md b/.opencode/skills/codebase-index/references/memory.md new file mode 100644 index 0000000..fa533ad --- /dev/null +++ b/.opencode/skills/codebase-index/references/memory.md @@ -0,0 +1,49 @@ +# Evidence Memory + +Load this when you use `--session`, when a packet contains `reused`, `stale` or +`memory`, or when you want to cite evidence for later. + +## What is guaranteed + +Evidence is identified by the exact bytes of a file span. `reused: true` is set +only when this session already received the same text — or the whole span — +and those bytes are unchanged now. Nothing is decided by query similarity, and +memory stores no source text. + +## Verdict states + +| state | still true? | do this | +|---|---|---| +| `valid` | yes | use it | +| `relocated` | yes — identical content moved inside the file | use it; cite the new lines | +| `changed` | no | reread the range before relying on it | +| `ambiguous` | no — identical content now occurs more than once | reread | +| `deleted` | no — file gone; a rename or move counts as deleted | search again | +| `excluded` | no — now ignored or secret-like | do not read it | +| `unreadable` | no | search again | + +## Reread even when evidence is valid + +- You are about to edit and need the exact current text. +- The earlier snippet was a skeleton or a signature and you need the body. +- Your context was cleared or compacted, or you cannot see the earlier snippet. +- A tool output was truncated. + +## Citing evidence across conversations + +`codebase-index verify --session --json` lists everything the session +received as `path:start-end@hash` references. Keep the references next to the +conclusions they support in notes or handoffs: + +```text +Refunds are capped at the invoice total [billing/refund.py:3-4@3f9a2c1b7d4e8a90] +``` + +Later, from any agent, check them before trusting the note: + +```bash +codebase-index verify "billing/refund.py:3-4@3f9a2c1b7d4e8a90" --json +``` + +`all_valid: true` means every cited span still holds exactly. Otherwise reread +the invalid spans and re-derive the conclusion; do not patch the old one. diff --git a/.opencode/skills/codebase-index/references/response-contract.md b/.opencode/skills/codebase-index/references/response-contract.md index 798f152..98af67f 100644 --- a/.opencode/skills/codebase-index/references/response-contract.md +++ b/.opencode/skills/codebase-index/references/response-contract.md @@ -16,6 +16,9 @@ Each result can contain: - `snippet` - `skeletonized` - `elided_lines` +- `stale` — only when the index text no longer matches the file +- `reused` — only with `--session`, when `snippet` was withheld because this + session already received that exact text from unchanged source `recommended_reads` is the read plan. Start with its first one to three entries and use exact line ranges. An entry with `truncated: true` was capped at the @@ -23,6 +26,11 @@ definition head (`max_read_lines`, default 120); `line_end_full` gives the real extent. Read the capped range first and continue only when the head is not enough. +With `--session`, the packet also carries `memory`: `session`, `reused`, +`tokens_saved`, and `invalidated` — references to evidence the session received +that has since changed. `available: false` means nothing was withheld. See +[memory.md](memory.md). + `pagination.has_more` and `pagination.next_offset` indicate additional results. Prefer a more specific command or a larger token budget before paging. diff --git a/.opencode/skills/codebase-index/scripts/cbx b/.opencode/skills/codebase-index/scripts/cbx index 94a9114..18c3ed0 100644 --- a/.opencode/skills/codebase-index/scripts/cbx +++ b/.opencode/skills/codebase-index/scripts/cbx @@ -4,7 +4,7 @@ # - Whitelists subcommands so the skill can never invoke destructive ones (clean/init/watch). set -euo pipefail -ALLOWED="search explain architecture symbol refs impact diff-impact path describe graph stats doctor update index" +ALLOWED="search explain architecture symbol refs impact diff-impact path describe verify graph stats doctor update index" sub="${1:-}" case " $ALLOWED " in diff --git a/.opencode/skills/codebase-index/scripts/cbx.ps1 b/.opencode/skills/codebase-index/scripts/cbx.ps1 index eee56ae..6419e37 100644 --- a/.opencode/skills/codebase-index/scripts/cbx.ps1 +++ b/.opencode/skills/codebase-index/scripts/cbx.ps1 @@ -10,7 +10,7 @@ param( $ErrorActionPreference = "Stop" $allowed = @( "search", "explain", "architecture", "symbol", "refs", "impact", "diff-impact", - "path", "describe", "graph", "stats", "doctor", "update", "index" + "path", "describe", "verify", "graph", "stats", "doctor", "update", "index" ) if ($allowed -notcontains $Subcommand) { diff --git a/AGENTS.md b/AGENTS.md index 65f18e5..0cd9881 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,10 +7,15 @@ Use the local codebase index before scanning repository files. Skill resources: `.codex/skills/codebase-index/SKILL.md` -Run `codebase-index search "" --json` for general questions, or use -`symbol`, `refs`, `impact`, and `graph` for symbol lookup, references, change -impact, and HTML graph export. Search/read commands auto-build the index when -it is missing; run `codebase-index update` when responses report stale data. +Run `codebase-index search "" --session --json` for general +questions, or use `symbol`, `refs`, `impact`, and `graph` for symbol lookup, +references, change impact, and HTML graph export. Use one session tag per +conversation: unchanged evidence you already received comes back as +`reused: true`, and evidence that changed is listed under `memory.invalidated`. +Before relying on something read earlier, run +`codebase-index verify --session --json`. Search/read commands auto-build +the index when it is missing; run `codebase-index update` when responses report +stale data. ## Versioning Policy diff --git a/CHANGELOG.md b/CHANGELOG.md index f65060c..1e6a1f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,66 @@ All notable changes to this project are documented here. The format is based on ## [Unreleased] +## [2.0.0] - 2026-09-14 + +Evidence release. Everything an agent reads through `codebase-index` is now identified +by the exact bytes of the span it came from, can be re-checked against the working tree +at any later moment, is not resent to a session that already holds it, and is reported +when it changes. 2.0.0 also includes the community-readiness work merged from #26 and the +1.10.0 ranking release, which was prepared but never published on its own; its notes are in +the [1.10.0 section of the changelog](https://github.com/denfry/codebase-index/blob/main/CHANGELOG.md#1100---2026-09-02). + +Measured by replaying this repository's own history in sessions of 5 to 87 tasks +(`tests/eval/results/2026-09-14-evidence-memory.md`): memory withheld **zero stale +snippets** at every session length while the locator-based alternative withheld +15–54, change notices had precision 1.000 and recall 0.87–0.97, restoring withheld +snippets reproduced the 1.10.0 packet on every task, and snippet tokens fell by +4–11%. The savings are modest on this corpus; correctness is the deliverable. Summary in +[docs/BENCHMARKS.md](docs/BENCHMARKS.md#evidence-memory), design in +[docs/MEMORY.md](docs/MEMORY.md). + ### Added +- **Evidence identity and verification.** Every delivered snippet carries a + `path:start-end@hash` reference whose hash covers the span's exact bytes under the + indexer's line model. `codebase-index verify [REF ...] [--session TAG] [--strict]` + and MCP `verify_evidence` re-check references against the working tree, read-only + and without an index, returning `valid`, `relocated`, `changed`, `ambiguous`, + `deleted`, `excluded` or `unreadable` per reference and `all_valid` overall. + References are untrusted input: absolute paths, drive letters, `..` and NUL are + rejected before any filesystem access. +- **Session-scoped evidence reuse.** `search`/`explain --session TAG` (MCP `session`) + name one agent context. A snippet the session already received from byte-identical + source comes back as `snippet: null, reused: true`; evidence the session received + that has since changed is listed once under `memory.invalidated`. The evidence hook + runs after ranking, budgeting and pagination are final, so memory can never change + which results are returned. Sessions are only ever named explicitly. +- **Stale marking without a session.** A result whose index text no longer matches the + working tree carries `stale: true`. Derived index text (config-key chunks, Markdown + section summaries) is never flagged: a mismatch counts only when the file's current + hash differs from the one the index was built from. +- **`memory.sqlite`**, a content-free ledger next to the index: hashes, paths, line + numbers, token counts and timestamps; session tags stored hashed. Separate from + `index.sqlite` so rebuilds and `clean` leave it alone and ledger writes never queue + behind an update. A newer schema is refused, a corrupt file is moved aside, lock + contention degrades to no-memory output. `memory gc` and `memory clear` are CLI-only + maintenance, deliberately absent from the skill wrappers and MCP, like `clean`. + `stats`, `doctor`, `index_stats` and `healthcheck` gain an additive `memory` block. + Config: `memory.enabled` (true), `memory.retention_days` (14), + `memory.max_deliveries` (50 000); `CBX_MEMORY=0` restores 1.10.0 output byte for byte. +- **Sequential real-history benchmark** (`tests/eval/memory_eval.py`) with an oracle + independent of memory's hashing, plus repository-lifecycle, security and + stale-context test suites driven by real git: branch switches, detached HEAD, + worktrees, dirty trees, renames, rebases, CRLF checkouts, and excluded content that + must never reach the ledger. +- **Upgrade test from a real 1.10.0 index.** The index schema stays at version 3; an + upgraded project keeps its index, a 1.x config loads with memory defaults, and + `memory.sqlite` attaches on the first `--session` call. +- **Skill: Find → Trace → Verify → Predict.** The agent wrappers pass one session tag + per conversation, run `verify` before relying on evidence gathered earlier, and + document `reused`, `stale` and `memory.invalidated` in `references/memory.md`. + `verify` joins the wrapper whitelists; `memory` does not. + - **Public baseline benchmark.** `tests/eval/run_baselines.py` compares the index with a disciplined `rg` + 80-line-window agent and with repo-map-style context on Flask, Gson and Fastify at pinned commits, using git-derived ground truth, one @@ -31,6 +89,15 @@ All notable changes to this project are documented here. The format is based on ### Changed +- **Indexing gates are shared with later working-tree reads.** Evidence validation + reads files long after they were indexed, so the walker and the validator now go + through one `PathGate`; a parity test asserts the gate admits exactly the files a + walk indexes. The resolved on-disk path is gated again, so a symlink or a + differently-cased path on a case-insensitive filesystem cannot reach a file the + walker would never have indexed. +- MCP `schema_version` stays **1**: every payload change is an added field. No CLI + command, flag or JSON field is removed or retyped. + - **Read plan is bounded.** `recommended_reads` entries are capped at `retrieval.max_read_lines` (default 120) and carry `truncated: true` plus `line_end_full` when capped. A symbol-aligned chunk can be a whole class; on the @@ -50,6 +117,10 @@ All notable changes to this project are documented here. The format is based on ### Fixed +- `codebase-index index` no longer crashes with `ValueError: ... is not in the subpath of ...` + when the repository contains a symlink that points outside the tree (for example Bazel's + `bazel-out` / `bazel-bin` convenience symlinks). Such entries are now skipped by the walker, + matching the gate's existing "resolves outside the repository" exclusion (#27). - **MCP server did not start on mcp 2.x.** The SDK renamed `FastMCP` to `MCPServer` and removed the old import path, so `codebase-index mcp` reported "needs the optional extra" even with the extra installed, and the MCP tests @@ -69,6 +140,141 @@ All notable changes to this project are documented here. The format is based on Flask-style `CHANGES.rst` was not covered. Both are fixed; `CHANGES*`, `HISTORY*`, `NEWS*` and `RELEASE_NOTES*` are refused as answers and excluded from the corpus. +## [1.10.0] - 2026-09-02 + +Never published on its own: tagged releases go from 1.9.0 straight to 2.0.0, which +includes everything in this section. + +Ranking release. 1.9.0's own diagnostics showed that a perfect reranker over the +candidate pool it already generated would score MRR 0.902 against the 0.577 actually +delivered — a ranking gap roughly three times larger than the remaining recall gap. +1.10.0 spends its entire budget on closing part of that gap, and adds the metrics +that make the gap visible. + +Measured over **420 queries across eight repositories** (Python ×2, Java ×3, +TypeScript/TSX ×2, PowerShell ×1) against a pinned 1.9.0: MRR +0.0188 (p=0.004), +nDCG@10 +0.0304 (p<0.001), MAP +0.0213 (p=0.001), recall@10 +0.0627 (p<0.001, 37 +wins / 0 losses), useful@budget +0.0292 (p=0.020), −19 tokens per query. The +candidate pool is unchanged, so every gain is reranking. No corpus regressed. Under +leave-one-repository-out the pooled gain is +0.0219 MRR with 7/8 folds improving and +0 regressing. Full tables in [docs/BENCHMARKS.md](docs/BENCHMARKS.md). + +### Added + +- **Query↔candidate name co-occurrence** (`retrieval/features.py`), the one new + ranking signal. Every retriever scores each query term independently and RRF sums + those independent verdicts, so nothing in the pipeline could distinguish a + candidate that matched *one* query term well from one that matched *three* terms in + a single name. On "graph resolution + traversal accessors" 1.9.0 ranked + `graph/retrieval.py` (one term) above `test_graph_accessors_resolve_and_walk` + (three); on "greedy token budgeting with redaction" it ranked `output/redact.py` + above `retrieval/budget.py`. The signal credits query terms for occurring + *together* in one name — file basename plus symbol, camel/snake split, directories + excluded — and only for terms beyond the first, since the first match is already + paid for by the retriever that surfaced the candidate. Cost is + `O(len(path) + len(symbol) + len(terms))` per candidate: no corpus statistics, no + posting-list scan, no model, no network. Ablatable via + `RetrievalTuning(name_cooccurrence=False)`. +- **Oracle / headroom metrics** in the eval harness (`oracle`, `cand_recall`, `eff`). + MRR alone cannot separate "retrieval never found it" from "the ranker buried it", + and those two failures share no fix. `oracle` is the MRR a perfect reranker would + achieve over the pool actually generated, `eff = MRR / oracle` is the fraction of + achievable quality delivered (0.639 → 0.660 in this release). These also make + ranking changes falsifiable in a new way: `-name_cooccurrence` moves eight quality + metrics while leaving `oracle` at ±0.0000, proving the gain is not disguised recall. +- **`search(..., explain=True)`** returns a `diagnostics` block with the pre-rerank + candidate pool and the final order, each candidate carrying source, symbol, score + and retriever agreement. This is what the oracle metrics are computed from, and + what turns "the ranking is wrong" into a decomposable failure. Measured at −0.8ms + p50 (inside noise); nothing is allocated when the flag is off. +- **`RetrievalTuning.v190()`** pins the previous release as the comparison column, so + "better than what we shipped last" cannot drift as the default changes. + `run_eval.py` now reports 1.7.0, 1.9.0 and the current default side by side. + +### Changed + +- **Pages are packed with distinct files** (`max_per_file` 3 → 1). The agent's unit + of decision is "which file do I open", so a 10-result page spending three slots on + three regions of one file offers seven choices, not ten. 1.9.0's page held 7.1 + distinct files on average, and of the queries whose answer was in the pool but + missing from the page, 45 of 57 had it past rank 10 — crowded out by repeat hits + rather than better candidates. Monotone over 1–5, so this is a plateau boundary, + not a fitted peak. Nothing is dropped: overflow hits keep their relative order at + the tail. recall@10 +0.045, nDCG@10 +0.015, unchanged token cost. +- **Name co-occurrence is discounted for test and generated sources** + (`name_cooccurrence_demoted_scale`, 0.5). Test function names are descriptive + sentences (`test_compactor_output_is_redacted`), so they harvest query-term + co-occurrences real identifiers never do. The value was chosen by splitting the + benchmark on whether its own ground truth is a test: across the 261 queries whose + answer is *not* a test the gain is flat at +0.020 MRR for every scale, so the whole + aggregate difference between 0.5 and 1.0 comes from the 159 test-answer queries — + an artifact of mining ground truth from commits, which touch tests. 0.5 is the only + setting that improves both partitions. +- **Benchmark corpora no longer index changelog files.** `gen_queries` documented + `CHANGELOG_EXCLUDES` as applied to the corpus but the harness never wired it in. A + git-derived query *is* a commit subject and a changelog entry paraphrases it + verbatim while never being an accepted answer, so every affected query carried an + unbeatable distractor that compressed all variants toward the same floor. + +### Performance + +- **Duplicate detection roughly halved.** It was 42% of the query path on the Java + corpus. Two independent fixes: the operator scan in `normalize_code_tokens` no + longer runs up to 24 `str.startswith` calls per punctuation character (1.71× + faster on 1600 real chunks, bit-identical token stream), and only the leading 2000 + characters of a body now decide duplication — two chunks agreeing for 2000 + characters are the same snippet. Recall, duplicate rate and useful-context are + identical; MRR within −0.0003 (p=0.51). + +### Fixed + +- **Non-ASCII identifiers can now earn name-level ranking credit.** The query side + parses Unicode correctly, so `расчёт_налога.py` produced matching query terms but + its own name components were silently dropped, making the file unrankable by name. +- **`.skill_version` is pinned to LF in `.gitattributes`.** `sync_skill_copies.py` + writes and byte-compares `"\n"`, but the file carried no EOL attribute, + so with `core.autocrlf=true` git materialised CRLF and `test_real_repo_is_in_sync` + failed on any fresh Windows clone — and again after any `git checkout` of that file. +- **Lint is clean repository-wide.** `skill/scripts/` was outside the CI lint scope + (`ruff check src tests`) and had accumulated three violations. + +### Verdicts on existing signals + +Every pre-existing signal was re-examined rather than inherited. `soft_lexical` +(−0.161 MRR when off) and `source_priors` (−0.029) remain load-bearing; +`file_agreement` (−0.010, p=0.048) and `dedup` (−0.0016, p=0.006) keep their places. + +- **`query_expansion` survives a deletion attempt, and the reason is a lesson.** On + the 420 git-derived queries the synonym vocabulary is worth nothing measurable + (MRR −0.0022, p=0.40 when removed), and the flag's apparent benefit turned out to + come from it also swapping the symbol retriever's tokenizer. It was removed — and + then restored, because a commit subject is written by someone looking at the + identifiers they just changed and so reuses the codebase's spelling, while a user + asking a question does not. On the 36 hand-written natural-language queries removal + cost −0.060 MRR. The git benchmark is structurally blind to morphology; both query + families are needed to make this call. +- **The intent classifier is inert on commit-style queries but kept.** It returns + `keyword` for 419 of 420 git-derived queries, and removing the layer entirely is + bit-identical on that benchmark. It fires on 30.6% of hand-written questions, where + removing it costs MRR, and the retriever *weights* it selects are worth +0.058 MRR + against uniform weights. Deleting it would optimise for the benchmark's phrasing. +- **`fuzzy_symbols` and `graph_source` remain measurably inert** on all eight corpora + (0/1 and 0/0 query changes respectively); both stay as-is rather than accumulating + new tuning. + +### Rejected + +Recorded so they are not re-attempted without a new hypothesis. A pairwise logistic +ranker fitted over 19 deterministic query↔candidate features selected exactly one +feature, and forward selection found no second feature clearing the noise floor, so +one explainable term ships instead of a model. Individually measured and rejected: +idf weighting of matched terms (pool-local and corpus-wide), substring matching, +prefix/stem-tolerant matching, ordered-subsequence matching, term proximity in the +chunk body, body-text coverage, zone-size normalisation, restricting the name zone to +the filename or symbol alone, and including the parent directory. `exact_symbol` was +found to have no discriminative power at all (AUC 0.500) but is left untouched, since +changing it is a separate experiment from adding a signal. + ## [1.9.0] - 2026-09-02 ### Added @@ -567,7 +773,10 @@ Pooled over 305 queries (Python, Java, TypeScript), v1.8.0 → 1.9.0: - Hooks example + `watch` mode for keeping the index fresh without blocking the edit loop (M8). - `doctor`, `stats`, `clean` diagnostics/maintenance commands. -[Unreleased]: https://github.com/denfry/codebase-index/compare/v1.8.0...HEAD +[Unreleased]: https://github.com/denfry/codebase-index/compare/v2.0.0...HEAD +[2.0.0]: https://github.com/denfry/codebase-index/compare/v1.9.0...v2.0.0 +[1.10.0]: https://github.com/denfry/codebase-index/compare/v1.9.0...abb67df +[1.9.0]: https://github.com/denfry/codebase-index/compare/v1.8.0...v1.9.0 [1.8.0]: https://github.com/denfry/codebase-index/compare/v1.7.0...v1.8.0 [1.7.0]: https://github.com/denfry/codebase-index/compare/v1.6.0...v1.7.0 [1.6.0]: https://github.com/denfry/codebase-index/compare/v1.5.0...v1.6.0 diff --git a/README.md b/README.md index 96340f7..5271de0 100644 --- a/README.md +++ b/README.md @@ -191,7 +191,7 @@ installed skills and the MCP server all call the same service layer, so behaviou cannot drift between surfaces. Deep dives: [Architecture](docs/ARCHITECTURE.md) · -[Retrieval](docs/RETRIEVAL.md) · [Schema](docs/SCHEMA.md) · +[Retrieval](docs/RETRIEVAL.md) · [Evidence memory](docs/MEMORY.md) · [Schema](docs/SCHEMA.md) · [Languages](docs/LANGUAGES.md) · [Skill design](docs/SKILL_DESIGN.md) ## Privacy and security @@ -225,14 +225,15 @@ silently presented as "no callers". Tiers and how to add a language: ## Project status -Current line: **1.9.x**, on PyPI, MIT. CI runs Linux, macOS and Windows on Python +Current line: **2.0.x**, on PyPI, MIT. CI runs Linux, macOS and Windows on Python 3.11–3.13 with an 80% coverage gate, golden snapshots for every CLI and MCP payload, a packaging smoke test on every PR, and a skill-copy drift check. What works today: hybrid retrieval with optional local vectors; Tree-sitter symbols and edges; `search`, `explain`, `symbol`, `refs`, `impact`, `diff-impact`, `path`, -`describe`, `architecture`, `graph`; token-budgeted, skeletonized packets; incremental -`update`, `watch`, and hooks; CLI, Claude Code plugin/skill, Codex, OpenCode and +`describe`, `architecture`, `graph`, `verify`; token-budgeted, skeletonized packets; +evidence memory that withholds byte-identical snippets within a session and reports +evidence that changed; incremental `update`, `watch`, and hooks; CLI, Claude Code plugin/skill, Codex, OpenCode and MCP delivery; a reproducible multi-repository benchmark. What does not exist yet: framework-aware typed edges (routes, DI, migrations), diff --git a/SECURITY.md b/SECURITY.md index 5d1e94f..21f1e86 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -11,8 +11,8 @@ Only the latest minor release line receives security fixes. | Version | Supported | |---|---| -| 1.9.x (latest) | Yes | -| < 1.9 | No — upgrade with `pip install -U codebase-index` | +| 2.0.x (latest) | Yes | +| < 2.0 | No — upgrade with `pip install -U codebase-index` | ## Reporting a vulnerability diff --git a/bin/cbx b/bin/cbx index d4fb7f6..fd4af7b 100644 --- a/bin/cbx +++ b/bin/cbx @@ -3,7 +3,7 @@ # from the venv provisioned by scripts/bootstrap.sh (located via the .venv-path pointer). set -euo pipefail -ALLOWED="search explain architecture symbol refs impact diff-impact path describe graph stats doctor update index" +ALLOWED="search explain architecture symbol refs impact diff-impact path describe verify graph stats doctor update index" sub="${1:-}" case " $ALLOWED " in *" ${sub} "*) : ;; diff --git a/bin/cbx.ps1 b/bin/cbx.ps1 index d841080..3d9a8ba 100644 --- a/bin/cbx.ps1 +++ b/bin/cbx.ps1 @@ -5,7 +5,7 @@ param( [Parameter(ValueFromRemainingArguments = $true)] [string[]]$Rest ) $ErrorActionPreference = "Stop" -$allowed = @("search", "explain", "architecture", "symbol", "refs", "impact", "diff-impact", "path", "describe", "graph", "stats", "doctor", "update", "index") +$allowed = @("search", "explain", "architecture", "symbol", "refs", "impact", "diff-impact", "path", "describe", "verify", "graph", "stats", "doctor", "update", "index") if ($allowed -notcontains $Subcommand) { Write-Error "cbx: refusing subcommand '$Subcommand'. Allowed: $($allowed -join ', ')" exit 2 diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 3f6579d..7bfc85a 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -101,8 +101,8 @@ codebase-index/ ├── graph/ # builder.py (edge resolution), expand.py (impact), analysis.py │ # (communities/god nodes), navigate.py (path/describe), export.py ├── storage/ # db.py (pragmas, schema, version guard), schema.sql, repo.py - ├── retrieval/ # intent.py, searchers.py, fusion.py, rerank.py, priors.py, - │ # lexical.py, fuzzy.py, diversity.py, skeleton.py, + ├── retrieval/ # intent.py, searchers.py, fusion.py, rerank.py, features.py, + │ # priors.py, lexical.py, fuzzy.py, diversity.py, skeleton.py, │ # tuning.py (ablation contract), budget.py, pipeline.py, types.py ├── embeddings/ # backend.py, noop.py (default), local.py, external.py — opt-in ├── output/ # markdown.py, json.py, redact.py @@ -132,7 +132,8 @@ CI fails if they drift (`--check`). (a future-versioned index asks for a rebuild rather than guessing), and typed accessors. FTS5 virtual tables and (optional) `sqlite-vec` vector tables live here. - **retrieval** — The query path. `intent.py` classifies the query; `searchers.py` runs the - relevant retrievers; `fusion.py` merges them with RRF; `rerank.py` reorders; `graph.expand` + relevant retrievers; `fusion.py` merges them with RRF; `features.py` extracts deterministic + query↔candidate features and `rerank.py` reorders on them; `graph.expand` pulls in related nodes; `budget.py` trims to a token budget. - **embeddings** — Opt-in only. A `Backend` protocol so vector providers are pluggable. Default is `noop` (disabled). Local models supported; external APIs require explicit config + a warning. diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 604237f..4b3c80f 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -126,6 +126,48 @@ one-signal-off ablations, pooled across corpora, with the same significance machinery. It cannot say the index beats an alternative; it says whether a change to the ranker helped. +`1.10.0` was measured over **420 queries across eight repositories** (Python ×2, +Java ×3, TypeScript/TSX ×2, PowerShell ×1) against a pinned `1.9.0` +(`RetrievalTuning.v190()`, so the comparison point cannot drift with the default): + +| Metric | 1.9.0 | 1.10.0 | Δ | 95% CI | p | W/L/T | +|---|---|---|---|---|---|---| +| MRR | 0.5767 | 0.5955 | +0.0188 | [+0.0061, +0.0321] | 0.004 | 44/14/362 | +| nDCG@10 | 0.5352 | 0.5656 | +0.0304 | [+0.0197, +0.0416] | <0.001 | 64/19/337 | +| MAP | 0.4654 | 0.4867 | +0.0213 | [+0.0104, +0.0331] | 0.001 | 64/19/337 | +| recall@5 | 0.6026 | 0.6212 | +0.0187 | [+0.0089, +0.0306] | <0.001 | 14/0/406 | +| recall@10 | 0.6306 | 0.6933 | +0.0627 | [+0.0421, +0.0847] | <0.001 | 37/0/383 | +| P@5 | 0.1971 | 0.2033 | +0.0062 | [+0.0030, +0.0099] | <0.001 | 14/2/404 | +| useful@budget | 0.5919 | 0.6210 | +0.0292 | [+0.0048, +0.0550] | 0.020 | 36/17/367 | +| hit@3 | 0.6690 | 0.6833 | +0.0143 | [+0.0000, +0.0286] | 0.107 | 8/2/410 | +| tokens/query | 1080 | 1061 | −19 | — | — | — | + +Two properties of that table matter more than the deltas: + +- **`oracle` and `cand_recall` are unchanged to four decimals.** The candidate pool + is identical, so every gain is reranking, not new recall. Reranking efficiency + (MRR / oracle) went 0.639 → 0.660, closing ~6% of the ranking headroom 1.9.0 left + on the table. +- **No corpus regressed.** An aggregate improvement is worthless if one large corpus + masks a regression elsewhere, so per-corpus MRR is checked on every run. + +Held-out validation, because hand-picked coefficients are still fitted parameters: +under leave-one-repository-out — both tuned numbers selected on seven corpora and +scored on the eighth — the pooled gain is **+0.0219 MRR, with 7/8 folds improving +and 0 regressing**. The shipped configuration is deliberately *more conservative* +than that selection would pick (see the `name_cooccurrence_demoted_scale` rationale +in `retrieval/tuning.py`), so these numbers under-claim what the benchmark alone +would support. + +Known cost, stated because the benchmark family that reveals it is the small one: on +the 36 hand-written natural-language queries MRR moved −0.028 (p=0.63; 1 win, 3 +losses, 32 ties). Three questions fell from rank 1 to rank 2–3 where a test file's +descriptive function name matches more query terms than the implementation's name. + +Per-corpus `oracle` ranges 0.795–1.000 and reranking efficiency 0.557–0.753, so the +largest remaining headroom is still ranking, not recall — see §10 of +[RETRIEVAL.md](RETRIEVAL.md). + `1.9.0` was measured over 305 queries across Python, Java and TypeScript corpora against `1.8.0`: MRR +0.027, MAP +0.028, nDCG@10 +0.024, recall@5 +0.031 (all p < 0.001), with p50 latency 78.6 ms → 51.2 ms. Two of the three corpora used @@ -136,6 +178,41 @@ Every ranking signal that ships has an ablation row. 1.9.0 removed two signals that could not demonstrate a benefit and rejected several plausible ones (IDF-weighted coverage, stemming, graph propagation, MMR, a file-length prior). +## Evidence memory + +`tests/eval/memory_eval.py` replays a repository's own history: for every git-derived +query the tree is checked out at the parent of the query's commit, the index is updated +incrementally, one retrieval call is made, and consecutive tasks form sessions of K +tasks. Every arm is computed from the same packet, so arms differ only in what they do +with evidence: **B** is the 1.10.0 packet, **C** is 2.0 evidence memory (withhold only +byte-identical evidence, report changes), **S** is the unsafe alternative (withhold by +locator without checking the source), **A** rereads whole files. The oracle keeps the +text each session was actually handed and is independent of memory's hashing. + +Logged run over this repository, 87 tasks (`tests/eval/results/2026-09-14-evidence-memory.md`): + +| K (tasks / session) | snippet tokens B → C | saved / task, 95% CI | C stale withheld | S saved | S stale withheld | change notices P / R | +|---|---|---|---|---|---|---| +| 5 | 98 222 → 94 358 (−3.9%) | 44 [13, 89] | 0 | −10.8% | 15 | 1.000 / 0.973 | +| 10 | 98 222 → 92 565 (−5.8%) | 65 [31, 112] | 0 | −14.7% | 20 | 1.000 / 0.925 | +| 25 | 98 222 → 92 268 (−6.1%) | 68 [35, 114] | 0 | −19.4% | 30 | 1.000 / 0.920 | +| all | 98 222 → 87 634 (−10.8%) | 122 [76, 179] | 0 | −34.5% | 54 | 1.000 / 0.871 | + +What this does and does not show: + +- **Correctness is the result.** Across every session length, memory withheld zero + stale snippets and every change notice was correct (precision 1.000). The unsafe + arm saves two to three times more tokens and pays for it with 15–54 stale + withholdings per run, six of which hide a gold answer at K = all. +- **Token savings are modest**, 4–11% of snippet tokens at 44–122 tokens per task on + this corpus. Restoring withheld snippets reproduced the B packet on every task (0 + page mismatches), and `useful@budget` is identical for B and C. +- **Cost:** memory adds 8–37 ms p50 per call (K = 5 … all) on a 31.6 ms p50 search; + the store stayed under 410 KB. +- One corpus, one language. The number to watch on other repositories is stale + withheld, which must stay at zero; the savings figure will vary with how often a + session revisits the same files. + ## Claims that must NOT be made Do not write, imply, or ship any of these until a run with published logs exists: diff --git a/docs/FAQ.md b/docs/FAQ.md index c381aa2..fde1586 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -160,7 +160,7 @@ Yes. Use any of these methods: ## Is it production-ready? -Yes, with the caveats below. The current line is **1.9.x** (see +Yes, with the caveats below. The current line is **2.0.x** (see [CHANGELOG.md](../CHANGELOG.md)). It ships: - Hybrid FTS5 / path / symbol retrieval with optional local embeddings; rank fusion that diff --git a/docs/MCP.md b/docs/MCP.md index 9cc6e84..ab7a9bf 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -41,7 +41,16 @@ The MCP server exposes the same retrieval contract as the CLI. | `architecture_overview` | Modules, god nodes, surprising connections, suggested questions | `architecture` | | `path_between` | Shortest dependency/call path between two symbols or files | `path` | | `describe_symbol` | Node card: definition, callers, callees, centrality, module | `describe` | -| `index_stats` | Return counts, language coverage, graph stats, freshness | `stats` | +| `index_stats` | Return counts, language coverage, graph stats, freshness, memory | `stats` | +| `verify_evidence` | Re-check `path:start-end@hash` references and/or a session's deliveries against the working tree (read-only) | `verify` | + +`search_code` and `explain_code` accept an optional `session` string. With a session, a +snippet the session already received from byte-identical source comes back as +`snippet: null, reused: true`, and evidence the session received that has since changed is +listed once under `memory.invalidated`. Without a session the payload is unchanged except +for `stale: true` on a result whose index text no longer matches the working tree. Memory +maintenance (`memory gc`, `memory clear`) is CLI-only, like `clean`. See +[MEMORY.md](MEMORY.md). ## Output contract @@ -69,7 +78,7 @@ branch on the contract without sniffing the shape: version. The current version is **1**. - `tool` (string) — the emitting tool name (`search_code`, `find_symbol`, `find_refs`, `impact_of`, `impact_of_diff`, `explain_code`, `architecture_overview`, - `path_between`, `describe_symbol`, `index_stats`, `healthcheck`). + `path_between`, `describe_symbol`, `index_stats`, `healthcheck`, `verify_evidence`). - The no-index / error path carries the same envelope plus an `"error"` field. Rules: @@ -165,7 +174,8 @@ same trust boundaries: - Done: `codebase-index mcp --root ` CLI entrypoint. - Done: `healthcheck`, `search_code`, `find_symbol`, `find_refs`, `impact_of`, `impact_of_diff`, `explain_code`, `architecture_overview`, `path_between`, - `describe_symbol`, and `index_stats` tools. + `describe_symbol`, `index_stats`, and `verify_evidence` tools; `session` on `search_code` + and `explain_code`; additive `memory` block in `index_stats` and `healthcheck`. - Done: focused tests for tool registration, missing-index behavior, config resolution, and run entrypoint. - Done: explicit `schema_version` + `tool` envelope on every structured tool payload (including the error path), asserted by `tests/test_mcp_server.py` and `tests/test_mcp_golden.py`. diff --git a/docs/MEMORY.md b/docs/MEMORY.md new file mode 100644 index 0000000..18bb09e --- /dev/null +++ b/docs/MEMORY.md @@ -0,0 +1,107 @@ +# Evidence memory + +Evidence memory is the 2.0 answer to one question: **is what the agent read earlier +still true?** It identifies every delivered snippet by the exact bytes of its span, +re-checks those bytes against the working tree on later calls, withholds snippets the +same session already holds, and reports evidence that has changed. It never decides +anything by query similarity and it stores no source text. + +## Evidence references + +A reference is `path:start-end@hash`. The hash covers the bytes of lines `start..end` +under the indexer's own line model (universal newlines); only line terminators are +normalised. References are printed by `verify` and can be kept next to the conclusions +they support: + +```text +Refunds are capped at the invoice total [billing/refund.py:3-4@3f9a2c1b7d4e8a90] +``` + +References are untrusted input. Absolute paths, drive letters, `..` segments and NUL +are rejected before any filesystem access, and every read goes through the same +`PathGate` the indexer uses, so a reference can never reach a file the walker would +refuse (ignored paths, dependency and build directories, secret filenames, oversized +or binary files, symlinks that resolve outside the repository). + +## Verdicts + +| state | still true? | meaning | +|---|---|---| +| `valid` | yes | bytes unchanged at the same lines | +| `relocated` | yes | identical bytes occur exactly once elsewhere in the same file | +| `changed` | no | bytes at the span differ and the text occurs nowhere else | +| `ambiguous` | no | identical bytes now occur more than once | +| `deleted` | no | file gone; a rename or move counts as deleted | +| `excluded` | no | the path is now ignored, secret-like or outside the repository | +| `unreadable` | no | the file could not be read | + +`all_valid` is true only when every verdict is `valid` or `relocated`; it is false for an +empty or unknown set. + +## Sessions + +A session is a tag the caller chooses, one per agent conversation. Sessions are never +inferred from a process, an environment variable or a time window, because none of +those identify what an agent still holds in context. + +```bash +codebase-index search "refund cap" --session auth-fix-1 --json +codebase-index explain "how are refunds capped" --session auth-fix-1 --json +codebase-index verify --session auth-fix-1 --json +codebase-index verify "billing/refund.py:3-4@3f9a2c1b7d4e8a90" --strict +``` + +With a session: + +- a snippet the session already received from byte-identical source is replaced by + `snippet: null, reused: true`. A skeleton is withheld only when the same skeleton or + the whole span was delivered before; +- evidence the session received earlier that has since changed is reported once under + `memory.invalidated`; +- ranking, budgeting and pagination are untouched. The evidence hook runs after they + are final, and restoring the withheld snippets reproduces the no-memory packet + exactly (asserted by tests and by the benchmark on every task). + +Without a session the packet is unchanged except for `stale: true` on a result whose +index text no longer matches the working tree. Index text that is derived rather than +copied (config-key chunks, Markdown section summaries) is never flagged: a mismatch +counts as stale only when the file's current hash differs from the one the index was +built from. + +## Storage + +`memory.sqlite` lives next to `index.sqlite` (override with `CBX_MEMORY_PATH`). It is a +separate file because rebuilds and `clean` delete the index, and so that ledger writes +never queue behind an update transaction. It holds hashes, paths, line numbers, token +counts and timestamps only; session tags are stored hashed. A newer schema is refused +without touching the file, a corrupt file is moved aside rather than deleted, lock +contention degrades to no-memory output instead of blocking, and GC only removes rows. + +Configuration (`.codeindex.json`, not part of `config_hash`): + +```json +{ "memory": { "enabled": true, "retention_days": 14, "max_deliveries": 50000 } } +``` + +`CBX_MEMORY=0` or `memory.enabled: false` disables everything and gives 1.10.0 output +byte for byte. + +Maintenance is CLI-only and is not exposed to the skill wrappers or MCP, like `clean`: + +```bash +codebase-index memory gc # drop expired sessions, compact +codebase-index memory clear --session TAG # forget one session +codebase-index memory clear --yes # forget everything +``` + +`stats`, `doctor`, MCP `index_stats` and `healthcheck` carry an additive `memory` block +(counts, schema, size; never paths or content). + +## What was measured + +The sequential real-history benchmark (`tests/eval/memory_eval.py`) replays a +repository's own commits: each git-derived query runs against the tree at the parent of +its commit, consecutive queries form sessions of K tasks, and the repository evolves +inside a session. The logged run over this repository is in +`tests/eval/results/2026-09-14-evidence-memory.md`; the summary is in +[BENCHMARKS.md](BENCHMARKS.md#evidence-memory). diff --git a/docs/RETRIEVAL.md b/docs/RETRIEVAL.md index de233b3..2da3e49 100644 --- a/docs/RETRIEVAL.md +++ b/docs/RETRIEVAL.md @@ -23,8 +23,7 @@ query ▼ [3] rank fusion ── Reciprocal Rank Fusion (RRF) across retriever result lists │ - ▼ -[4] rerank ── feature-based score (symbol-kind, path, source role, centrality) +[4] rerank ── query↔candidate features: name co-occurrence, symbol, path, source role, centrality │ ▼ [5] optional graph expansion ── pull in imports/callers/callees per intent (bounded) @@ -119,10 +118,13 @@ there. Set `RetrievalTuning(file_agreement=False)` to recover plain locator-only ## 4. Reranking (`retrieval/rerank.py`) A lightweight, explainable feature score (no external model required) layered on the fused order. -Every term is bounded, so reranking reorders near-neighbours rather than overruling retrieval: +Most terms are bounded tiebreakers, so they reorder near-neighbours rather than overruling +retrieval. The exception is name co-occurrence, which is deliberately large enough to move a +candidate several places — see below for why, and for the evidence that it should. | Feature | Effect | Intuition | |---|---:|---| +| Query terms co-occurring in one name | ≤ +1.80 | several query terms in one name is qualitatively better evidence than one term matched well | | Exact symbol match | +0.20 | the user named a specific symbol | | Symbol definition kind | +0.05 | a `def`/`class` outranks an incidental mention | | Symbol name among query terms | +0.05 | the name was asked for, not just matched | @@ -132,6 +134,56 @@ Every term is bounded, so reranking reorders near-neighbours rather than overrul | Source role prior | −0.25…+0.08 | see below | | Generated, or test on a non-test query | −0.15 | supporting evidence, not the answer | +### Name co-occurrence (`retrieval/features.py`) + +Every retriever scores each query term independently, and RRF sums those independent verdicts. That +structure cannot distinguish a candidate which matched **one** query term very well from one which +matched **three** terms in a single name — and on the 1.9.0 benchmark that confusion was the single +largest reranking loss. Two measured examples, both ranked wrong by 1.9.0: + +| Query | 1.9.0 winner | correct answer, ranked below it | +|---|---|---| +| "graph resolution + traversal accessors" | `graph/retrieval.py` (`graph`) | `test_graph_accessors_resolve_and_walk` (`graph` + `accessors` + `resolve`) | +| "greedy token budgeting with redaction" | `output/redact.py` (`redact`) | `retrieval/budget.py` (`budget` + `token`) | + +The fix is an interaction term, not another per-term bonus. Let `zone(d)` be the identifier +components of the candidate's *name* — file basename plus symbol, camel/snake split — and `m` the +number of salient query terms appearing in it: + +``` +cooccurrence(d) = max(0, m - 1) / (n_terms - 1) # 0 when m < 2 +score(d) += w · cooccurrence(d) · (scale if demoted else 1) +``` + +Only terms **beyond the first** earn credit: a single matched term is already fully paid for by the +retriever that surfaced the candidate, so crediting it again would merely re-weight lexical +matching, which is not what was missing. Directories are excluded from the zone — +`src/main/java/net/...` is shared by hundreds of files, so it adds co-occurrence noise to all of +them and evidence to none. + +`w = 1.8` sits in the interior of a plateau. Pooled MRR rises to ≈1.8 and then flattens, and past +that point per-query wins stay flat while losses nearly double (37W/19L at 1.8 against 39W/32L at +4.0), because a larger bonus turns the feature into the primary sort key and reduces fusion to a +tiebreak. Every value in 1.0–4.0 leaves all eight benchmark corpora at or above 1.9.0. + +`scale = 0.5` discounts — rather than withholds — the bonus for test and generated sources. Test +function names are descriptive sentences (`test_compactor_output_is_redacted`), so they harvest +query-term co-occurrences that real identifiers never do, and a bonus reaching +1.8 is not +counterbalanced by a flat −0.15 demotion calibrated when the largest name bonus was +0.05. The +value was chosen by splitting the benchmark on whether its *own ground truth* is a test: across the +261 queries whose answer is not a test, the gain is flat at +0.020 MRR for every scale, so the +entire aggregate difference between 0.5 and 1.0 comes from the 159 test-answer queries — an +artifact of mining ground truth from commits, which touch tests. 0.5 is the only setting that +improves both partitions. + +Variants that were measured and **rejected**, each failing to beat this one on held-out +repositories: idf weighting of the matched terms (pool-local and corpus-wide); substring instead of +component matching; prefix/stem-tolerant matching (`redact`/`redacted`); ordered-subsequence +matching; term proximity within the chunk body; body-text coverage; zone-size ("tightness") +normalisation; and restricting the zone to the filename or the symbol alone. A pairwise logistic +ranker fitted over 19 candidate features selected this one, and forward selection found no second +feature clearing the noise floor — so one feature ships, not a model. + ### Source role priors (`retrieval/priors.py`) | Role | Prior | Rationale | @@ -164,19 +216,44 @@ When enabled, it is bounded by depth and node cap: Expanded nodes retain edge confidence and receive distance-decayed scores so seeds stay on top. -## 6. Diversity and duplicate control +## 6. Diversity, page packing, and duplicate control `retrieval.diversity` provides bounded MMR selection and SimHash near-duplicate suppression. +### Page packing (`max_per_file`) + +The agent's unit of decision is "which file do I open", so a 10-result page that spends three +slots on three regions of one file offers seven choices, not ten. Measured across eight +repositories, 1.9.0's page held **7.1 distinct files** on average, and of the queries whose answer +was in the candidate pool but missing from the page, 45 of 57 had it sitting past rank 10 — crowded +out by repeat hits rather than by better candidates. + +`max_per_file = 1` keeps one hit per file in place and pushes the rest to the tail. Nothing is +dropped, so a file with several relevant regions still surfaces them below the first page. The +parameter is monotone over 1–5 (1 > 2 > 3 > 4 > 5), so this is a plateau boundary rather than a +fitted peak: recall@10 +0.045 and nDCG@10 +0.015 against 1.9.0's value of 3, at unchanged token +cost, with no metric and no corpus regressing. + +### MMR + MMR is disabled in the shipped default: it moved no ranking metric on the benchmark and roughly doubled p50 latency. Callers that need broader snippet coverage can enable `RetrievalTuning(mmr=True)`. -SimHash duplicate suppression stays on, but on noise grounds rather than ranking grounds: it does -not move MRR, and it takes the duplicate rate of returned snippets from ~1.6% to ~0%. The -over-fetch that feeds selection is now an explicit `candidate_pool_multiplier` rather than an -implicit side effect of enabling dedup — an earlier ablation credited dedup with a quality win that -was really the wider pool doing the work. +### Duplicate suppression + +SimHash duplicate suppression stays on, but on noise grounds rather than ranking grounds: it moves +MRR by −0.0016 (p=0.006) and takes the duplicate rate of returned snippets to ~0%. The over-fetch +that feeds selection is an explicit `candidate_pool_multiplier` rather than an implicit side effect +of enabling dedup — an earlier ablation credited dedup with a quality win that was really the wider +pool doing the work. + +Only the leading 2000 characters of a candidate body decide duplication. Fingerprinting whole chunk +bodies made this the most expensive stage of the query path — 42% of it on the Java corpus — to +answer a question the first ~50 lines already answer: two chunks that agree for 2000 characters are +the same snippet. The bound leaves recall, duplicate rate and useful-context identical and MRR +within −0.0003 (p=0.51). Combined with a rewritten operator scan in the tokeniser (1.71× faster, +bit-identical output), duplicate control costs roughly half what it did in 1.9.0. ## 7. Token budgeting (`retrieval/budget.py`) @@ -238,3 +315,26 @@ Default token budget: 1500 for `search`, 2200 for `explain`, configurable per pr The Markdown renderer (`output/markdown.py`) prints the same data as a tight table + fenced snippets so it's compact in Claude's context. See SKILL.md for how Claude is told to read it. + +## 10. Ranking diagnostics (`--explain`) + +Every result already carries a `reason` string, but diagnosing *why the ranking was wrong* needs +the order before reranking as well as after. `search(..., explain=True)` adds a `diagnostics` +block containing the pre-rerank candidate pool and the final order, each with per-candidate +source, symbol, score and retriever agreement. Building it costs about thirty dict literals per +query — measured at −0.8ms p50, i.e. inside the noise of a 40ms query — and nothing is allocated +when the flag is off. + +This is what the evaluation harness uses to compute the **oracle** metrics, and it is what turns +"MRR is low" into an actionable decomposition: + +| Metric | Question it answers | +|---|---| +| `oracle` | what MRR a *perfect* reranker would score over the pool actually generated | +| `1 - oracle` | the share of queries only better **recall** can ever fix | +| `eff` = MRR / oracle | the share of achievable ranking quality actually delivered | + +At 1.9.0 that split read `oracle = 0.902`, `MRR = 0.577`, `eff = 0.639`: a third of the answers +already sitting in the candidate pool were ranked below something else, an error mode roughly +three times larger than the remaining 0.098 of recall headroom. That measurement is why 1.10.0 +spent its effort on the reranker instead of adding retrievers. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 57ae43b..823b786 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -1,7 +1,7 @@ # Roadmap `codebase-index` is becoming the evidence layer between a coding agent and a -repository: **Find implementations, Trace behavior, Predict change impact.** +repository: **Find implementations, Trace behavior, Verify evidence, Predict change impact.** This document separates shipped capability from forward work. Roadmap entries are not product claims until they appear under `[Unreleased]` or a tagged @@ -32,6 +32,13 @@ release in `CHANGELOG.md`. - Graph-coverage honesty for partially supported languages. - Diagnostics that surface stale indexes and incomplete graph extraction. +### Verify + +- Evidence references (`path:start-end@hash`) minted for every delivered snippet. +- `verify` and MCP `verify_evidence`: byte-identity verdicts against the working tree. +- Session-scoped reuse: byte-identical snippets withheld, changed evidence reported. +- Content-free `memory.sqlite`, gated by the same path rules as the indexer. + ### Delivery and trust - CLI, Claude Code plugin/skill, Codex CLI, OpenCode, and stdio MCP. diff --git a/docs/installer.md b/docs/installer.md index fd8bd05..abf2c1f 100644 --- a/docs/installer.md +++ b/docs/installer.md @@ -110,7 +110,7 @@ pwsh ./install.ps1 -Target claude -InstallDir "D:\skills\codebase-index" **Pin to a branch or tag** (reproducibility and safety): ```sh -sh install.sh --branch v1.9.0 +sh install.sh --branch v2.0.0 ``` --- diff --git a/docs/superpowers/specs/2026-09-10-evidence-memory-design.md b/docs/superpowers/specs/2026-09-10-evidence-memory-design.md new file mode 100644 index 0000000..baf97ce --- /dev/null +++ b/docs/superpowers/specs/2026-09-10-evidence-memory-design.md @@ -0,0 +1,365 @@ +# Evidence memory — 2.0 design + +Status: Phase B design, written before implementation. Inputs: `research/2.0-audit.md`, +`research/experiments.md`. Every decision below names the evidence or requirement it +rests on; alternatives that were considered and rejected are listed in §15. + +## 1. Goal + +Let a coding agent stop re-receiving repository evidence it already holds, **only while +that evidence is provably unchanged**, and tell it exactly which evidence it holds has +become stale. Make any piece of evidence citable by a short reference that any agent can +re-verify later. + +Non-goals for 2.0: storing agent conclusions or prose; semantic or query-keyed caching; +changing ranking; graph/co-change expansion; cross-repository memory; an LLM in the loop. + +## 2. The central observation + +The research measured that 42% of evidence tokens repeat across a workload. A repeated +atom can only be *withheld* from a consumer that still holds the earlier copy in its +context window. Across independent contexts (a new session, a subagent, a different +tool) the text must be sent again — no store can change that. So 2.0 separates two +things the research merged: + +| Value | Where it applies | Mechanism | +|---|---|---| +| Token reuse | Within one agent context | Session ledger: omit a snippet already delivered to this context, if its source bytes are unchanged | +| Validity | Everywhere, any time later | Content-addressed evidence references re-verified against the working tree | + +## 3. Concepts + +**Evidence atom.** A contiguous span of one file, identified by the exact bytes it held +when it was delivered: `(repo, path, span_sha256)`. Line numbers are a locator, not +identity. + +**Evidence reference.** A self-describing, verifiable string: +`:-@`, e.g. +`src/auth/service.py:42-81@3f9a2c1b7d4e8a90`. It can be stored anywhere (agent notes, a +PR description, another tool's memory) and verified without the memory database. + +**Session.** An opaque tag naming *one agent context*. Snippets are only ever withheld from +the session they were delivered to. Tags are supplied explicitly by the caller. + +**Delivery.** A snippet that was actually placed in a packet (inside the token budget) +and verified against the working tree at delivery time. + +**Verdict states.** + +| State | Valid? | Meaning | +|---|---|---| +| `valid` | yes | Bytes at the recorded locator are identical | +| `relocated` | yes | Identical bytes occur exactly once elsewhere in the same file | +| `changed` | no | The bytes no longer occur in the file | +| `ambiguous` | no | The bytes occur more than once in the file; identity cannot be established | +| `deleted` | no | The file no longer exists | +| `excluded` | no | The path is now rejected by the discovery gates (ignore/secret/binary/size) | +| `unreadable` | no | I/O error | + +Every uncertain case resolves to *invalid* (requirement: prefer false invalidation to +stale reuse). + +## 4. Identity and the line model + +The indexer reads text with `read_text(errors="ignore")` (universal newlines) and chunks +with `str.splitlines()` (audit §4.2). Hashing uses the same line model so that a snippet +and its hashed span are the same lines: + +``` +raw = file bytes +text = raw.decode("utf-8", "surrogateescape") # invalid bytes stay visible +text = text.replace("\r\n", "\n").replace("\r", "\n") # universal newlines +lines = text.splitlines() # chunker line model +span = "\n".join(lines[start-1:end]) +sha = sha256(span.encode("utf-8", "surrogateescape")) +``` + +Only line terminators are normalised (a CRLF checkout of identical code is identical +evidence). Whitespace, comments and case are **not** normalised: a strict hash cannot +confuse two different programs, and fuzzy similarity is never used as proof (§15). +`surrogateescape` means a change to an undecodable byte still changes the hash. + +Repository scope: `repo_id = sha256("repo:" + canonical root)`, where the canonical root is +the resolved absolute POSIX path, case-folded on Windows. Rows from another `repo_id` are +never read. Moving a checkout directory therefore starts a fresh scope (a false +invalidation, never a false reuse). + +## 5. Validation + +``` +validate(ref, root, gates, hint=None): + if gates reject path: return excluded # never open excluded files + if path missing: return deleted + lines = line_model(read(path)) + if sha(lines[start-1:end]) == ref.sha: return valid(start, end) + candidates = hint.first_line_sha ? {i : sha(lines[i]) == first_line_sha} + : bounded scan of all starts (≤ 200k line-steps) + matches = {i in candidates : sha(lines[i:i+n]) == ref.sha} + |matches| == 1 → relocated(i+1, i+n); > 1 → ambiguous; 0 → changed + (bounded scan exhausted → changed, conservatively) +``` + +A reference's hash prefix (64 bits) is compared as a prefix; stored rows compare the full +256-bit hash. + +**Cross-file moves are not reused.** `src/a.py → src/core/a.py` yields `deleted` for the +old reference. The path is part of what the agent was told (imports and module identity +change with it), and the new location is found by ordinary retrieval. Identical short +text in an unrelated file must never validate a reference (requirement 38). + +**Symbol evolution** follows from byte identity: a rename, signature change, body change or +doc change inside the span is `changed`; a split or merge changes the bytes and is +`changed`; code that only shifted inside its file is `relocated`. No attempt is made to +decide semantic equivalence. + +## 6. Where evidence is minted — and per-result verification + +`retrieval.pipeline.search` gains an optional `evidence` hook invoked after budgeting and +pagination with the delivered page and the candidates behind it. `None` (the default for +library callers) leaves the function byte-identical. + +For each result that carries a snippet, the hook reads the working-tree file (through the +gates), computes the span hash, and checks that the index content the snippet was built +from is present in the current span (equality for chunk results; containment for +signature/doc results). This closes audit finding §4.3 per result: + +- consistent → the snippet reflects current bytes; it is eligible to be a delivery; +- inconsistent → the result is marked `"stale": true` (index entry older than the file) and + is never recorded or withheld. + +With a fresh index nothing is added to the packet, so default output is unchanged. + +## 7. Reuse: session-scoped withholding + +With a session tag, for each verified result: + +``` +atom = (repo_id, path, span_sha) +known = ledger[session] has atom with (full == true or snippet_sha == sha(snippet)) +if known: snippet → null, result.reused = true, tokens_saved += token_est +else: deliver snippet; ledger[session] += (atom, snippet_sha, full) +``` + +- `full` is true when the delivered snippet text is the entire span. A skeleton or a bare + signature only covers part of the span, so it can only satisfy a later request for the + *same* snippet text. Query-dependent skeletons therefore never hide lines the agent did + not see. +- **Budget accounting is unchanged**: a withheld snippet still consumes its budget in + `apply_budget`, so the page, the ranks, which results carry snippets and + `recommended_reads` are identical to the no-memory packet. The only differences are + `snippet: null` + `reused: true` on withheld results and a `memory` block. Tokens saved + are therefore pure savings with identical coverage, which keeps attribution exact. + Reinvesting freed budget is a separate, later experiment. +- A session with an empty ledger produces the no-memory packet plus the `memory` block. + +**Invalidation notices.** On each session call, deliveries not yet reported invalid are +re-validated (using the stored first-line hash for relocation). Newly invalid ones are +listed once: + +```json +"memory": { + "session": "auth-fix", + "reused": 2, "tokens_saved": 412, + "invalidated": [{"ref": "src/auth/service.py:42-81@3f9a2c1b7d4e8a90", "state": "changed"}] +} +``` + +**Why sessions are explicit.** An MCP server process outlives `/clear` and survives context +compaction; a CLI process knows nothing about the caller; a time window would let two +parallel agents withhold evidence from each other; subagents have fresh contexts but may +inherit environment variables. None of these identify an agent context, so the caller +names it (`--session`, `CBX_SESSION`, MCP `session`). Documented rule: a session tag is +used by exactly one context and is replaced after `/clear`, compaction, or whenever the +agent cannot see earlier snippets. + +## 8. Claims depend on evidence; memory does not store claims + +A conclusion ("auth uses JWT and Redis") belongs to the agent. 2.0 makes its dependency +set checkable instead of caching it: the agent cites references next to its note, and +`verify` reports whether every dependency still holds (`all_valid`). The store keeps +atoms independent of sessions, so a later `claims → atoms` table can reference them +without redesign. No conclusion text, prompt, or reasoning is stored. + +## 9. Storage + +A separate SQLite file, `.claude/cache/codebase-index/memory.sqlite` (override +`CBX_MEMORY_PATH`; when only `CBX_DB_PATH` is set, next to that DB). + +Why not tables in `index.sqlite`: `index --rebuild`, schema-triggered rebuilds and `clean` +delete that file (audit §4.1), which would destroy memory on every rebuild (requirement +7.6); and search-time ledger writes would contend with `update`'s long write transaction +on the same WAL. A second file in the same cache directory reuses the existing SQLite +pattern with none of those couplings, and leaves the 1.x index schema untouched, so +upgrading requires no reindex. + +Schema v1 (content-free — no source text is ever stored): + +```sql +meta(key PRIMARY KEY, value) -- schema_version, last_gc_at +atoms(id PK, repo_id, path, span_sha, line_count, first_line_sha, + first_seen_at, last_state, last_checked_at, + UNIQUE(repo_id, path, span_sha)) +sessions(id PK, repo_id, tag_sha, created_at, last_used_at, + tokens_delivered, tokens_saved, reused, UNIQUE(repo_id, tag_sha)) +deliveries(session_id → sessions ON DELETE CASCADE, atom_id → atoms ON DELETE CASCADE, + snippet_sha, full, line_start, line_end, token_est, delivered_at, + invalid_state NULL, PRIMARY KEY(session_id, atom_id, snippet_sha)) +``` + +- Session tags are stored as `sha256(repo_id + tag)`; the tag itself is never persisted. +- **Migrations**: ordered `MIGRATIONS[v] → callable`, applied in one `BEGIN IMMEDIATE` + transaction; `user_version`-style guard in `meta`. A newer on-disk version disables + memory for that process (search still works, reported by `stats`/`doctor`), it never + downgrades. +- **Corruption**: `sqlite3.DatabaseError` on open → the file is renamed to + `memory.sqlite.corrupt-` (preserved, not deleted), a fresh store is + created, and `doctor` reports it. +- **Concurrency**: WAL, `busy_timeout` 2000 ms, short `BEGIN IMMEDIATE` writes, idempotent + upserts. A lock timeout degrades that call to "no withholding, no ledger write" — the + packet is still correct, only larger. +- **Bounded growth**: `memory.retention_days` (14) removes sessions not used within the + window; orphan atoms are removed; `memory.max_deliveries` (50 000) drops the least + recently used sessions beyond the cap. GC runs at most daily on a session write, and via + `memory gc`. GC only deletes rows; it cannot make evidence valid, because validity is + computed from the working tree on every check. + +## 10. Branches, worktrees, dirty trees + +Validity is byte equality with the **current working tree**, never a commit. Consequences: + +- Dirty, staged and unstaged edits are what is validated against; `HEAD` is irrelevant. +- `git checkout`/rebase: evidence whose bytes are identical on the new branch stays valid + (it is the same text); anything else is `changed`/`deleted`. +- Each worktree has its own cache directory, hence its own memory; sharing across worktrees + is not attempted. +- New files have no prior evidence; deleted files yield `deleted`; renames yield `deleted` + for the old path. + +## 11. Public surface + +| Surface | Addition | +|---|---| +| `search`, `explain` | `--session TAG` (env `CBX_SESSION`) | +| `verify` (new, read-only) | `codebase-index verify [REF ...] [--session TAG] [--strict] [--json]`: verdict per reference or per session delivery, `all_valid`; `--strict` exits 1 when anything is invalid | +| `memory gc`, `memory clear` (new, maintenance) | Explicit GC; delete one session or all memory (`--yes`) | +| `stats`, `doctor` | Additive `memory` block / `memory_store` finding | +| MCP | `session` parameter on `search_code`/`explain_code`; `verify_evidence(refs, session)`; memory block in `index_stats`/`healthcheck`. No GC/clear over MCP (same policy as `clean`). | +| Skill | `verify` allowed; `memory` not allowed (like `clean`) | + +MCP `schema_version` stays 1: every change is an additive field. No CLI command, flag or +JSON field is removed or retyped. + +Config: `memory.enabled` (true), `memory.retention_days` (14), `memory.max_deliveries` +(50 000); env `CBX_MEMORY=0` disables everything, giving 1.10.0 output byte-for-byte. Not +part of `config_hash`. + +## 12. Security + +- The store holds hashes, paths, line numbers, token counts and timestamps. It holds **no + source text**, so no secret can be persisted by it through any path. +- Gates (ignore files, built-in denylist, secret filenames, size, NUL sniff) run before any + working-tree read performed by validation; an excluded path is reported `excluded`, + never opened, and its rows are purged by GC. +- Snippet content still comes only from the index and still passes redaction; memory adds + no new content path to the agent. +- Session tags are hashed; prompts, queries and conclusions are not stored. +- Everything stays local; no new dependency, no network. + +## 13. Failure modes + +| Failure | Behaviour | +|---|---| +| Memory DB locked / read-only / corrupt / newer schema | Packet without withholding; reason in `stats`/`doctor`; corrupt file preserved | +| Index stale for a result | `stale: true`, snippet delivered, no ledger write | +| Packet truncated by the agent harness after a ledger write | Later call may withhold what was never seen. Mitigation: default budgets stay far below harness limits; documented rule to start a new session when output was truncated | +| Session tag reused by another context | Withholding from a context that lacks the text. Mitigation: documented one-context rule; tags are caller-chosen, never inferred | +| Hash prefix collision on a bare reference | 64-bit prefix; stored rows use full 256-bit hashes | +| Parallel calls in one session | Both may deliver (duplicate tokens); neither withholds unseen text | + +## 14. Guarantees and tests + +Invariants, each backed by a test: + +1. Same bytes → same `span_sha`, independent of line endings and position. +2. Changed bytes → never `valid`/`relocated` under the old reference. +3. Repository A rows are invisible to repository B. +4. Deleted or excluded files → never valid; excluded files never opened. +5. `CBX_MEMORY=0` → payload byte-identical to 1.10.0 for every query in the eval sets. +6. Memory enabled, no session, fresh index → byte-identical to (5). +7. Session with empty ledger → (5) plus a `memory` block; after re-filling withheld + snippets, identical to (5). +8. GC never changes a verdict; after GC the set of withheld snippets can only shrink. +9. Rebuilding the index with unchanged sources keeps every reference valid. +10. Excluded content (`.env`, keys, ignored paths) never reaches `memory.sqlite` + (byte scan of the DB file). +11. Deterministic stale-context scenario (§16.2): a query-keyed cache serves the stale fact; + evidence memory reports `changed` and re-delivers current text. + +Lifecycle tests use real git repositories: edit, insert-above (relocation), rename, delete, +move across files, branch switch, detached HEAD, worktree, dirty/staged/unstaged, new +file, rebase. Randomised property tests use a seeded `random.Random` (no new dependency). + +## 15. Rejected alternatives + +| Alternative | Reason | +|---|---| +| Query-similarity (semantic) cache | 20.5–69.1% stale on real history; the key omits the dependency | +| Store snippet text in memory | Duplicates the index, creates a second secret-retention surface, unnecessary: validation needs only hashes | +| Memory tables inside `index.sqlite` | Destroyed by rebuild/`clean`; write contention with `update` | +| Automatic session per MCP process / time window | Does not identify an agent context (`/clear`, compaction, parallel agents) | +| Normalised (whitespace/comment-insensitive) identity | Can equate different programs; similarity is never proof | +| Cross-file relocation by content | Path is part of the evidence; short identical text in unrelated files would validate | +| Whole-file atoms | 1.36× less surviving reuse at 10 commits (research H4) | +| Reinvest withheld budget into more results | Changes the page and confounds attribution; deferred to a separate experiment | +| Contract slices / accretion / co-change in the default packet | Negative slot economics and lower useful-per-token (research Exp. 1–7) | +| Emitting a reference on every result by default | ~5–9% packet tokens for a feature most calls do not use; references are available from `verify --session` | + +## 16. Benchmark plan (Phase D) + +### 16.1 Sequential real-history replay (`tests/eval/memory_eval.py`) + +For each corpus: a `git clone --shared --no-checkout` into a temp dir (source repositories +are never modified); queries ordered oldest-first by their commit; for task *i* the tree is +checked out at the **parent** of the task's commit, the index is updated incrementally, +and the retrieval call is made. Tasks are grouped into sessions of *K* consecutive tasks +(K ∈ {1, 5, 10, 25, all}). The repository really evolves between tasks. + +Arms, all on identical packets: + +| Arm | Description | +|---|---| +| A | Read the full files of every delivered result (the "just reread the file" baseline) | +| A-mem | A with file-hash memory: skip a file already read in the session and unchanged | +| B | 1.10.0 packets, no memory | +| S | Unsafe dedup: withhold any result whose `(path, start, end)` was delivered in the session, no validation (models "trust what you saw") | +| C | 2.0 validated session memory | + +The oracle is independent of the memory code: the harness keeps every snippet text it has +handed to each session, and a withheld snippet counts as **stale** unless the text B would +deliver now equals a text previously delivered to that session (or is contained in a +previously delivered full span). + +Metrics (defined in `docs/MEMORY.md`): evidence/token reuse rate, validated and stale reuse +rate, invalidation precision/recall, survival by commit distance, unique-atom ratio, +repeat-read reduction (A vs A-mem), useful@budget per arm (stale evidence counts as absent), +packet tokens (serialised JSON / 4, so notice and marker overhead is billed), latency. +Paired bootstrap CIs over tasks. + +### 16.2 Deterministic stale-context test + +T0 deliver evidence E proving fact X · T1 unrelated commit (E still `valid`, withheld) · T2 +edit E so X is false · T3 semantically similar query. Expected: a query-keyed cache returns +the T0 packet (stale X); memory reports `changed` for E and delivers the current text. + +### 16.3 Regression and performance + +The 8-corpus retrieval eval re-run back-to-back with the pinned 1.10.0 commit (identical +ranking expected by construction); latency p50/p95 with memory off, on without session, and +on with session; `verify` latency; GC time; `memory.sqlite` size after the replay. + +## 17. Decisions deferred to measurement + +- Whether withholding produces material savings at realistic session lengths (if not, 2.0 + ships verification only and says so — or is not released as 2.0). +- The primary session length reported (all K are published). +- Whether a notice's token cost ever outweighs the tokens it saves. diff --git a/requirements.lock b/requirements.lock index 6843b5b..eb8619a 100644 --- a/requirements.lock +++ b/requirements.lock @@ -1,3 +1,3 @@ -codebase-index @ https://github.com/denfry/codebase-index/archive/refs/tags/v1.9.0.tar.gz +codebase-index @ https://github.com/denfry/codebase-index/archive/refs/tags/v2.0.0.tar.gz tree-sitter==0.25.2 tree-sitter-language-pack==1.8.1 diff --git a/research/2.0-audit.md b/research/2.0-audit.md new file mode 100644 index 0000000..56ed1b7 --- /dev/null +++ b/research/2.0-audit.md @@ -0,0 +1,175 @@ +# codebase-index 2.0 — Phase A repository audit + +Audit date: 2026-09-10. Scope: the whole repository at `release/1.10.0` (committed as +`90afa71 release: v1.10.0` before any 2.0 work, so 2.0 has a clean, attributable base), +plus the untracked `research/` spike. No production behaviour was changed during this +phase. + +## 1. Current state + +| Item | Finding | +|---|---| +| Version in tree | `1.10.0` (`src/codebase_index/__init__.py`, plugin manifest, skill stamps, lock tag) | +| Last tag on `main` | `v1.9.0` (merge `b48ed14`); 1.10.0 was staged locally, never merged or tagged | +| Python | `>=3.11`; CI matrix Ubuntu/macOS/Windows × 3.11/3.12/3.13 | +| Runtime deps | typer, rich, pydantic, pathspec, tree-sitter (pinned), tree-sitter-language-pack (pinned) | +| Tests | 536 collected, all pass (4 platform skips); `ruff check src tests` clean; `mypy` clean (61 files) | +| Research spike | `research/` untracked; `research/data/` git-ignored (private corpora commit subjects + SQLite indexes) | +| Working tree anomaly | `tests/fixtures/sample_repo/node_modules/leftpad/index.js` was deleted (unstaged). It is the fixture that proves the dependency-dir gate; `test_discovery` passed vacuously without it. Restored from HEAD. | + +## 2. Architecture map + +``` +discovery/ (walker, ignore, classify) gates: ignore files, built-in denylist, + │ secret filenames, NUL sniff, size + ▼ +parsers/ (tree-sitter Tier A/B, line chunker) chunks (symbol bodies / 80-line windows / doc), + │ symbols, raw edges + ▼ +indexer/pipeline.py build_index / update_index (mtime+size fast path, sha256 confirm) + │ graph/builder.py resolves edges globally + ▼ +storage/ index.sqlite (WAL): files, symbols, chunks(+FTS5 triggers), edges, modules, meta, + optional vec_* tables. SCHEMA_VERSION = 3. + ▲ + │ reads +retrieval/pipeline.search: intent → retrievers(fts, symbol, path, [vector], [graph]) → RRF + (+file agreement) → rerank (priors, name co-occurrence) → dedup → per-file packing + → confidence → skeleton compactor → apply_budget (greedy, redact) → paginate + ▲ +service.py (shared by CLI and MCP: db/config resolution, search/diff-impact/architecture/stats) + ├── cli.py (Typer) ── skill wrappers (skill_template/scripts/cbx, cbx.ps1; bin/cbx for plugin) + └── mcp/server.py (FastMCP, stdio; every payload in {schema_version: 1, tool} envelope) +``` + +Freshness (`indexer/freshness.py`) is a single index-level signal: git fast path (HEAD equals +indexed head and a clean tree) or a full walk diffing `(mtime, size, sha256)`. + +## 3. Reusable components for 2.0 + +| Component | Why it matters for evidence memory | +|---|---| +| `service.py` | One place to integrate memory for both CLI and MCP, preventing surface drift. | +| `storage/db.py` `Database` | WAL + pragmas + version-guard pattern to reuse for a memory store. | +| `discovery.walker/classify/ignore` | Gates must be re-applied before any evidence is read or retained. | +| `retrieval/budget.apply_budget` | The exact point where a snippet is *delivered* — the natural evidence mint point. | +| `retrieval.pipeline.search(explain=…)` | Precedent for an opt-in, zero-cost-when-off extension of the payload. | +| `tests/eval/metrics.py` | Paired bootstrap CI + permutation p (seeded) — reused verbatim by any new benchmark. | +| `tests/eval/gen_queries.py` output | Each query carries its `commit`, which makes a real-history replay benchmark possible. | +| `tests/golden_utils.py` | Golden normalisation for CLI/MCP contract snapshots. | +| `research/nucleus/memory.py` | Survival measurement via `git diff --unified=0` hunks (validated H3/H4 method). | +| `scripts/sync_skill_copies.py` | Single source for skill text + version stamps across five committed copies. | + +## 4. Technical debt found (verified by reading the code) + +These are pre-existing and independent of memory; the ones that touch 2.0 correctness or +documentation honesty are called out. + +1. **No schema migration framework.** `Database._guard_version` tolerates older schemas on + open; `index`/`update` *delete the DB file* when `schema_version < SCHEMA_VERSION` + (`cli.py` `index`/`update`). Anything stored in `index.sqlite` is therefore destroyed by + a rebuild or a schema bump. **Direct constraint on where memory may live.** +2. **Two line-numbering models inside the indexer.** Chunkers slice with `str.splitlines()` + (`parsers/line_chunker.py`, `parsers/symbol_chunks.py`), tree-sitter symbol rows count + `\n`, doc chunks count `\n`. They agree except for files containing `\f`, `\v`, + `\x1c-\x1e`, `\x85`, `
`, `
`. Evidence hashing must follow the chunkers' + model so a snippet and its hashed span are the same bytes. +3. **Search trusts index content without checking the working tree.** Snippets come from + `chunks.content`; the only staleness signal is index-level. `_git_clean_at` reports + fresh when HEAD equals the indexed head and the tree is clean, which is wrong for an + index built from a dirty tree that was later reverted (by inspection). +4. **`index` and `update` hardcode the DB path** (`cli.py`), ignoring `CBX_DB_PATH`, while + every read path honours it via `service.resolve_db`. +5. **Documentation claims more security than is implemented.** + - Redaction: code implements private keys, AWS access keys, and assigned secrets + (`output/redact.py`). JWTs, bearer tokens, connection strings, and Slack tokens are + documented but not implemented. Markers are `<>`, docs say `«…»`. + - Secret filenames: code excludes `.env`, `.env.*`, `id_rsa`, `id_ed25519`, + `credentials.json`, `service-account.json`, `secrets.json`, `*.pem|key|p12|pfx`. + Docs list `*.crt`, `*.keystore`, `credentials*`, `secrets*`, `id_rsa*` (the prefix + forms would also exclude legitimate source such as `secrets.py`). + - Generated files are *indexed and demoted* (`is_generated`), not excluded; `*.lock`, + `*.pb.go`, `*_pb2.py` are not recognised. `vendor/` is not in the denylist. Binary + detection is a NUL sniff only. + - `doctor` implements 5 checks (cache ignored, hooks, freshness, symbol extraction, + graph coverage); the secret-leak scan, endpoint report, permissions check and + allowed-tools diff are documented but absent. +6. **Plugin wrapper whitelist is stale.** `bin/cbx` omits `architecture`, `diff-impact`, + `path`, `describe`, which the plugin skill tells the agent to run. +7. **`docs/DATABASE_SCHEMA.md` describes a schema that does not exist** (`language`, + `content_hash`, `chunk_index`, `summaries` table…); `docs/SCHEMA.md` is the accurate one. +8. **The commit-derived retrieval benchmark retrieves at HEAD**, i.e. after the change the + query describes. Symmetric across variants, so deltas are valid, but absolute scores are + optimistic. A replay benchmark can retrieve at the parent commit instead. + +## 5. Research artifacts and the constraints they impose + +| Result (`research/experiments.md`) | Constraint on 2.0 | +|---|---| +| Anchor-and-expand: MRR −0.0073 (p<0.001); recovery 0.0310 < eviction cost 0.0347 | No default graph/co-change expansion that spends rank slots. | +| Contract slices appended: useful@budget +0.0056 (p=0.030) for +6.4% tokens; useful/token 5.22e-4 vs 5.51e-4 | Not shipped as a retrieval feature; `hybrid13` page-matched control is mandatory for any future attempt. | +| LORO weight fit: in-sample +46%, held-out −0.0024, 1/8 folds | No fitted weights; conservative defaults only. | +| Span survival 0.795 / 0.521 / 0.309 at 1/10/20 commits; semantic-cache stale 20.5% / 47.9% / 69.1% | Reuse must be keyed on evidence identity and validated against current bytes, never on query similarity. | +| Span-keyed retains 1.36× file-keyed reuse at h=10 | Evidence granularity should be the delivered span, not the file. | +| 1738 atom reads → 1098 distinct; 42.0% of evidence tokens repeated | Upper bound for dedup when earlier evidence is still in the consumer's context. | +| Whole-conclusion reuse unmeasured; LSA is not a neural encoder; ground truth is a proxy | No claims about conclusion caching, embeddings, or agent success. | +| Prior art: invalidation contracts (2609.00243), PlanFence (2609.03340), Bazel/Nix | No mechanism-novelty claim; contribution is a local, tested implementation plus measurements. | + +The research measured atom *sharing*; it did not measure where the repeated tokens can +actually be withheld. A repeated atom saves tokens only if the consumer still holds the +earlier copy. That is the central design question for Phase B. + +## 6. Baselines measured for the 2.0 regression boundary + +### Retrieval (tests/eval/run_eval.py, 420 queries, 8 corpora, re-run today) + +Raw output: `tests/eval/results/1.10.0-retrieval-8corpus.txt`. + +| Metric | 1.9.0 (pinned) | 1.10.0 default | Δ | p | +|---|---|---|---|---| +| MRR | 0.560 | 0.580 | +0.0200 | 0.002 | +| recall@10 | 0.623 | 0.687 | +0.0631 | <0.001 | +| nDCG@10 | 0.522 | 0.554 | +0.0316 | <0.001 | +| useful@budget | 0.583 | 0.613 | +0.0296 | 0.011 | +| tokens/query | 1099 | 1084 | −16 | — | +| p50 / p95 ms | 48.6 / 95.0 | 53.0 / 102.3 | | | + +Absolute values are below the published 1.10.0 table (MRR 0.5955) because several sibling +corpora are live working trees that changed after 2026-09-02 (two have >100 uncommitted +files). The paired delta against 1.9.0 reproduces (+0.020 vs +0.019 published). **Any 2.0 +retrieval comparison must be run back-to-back against this pinned commit on the same trees.** + +### Performance (this repository, 311 files; `tests/eval/results/1.10.0-perf.json`) + +| Measure | 1.10.0 | +|---|---| +| Cold index | 4.32 s (5 636 chunks, 1 242 symbols) | +| Index DB size | 5.8 MB | +| No-op `update` | 0.139 s | +| One-file `update` | 0.251 s | +| Query latency (123 queries × 3, in-process) | p50 38.1 ms, p95 59.8 ms | + +## 7. 2.0 integration points + +- `retrieval/pipeline.search`: after budgeting/pagination, the delivered page and its + candidates are known — an opt-in hook here can validate and annotate without touching + ranking. +- `service.search_payload`: session resolution, memory store access, config gating. +- CLI `search`/`explain` options; MCP `search_code`/`explain_code` parameters. +- `stats`, `doctor`, MCP `index_stats`/`healthcheck`: additive diagnostics. +- Skill template + both wrapper whitelists (`skill_template/scripts/cbx{,.ps1}`, `bin/cbx`), + `tests/test_packaging.py` asserts the whitelist string. +- `Config`: a new section must stay out of `config_hash` (it is not indexing-relevant). + +## 8. Compatibility risks + +| Risk | Mitigation to design for | +|---|---| +| CLI/MCP goldens (`tests/golden/*.json`) freeze payload shape | Default packets must be byte-identical when no memory feature is engaged. | +| MCP `schema_version` 1 contract | Only additive fields; no removals or type changes. | +| Index schema bump forces a full rebuild for every 1.x user | Avoid touching index tables. | +| Storing new state in `index.sqlite` would be deleted by `index --rebuild` / `clean` | Store memory separately. | +| Concurrent `update` (long write txn) vs search-time writes | Separate WAL file, short transactions, busy timeout, degrade instead of fail. | +| Parallel agent tool calls against one DB | Idempotent upserts; never lose information on a race. | +| Windows: case-insensitive paths, CRLF checkouts, file locking | Canonical repo identity, CRLF-insensitive hashing, no long-held locks. | +| Skill auto-update rewrites installed copies on version change | Sync script and stamps must move together to 2.0.0. | diff --git a/research/README.md b/research/README.md new file mode 100644 index 0000000..0c61b82 --- /dev/null +++ b/research/README.md @@ -0,0 +1,125 @@ +# NUCLEUS — a research spike on necessity-driven context for coding agents + +**The question.** Can an agent be given not "similar data" but a *computed minimal set +of knowledge necessary for the task at hand*? + +**The answer this work supports.** Partly, and not the part I expected. Computing the +required *set* by relation-based completion is measurable, real, and **not worth its +tokens** — it is dominated by simply returning more results. Computing what a +conclusion *depended on*, so that work can be reused soundly, is worth a great deal. + +Everything below is measured on 420 queries across eight real repositories in four +languages, with paired significance tests, leave-one-repository-out fitting, and a +no-lookahead guarantee enforced by the data structure rather than by discipline. + +--- + +## Headline results + +| Claim | Verdict | Evidence | +|---|---|---| +| Required-set members are related by computable relations | **True** | 71.8% of 900 anchor→target gold pairs, no lookahead | +| Structure and history are non-redundant | **True** | edges contribute 5.3% uniquely, co-change 9.8% | +| Anchor + completion beats a hybrid ranker | **False** | dominated by `hybrid13`; 5% *worse* useful-per-token | +| Relation weights can be fitted | **False** | LORO: in-sample +46%, held-out −0.0024, 1/8 folds improve | +| The completion score is calibrated | **True** | precision 0.073 → 0.353 from "always fire" to "top 10%" | +| Query-keyed ("semantic") caches are unsound for code | **True** | **47.9% stale after 10 commits**, 20.5% after one | +| Finer evidence granularity retains more valid reuse | **True, weaker than predicted** | 1.36× at h=10 (predicted 2–3×), rising to 1.45× at h=20 | +| Agent work is heavily shareable | **True** | **42.0%** of tokens never re-sent across a 420-task workload | + +## The one number that killed the retrieval plane + +``` +accretion recovers gold the baseline missed : 0.0310 per query +cost of evicting even the cheapest rank slot: 0.0347 per query +``` + +Every completion slot is net-negative. Removing the eviction (appending contract +slices instead) makes it positive but not efficient: **+0.9% answer for +6.4% tokens**, +i.e. 5.22e-4 useful-per-token against the baseline's 5.51e-4. + +## The one number that saved the memory plane + +``` +conclusions whose evidence had changed after 10 commits: 47.9% +what a query-keyed cache would do with them : serve all of them +what an evidence-keyed cache does with them : cannot address them at all +``` + +Soundness here is free — a property of the key construction, not a tuned threshold. + +--- + +## Documents + +| File | Stage | Contents | +|---|---|---| +| `related-work.md` | 1 | 15 approach families, the *fundamental* limit of each, and the single shared assumption underneath them | +| `hypotheses.md` | 2–3 | 13 hypotheses with formal models; the go/no-go diagnostic | +| `selection.md` | 4 | Scoring matrix, TOP-3, what was rejected and why | +| `architecture.md` | 5 | NUCLEUS: abstraction, data model, algorithms, pseudocode, Big-O | +| `experiments.md` | 6–7 | Full protocol, 8 experiments, ablations, and the attempt to destroy the result | +| `novelty.md` | 8 | Prior-art check. **Both mechanisms turned out to have close prior art**; what remains is empirical | + +## Code + +``` +research/ + nucleus/ + relations.py typed relation union; CoChangeModel with a structural no-lookahead guarantee + search.py nucleate -> accrete -> select; reproduces the shipped pipeline when accretion is off + baselines.py BM25, LSA dense (numpy-only randomized SVD), hybrid RAG, product hybrid, forced-PPR graph + memory.py evidence-keyed memo, span/file granularity, exact invalidation + evalrun.py benchmark runner; reuses tests/eval/metrics.py verbatim + build_indexes.py one persistent index per corpus + diagnose_structure.py Experiment 0: is the premise true at all + diagnose_accretion.py generation vs ranking vs displacement decomposition + diagnose_slots.py slot economics: gain vs eviction cost per rank + diagnose_calibration.py precision as a function of score + fit_weights.py LORO weight fit (result: does not generalise) + fit_gate.py LORO gate selection (result: tau=0.6, 7/8 folds, on a plateau) + experiment_memory.py H3/H4: survival, unsoundness, cross-task sharing + test_nucleus.py correctness, incl. equivalence-to-incumbent and no-lookahead +``` + +## Reproducing + +```bash +# 1. query sets from git history (leak-free: subjects live in metadata, not the corpus) +python tests/eval/gen_queries.py --repo --out research/data/.yml + +# 2. one index per corpus, shared by every system +PYTHONPATH=. python research/build_indexes.py --corpus ":" ... + +# 3. correctness first -- the equivalence test is what makes deltas attributable +PYTHONPATH=. python -m pytest research/test_nucleus.py --no-cov -q + +# 4. the experiments +PYTHONPATH=. python research/diagnose_structure.py --corpus "|||" ... +PYTHONPATH=. python -m research.nucleus.evalrun --systems bm25,dense,rag,graph,hybrid,hybrid13,nucleus13 +PYTHONPATH=. python -m research.nucleus.evalrun --systems hybrid13,nucleus13 --ablate --baseline-label nucleus13 +PYTHONPATH=. python research/diagnose_slots.py +PYTHONPATH=. python research/fit_weights.py --rebuild --c 1 +PYTHONPATH=. python research/experiment_memory.py +``` + +External corpora are not vendored — shipping someone else's source to run a benchmark +is not reproducible either. The generator is the reproducible part: point it at any +git repository and the protocol is identical. + +## Honest limits + +- **Ground truth is a proxy.** "Files the commit touched" ≠ "files the agent needed". + The right instrument is a `ddmin` oracle over an executable verifier (H6); it needs + an LLM agent loop unavailable here. Every number inherits this proxy — on both sides + of every comparison. +- **The dense baseline is LSA, not a neural code encoder.** No + `sentence-transformers`, GPU, or network in this environment. **No claim of the form + "beats embeddings" is made anywhere.** The load-bearing comparison is the paired one + against NUCLEUS's own anchor stage. +- **Whole-conclusion memo hit rate is unmeasured.** The 42% figure is atom-level + sharing. Exact-set keying is brittle and the "agents don't redo work" claim is not + fully tested. This is the largest hole. +- **Neither mechanism is novel.** See `novelty.md`. The contribution is measurement + and falsification, not invention. +- **Multi-agent routing (H11) is designed, not measured.** No claim is made for it. diff --git a/research/__init__.py b/research/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/research/architecture.md b/research/architecture.md new file mode 100644 index 0000000..434ec9f --- /dev/null +++ b/research/architecture.md @@ -0,0 +1,185 @@ +# Stage 5 — NUCLEUS: architecture, algorithms, complexity + +> **Status after Stage 6–7. Read this first.** +> The **retrieval plane (H1/H2) is refuted** on this benchmark: it is dominated by the +> trivial baseline of returning more results, and is *less* token-efficient. The +> **memory plane (H3/H4) is validated** and is the part worth keeping. The +> architecture is documented in full anyway, because a design that was measured and +> rejected is a result, and because the refutation only makes sense against the +> design it refutes. Numbers: `experiments.md`. + +**NUCLEUS** — *Necessity-driven Unified Closure over Lexical, Evolutionary and +Structural relations.* + +--- + +## 1. Core abstraction: the evidence atom + +Everything in the system is an operation on one object. + +``` +Atom = (id, kind, span, content_hash, obligations) + id stable identity: (path, line_start, line_end) + kind file | span | symbol | conclusion + content_hash hash of the bytes this atom currently holds + obligations set of identifiers the atom defines or references +``` + +Three properties fall out of this and are the whole reason for the choice: + +- **Content addressing gives deduplication for free.** Two agents that read the same + region name it identically, so a shared store holds it once. Measured: 1738 atom + reads across a 420-task workload collapse to 1098 distinct atoms — **42% of tokens + never re-sent**. +- **Content addressing gives *exact* invalidation.** A conclusion records the atoms it + consumed; it is valid iff every one of those hashes is unchanged. Not "probably + fresh", not "recent enough" — decidable. +- **Obligations give a coverage objective.** A set of atoms can be scored by what it + covers rather than by how similar its members are to a query. + +## 2. Data model + +``` +atoms(id, kind, path, line_start, line_end, content_hash, token_est) +obligations(atom_id, symbol) -- bipartite atom x obligation +relations(src, dst, type, weight) -- typed, undirected at file level + -- type in {edge, cochange, testlink, + -- stem, dir} +memo(key, task_class, result, evidence[], created_at) + -- key = H(sorted atom content hashes) +``` + +`relations` is the union of two very different sources, and the diagnostic in +`hypotheses.md` is what justifies carrying both: static edges connect 30.7% of gold +co-members, history connects 30.4%, and each contributes *uniquely* (5.3% / 9.8%). + +## 3. Indexing algorithm + +``` +INDEX(repo): + for each file f: # O(N) files + parse f -> symbols, spans # tree-sitter, O(|f|) + atoms += spans; obligations += symbols + resolve edges globally # O(E α(N)) with a suffix map + fold git history into co-change counts # O(C · k²), k = files/commit + build name/dir/stem inverted maps # O(N · t), t = tokens per name +``` + +`k` is bounded by `MAX_COMMIT_FILES = 40`, so the history fold is linear in commits, +not quadratic in repository size. This matters: it is what makes the evolutionary +relation affordable on a repository with a long history. + +## 4. Retrieval algorithm + +``` +NUCLEUS_SEARCH(q, B): # B = token budget + # -- NUCLEATE ------------------------------------------------------ + A <- incumbent_hybrid(q) # unchanged, deliberately + A0 <- distinct_paths(A[:anchor_head]) # anchors + + # -- ACCRETE ------------------------------------------------------- + contrib <- {} + for i, a in enumerate(A0): + alpha <- 1 / (1 + decay·i) + for c in neighbours(a): # relation-local, not a scan + for r in RELATIONS: + contrib[c][r] += alpha · rel_r(a, c) + for c in contrib: # hub damping + contrib[c] /= sqrt(1 + degree(c)) + score(c) <- Σ_r θ_r · contrib[c][r] # linear in θ by construction + C <- { c : score(c) ≥ τ } + + # -- SELECT (budgeted greedy coverage) ------------------------------ + covered <- obligations(A0) ∪ terms(q) + S <- {} + while |S| < max_completions and C ≠ ∅: + c* <- argmax_{c∈C} score(c) · sqrt(1 + |obligations(c) \ covered|) + S <- S ∪ {c*}; covered <- covered ∪ obligations(c*); C <- C \ {c*} + + # -- EMIT ----------------------------------------------------------- + render each c ∈ S as a CONTRACT SLICE (signatures, not bodies) + return A ++ S under budget B +``` + +Two design decisions are load-bearing and both came from measurement, not taste: + +**Score is linear in θ.** `contrib` is per-relation, so re-weighting never re-walks the +graph. That is what made leave-one-repository-out fitting affordable (`fit_weights.py`) +— and therefore what made it possible to *discover* that the fit does not generalise. + +**Completions are appended, never inserted.** The first implementation inserted at +rank 4 and lost significantly (MRR −0.0073, p<0.001). `diagnose_slots.py` explains why: +a completion has precision 0.073, while the rank-4 slot it displaces holds gold with +probability 0.096. Appending removes the eviction entirely, so a completion must only +justify its own tokens. + +## 5. Memory algorithm (the part that survived) + +``` +REMEMBER(task, evidence, result): + key <- H( sort{ (atom.id, atom.content_hash) for atom in evidence } ‖ H(task_class) ) + memo[key] <- (result, evidence) + +RECALL(task, evidence): + key <- H( ... ) # same construction + return memo.get(key) # a hit is sound by construction +``` + +There is no eviction policy, no TTL, no similarity threshold, and no staleness +heuristic. An entry whose evidence changed is not evicted — it simply becomes +unreachable, because the key that would address it can no longer be constructed. This +is the Bazel/Nix action-cache discipline applied to agent reasoning. + +**Granularity is the only tuning knob, and it is the one that matters.** Keying on +whole files is the obvious implementation and costs 26% of the achievable reuse at a +10-commit horizon (survival 0.383 vs 0.521 span-keyed). + +## 6. Complexity + +| Operation | Cost | Notes | +|---|---|---| +| Index construction | `O(Σ|f| + E·α(N) + C·k²)` | parse + global edge resolve + history fold; `k ≤ 40` | +| Insert / update one file | `O(|f| + deg(f) + k²)` | re-parse, re-resolve its edges, re-fold its commits | +| Accretion (per query) | `O(|A₀| · d̄ · \|R\|)` | `d̄` = mean relation degree; **no corpus scan** | +| Coverage selection | `O(m · max_completions · \|O\|)` | `m` = gated candidates; greedy, `(1−1/e)` | +| Retrieval total | anchor stage + ~1 ms | measured: p50 41.3 ms → 56.2 ms | +| Memo lookup | `O(\|E\|)` hash + `O(1)` | `\|E\|` = evidence atoms, measured mean 4.1 | +| Memo invalidation | `O(1)` amortised | no scan: stale keys are simply never formed | +| Memory usage | `O(N + E + P)` | `P` = distinct co-change pairs, bounded by `C·k²` | +| Multi-agent sync | `O(1)` per atom | content-addressed ⇒ no coordination, no consensus | + +The `O(1)` synchronisation is worth naming: because every atom and every memo entry is +named by a hash of its content, two agents never need to agree on anything. There is +no invalidation broadcast, no version vector, and no leader. It is the CRDT-like +property that content addressing buys. + +## 7. Fault tolerance and incremental update + +- **Partial index is safe.** Missing relations reduce accretion recall; they cannot + produce a wrong answer, because relations only *propose* candidates. +- **Corrupt/absent history degrades to structure-only.** `CoChangeModel` with no + commits yields zero co-change weight and the system still runs (asserted in + `test_nucleus.py`). +- **The memo cannot serve a stale entry** even after a crash mid-write: a partially + written entry has a key nothing will construct. +- **Incremental update** is per-file; the co-change fold is append-only, which is also + what enforces the no-lookahead property in evaluation. + +## 8. Agent communication and distributed operation (designed, not measured) + +Routing by obligation ownership (H11): a task's implicated obligation set determines +which shard owns its atoms, so assignment is graph partitioning of the atom×obligation +bipartite graph minimising cut. Two agents whose obligation sets intersect are +predicted to conflict *before* either writes. This is specified but **not measured** — +it needs a live multi-agent workload, and no claim is made for it here. + +## 9. What the architecture got right and wrong + +Right: content addressing as the single abstraction; separating anchor budget from +completion budget; keeping the score linear so it could be fitted and falsified; +contract slices instead of bodies. + +Wrong: the central bet. Conditional completion assumed the incumbent's tail slots were +weak. They are not — a tuned hybrid ranker's rank-6..10 results carry gold at +0.021–0.035, and the relation union's top candidate carries it at 0.073 only when it +fires at all, on 42% of queries. The margin is real but too thin to pay for its tokens. diff --git a/research/build_indexes.py b/research/build_indexes.py new file mode 100644 index 0000000..0dcd2c4 --- /dev/null +++ b/research/build_indexes.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Build one persistent index per corpus, shared by every experiment. + +Rebuilding per variant would make ablation deltas measure indexing variance +instead of retrieval, and re-indexing 2k-file Java repos for each of a dozen +configurations is the difference between a benchmark that runs and one that does +not. The exclude list is taken verbatim from the shipped eval harness so the +research runs grade themselves on exactly the corpus the product's own benchmark +uses. +""" + +from __future__ import annotations + +import argparse +import time +from pathlib import Path + +from codebase_index.config import Config +from codebase_index.indexer.pipeline import build_index +from codebase_index.storage.db import Database + +from tests.eval.harness import CORPUS_EXCLUDES + +INDEX_DIR = Path(__file__).parent / "data" / "index" + + +def build(name: str, root: Path, *, force: bool = False) -> tuple[int, float]: + INDEX_DIR.mkdir(parents=True, exist_ok=True) + db_path = INDEX_DIR / f"{name}.sqlite" + if db_path.exists() and not force: + db = Database(db_path).open() + n = db.conn.execute("SELECT COUNT(*) FROM files").fetchone()[0] + db.close() + return n, 0.0 + for suffix in ("", "-wal", "-shm"): + p = Path(str(db_path) + suffix) + if p.exists(): + p.unlink() + cfg = Config() + cfg.root = str(root) + cfg.embeddings.enabled = False + cfg.extra_ignore = [*cfg.extra_ignore, *CORPUS_EXCLUDES] + start = time.perf_counter() + db = Database(db_path).open() + build_index(cfg, db, root=root) + elapsed = time.perf_counter() - start + n = db.conn.execute("SELECT COUNT(*) FROM files").fetchone()[0] + sym = db.conn.execute("SELECT COUNT(*) FROM symbols").fetchone()[0] + edg = db.conn.execute("SELECT COUNT(*) FROM edges WHERE resolved = 1").fetchone()[0] + db.close() + print(f" {name}: {n} files, {sym} symbols, {edg} resolved edges, {elapsed:.1f}s") + return n, elapsed + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--corpus", action="append", required=True, help="name:repo_path") + ap.add_argument("--force", action="store_true") + args = ap.parse_args() + for spec in args.corpus: + name, root = spec.split(":", 1) + print(f"[{name}] indexing {root}") + build(name, Path(root).resolve(), force=args.force) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/research/data/.gitignore b/research/data/.gitignore new file mode 100644 index 0000000..a4ed2ec --- /dev/null +++ b/research/data/.gitignore @@ -0,0 +1,8 @@ +# Generated benchmark artifacts. Deliberately not committed: +# * the query YAMLs contain commit subjects mined from private sibling repositories; +# * the SQLite indexes are large binaries regenerable in seconds. +# `research/README.md` documents the two commands that rebuild everything here, which +# is the reproducible part -- vendoring someone else's source to run a benchmark is +# not reproducible either. +* +!.gitignore diff --git a/research/diagnose_accretion.py b/research/diagnose_accretion.py new file mode 100644 index 0000000..f542611 --- /dev/null +++ b/research/diagnose_accretion.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Why does accretion lose? Split the failure into generation vs ranking vs displacement. + +The pooled benchmark says NUCLEUS(default) is worse than the incumbent. Three very +different causes produce that same number, and they have opposite fixes: + + generation the gold file the baseline missed is never even proposed as a + completion candidate -> the relation union is too narrow + ranking it is proposed but scored below the completions actually inserted + -> the relation weights are wrong + displacement the completions inserted are fine, but they evict baseline results + that were themselves gold -> the slot policy is too aggressive + +This measures all three on the same sweep, with the same no-lookahead history. +""" + +from __future__ import annotations + +import argparse +import sqlite3 +import statistics +from collections import Counter +from pathlib import Path + +from codebase_index.retrieval.tuning import RetrievalTuning +from research.nucleus import baselines +from research.nucleus.evalrun import CORPORA, DATA, load_queries +from research.nucleus.relations import CoChangeModel, RelationGraph, load_static_edges +from research.nucleus.search import NucleusParams, ObligationIndex, accrete + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--corpus", action="append", default=None) + ap.add_argument("--anchor-head", type=int, default=3) + ap.add_argument("--min-score", type=float, default=0.0, + help="0 disables the gate so generation can be measured") + args = ap.parse_args() + + corpora = CORPORA if not args.corpus else [c for c in CORPORA if c[0] in set(args.corpus)] + + tot = Counter() + acc_ranks: list[int] = [] + prec_at = Counter() + n_slots_eval = 0 + + for name, repo in corpora: + qs, idx = DATA / f"{name}.yml", DATA / "index" / f"{name}.sqlite" + if not qs.exists() or not idx.exists(): + continue + queries = load_queries(qs, Path(repo)) + conn = sqlite3.connect(idx) + conn.row_factory = sqlite3.Row + files = [r[0].replace("\\", "/") for r in conn.execute("SELECT path FROM files")] + cochange = CoChangeModel.from_repo(Path(repo)) + graph = RelationGraph(files, static=load_static_edges(idx), cochange=cochange) + obl = ObligationIndex(conn) + params = NucleusParams(anchor_head=args.anchor_head, min_score=args.min_score) + + for q in queries: + cochange.advance_to(q.position) + cands, _pool = baselines.product_candidates( + conn, q.query, limit=10, tuning=RetrievalTuning()) + base_files: list[str] = [] + for c in cands[:10]: + p = c.path.replace("\\", "/") + if p not in base_files: + base_files.append(p) + gold = set(q.expected_files) + if not gold: + continue + tot["queries"] += 1 + + missed = gold - set(base_files) + tot["gold_total"] += len(gold) + tot["gold_missed_by_baseline"] += len(missed) + + head = base_files[: args.anchor_head] + anchors = list(dict.fromkeys(head)) + if not anchors: + continue + ranked = accrete(conn, anchors=anchors, graph=graph, obligations=obl, + params=params, exclude=set(head), query_terms=set()) + ranked_paths = [p for p, _s, _pa in ranked] + rank_of = {p: i for i, p in enumerate(ranked_paths)} + + for gfile in missed: + if gfile in rank_of: + tot["missed_generated"] += 1 + acc_ranks.append(rank_of[gfile]) + else: + tot["missed_not_generated"] += 1 + + # Precision of the slots accretion would actually claim. + for k in (1, 3, 5): + top = ranked_paths[:k] + if top: + prec_at[f"hits@{k}"] += sum(1 for p in top if p in gold) + prec_at[f"slots@{k}"] += len(top) + if ranked_paths: + n_slots_eval += 1 + + # Displacement: baseline results at ranks anchor_head..10 that are gold + # and would be pushed past rank 10 by 3 insertions. + tail = base_files[args.anchor_head:] + evicted = tail[max(0, len(tail) - 3):] + tot["evicted_gold"] += sum(1 for p in evicted if p in gold) + tot["evicted_total"] += len(evicted) + conn.close() + + print("\n=== accretion failure decomposition ===") + print(f"queries {tot['queries']}") + print(f"gold files (total) {tot['gold_total']}") + print(f"gold missed by baseline top-10 {tot['gold_missed_by_baseline']}" + f" ({tot['gold_missed_by_baseline']/max(1,tot['gold_total']):.1%} of gold)") + gen = tot["missed_generated"] + nogen = tot["missed_not_generated"] + print(f" ... proposed by accretion {gen}" + f" ({gen/max(1,gen+nogen):.1%}) <- generation recall") + print(f" ... never proposed {nogen} ({nogen/max(1,gen+nogen):.1%})") + if acc_ranks: + acc_ranks.sort() + print(f" rank of proposed gold within accretion list: " + f"median={statistics.median(acc_ranks):.0f} " + f"p25={acc_ranks[len(acc_ranks)//4]} p75={acc_ranks[3*len(acc_ranks)//4]}") + for k in (1, 3, 5, 10, 20): + print(f" within top-{k:<2}: {sum(1 for r in acc_ranks if r < k)}" + f"/{len(acc_ranks)} ({sum(1 for r in acc_ranks if r < k)/len(acc_ranks):.1%})") + print("\nprecision of the slots accretion claims (vs full gold set):") + for k in (1, 3, 5): + h, s = prec_at[f"hits@{k}"], prec_at[f"slots@{k}"] + print(f" P@{k} = {h}/{s} = {h/max(1,s):.3f}") + print(f"\ndisplacement: of the last 3 baseline slots, " + f"{tot['evicted_gold']}/{tot['evicted_total']} = " + f"{tot['evicted_gold']/max(1,tot['evicted_total']):.3f} are gold") + print(" ^ accretion must beat THIS precision to be worth a slot") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/research/diagnose_calibration.py b/research/diagnose_calibration.py new file mode 100644 index 0000000..bdc3b85 --- /dev/null +++ b/research/diagnose_calibration.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Is the accretion score calibrated? The last chance for a positive-EV design. + +Accretion loses on average (gain 0.031 < cost 0.035 per slot). Averages hide +selectivity: if a *high* accretion score reliably predicts gold, a gated accretion +that fires rarely could still beat the slot it spends, even though firing always does +not. This measures precision as a function of the score, using the cached +contributions (no retrieval re-run needed). + +Decision rule: accretion is worth keeping iff some score threshold yields +precision > 0.035 (the rank-10 gold density) on a non-trivial share of queries. +""" +from __future__ import annotations +import json, sys +from pathlib import Path + +CACHE = Path("research/data/contrib_cache.json") +RELATIONS = ["edge", "cochange", "testlink", "stem", "dir"] +COST = 0.0347 # measured P(gold) at baseline rank 10 -- the cheapest slot + +def main() -> int: + cache = json.loads(CACHE.read_text(encoding="utf-8")) + w = {r: 1.0 for r in RELATIONS} + rows = [] # (top1_score, is_gold, n_candidates) + for corpus, qs in cache.items(): + for row in qs: + cands, missed = row["cands"], set(row["missed"]) + if not cands: + continue + scored = sorted( + ((sum(w[k]*v for k, v in acc.items()), p) for p, acc in cands.items()), + key=lambda t: (-t[0], t[1])) + s, p = scored[0] + rows.append((s, 1 if p in missed else 0, len(cands))) + rows.sort(key=lambda t: -t[0]) + n = len(rows) + print(f"queries with >=1 accretion candidate: {n}") + print(f"median candidate-list size: {sorted(r[2] for r in rows)[n//2]}") + print(f"\n{'threshold':>10} {'fires':>7} {'fire%':>7} {'hits':>5} {'precision':>10} {'vs cost':>9}") + print("-" * 56) + for frac in (0.02, 0.05, 0.10, 0.20, 0.30, 0.50, 0.75, 1.00): + k = max(1, int(n * frac)) + top = rows[:k] + hits = sum(t[1] for t in top) + prec = hits / k + print(f"{top[-1][0]:>10.3f} {k:>7} {frac:>6.0%} {hits:>5} {prec:>10.4f} " + f"{prec - COST:>+9.4f}") + # Expected net gold per query at each threshold, versus spending nothing. + print(f"\nnet expected gold per query (gain - cost), all {n} queries as denominator:") + for frac in (0.02, 0.05, 0.10, 0.20, 0.50, 1.00): + k = max(1, int(n * frac)) + top = rows[:k] + hits = sum(t[1] for t in top) + net = (hits - COST * k) / n + print(f" fire on top {frac:>4.0%}: net = {net:+.5f} gold/query") + return 0 + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/research/diagnose_slots.py b/research/diagnose_slots.py new file mode 100644 index 0000000..122c702 --- /dev/null +++ b/research/diagnose_slots.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Slot economics: what does a completion slot cost, and is any slot cheap enough? + +Accretion buys recall by spending a rank slot. Whether that trade is ever positive is +an arithmetic question that can be answered without running the retriever end to end: + + gain(slot) = P(top accretion candidate is gold the baseline top-10 missed) + cost(slot) = P(the baseline result evicted from that slot was gold) + +If gain < cost at every slot, no amount of tuning saves the design and it should be +reported as refuted. If some slots are free -- because the baseline did not fill them +-- then accretion has a positive-expected-value niche and the design survives in a +narrower form than proposed. +""" + +from __future__ import annotations + +import argparse +import sqlite3 +from collections import Counter +from pathlib import Path + +from codebase_index.retrieval.tuning import RetrievalTuning +from research.nucleus import baselines +from research.nucleus.evalrun import CORPORA, DATA, load_queries +from research.nucleus.relations import CoChangeModel, RelationGraph, load_static_edges +from research.nucleus.search import NucleusParams, ObligationIndex, accrete + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--anchor-head", type=int, default=3) + args = ap.parse_args() + + rank_gold = Counter() # baseline rank -> # gold + rank_seen = Counter() # baseline rank -> # queries that filled it + nfiles_hist = Counter() + recovery_at = Counter() + n_queries = 0 + free_slot_queries = 0 + free_slot_recovery = 0 + + for name, repo in CORPORA: + qs, idx = DATA / f"{name}.yml", DATA / "index" / f"{name}.sqlite" + if not qs.exists() or not idx.exists(): + continue + queries = load_queries(qs, Path(repo)) + conn = sqlite3.connect(idx) + conn.row_factory = sqlite3.Row + files = [r[0].replace("\\", "/") for r in conn.execute("SELECT path FROM files")] + cochange = CoChangeModel.from_repo(Path(repo)) + graph = RelationGraph(files, static=load_static_edges(idx), cochange=cochange) + obl = ObligationIndex(conn) + params = NucleusParams(anchor_head=args.anchor_head, min_score=0.0) + + for q in queries: + cochange.advance_to(q.position) + cands, _ = baselines.product_candidates( + conn, q.query, limit=10, tuning=RetrievalTuning()) + base_files: list[str] = [] + for c in cands[:10]: + p = c.path.replace("\\", "/") + if p not in base_files: + base_files.append(p) + gold = set(q.expected_files) + if not gold: + continue + n_queries += 1 + nfiles_hist[len(base_files)] += 1 + for i, p in enumerate(base_files, start=1): + rank_seen[i] += 1 + if p in gold: + rank_gold[i] += 1 + + missed = gold - set(base_files) + head = base_files[: args.anchor_head] + anchors = list(dict.fromkeys(head)) + if not anchors: + continue + ranked = accrete(conn, anchors=anchors, graph=graph, obligations=obl, + params=params, exclude=set(head), query_terms=set()) + paths = [p for p, _s, _pa in ranked] + for c in (1, 2, 3): + recovery_at[c] += sum(1 for p in paths[:c] if p in missed) + if len(base_files) < 10: + free_slot_queries += 1 + free_slot_recovery += sum(1 for p in paths[:1] if p in missed) + conn.close() + + print(f"\n=== baseline gold density by rank (n={n_queries} queries) ===") + print(f"{'rank':>5} {'filled':>7} {'gold':>6} {'P(gold)':>9}") + for i in range(1, 11): + s, g = rank_seen[i], rank_gold[i] + print(f"{i:>5} {s:>7} {g:>6} {g/max(1,s):>9.4f}") + + print(f"\n=== how many distinct files the baseline actually returns ===") + for k in sorted(nfiles_hist): + print(f" {k:>2} files: {nfiles_hist[k]:>4} queries " + f"({nfiles_hist[k]/max(1,n_queries):>6.1%})") + under = sum(v for k, v in nfiles_hist.items() if k < 10) + print(f" fewer than 10: {under}/{n_queries} = {under/max(1,n_queries):.1%}") + + print("\n=== the trade ===") + for c in (1, 2, 3): + print(f" recovery@{c} (gold the baseline missed, per query): " + f"{recovery_at[c]/max(1,n_queries):.4f}") + last = rank_seen[10] + print(f" cost of evicting rank 10: P(gold) = {rank_gold[10]/max(1,last):.4f}") + print(f" cost of evicting rank 4: P(gold) = {rank_gold[4]/max(1,rank_seen[4]):.4f}") + print(f"\n queries with a FREE slot (<10 files returned): {free_slot_queries}" + f" ({free_slot_queries/max(1,n_queries):.1%})") + print(f" gold recovered into free slots by top-1 accretion: {free_slot_recovery}" + f" ({free_slot_recovery/max(1,free_slot_queries):.4f} per such query)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/research/diagnose_structure.py b/research/diagnose_structure.py new file mode 100644 index 0000000..4118f76 --- /dev/null +++ b/research/diagnose_structure.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +"""Go/no-go diagnostic for necessity-based retrieval. + +The premise of "compute the required set" rather than "rank similar documents" is +that the members of a required set are *related to each other* by something +computable. If a commit's files are mutually unrelated by every cheap relation, +then no completion mechanism can ever recover the members that lexical retrieval +misses, and the whole research direction is dead. + +This script measures that premise before anything is built. For every +multi-file ground-truth answer it asks: given one gold file (the "anchor"), is +each *other* gold file reachable by + + * dir - same directory + * stem - shared identifier tokens in the file name + * testlink - one is the test/spec of the other (or vice versa) + * edge - a resolved import/call/reference edge links the two files in + the code graph (direction ignored) + * cochange - the two files changed together in history STRICTLY BEFORE the + query's own commit (no lookahead, so the number is what a + deployed system could actually have known) + +Reported as *reachability*: the fraction of non-anchor gold files that at least +one relation connects to the anchor set. That is the ceiling on what any +completion stage can add. +""" + +from __future__ import annotations + +import argparse +import re +import sqlite3 +import subprocess +import sys +from collections import Counter, defaultdict +from pathlib import Path + +import yaml + +_WORD = re.compile(r"[A-Za-z][A-Za-z0-9]*") +_TEST_MARK = re.compile(r"(?:^|[._-])(?:test|tests|spec|specs)(?:$|[._-])", re.I) + + +def git(repo: Path, *args: str) -> str: + p = subprocess.run( + ["git", "-C", str(repo), *args], + capture_output=True, text=True, encoding="utf-8", errors="replace", + ) + if p.returncode != 0: + raise RuntimeError(f"git {' '.join(args)}: {p.stderr.strip()}") + return p.stdout + + +def commit_order(repo: Path) -> dict[str, int]: + """sha12 -> position, 0 = newest. Larger position == older.""" + shas = [s.strip() for s in git(repo, "log", "--format=%H").splitlines() if s.strip()] + return {s[:12]: i for i, s in enumerate(shas)} + + +def commit_filesets(repo: Path) -> list[tuple[str, list[str]]]: + """[(sha12, [paths])] newest first, merges excluded (matching gen_queries).""" + raw = git(repo, "log", "--no-merges", "--name-only", + "--pretty=format:%x01%H", "--diff-filter=ACMR") + out: list[tuple[str, list[str]]] = [] + for rec in raw.split("\x01"): + rec = rec.strip("\n") + if not rec: + continue + head, _, body = rec.partition("\n") + files = [ln.strip() for ln in body.splitlines() if ln.strip()] + if files: + out.append((head.strip()[:12], files)) + return out + + +def stem_tokens(path: str) -> set[str]: + base = path.rsplit("/", 1)[-1] + base = base.split(".", 1)[0] + # camelCase / snake_case / kebab-case -> lowercase tokens + parts = re.sub(r"([a-z0-9])([A-Z])", r"\1 \2", base).replace("_", " ").replace("-", " ") + return {w.lower() for w in _WORD.findall(parts) if len(w) > 2} + + +def core_stem(path: str) -> str: + """File stem with test/spec markers stripped, for test<->impl pairing.""" + base = path.rsplit("/", 1)[-1].split(".", 1)[0] + base = re.sub(r"^(?:test|spec)[._-]?", "", base, flags=re.I) + base = re.sub(r"[._-]?(?:test|tests|spec|specs)$", "", base, flags=re.I) + return base.lower() + + +def is_testish(path: str) -> bool: + return bool(_TEST_MARK.search(path)) or "/test" in path.lower() + + +def static_edges(index_path: Path) -> set[frozenset[str]]: + """Undirected file-level adjacency from resolved import/call/reference edges. + + Symbol targets are lifted to their defining file: the unit of the ground truth + is a file, so a call edge into a symbol is evidence about that symbol's file. + Self-loops are dropped (a file importing itself carries no completion signal). + """ + if not index_path or not index_path.exists(): + return set() + conn = sqlite3.connect(index_path) + try: + rows = conn.execute( + """ + SELECT src.path AS a, + CASE WHEN e.dst_kind = 'file' THEN df.path ELSE sf.path END AS b + FROM edges AS e + JOIN files AS src ON src.id = e.file_id + LEFT JOIN files AS df ON e.dst_kind = 'file' AND df.id = e.dst_id + LEFT JOIN symbols AS s ON e.dst_kind = 'symbol' AND s.id = e.dst_id + LEFT JOIN files AS sf ON sf.id = s.file_id + WHERE e.resolved = 1 + """ + ).fetchall() + except sqlite3.Error: + return set() + finally: + conn.close() + out: set[frozenset[str]] = set() + for a, b in rows: + if a and b and a != b: + out.add(frozenset((a, b))) + return out + + +def related(a: str, b: str, cochange: set[frozenset[str]], + edges: set[frozenset[str]]) -> set[str]: + rels: set[str] = set() + da, db = a.rsplit("/", 1)[0] if "/" in a else "", b.rsplit("/", 1)[0] if "/" in b else "" + if da == db: + rels.add("dir") + ta, tb = stem_tokens(a), stem_tokens(b) + if ta & tb: + rels.add("stem") + if (is_testish(a) != is_testish(b)) and core_stem(a) == core_stem(b) and core_stem(a): + rels.add("testlink") + if frozenset((a, b)) in edges: + rels.add("edge") + if frozenset((a, b)) in cochange: + rels.add("cochange") + return rels + + +def analyse(repo: Path, queries_path: Path, name: str, + index_path: Path | None = None) -> dict: + edges = static_edges(index_path) if index_path else set() + records = yaml.safe_load(queries_path.read_text(encoding="utf-8")) or [] + order = commit_order(repo) + history = commit_filesets(repo) + + # Pre-index history by position so the temporal filter is a slice, not a scan. + hist_by_pos: list[tuple[int, list[str]]] = [] + for sha, files in history: + pos = order.get(sha) + if pos is not None: + hist_by_pos.append((pos, files)) + + sizes = Counter() + rel_hits: Counter[str] = Counter() + n_targets = 0 + n_reachable = 0 + n_multi = 0 + cochange_only = 0 + edge_only = 0 + unreachable_examples: list[tuple[str, str, str]] = [] + hist_depth: list[int] = [] + + for rec in records: + gold = list(dict.fromkeys(rec.get("expected_files", []))) + sizes[len(gold)] += 1 + if len(gold) < 2: + continue + n_multi += 1 + qpos = order.get(rec.get("commit", ""), None) + if qpos is None: + continue + # Co-change pairs observable strictly before this commit (pos > qpos). + prior = [f for pos, f in hist_by_pos if pos > qpos] + hist_depth.append(len(prior)) + pairs: set[frozenset[str]] = set() + for files in prior: + if len(files) > 40: # sweeping commits couple everything; ignore + continue + for i, x in enumerate(files): + for y in files[i + 1:]: + pairs.add(frozenset((x, y))) + + # Every gold file gets a turn as the anchor; the rest are targets. + for anchor in gold: + for target in gold: + if target == anchor: + continue + n_targets += 1 + rels = related(anchor, target, pairs, edges) + if rels: + n_reachable += 1 + for r in rels: + rel_hits[r] += 1 + if rels == {"cochange"}: + cochange_only += 1 + if rels == {"edge"}: + edge_only += 1 + elif len(unreachable_examples) < 5: + unreachable_examples.append((rec["query"][:60], anchor, target)) + + return { + "corpus": name, + "queries": len(records), + "multi_file_queries": n_multi, + "size_hist": dict(sorted(sizes.items())), + "anchor_target_pairs": n_targets, + "reachable": n_reachable, + "reachability": (n_reachable / n_targets) if n_targets else 0.0, + "by_relation": {k: v / n_targets for k, v in rel_hits.items()} if n_targets else {}, + "cochange_only_share": (cochange_only / n_targets) if n_targets else 0.0, + "edge_only_share": (edge_only / n_targets) if n_targets else 0.0, + "n_static_edges": len(edges), + "median_prior_commits": ( + sorted(hist_depth)[len(hist_depth) // 2] if hist_depth else 0 + ), + "unreachable_examples": unreachable_examples, + } + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--corpus", action="append", required=True, + help="name|repo_path|queries.yml[|index.sqlite]") + args = ap.parse_args() + + rows = [] + for spec in args.corpus: + parts = spec.split("|") + name, repo, qs = parts[0], parts[1], parts[2] + idx = Path(parts[3]) if len(parts) > 3 else None + try: + rows.append(analyse(Path(repo), Path(qs), name, idx)) + except Exception as exc: # a corpus that cannot be read must not hide the rest + print(f"[{name}] FAILED: {type(exc).__name__}: {exc}", file=sys.stderr) + + print(f"\n{'corpus':22} {'q':>4} {'multi':>6} {'pairs':>6} {'reach':>7} " + f"{'dir':>6} {'stem':>6} {'test':>6} {'coch':>6} {'coch-only':>9} {'hist':>5}") + print("-" * 100) + tot_pairs = tot_reach = 0 + agg_rel: Counter[str] = Counter() + agg_conly = 0 + agg_eonly = 0 + for r in rows: + b = r["by_relation"] + print(f"{r['corpus']:22} {r['queries']:>4} {r['multi_file_queries']:>6} " + f"{r['anchor_target_pairs']:>6} {r['reachability']:>7.3f} " + f"{b.get('dir',0):>6.3f} {b.get('stem',0):>6.3f} {b.get('testlink',0):>6.3f} " + f"{b.get('edge',0):>6.3f} {b.get('cochange',0):>6.3f} " + f"{r['edge_only_share']:>7.3f} {r['cochange_only_share']:>7.3f} " + f"{r['median_prior_commits']:>5}") + tot_pairs += r["anchor_target_pairs"] + tot_reach += r["reachable"] + for k, v in b.items(): + agg_rel[k] += v * r["anchor_target_pairs"] + agg_conly += r["cochange_only_share"] * r["anchor_target_pairs"] + agg_eonly += r["edge_only_share"] * r["anchor_target_pairs"] + if tot_pairs: + print("-" * 112) + print(f"{'POOLED':22} {'':>4} {'':>6} {tot_pairs:>6} {tot_reach/tot_pairs:>7.3f} " + f"{agg_rel['dir']/tot_pairs:>6.3f} {agg_rel['stem']/tot_pairs:>6.3f} " + f"{agg_rel['testlink']/tot_pairs:>6.3f} {agg_rel['edge']/tot_pairs:>6.3f} " + f"{agg_rel['cochange']/tot_pairs:>6.3f} " + f"{agg_eonly/tot_pairs:>7.3f} {agg_conly/tot_pairs:>7.3f}") + + print("\ngold-set size distribution (pooled):") + total_sizes: Counter = Counter() + for r in rows: + for k, v in r["size_hist"].items(): + total_sizes[k] += v + n_all = sum(total_sizes.values()) + for k in sorted(total_sizes): + print(f" |gold|={k}: {total_sizes[k]:>4} ({total_sizes[k]/n_all:.1%})") + multi = sum(v for k, v in total_sizes.items() if k >= 2) + print(f" multi-file share: {multi}/{n_all} = {multi/n_all:.1%}") + + print("\nunreachable examples (query | anchor | target):") + for r in rows[:3]: + for q, a, t in r["unreachable_examples"][:3]: + print(f" [{r['corpus']}] {q!r}\n {a}\n -> {t}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/research/experiment_memory.py b/research/experiment_memory.py new file mode 100644 index 0000000..24c94ba --- /dev/null +++ b/research/experiment_memory.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""H3/H4: soundness and survival of evidence-keyed agent memory. + +Two questions, both answered against real history with no model in the loop: + + 1. How often would a query-keyed ("semantic") cache serve a stale answer? That is + exactly `1 - survival`, because such a cache reuses on question identity and + therefore reuses every entry whose evidence has since moved. + + 2. Does evidence granularity change survival multiplicatively (H4)? File-keyed and + span-keyed memories are both perfectly sound; they differ only in how much valid + reuse they retain, and the gap is the cost of the lazy implementation choice. + +Conclusions are not synthetic: each benchmark query is run through the shipped +retriever, and the spans it actually returns are recorded as that conclusion's +evidence. This is what an agent would really have read. +""" + +from __future__ import annotations + +import argparse +import sqlite3 +import statistics +from pathlib import Path + +from codebase_index.retrieval.tuning import RetrievalTuning +from research.nucleus import baselines +from research.nucleus.evalrun import BUDGET, CORPORA, DATA, load_queries +from research.nucleus.memory import Conclusion, Span, changed_regions, survives + +HORIZONS = (1, 2, 5, 10, 20, 50) + + +def collect(name: str, repo: Path, top_k: int) -> list[Conclusion]: + qs, idx = DATA / f"{name}.yml", DATA / "index" / f"{name}.sqlite" + if not qs.exists() or not idx.exists(): + return [] + queries = load_queries(qs, repo) + conn = sqlite3.connect(idx) + conn.row_factory = sqlite3.Row + out: list[Conclusion] = [] + for q in queries: + cands, _ = baselines.product_candidates( + conn, q.query, limit=top_k, tuning=RetrievalTuning()) + payload = baselines.finalize(cands, query=q.query, token_budget=BUDGET, + limit=top_k) + spans, tokens = [], 0 + for r in payload["results"]: + if not r.get("snippet"): + continue # not placed in context, so not evidence + spans.append(Span(r["path"].replace("\\", "/"), int(r["line_start"]), + int(r["line_end"]), int(r.get("token_est") or 0))) + tokens += int(r.get("token_est") or 0) + if spans: + out.append(Conclusion(task=q.query, spans=tuple(spans), tokens=tokens)) + conn.close() + return out + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--top-k", type=int, default=5, + help="results an agent is assumed to actually read") + ap.add_argument("--corpus", action="append", default=None) + args = ap.parse_args() + + corpora = CORPORA if not args.corpus else [c for c in CORPORA if c[0] in set(args.corpus)] + + rows: dict[str, dict[int, tuple[float, float, int]]] = {} + pooled: dict[int, list[tuple[bool, bool]]] = {h: [] for h in HORIZONS} + span_counts: list[int] = [] + file_counts: list[int] = [] + + for name, repo in corpora: + repo_p = Path(repo) + cons = collect(name, repo_p, args.top_k) + if not cons: + continue + span_counts += [len(c.spans) for c in cons] + file_counts += [len(c.files) for c in cons] + rows[name] = {} + for h in HORIZONS: + changed, hunks = changed_regions(repo_p, h) + if not changed and h > 1: + # Shallow history: no commit at that depth. Skip rather than report + # a spurious 100% survival. + continue + sf = [survives(c, changed, hunks, granularity="file") for c in cons] + ss = [survives(c, changed, hunks, granularity="span") for c in cons] + rows[name][h] = (statistics.fmean(sf), statistics.fmean(ss), len(cons)) + pooled[h] += list(zip(sf, ss)) + + print(f"\n=== evidence survival (top-{args.top_k} results treated as evidence) ===") + print("survival = fraction of conclusions whose evidence was NOT disturbed") + print("unsound = what a query-keyed semantic cache would serve stale = 1 - survival\n") + print(f"{'corpus':22} {'h':>4} {'n':>5} {'file-keyed':>11} {'span-keyed':>11} " + f"{'span/file':>10} {'semantic unsound':>17}") + print("-" * 90) + for name, hs in rows.items(): + for h, (sf, ss, n) in sorted(hs.items()): + ratio = (ss / sf) if sf > 0 else float("nan") + print(f"{name:22} {h:>4} {n:>5} {sf:>11.3f} {ss:>11.3f} {ratio:>10.2f} " + f"{1-ss:>17.3f}") + print("-" * 90) + print(f"{'POOLED':22} {'h':>4} {'n':>5} {'file-keyed':>11} {'span-keyed':>11} " + f"{'span/file':>10} {'semantic unsound':>17}") + for h in HORIZONS: + pairs = pooled[h] + if not pairs: + continue + sf = statistics.fmean([1.0 if a else 0.0 for a, _ in pairs]) + ss = statistics.fmean([1.0 if b else 0.0 for _, b in pairs]) + ratio = (ss / sf) if sf > 0 else float("nan") + print(f"{'':22} {h:>4} {len(pairs):>5} {sf:>11.3f} {ss:>11.3f} {ratio:>10.2f} " + f"{1-ss:>17.3f}") + + if span_counts: + print(f"\nevidence footprint: {statistics.fmean(span_counts):.1f} spans across " + f"{statistics.fmean(file_counts):.1f} files per conclusion") + + # --- multi-agent sharing -------------------------------------------------- + # Content addressing gives dedup for free: two agents that read the same span + # name it with the same key. This measures how much of a realistic workload is + # actually shared, i.e. the ceiling on what a shared memory can save. + print("\n=== evidence sharing across the task workload ===") + print(f"{'corpus':22} {'tasks':>6} {'atoms':>7} {'distinct':>9} {'shared%':>8} " + f"{'tok gross':>10} {'tok dedup':>10} {'saving':>7}") + print("-" * 88) + tot = [0, 0, 0, 0, 0] + for name, repo in corpora: + cons = collect(name, Path(repo), args.top_k) + if not cons: + continue + seen: dict[tuple[str, int, int], int] = {} + tok: dict[tuple[str, int, int], int] = {} + gross = 0 + for c in cons: + for sp in c.spans: + k = (sp.path, sp.start, sp.end) + seen[k] = seen.get(k, 0) + 1 + tok[k] = sp.tokens + gross += sp.tokens + atoms = sum(seen.values()) + distinct = len(seen) + shared = sum(1 for v in seen.values() if v > 1) + dedup = sum(tok.values()) + print(f"{name:22} {len(cons):>6} {atoms:>7} {distinct:>9} " + f"{shared/max(1,distinct):>7.1%} {gross:>10} {dedup:>10} " + f"{1-dedup/max(1,gross):>6.1%}") + tot[0] += len(cons); tot[1] += atoms; tot[2] += distinct + tot[3] += gross; tot[4] += dedup + print("-" * 88) + print(f"{'POOLED':22} {tot[0]:>6} {tot[1]:>7} {tot[2]:>9} {'':>8} " + f"{tot[3]:>10} {tot[4]:>10} {1-tot[4]/max(1,tot[3]):>6.1%}") + print("\n 'saving' = tokens a content-addressed shared memory never re-sends,") + print(" because a second agent asking a different question reads an atom the") + print(" first agent already materialised. Dedup, not compression.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/research/experiments.md b/research/experiments.md new file mode 100644 index 0000000..eff5e28 --- /dev/null +++ b/research/experiments.md @@ -0,0 +1,313 @@ +# Stage 6–7 — Experiments, ablations, and the refutation + +## Protocol + +**Corpora.** Eight real repositories, four languages (Python ×2, Java ×3, +TypeScript/TSX ×2, PowerShell ×1), 2 264 indexed files, 420 queries. Identical to the +set `docs/BENCHMARKS.md` reports 1.10.0 against, so the numbers here sit next to the +product's own published numbers. + +**Ground truth.** Commit subject → files that commit changed, mined by the shipped +`tests/eval/gen_queries.py`. Leak-free: the query text lives in git metadata, not in +the indexed corpus. Changelog-like files are excluded as both documents and answers. + +**Metrics.** `tests/eval/metrics.py` verbatim — never reimplemented. Every non-baseline +row carries a paired bootstrap 95% CI and a paired permutation p-value, both seeded. + +**Indexing.** One index per corpus, built once, shared by every system, so deltas +measure retrieval and never indexing variance. + +**Temporal discipline.** The co-change relation could trivially leak, since ground +truth is mined from commits. `CoChangeModel` is append-only and queries are swept +oldest-first, so a query's own commit and every later commit are structurally +unreachable — `test_cochange_model_refuses_lookahead` asserts that rewinding raises. + +**Attribution.** `NucleusParams(accrete=False)` reproduces +`codebase_index.retrieval.pipeline.search` result-for-result +(`test_accretion_off_reproduces_shipped_pipeline`, 5 queries × exact snippet match). +Without that equivalence every "NUCLEUS vs incumbent" delta would be confounded with +an incidental reimplementation. + +--- + +## Experiment 0 — Is the premise even true? + +Before building anything: are the members of a required set related to each other by +anything computable? Over 900 anchor→target gold pairs, no lookahead: + +| relation | covers | uniquely | +|---|---|---| +| stem (shared name tokens) | 35.8% | — | +| static edge (import/call/ref) | 30.7% | 5.3% | +| co-change (strictly prior history) | 30.4% | 9.8% | +| same directory | 23.8% | — | +| test ↔ impl | 14.4% | — | +| **union** | **71.8%** | | + +The premise holds, and structure and history are **not** redundant. This is what +justified proceeding, and it is also what makes the eventual negative result +interesting rather than trivial: the signal is genuinely there. + +Gold-set sizes: 53.1% single-file, 46.9% multi-file (2–4). Completion can only ever +help the 46.9%. + +--- + +## Experiment 1 — First implementation: a significant loss + +Default configuration (3 completions inserted after a 3-result anchor head): + +| vs `hybrid` | Δ | 95% CI | p | +|---|---|---|---| +| MRR | **−0.0073** | [−0.0108, −0.0038] | **<0.001** | +| P@5 | **−0.0102** | [−0.0176, −0.0029] | **0.009** | +| recall@10 | −0.0105 | [−0.0312, +0.0103] | 0.330 | +| tokens | **+108** | — | — | + +A clear loss. The interesting part is *why*, and three very different causes produce +the same number. + +## Experiment 2 — Decomposing the failure + +`diagnose_accretion.py`, 420 queries: + +``` +gold missed by baseline top-10 266 (36.6% of all gold) + ... proposed by accretion 142 (53.4%) <- generation works + ... never proposed 124 (46.6%) +rank of proposed gold within the accretion list: median 17, p25 5, p75 41 + within top-1 9.2% top-3 16.2% top-5 23.9% top-10 36.6% +``` + +So generation is not the bottleneck — **ranking inside the accretion list is**. Then +the economics, from `diagnose_slots.py`: + +``` +baseline gold density by rank: r1 0.467 r2 0.233 r3 0.122 r4 0.096 + r5 0.066 r6 0.021 r7 0.027 r8 0.025 + r9 0.034 r10 0.035 + +accretion recovers (gold the baseline missed): @1 0.0310 @2 0.0476 @3 0.0548 +cost of evicting the cheapest slot (rank 10): 0.0347 +``` + +**Every completion slot is net-negative**: 0.0310 gained < 0.0347 lost, and the second +and third slots are worse (+0.0166 and +0.0072 against 0.0339 and 0.0247). Inserting at +rank 4 was worse still — it displaced a slot holding gold at 0.096, which is why MRR +and P@5 fell while recall barely moved. + +The "free slot" escape is empty: only 17.6% of queries return fewer than 10 files, and +accretion placed exactly **1** gold file into all of them combined. + +## Experiment 3 — Fitting the weights: overfits, reported as such + +The weights had never been fitted (all 1.0). Leave-one-repository-out, +objective `recovery@1`: + +| | uniform | fitted | Δ | +|---|---|---|---| +| in-sample | 0.0310 | 0.0452 | +46% | +| **held-out (pooled)** | **0.0310** | **0.0286** | **−0.0024** | + +1 of 8 folds improved, 2 regressed, and the coefficients swing wildly across folds +(`cochange` 0.91–4.21, `edge` 0.00–0.28). **The fit does not generalise**, so uniform +weights are used everywhere below. A 46% in-sample gain that reverses out of sample is +exactly the artifact leave-one-out exists to catch. + +## Experiment 4 — Calibration: the signal is real and selective + +Accretion proposes nothing on 58% of queries. Conditioned on firing (n=177): + +| fire on top | threshold | precision | vs slot cost 0.035 | +|---|---|---|---| +| 2% | 2.095 | 0.667 | **19×** | +| 10% | 1.547 | 0.353 | **10×** | +| 20% | 1.321 | 0.200 | 5.8× | +| 50% | 0.826 | 0.125 | 3.6× | +| 100% | 0.124 | 0.073 | 2.1× | + +The score is **well calibrated** — precision rises monotonically and steeply. Gate +selected under LORO: τ = 0.600 in 7 of 8 folds, on a plateau (net gold/query flat at +0.021–0.024 for τ ∈ [0, 1.1]). This is a plateau interior, not a peak. + +## Experiment 5 — Fixing the slot policy: a null result + +Tail placement, one gated completion: + +| vs `hybrid` | Δ | 95% CI | p | +|---|---|---|---| +| recall@10 | +0.0062 | [−0.0085, +0.0212] | 0.431 | +| MRR | −0.0005 | [−0.0019, +0.0007] | 0.732 | +| useful@budget | +0.0038 | [−0.0062, +0.0145] | 0.501 | + +Nothing significant either way. The regression is repaired; no gain replaces it. + +## Experiment 5b — Comparison against the required baseline families + +All seven systems, same 420 queries, same indexes, same 1500-token budget, same +`apply_budget` and compactor, so token accounting is symmetric: + +| system | recall@5 | recall@10 | MRR | nDCG@10 | useful@budget | tokens | useful/token | p50 ms | +|---|---|---|---|---|---|---|---|---| +| `bm25` (FTS5 Okapi) | 0.452 | 0.497 | 0.368 | 0.370 | 0.423 | 1184.6 | 3.57e-4 | 13.5 | +| `dense` (LSA-160) | 0.351 | 0.386 | 0.243 | 0.262 | 0.333 | 741.3 | 4.49e-4 | 6.3 | +| `rag` (BM25 ⊕ dense, RRF) | 0.448 | 0.512 | 0.324 | 0.351 | 0.426 | 1131.7 | 3.76e-4 | 15.1 | +| `graph` (PPR, forced on) | 0.616 | 0.698 | 0.587 | 0.563 | 0.617 | 1083.2 | **5.70e-4** | 63.5 | +| `hybrid` (shipped 1.10.0) | 0.618 | 0.693 | 0.583 | 0.559 | 0.617 | 1093.0 | 5.64e-4 | 45.9 | +| `hybrid13` | 0.629 | 0.695 | 0.589 | 0.564 | 0.644 | 1168.4 | 5.51e-4 | 55.9 | +| `nucleus13` | 0.629 | **0.696** | **0.590** | **0.565** | **0.649** | 1243.0 | 5.22e-4 | 61.4 | + +Three things worth naming: + +- **The dense baseline is LSA, not a neural encoder** (no `sentence-transformers`, GPU + or network here). It is weak — MRR 0.243 — and I therefore make **no claim of + beating "embeddings"**. Its role is to keep the hybrid-RAG row honest, not to stand + in for a modern code encoder. +- **`graph` with PPR forced on is essentially tied with `hybrid`** (MRR +0.004, ns) but + raises `cand_recall` +0.049 (p<0.001). Structural expansion genuinely widens the + pool and genuinely fails to convert that into ranking gains — independent + corroboration of this repository's decision to ship `graph_source=False`, and the + first sign that Experiment 6 was going to come out the way it did. +- **Token efficiency falls monotonically with page size** (5.70 → 5.64 → 5.51 → 5.22 + e-4). "Useful per token" alone always favours returning less, so it cannot be used + as a lone objective; the honest object is the recall-versus-tokens curve, on which + NUCLEUS sits *below* the baseline. + +## Experiment 6 — The decisive comparison + +If the constraint is tokens rather than rank slots, completions can be *appended* as +contract slices (signatures, not bodies) instead of evicting. But then the baseline +must be allowed the same page growth, or "bigger page wins" is indistinguishable from +"better page wins". Hence `hybrid13`. + +| system | recall@10 | recall@15 | MRR | useful@budget | tokens | useful/token | +|---|---|---|---|---|---|---| +| `hybrid` (limit 10) | 0.693 | — | 0.583 | 0.617 | 1093 | 5.65e-4 | +| `hybrid13` (limit 13) | 0.695 | 0.723 | 0.589 | **0.644** | 1168 | **5.51e-4** | +| `nucleus` (10 + 3) | 0.694 | 0.717 | 0.584 | 0.629 | 1189 | 5.29e-4 | +| `nucleus13` (13 + 3) | 0.696 | **0.741** | 0.590 | **0.649** | 1243 | 5.22e-4 | + +Paired, against the page-matched baseline: + +| `nucleus` vs `hybrid13` | Δ | p | | `nucleus13` vs `hybrid13` | Δ | p | +|---|---|---|---|---|---|---| +| useful@1000 | **−0.0171** | **0.038** | | useful@budget | +0.0056 | **0.030** | +| cand_recall | **−0.0190** | **<0.001** | | useful@500 | +0.0062 | **0.029** | +| useful@budget | −0.0149 | 0.075 | | recall@15 | +0.0181 | **<0.001** | +| MRR | −0.0051 | 0.343 | | MAP | +0.0025 | **<0.001** | + +**This is the refutation.** + +1. `nucleus` (spending slots) is **worse** than simply returning three more results. +2. `nucleus13` (spending tokens) does beat the page-matched baseline significantly — + but by +0.0056 useful@budget for **+75 tokens (+6.4%)**. Its token efficiency is + **5.22e-4 vs 5.51e-4 useful-per-token: NUCLEUS is 5% *less* efficient than the + baseline it beats.** + +On the axis the whole research programme was aimed at — more answer per token — the +mechanism is negative. A +0.9% relative gain bought with +6.4% more tokens is not +minimal sufficient context; it is a slightly bigger context. + +## Experiment 7 — Ablations + +Every component ablated independently against `nucleus13` (420 queries): + +| variant | useful@budget Δ | tokens Δ | +|---|---|---| +| `-accretion` (= `hybrid13`) | −0.006 | −74.7 | +| `-cochange` | 0.000 | −0.5 | +| `-edges` | +0.001 | −16.7 | +| `-testlink` | −0.001 | −2.6 | +| `-stem` | −0.002 | −65.9 | +| `-dir` | 0.000 | −4.4 | +| `-coverage_select` (H2 off) | −0.001 | −4.0 | +| `-hub_penalty` | +0.009 | +67.6 | +| `-compact_completions` (H8 off) | −0.003 | +14.7 | +| `combine=max` | +0.003 | −17.6 | +| `gate=0.0` | +0.001 | +77.1 | +| `slots=1` | −0.001 | −36.5 | + +**No single relation carries the effect.** Every individual ablation moves +useful@budget by ≤0.003 — within noise on this query set. The only ablation that moves +anything is removing accretion wholesale. H2's coverage selection contributes −0.001, +i.e. nothing measurable: the elegant submodular objective is doing no work here, +because with `max_completions = 3` from a gated pool there is rarely a redundancy to +resolve. + +`-compact_completions` costs −0.003 useful@budget while *adding* 15 tokens, which is +the one place H8 (contracts instead of bodies) shows its expected sign. + +## Experiment 8 — Memory plane (H3/H4): validated + +Evidence = the spans the shipped retriever actually returns for each of the 420 +queries. Survival measured retrospectively against real history via +`git diff --unified=0 HEAD~h HEAD` new-side hunks, which are already in HEAD +coordinates — exact overlap, no line mapping, no re-indexing. + +| horizon (commits) | file-keyed survival | span-keyed survival | span/file | **semantic-cache unsound rate** | +|---|---|---|---|---| +| 1 | 0.729 | 0.795 | 1.09 | **20.5%** | +| 2 | 0.624 | 0.740 | 1.19 | 26.0% | +| 5 | 0.474 | 0.629 | 1.33 | 37.1% | +| 10 | 0.383 | 0.521 | 1.36 | **47.9%** | +| 20 | 0.213 | 0.309 | 1.45 | 69.1% | +| 50 | 0.099 | 0.130 | 1.32 | 87.0% | + +Two results: + +**Query-keyed caches are unsound at a rate that makes them unusable for code.** A +semantic cache reuses on question identity, so it would serve every one of these +entries; after 10 commits **47.9% of them are stale**, and after a *single* commit, +20.5%. There is no similarity threshold that fixes this, because the failure is not in +the similarity estimate — the cache key omits the dependency entirely. Evidence keying +drives this to zero by construction, not by tuning. + +**H4 holds, but weaker than predicted.** I predicted a large multiplicative retention +gain from finer granularity. Measured: **1.36× at h=10**, rising to 1.45× at h=20 — +real, and growing with horizon as predicted, but not the 2–3× I claimed. Per-repo +variance is large (1.00 for `denfry.github.io` and `DevGraph`, 2.75 for +`codebase-index` at h=10) and tracks whether commits touch files at a granularity +finer than the whole file. The mean evidence footprint is 4.1 spans across 4.1 files — +one span per file — which caps how much span-keying can possibly recover. + +**Cross-agent sharing.** Over the 420-task workload, 1738 atom reads collapse to 1098 +distinct atoms: **42.0% of tokens are never re-sent** by a content-addressed shared +memory (range 13.1% `TerraForge` – 65.4% `denfry.github.io`). This is deduplication, +not compression, and it is exact. + +--- + +## Stage 7 — Trying to destroy the surviving result + +**Adversarial corpora.** `denfry.github.io` (a website: 48 files, 30 symbols) and +`WinCleaner` (PowerShell: 39 files, **0** symbols, **0** edges) were kept in +deliberately. They break the retrieval plane (reachability 0.333 and structure-free) +and they do *not* break the memory plane, whose mechanism needs no symbols at all. +`PoliternalParkour` is the pathological case for memory: survival 0.203 at h=1, +because its commits are enormous — exactly the "highly dynamic codebase" failure mode. +Reported, not hidden. + +**Is the survival result just "code changes"?** Partly, and that is the point: the +contribution is the *magnitude* on real repositories and the demonstration that +granularity is a lever. But it means the numbers are corpus-specific and would differ +on a slow-moving repository. They are not a universal constant. + +**Does the memo help if evidence sets never repeat exactly?** This is the strongest +objection. Exact-set keying is brittle: two agents must read *identically* to share an +entry. The 42% dedup figure is measured at *atom* granularity (sub-conclusion sharing), +which is the level where reuse actually occurs; whole-conclusion reuse will be far +rarer. **I did not measure whole-conclusion hit rate under a realistic multi-agent +workload, and the architecture's `MemoStore.get_or_compute` is therefore untested at +the level that matters most for the "don't redo work" claim.** That is the largest +remaining hole in this work. + +**Is the retrieval refutation an artifact of file-level ground truth?** Plausibly, and +this is the fairest defence of H1. The benchmark scores at file granularity, so a +completion that supplies exactly the right *function* in a file the baseline already +returned earns nothing. Symbol-level ground truth might change the verdict. It would +not change the token-efficiency arithmetic, which is granularity-independent. + +**Threat I cannot close.** Ground truth is "files the commit touched", a proxy for +"files the agent needed". H6 (a `ddmin` oracle over an executable verifier) is the +right instrument and needs an LLM agent loop I do not have here. Every number above +inherits that proxy, on both sides of every comparison. diff --git a/research/fit_gate.py b/research/fit_gate.py new file mode 100644 index 0000000..5fa75d7 --- /dev/null +++ b/research/fit_gate.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Select the accretion gate under leave-one-repository-out. + +The calibration curve is broad and monotone, so a threshold can be chosen from a +*plateau* rather than a peak -- the same discipline `retrieval/tuning.py` applies to +the product's own coefficients. Objective is net expected gold per query: + + net(tau) = ( hits@1(score >= tau) - COST * n_fire(tau) ) / n_queries + +with COST = 0.0347, the measured gold density of the rank-10 slot a completion evicts. +""" +from __future__ import annotations +import json +from pathlib import Path + +CACHE = Path("research/data/contrib_cache.json") +RELATIONS = ["edge", "cochange", "testlink", "stem", "dir"] +COST = 0.0347 + +def rows_for(qs, w): + out = [] + for row in qs: + cands, missed = row["cands"], set(row["missed"]) + if not cands: + continue + best = max(((sum(w[k]*v for k, v in acc.items()), p) for p, acc in cands.items()), + key=lambda t: (t[0], t[1])) + out.append((best[0], 1 if best[1] in missed else 0)) + return out + +def net(rows, n_queries, tau): + fire = [r for r in rows if r[0] >= tau] + return (sum(r[1] for r in fire) - COST * len(fire)) / max(1, n_queries) + +def main() -> int: + cache = json.loads(CACHE.read_text(encoding="utf-8")) + w = {r: 1.0 for r in RELATIONS} + names = list(cache) + grid = [0.0, 0.2, 0.4, 0.6, 0.7, 0.826, 0.9, 1.0, 1.1, 1.3, 1.5, 2.0] + + print(f"{'held-out':22} {'tau*':>6} {'net(tau*)':>10} {'net(0)':>10} {'delta':>9}") + print("-" * 62) + tot_g = tot_0 = 0.0; N = 0 + taus = [] + for held in names: + tr = [r for k in names if k != held for r in cache[k]] + te = cache[held] + tr_rows, te_rows = rows_for(tr, w), rows_for(te, w) + tau = max(grid, key=lambda t: net(tr_rows, len(tr), t)) + g, z = net(te_rows, len(te), tau), net(te_rows, len(te), 0.0) + taus.append(tau) + tot_g += g*len(te); tot_0 += z*len(te); N += len(te) + print(f"{held:22} {tau:>6.3f} {g:>10.5f} {z:>10.5f} {g-z:>+9.5f}") + print("-" * 62) + print(f"{'POOLED (held-out)':22} {'':>6} {tot_g/N:>10.5f} {tot_0/N:>10.5f} {(tot_g-tot_0)/N:>+9.5f}") + print(f"chosen tau across folds: min={min(taus):.3f} median={sorted(taus)[len(taus)//2]:.3f} max={max(taus):.3f}") + + all_rows = rows_for([r for k in names for r in cache[k]], w) + nq = sum(len(cache[k]) for k in names) + print(f"\nplateau on all corpora (net gold/query vs tau):") + for t in grid: + fire = [r for r in all_rows if r[0] >= t] + print(f" tau={t:>5.3f} fires={len(fire):>4} hits={sum(r[1] for r in fire):>3} " + f"net={net(all_rows, nq, t):+.5f}") + return 0 + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/research/fit_weights.py b/research/fit_weights.py new file mode 100644 index 0000000..8112652 --- /dev/null +++ b/research/fit_weights.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +"""Fit the relation weights under leave-one-repository-out. + +The first benchmark run used weights of 1.0 for everything, which is not a +configuration so much as an absence of one, and the accretion ranking was +correspondingly poor (median rank 17 for gold that *was* generated). Hand-picking +coefficients on the same corpora that report the result is how a benchmark gets gamed +by accident, so weights are selected exactly the way this repository selects its +ranking parameters: choose on N-1 corpora, score on the held-out one, pool the folds. + +Objective: `recovery@C` -- the mean number of gold files, per query, that the top-C +accretion candidates recover *from among the gold the baseline's top-10 already +missed*. That is precisely the quantity a completion slot exists to buy, and it maps +monotonically onto recall@10. + +Because `contributions()` is linear in the weights, the whole fit runs over a cached +table of per-relation contributions: the graph is walked once, then tens of thousands +of weight vectors are scored as dot products. +""" + +from __future__ import annotations + +import argparse +import json +import sqlite3 +from dataclasses import replace +from pathlib import Path + +from codebase_index.retrieval.tuning import RetrievalTuning +from research.nucleus import baselines +from research.nucleus.evalrun import CORPORA, DATA, load_queries +from research.nucleus.relations import ( + CoChangeModel, RelationGraph, RelationWeights, load_static_edges, +) +from research.nucleus.search import NucleusParams, ObligationIndex, contributions + +RELATIONS = ["edge", "cochange", "testlink", "stem", "dir"] +CACHE = DATA / "contrib_cache.json" + + +def build_cache(anchor_head: int = 3) -> dict: + """corpus -> [ {gold_missed: [...], cands: {path: {rel: value}}} ] per query.""" + cache: dict[str, list] = {} + for name, repo in CORPORA: + qs, idx = DATA / f"{name}.yml", DATA / "index" / f"{name}.sqlite" + if not qs.exists() or not idx.exists(): + continue + queries = load_queries(qs, Path(repo)) + conn = sqlite3.connect(idx) + conn.row_factory = sqlite3.Row + files = [r[0].replace("\\", "/") for r in conn.execute("SELECT path FROM files")] + cochange = CoChangeModel.from_repo(Path(repo)) + graph = RelationGraph(files, static=load_static_edges(idx), cochange=cochange) + params = NucleusParams(anchor_head=anchor_head, min_score=0.0) + rows = [] + for q in queries: + cochange.advance_to(q.position) + cands, _ = baselines.product_candidates( + conn, q.query, limit=10, tuning=RetrievalTuning()) + base_files: list[str] = [] + for c in cands[:10]: + p = c.path.replace("\\", "/") + if p not in base_files: + base_files.append(p) + gold = set(q.expected_files) + missed = sorted(gold - set(base_files)) + head = base_files[:anchor_head] + anchors = list(dict.fromkeys(head)) + if not anchors or not missed: + # Queries whose gold the baseline already has contribute nothing to + # the objective: there is nothing left for a completion to recover. + # They still matter for *cost*, which the gate (fitted separately) + # and the end-to-end benchmark account for. + rows.append({"missed": missed, "cands": {}}) + continue + contrib = contributions(anchors=anchors, graph=graph, params=params, + exclude=set(head)) + rows.append({"missed": missed, + "cands": {p: {k: round(v, 6) for k, v in acc.items()} + for p, acc in contrib.items()}}) + cache[name] = rows + conn.close() + print(f"[{name}] cached {len(rows)} queries") + return cache + + +def recovery(rows: list, w: dict[str, float], *, c: int) -> float: + """Mean gold files recovered per query by the top-`c` accretion candidates.""" + total = 0.0 + for row in rows: + cands = row["cands"] + if not cands: + continue + missed = set(row["missed"]) + if not missed: + continue + scored = sorted( + ((sum(w.get(k, 0.0) * v for k, v in acc.items()), p) for p, acc in cands.items()), + key=lambda t: (-t[0], t[1]), + ) + total += sum(1 for _s, p in scored[:c] if p in missed) + return total / max(1, len(rows)) + + +def coordinate_ascent(train: list, *, c: int, passes: int = 4) -> dict[str, float]: + grid = [0.0, 0.15, 0.3, 0.5, 0.75, 1.0, 1.5, 2.0, 3.0, 4.0] + w = {r: 1.0 for r in RELATIONS} + best = recovery(train, w, c=c) + for _ in range(passes): + improved = False + for r in RELATIONS: + cur = w[r] + for v in grid: + if v == cur: + continue + trial = dict(w) + trial[r] = v + # L1-normalise so the fitted vector's *scale* stays comparable and the + # gate threshold keeps meaning the same thing across folds. + s = sum(trial.values()) or 1.0 + trial = {k: val * len(RELATIONS) / s for k, val in trial.items()} + score = recovery(train, trial, c=c) + if score > best + 1e-9: + best, w, improved = score, trial, True + if not improved: + break + return w + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--rebuild", action="store_true") + ap.add_argument("--c", type=int, default=1, help="completion slots to fit for") + args = ap.parse_args() + + if args.rebuild or not CACHE.exists(): + cache = build_cache() + CACHE.write_text(json.dumps(cache), encoding="utf-8") + else: + cache = json.loads(CACHE.read_text(encoding="utf-8")) + + names = list(cache) + uniform = {r: 1.0 for r in RELATIONS} + + print(f"\n=== leave-one-repository-out, objective = recovery@{args.c} ===") + print(f"{'held-out':22} {'uniform':>9} {'fitted':>9} {'delta':>9} weights chosen on the rest") + print("-" * 108) + tot_u = tot_f = 0.0 + n = 0 + folds = [] + for held in names: + train = [r for k in names if k != held for r in cache[k]] + test = cache[held] + w = coordinate_ascent(train, c=args.c) + u, f = recovery(test, uniform, c=args.c), recovery(test, w, c=args.c) + folds.append((held, u, f, w)) + wq = len(test) + tot_u += u * wq + tot_f += f * wq + n += wq + print(f"{held:22} {u:>9.4f} {f:>9.4f} {f-u:>+9.4f} " + + " ".join(f"{k}={v:.2f}" for k, v in w.items())) + print("-" * 108) + print(f"{'POOLED (held-out)':22} {tot_u/n:>9.4f} {tot_f/n:>9.4f} {(tot_f-tot_u)/n:>+9.4f}") + print(f"folds improved: {sum(1 for _h,u,f,_w in folds if f > u)}/{len(folds)}; " + f"regressed: {sum(1 for _h,u,f,_w in folds if f < u)}") + + full = coordinate_ascent([r for k in names for r in cache[k]], c=args.c) + print("\nweights fitted on ALL corpora (what would ship, reported as fitted):") + print(" " + " ".join(f"{k}={v:.3f}" for k, v in full.items())) + print(f" in-sample recovery@{args.c}: " + f"{recovery([r for k in names for r in cache[k]], full, c=args.c):.4f} " + f"(uniform {recovery([r for k in names for r in cache[k]], uniform, c=args.c):.4f})") + + # Stability across folds is the honest read on whether these weights mean + # anything: a coefficient that swings wildly between folds is noise. + print("\nper-relation weight across folds (min / median / max):") + for r in RELATIONS: + vals = sorted(w[r] for _h, _u, _f, w in folds) + print(f" {r:10} {vals[0]:.2f} / {vals[len(vals)//2]:.2f} / {vals[-1]:.2f}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/research/hypotheses.md b/research/hypotheses.md new file mode 100644 index 0000000..ff65081 --- /dev/null +++ b/research/hypotheses.md @@ -0,0 +1,284 @@ +# Stage 2–3 — Thirteen hypotheses + +Each entry states what is proposed, why the field has not already done it, and a +formal model. They are deliberately drawn from different parent disciplines; the +selection matrix in `selection.md` then cuts them down by feasibility *on evidence +available here*, not by appeal. + +Notation: corpus of atoms `A`; task `t`; retrieved set `S ⊆ A`; token cost +`c: A → ℕ`; budget `B`; gold/necessary set `N(t) ⊆ A`. + +--- + +## H1 — Retrieval as conditional set completion ("nucleate and accrete") + +**Hypothesis.** Replace `top-k argmax_d P(d|q)` with a two-stage set construction: +find a small high-precision *anchor* set `A₀` from the query, then **complete** it +using relations conditioned on the anchor rather than on the query: + +``` +A₀ = argtop_a P(a | q) (lexical/symbol; high precision, low recall) +S = A₀ ∪ argtop_c P(c ∈ N(t) | A₀, c ∉ A₀) (completion; query-independent) +``` + +**Why it may work / why it is not already done.** Every mainstream system scores +documents *independently given the query*, so the factor `P(c | A₀)` — by far the +strongest available signal for "needed together" — is structurally inexpressible. +Where systems do inject structure (PPR, graph retrievers) they inject it as *another +pointwise opinion into the same fusion*, which forces architectural neighbours to +compete with direct hits for the same ranked slots. This repository ran that +experiment and shipped `graph_source = False` because it lost MRR. The hypothesis is +that the signal was fine and the **composition** was wrong. + +**Mathematical intuition.** The true objective is a set posterior which does not +factorise: + +``` +P(N(t) = S | q) ≠ Π_{d∈S} P(d ∈ N(t) | q) +``` + +Model the dependence with a first-order chain (Chow–Liu style tree over the +change-set): each non-anchor member attaches to some already-selected member, + +``` +P(S | q) ≈ max_{a∈A₀} P(a | q) · Π_{c ∈ S\A₀} max_{s ∈ S, s≺c} P(c | s) +``` + +so `log P(S|q)` decomposes into an anchor term plus **pairwise completion terms** +`log P(c|s)` estimated over a typed relation union. Empirically (see +`diagnose_structure.py`) `P(∃ relation | c,s both gold) = 0.718` pooled across eight +repositories, with no temporal lookahead, versus a base rate of `O(1/|A|)` for a +random pair — a likelihood ratio of roughly `10²–10³`. + +--- + +## H2 — Selection as budgeted maximum coverage over latent obligations + +**Hypothesis.** Replace "take top-k, then MMR for diversity" with an explicit +**budgeted maximum coverage** program. Posit latent *obligations* `O` (things the task +must account for: a symbol, a contract, a config key). Each atom `a` covers +`cov(a) ⊆ O`; the task implicates `O(t) ⊆ O`. Then + +``` +maximise f(S) = | O(t) ∩ ⋃_{a∈S} cov(a) | subject to Σ_{a∈S} c(a) ≤ B +``` + +**Why it may work.** `f` is monotone submodular, so greedy by *coverage gain per +token* is a `(1 − 1/e)`-approximation (Nemhauser; Khuller–Moss–Naor for the budgeted +variant). More importantly it is the *correct* objective: MMR maximises pairwise +dissimilarity, which is a proxy that actively misfires here — two files implementing +the same contract are highly similar and both necessary, exactly the pair MMR +suppresses. No mainstream retriever optimises coverage of task obligations because +obligations are never materialised. + +**Mathematical intuition.** Diminishing returns: `f(S∪{a}) − f(S)` is non-increasing +in `S`. Greedy picks `argmax_a (f(S∪{a}) − f(S)) / c(a)`. Contrast with top-k, which +picks `argmax_a score(a)` — identical only when `cov` are disjoint and equal-cost, +i.e. never. + +--- + +## H3 — Evidence-keyed memoisation: agent reasoning as a content-addressed build + +**Hypothesis.** Store every agent conclusion keyed not by its *question* but by a hash +of the **evidence it consumed**: + +``` +key(r) = H( sort{ (atom_id, content_hash(atom)) : atom ∈ evidence(r) } ‖ H(prompt_class) ) +``` + +A cached conclusion is reusable **iff every atom in its key still hashes to the same +value**. Otherwise it is not "probably stale" — it is *definitely* invalid, and is +evicted deterministically. + +**Why it may work / why it is not already done.** Semantic caches key on query +similarity, which carries zero information about whether the world changed. Build +systems (Bazel, Nix, `ccache`) solved exactly this problem for compilation two decades +ago with content-addressed action caches, and self-adjusting computation (Acar et al.) +solved it for general programs. Agent memory has not adopted it because agent +conclusions are usually stored as prose with no recorded provenance — the dependency +set is thrown away at write time. It is an *engineering omission that has a +correctness consequence*, which is the most promising kind. + +**Mathematical intuition.** Let `V(r,τ)` be the event "r is still valid at time τ". +Under evidence keying, `P(unsound reuse) = 0` exactly (a hash collision aside), while a +similarity cache has `P(unsound reuse) = P(evidence changed | q ≈ q_cached) > 0` and +*uncorrelated with the threshold*. The interesting quantity is therefore not +soundness — which is free — but **survival**: `P(V(r,τ))` as a function of evidence +granularity. That is H4. + +--- + +## H4 — The granularity law of memory survival + +**Hypothesis.** Reuse survival is governed by the *measure* of the evidence footprint, +not its semantic breadth. If a conclusion depends on `m` atoms each independently +invalidated at per-commit rate `λ`, survival after `n` commits is + +``` +P(valid after n commits) ≈ (1 − λ)^{m·n} +``` + +so shrinking atom granularity (file → symbol) shrinks `λ` roughly in proportion to the +fraction of the file a typical commit touches. **Prediction: symbol-level evidence keys +retain multiplicatively more valid reuse than file-level keys**, and the multiplier is +measurable directly from git history with no model in the loop. + +**Why it matters.** It converts "choose your memory granularity" from taste into an +estimable quantity, and it predicts that the naive choice (file-level provenance, +which is what any straightforward implementation would do) is the expensive one. + +--- + +## H5 — Retrieval as entropy minimisation over the answer set + +**Hypothesis.** Choose the next atom that maximally reduces uncertainty about the +answer: `a* = argmax_a I(Y ; a | S)` where `Y` is the answer random variable. + +**Mathematical intuition.** `H(Y|S)` decreasing is the ideal stopping criterion: +retrieve until `H(Y|S) < ε`, which yields *adaptive* `k` — few atoms for a sharp +question, many for a diffuse one. Beautiful, and it subsumes H2 (coverage is a +tractable surrogate for mutual information under a set-cover likelihood). + +**Why it is hard here.** Estimating `I(Y;a|S)` needs either a scoring LLM in the loop +(cost, non-determinism, no API in this environment) or a strong proxy. Kept as theory: +it is the *justification* for H2's objective rather than an independently testable +mechanism. + +--- + +## H6 — A delta-debugging oracle for minimal sufficient context + +**Hypothesis.** Ground truth for "minimal necessary context" can be *computed*, not +guessed: given a task with an executable verifier (test suite), run `ddmin` over +candidate context sets to find a 1-minimal `S` such that the agent still succeeds. +Then distil a cheap predictor from those labels. + +**Why it may work.** Every code-retrieval benchmark today uses a weak proxy label +("the files the commit touched"). ddmin would produce true minimal sufficient sets and +expose how wrong the proxy is. + +**Why it is not selected.** Requires `O(|S| log |S|)` *agent executions per query* with +a working LLM and per-repo test harness. Out of reach in this environment; recorded as +the highest-value follow-up in `further-work`. + +--- + +## H7 — Context prefetching from agent access traces + +**Hypothesis.** An agent's file-access sequence is a memory reference stream. Apply +correlation/Markov prefetching: `P(next atom | last k atoms)`, prefetch on anchor +resolution, and bound the achievable gain with a Belady-style offline optimum. + +**Why it may work.** Hardware prefetching theory is mature and directly transferable; +"working set" and reuse-distance (Mattson stack distance) give principled context-window +sizing. **Why it is not selected now:** it needs a corpus of real agent traces, which I +do not have. Note the co-change relation in H1 is a *degenerate offline form* of this +(commits as access traces), which is the part that is testable today. + +--- + +## H8 — Obligation-level granularity: index contracts, return slices + +**Hypothesis.** The retrievable unit should be an *obligation* (signature + +pre/post-conditions + error modes), with full bodies returned only for atoms the task +must modify. Expected 3–10× token reduction on "how do I call X" tasks. + +**Mathematical intuition.** `tokens(signature) / tokens(body) ≈ 0.05–0.2`. If a +fraction `ρ` of retrieved atoms are needed only as *callees* (understand the contract) +rather than *editees*, the bill falls to `ρ·0.1 + (1−ρ)` of baseline. + +**Status.** Compatible with, and partially already implemented by, this repository's +`retrieval/skeleton.py` compactor. Folded into the architecture as a cost model rather +than claimed as novel. + +--- + +## H9 — Index the derivative, not the state + +**Hypothesis.** Make the indexed document a *change* (commit: subject + diff + +co-changed set), not a file. Retrieval then answers "what changed like this before", +returning both the precedent and the files it touched. + +**Why it may work.** Causal/intentional information ("why") lives in change records +and is absent from the tree. **Risk:** benchmark circularity — the ground truth here is +itself derived from commits, so indexing commits would let the system read the answer. +Only admissible under a strict temporal split; that discipline is adopted for the +co-change relation and this hypothesis is otherwise deferred. + +--- + +## H10 — Self-organising index (Hebbian re-weighting) + +**Hypothesis.** Relation weights adapt online: atoms co-retrieved in successful tasks +strengthen their link, `w ← w + η(1−w)`; unsuccessful, decay. The index reorganises +toward the workload. + +**Mathematical intuition.** Stochastic approximation on the completion model of H1 — +`P(c|s)` becomes an online-estimated parameter. Convergence under Robbins–Monro +conditions; the risk is a rich-get-richer collapse without exploration. + +**Status.** A natural extension of H1 once feedback exists. Not testable without a +live agent workload; the architecture leaves the parameter slot open for it. + +--- + +## H11 — Routing by obligation ownership + +**Hypothesis.** Route multi-agent work by *which obligations a task implicates*, not by +task-description similarity. Two agents whose implicated obligation sets intersect are +on a collision course and must share memory or serialise. + +**Mathematical intuition.** Predicted conflict = `|O(t₁) ∩ O(t₂)| > 0`. Assignment +becomes graph partitioning of the obligation hypergraph minimising cut (shared +obligations) — the classic distributed-systems objective, with obligations as the +shared state. + +**Status.** Designed into the architecture; measurable only in simulation here. + +--- + +## H12 — Negative results as first-class atoms + +**Hypothesis.** Store *refuted* hypotheses and failed attempts with their evidence +keys. A second agent asking a question whose evidence key matches a recorded failure +gets the refutation instead of repeating the search. + +**Why it may work.** Failed work is the largest silent cost in multi-agent systems and +is never recorded because it produces no artifact. Under H3's keying it is free: a +failure is just a conclusion with a negative polarity. + +--- + +## H13 — Speculative closure precomputation + +**Hypothesis.** Precompute and cache completion closures for the `O(k)` most likely +near-future tasks (derived from open diffs, recent commits, TODOs), so the closure is +warm when the task arrives. + +**Mathematical intuition.** Amortises closure cost; value = `hit_rate × +closure_latency`. Only worthwhile if closure latency is a bottleneck — measured below, +it is ~1 ms, so this is premature. **Rejected on measurement**, which is the correct +reason to reject something. + +--- + +## What the diagnostic already tells us + +Before any selection, `diagnose_structure.py` (run on 900 anchor→target gold pairs, +eight repositories, strict temporal split) reports: + +| relation | covers | uniquely | +|---|---|---| +| stem (shared name tokens) | 35.8% | — | +| static edge (import/call/ref) | 30.7% | 5.3% | +| co-change (history before the query commit) | 30.4% | 9.8% | +| same directory | 23.8% | — | +| test↔impl link | 14.4% | — | +| **union** | **71.8%** | | + +Two facts decide Stage 4. First, the union is far above any single relation, so the +completer must be **multi-relation**. Second, structure and history each contribute +*uniquely* (5.3% and 9.8%), so neither subsumes the other and a system with only one of +them leaves measurable recall on the table. H1 is therefore not merely plausible — its +central premise is already measured, on the same corpora the final benchmark uses. diff --git a/research/novelty.md b/research/novelty.md new file mode 100644 index 0000000..a377c9e --- /dev/null +++ b/research/novelty.md @@ -0,0 +1,98 @@ +# Stage 8 — Novelty assessment + +The brief forbids claiming "nobody has done this" without a real check. I ran one, and +the honest answer is uncomfortable: **neither of the two mechanisms I designed is +novel.** Both have close, recent prior art that I was unaware of when designing them. +What survives as a contribution is narrower and mostly empirical. + +Searches run 2026-09-09 across arXiv/Scholar-indexed literature and the code-retrieval +ecosystem. Phrasing throughout is "novel relative to the reviewed literature" or, where +prior art was found, no novelty claim at all. + +--- + +## The retrieval plane (H1 + H2) — **not novel** + +| Prior work | Overlap | +|---|---| +| **RepoHyper** (Search-Expand-Refine on a Repo-level Semantic Graph, arXiv 2403.06095) | This is essentially my architecture. "Search-then-Expand" = nucleate-then-accrete; a link predictor refines the expanded set = my scored relation union. Published 2024. | +| **CoCoMIC** | Method-level dependency graph for cross-file completion; the anchor-plus-dependency-context pattern. | +| **GRACE** (2509.05980), **RANGER** (2509.25257) | Graph-guided / graph-enhanced repository-level retrieval, same family. | +| **PACMS** (2606.20047) | "Recasts agent context assembly as budget-constrained submodular selection", facility-location coverage, and reports it beats both MMR-style diversification and pure relevance. This is H2, done first and done better. | +| **Context-Picker** (2512.14465) | "Shifts the paradigm from similarity-based ranking to minimal sufficient subset selection", mining minimal sufficient sets via leave-one-out. This is both my framing *and* my rejected H6 oracle. | +| **Zimmermann et al. ROSE** (ICSE'04), **Ying et al.** (TSE'04) | Co-change association-rule mining — the evolutionary relation, twenty years prior. | +| **Aider repo-map** | PageRank over the symbol graph for LLM context. | + +I independently rederived a design that the field converged on, and the literature is +ahead of it on the selection objective in particular. That is worth stating plainly +rather than hedging. + +## The memory plane (H3 + H4) — **not novel as a mechanism** + +| Prior work | Overlap | +|---|---| +| **Invalidation Contracts for Cross-Episode Agent Memory** (2609.00243) | Version stamps + cacheability hints attached to cached agent conclusions so stale entries can be evicted without trial and error. Same problem, same diagnosis. | +| **Fresh Memory, Stale Plans: Dependency-Scoped Validation for Distributed LLM-Agent Memory** (2609.03340) | PlanFence: plans cite the exact records they used; the executor validates only records that can affect the pending action. This is dependency-keyed validity for agent memory. | +| **From Agent Traces to Trust: Evidence Tracing and Execution Provenance** (2606.04990) | Provenance over retrieved passages, tool outputs, memory items. | +| **Bazel / Nix / ccache; Adapton, self-adjusting computation** | Content-addressed action caches with dependency-keyed invalidation — the actual ancestor, decades old. | + +The transplant of build-system content addressing to agent memory, which I framed as +the novel step, was published at least twice in 2026 before this work. + +--- + +## What is left, stated conservatively + +Novel relative to the reviewed literature, and all of it **empirical rather than +mechanistic**: + +1. **A quantified refutation of anchor-and-expand retrieval under a page-matched + baseline.** The graph-expansion literature (RepoHyper, GRACE, RANGER) reports gains + against a fixed-size retrieval baseline. I find that on commit-derived ground truth + across eight repositories, the gain is dominated by the trivial control of + *returning more results*, and that the mechanism is 5% **less** token-efficient than + the baseline it beats. I did not find this control reported in the reviewed papers. + It is cheap, and it should be standard. + +2. **Slot economics as a decision procedure.** Expressing an expansion mechanism's + value as `gain(slot) vs P(gold at the displaced rank)` — measured here as 0.031 + vs 0.035 — turns "should we add graph expansion" into arithmetic. This framing, and + the per-rank gold-density table it needs, I did not find in the reviewed literature. + +3. **Magnitude of query-keyed cache unsoundness on real code history**: 20.5% stale + after 1 commit, 47.9% after 10, 69.1% after 20. The invalidation-contract papers + argue the problem exists; I have not seen it *measured* against version history on + real repositories. + +4. **The granularity/retention measurement**: span-keyed evidence retains 1.36× more + valid reuse than file-keyed at a 10-commit horizon, growing to 1.45× at 20, with + the multiplier tracking file size versus commit size. Apparently novel as a + measurement; the underlying idea (finer dependencies invalidate less) is folklore. + +5. **42.0% cross-task evidence deduplication** on a 420-task workload over eight + repositories — a concrete ceiling for shared agent memory. + +6. **A reproducible negative-result protocol**: no-lookahead history enforced by the + data structure rather than by discipline, equivalence-to-incumbent asserted by test, + and leave-one-repository-out applied to a fit that then *failed* to generalise and + was reported anyway. + +## What I would have done differently knowing the prior art + +Read PACMS and Context-Picker before Stage 5. PACMS in particular reports the +submodular selection result I set out to obtain, which would have redirected the +effort toward the memory plane — where the measurements turned out to be worth having — +several hours earlier. + +## Sources + +- [RepoHyper: Search-Expand-Refine on Semantic Graphs](https://arxiv.org/abs/2403.06095) +- [PACMS: Submodular Context Selection as a Pluggable Engine for LLM Agents](https://arxiv.org/html/2606.20047) +- [Context-Picker: Dynamic Context Selection Using Multi-stage RL](https://arxiv.org/html/2512.14465) +- [GRACE: Graph-Guided Repository-Aware Code Completion](https://arxiv.org/pdf/2509.05980) +- [RANGER: Repository-level Agent for Graph-Enhanced Retrieval](https://arxiv.org/html/2509.25257) +- [Invalidation Contracts for Cross-Episode Agent Memory](https://arxiv.org/html/2609.00243) +- [Fresh Memory, Stale Plans: Dependency-Scoped Validation](https://arxiv.org/html/2609.03340) +- [From Agent Traces to Trust: Evidence Tracing and Execution Provenance](https://arxiv.org/html/2606.04990v1) +- [Enhancing Software Maintenance: Learning to Rank for Co-changed Method Identification](https://arxiv.org/pdf/2411.19099) +- [Retrieval-Augmented Code Generation: A Survey with Focus on Repository-Level Approaches](https://arxiv.org/html/2510.04905v1) diff --git a/research/nucleus/__init__.py b/research/nucleus/__init__.py new file mode 100644 index 0000000..1d4e9b7 --- /dev/null +++ b/research/nucleus/__init__.py @@ -0,0 +1,6 @@ +"""NUCLEUS research prototype: necessity-driven closure retrieval + evidence memory. + +Kept out of `src/codebase_index/` on purpose. This is a research spike measured +against the shipped pipeline; promoting any part of it into the product is a +separate decision that the numbers in `research/experiments.md` have to earn. +""" diff --git a/research/nucleus/baselines.py b/research/nucleus/baselines.py new file mode 100644 index 0000000..d2c0a8e --- /dev/null +++ b/research/nucleus/baselines.py @@ -0,0 +1,313 @@ +"""Comparison systems, all emitting the shipped `Candidate` shape. + +Every baseline goes through the same `apply_budget` with the same budget and the same +compactor, so `tokens` and `useful@budget` are measured symmetrically. A comparison +where one side pays for its context and the other does not is not a comparison. + + bm25 FTS5 Okapi BM25 over the same chunks. Pure lexical, no reranking. + dense LSA: tf-idf -> randomized truncated SVD -> cosine. Classical dense + retrieval. NOT a neural code encoder -- see the honesty note below. + rag bm25 + dense fused by RRF. The standard hybrid-RAG configuration. + hybrid the shipped 1.10.0 pipeline. The strongest incumbent available here. + graph the shipped pipeline with `graph_source=True` AND PPR expansion forced + on for every query. Forcing matters: commit-subject queries almost all + classify as KEYWORD intent, whose plan sets `graph_strategy="none"`, so + an unforced graph baseline would silently never activate and I would be + beating a system that was switched off. + +Honesty note on `dense` +----------------------- +No `sentence-transformers`, GPU, or network access exists in this environment, so the +dense baseline is Latent Semantic Analysis, not a modern neural encoder. LSA is a +legitimate dense retriever and the original one, but a code-tuned neural encoder would +very likely score higher. Therefore **no claim of the form "NUCLEUS beats embeddings" +is made anywhere in this work**. The load-bearing comparison is the paired one against +NUCLEUS's own anchor stage, which isolates accretion from retriever quality and would +hold on top of a better encoder too. +""" + +from __future__ import annotations + +import math +import re +import sqlite3 +from dataclasses import dataclass +from typing import Callable, Optional + +import numpy as np + +from codebase_index.retrieval.budget import apply_budget +from codebase_index.retrieval.diversity import deduplicate +from codebase_index.retrieval.fusion import fuse +from codebase_index.retrieval.intent import detect_intent +from codebase_index.retrieval.pipeline import _diversify, _run_retrievers +from codebase_index.retrieval.rerank import rerank +from codebase_index.retrieval.skeleton import make_compactor +from codebase_index.retrieval.tuning import DEFAULT_TUNING, RetrievalTuning +from codebase_index.retrieval.types import Candidate + +_TOK = re.compile(r"[A-Za-z][A-Za-z0-9_]*") + + +def _subtokens(word: str) -> list[str]: + parts = re.sub(r"([a-z0-9])([A-Z])", r"\1 \2", word).replace("_", " ").split() + return [p.lower() for p in parts if len(p) >= 3] + + +def tokenize(text: str, *, cap: int = 4000) -> list[str]: + out: list[str] = [] + for w in _TOK.findall(text or "")[:cap]: + lw = w.lower() + if len(lw) >= 3: + out.append(lw) + subs = _subtokens(w) + if len(subs) > 1: + out.extend(subs) + return out + + +def finalize( + candidates: list[Candidate], *, query: str, token_budget: int, limit: int, + pool_paths: Optional[list[str]] = None, +) -> dict: + """Shared tail: budget + payload, identical for every system under test. + + `pool_paths` is the *pre-truncation candidate pool*, which the oracle metrics are + measured against. It must mean the same thing for every system or `oracle` and + `cand_recall` become incomparable across the table; when a system does not + distinguish a pool from its ranking, its full untruncated candidate list is the + honest answer. + """ + plan = detect_intent(query) + ranked = candidates[:limit] + compactor = make_compactor(intent=plan.intent, query=query, enabled=True, + min_reduction=0.25) + results, recommended = apply_budget(ranked, token_budget=token_budget, + compactor=compactor) + if pool_paths is None: + seen: list[str] = [] + for c in candidates: + if c.path not in seen: + seen.append(c.path) + pool_paths = seen + return { + "query": query, "intent": plan.intent.value, "results": results, + "recommended_reads": recommended, "confidence": "medium", + "diagnostics": {"pool": [{"path": p} for p in pool_paths]}, + } + + +# --- BM25 -------------------------------------------------------------------- + + +def _match_query(query: str) -> str: + terms = {t.lower() for t in _TOK.findall(query) if len(t) >= 3} + if not terms: + return "" + return " OR ".join(f'"{t}"' for t in sorted(terms)) + + +def bm25_candidates(conn: sqlite3.Connection, query: str, *, limit: int) -> list[Candidate]: + match = _match_query(query) + if not match: + return [] + try: + rows = conn.execute( + """ + SELECT f.path, c.line_start, c.line_end, c.content, c.token_est, + bm25(fts_chunks) AS s + FROM fts_chunks + JOIN chunks c ON c.id = fts_chunks.rowid + JOIN files f ON f.id = c.file_id + WHERE fts_chunks MATCH ? + ORDER BY s + LIMIT ? + """, + (match, limit * 8), + ).fetchall() + except sqlite3.Error: + return [] + out = [] + for path, ls, le, content, tok, s in rows: + out.append(Candidate( + path=path.replace("\\", "/"), line_start=ls, line_end=le, source="fts", + score=-float(s), content=content, token_est=int(tok or 0), reason="bm25", + )) + return out + + +# --- dense (LSA) ------------------------------------------------------------- + + +@dataclass +class LSAIndex: + """tf-idf -> randomized truncated SVD. numpy only, no scipy/sklearn available.""" + + vocab: dict[str, int] + idf: np.ndarray + doc_emb: np.ndarray # (n_docs, k), L2-normalised + V: np.ndarray # (n_terms, k) + meta: list[tuple[str, int, int, str, int]] + + @classmethod + def build(cls, conn: sqlite3.Connection, *, k: int = 160, seed: int = 20260909) -> "LSAIndex": + rows = conn.execute( + """ + SELECT f.path, c.line_start, c.line_end, c.content, c.token_est + FROM chunks c JOIN files f ON f.id = c.file_id + ORDER BY c.id + """ + ).fetchall() + meta = [(r[0].replace("\\", "/"), int(r[1]), int(r[2]), r[3] or "", int(r[4] or 0)) + for r in rows] + n = len(meta) + if n == 0: + return cls({}, np.zeros(0), np.zeros((0, k)), np.zeros((0, k)), meta) + + df: dict[str, int] = {} + per_doc: list[dict[str, int]] = [] + for _, _, _, content, _ in meta: + tf: dict[str, int] = {} + for t in tokenize(content): + tf[t] = tf.get(t, 0) + 1 + per_doc.append(tf) + for t in tf: + df[t] = df.get(t, 0) + 1 + + # min_df 2 removes hapax noise; the upper bound removes terms so common they + # carry no discrimination (language keywords, license headers). + hi = max(3, int(0.30 * n)) + vocab = {t: i for i, t in enumerate( + sorted(t for t, d in df.items() if 2 <= d <= hi) + )} + if not vocab: + return cls({}, np.zeros(0), np.zeros((n, k)), np.zeros((0, k)), meta) + idf = np.zeros(len(vocab), dtype=np.float32) + for t, i in vocab.items(): + idf[i] = math.log(1.0 + n / df[t]) + + rows_i: list[int] = [] + cols_i: list[int] = [] + vals_f: list[float] = [] + for d, tf in enumerate(per_doc): + acc = [] + for t, c in tf.items(): + j = vocab.get(t) + if j is not None: + acc.append((j, (1.0 + math.log(c)) * idf[j])) + if not acc: + continue + norm = math.sqrt(sum(v * v for _, v in acc)) or 1.0 + for j, v in acc: + rows_i.append(d) + cols_i.append(j) + vals_f.append(v / norm) + R = np.asarray(rows_i, dtype=np.int32) + C = np.asarray(cols_i, dtype=np.int32) + Vl = np.asarray(vals_f, dtype=np.float32) + m = len(vocab) + k = min(k, max(2, min(n, m) - 1)) + + rng = np.random.default_rng(seed) + + def A_dot(X: np.ndarray) -> np.ndarray: # (m,k) -> (n,k) + out = np.zeros((n, X.shape[1]), dtype=np.float32) + for s in range(0, len(R), 200_000): + e = s + 200_000 + np.add.at(out, R[s:e], Vl[s:e, None] * X[C[s:e]]) + return out + + def AT_dot(X: np.ndarray) -> np.ndarray: # (n,k) -> (m,k) + out = np.zeros((m, X.shape[1]), dtype=np.float32) + for s in range(0, len(R), 200_000): + e = s + 200_000 + np.add.at(out, C[s:e], Vl[s:e, None] * X[R[s:e]]) + return out + + Omega = rng.standard_normal((m, k)).astype(np.float32) + Y = A_dot(Omega) + Y = A_dot(AT_dot(Y)) # one power iteration + Q, _ = np.linalg.qr(Y) + BT = AT_dot(Q) # (m,k) == B^T + Ub, Sb, Vbt = np.linalg.svd(BT, full_matrices=False) + V = Ub # (m,k) right singular vecs of A + doc = A_dot(V) + norms = np.linalg.norm(doc, axis=1, keepdims=True) + doc = doc / np.maximum(norms, 1e-8) + return cls(vocab, idf, doc.astype(np.float32), V.astype(np.float32), meta) + + def query(self, text: str, *, limit: int) -> list[Candidate]: + if not self.vocab or self.doc_emb.size == 0: + return [] + tf: dict[str, int] = {} + for t in tokenize(text): + tf[t] = tf.get(t, 0) + 1 + vec = np.zeros(len(self.vocab), dtype=np.float32) + for t, c in tf.items(): + j = self.vocab.get(t) + if j is not None: + vec[j] = (1.0 + math.log(c)) * self.idf[j] + nrm = float(np.linalg.norm(vec)) + if nrm <= 0: + return [] + q = (vec / nrm) @ self.V + qn = float(np.linalg.norm(q)) + if qn <= 0: + return [] + sims = self.doc_emb @ (q / qn) + take = min(limit * 8, sims.shape[0]) + idx = np.argpartition(-sims, take - 1)[:take] + idx = idx[np.argsort(-sims[idx])] + out = [] + for i in idx: + path, ls, le, content, tok = self.meta[int(i)] + out.append(Candidate( + path=path, line_start=ls, line_end=le, source="vector", + score=float(sims[int(i)]), content=content, token_est=tok, + reason="lsa cosine", + )) + return out + + +# --- shipped pipeline variants ---------------------------------------------- + + +def product_candidates( + conn: sqlite3.Connection, query: str, *, limit: int, tuning: RetrievalTuning, + force_graph: bool = False, +) -> tuple[list[Candidate], list[str]]: + """Returns (ranked candidates, pre-rerank pool paths).""" + plan = detect_intent(query) + pool_mult = max(1, tuning.candidate_pool_multiplier) + pool_limit = limit if pool_mult == 1 else max(limit * pool_mult, 20) + lists, weights = _run_retrievers( + conn, query, mode="hybrid", limit=pool_limit, weights=plan.weights, + backend=None, tuning=tuning, graph_depth=tuning.graph_depth, + graph_node_cap=tuning.graph_node_cap, + graph_strategy=( + ("both" if force_graph else plan.graph_strategy) + if tuning.graph_source else "none" + ), + ) + fused = fuse(lists, weights=weights, k=tuning.rrf_k, + file_agreement=tuning.file_agreement_weight if tuning.file_agreement else 0.0) + pool_paths: list[str] = [] + for c in fused: + p = c.path.replace("\\", "/") + if p not in pool_paths: + pool_paths.append(p) + ranked = rerank(fused, query=query, intent=plan.intent, tuning=tuning) + if tuning.dedup: + ranked = deduplicate(ranked, hamming_distance=tuning.dedup_hamming) + ranked = _diversify(ranked, per_file=tuning.max_per_file) + return ranked, pool_paths + + +def rrf_hybrid(a: list[Candidate], b: list[Candidate], *, k: int = 60) -> list[Candidate]: + return fuse({"fts": a, "vector": b}, weights={"fts": 1.0, "vector": 1.0}, k=k, + file_agreement=0.0) + + +# --- graph PPR baseline uses the shipped pipeline with graph_source on ------- + +GRAPH_TUNING = RetrievalTuning(graph_source=True) diff --git a/research/nucleus/evalrun.py b/research/nucleus/evalrun.py new file mode 100644 index 0000000..1075f2e --- /dev/null +++ b/research/nucleus/evalrun.py @@ -0,0 +1,382 @@ +"""Benchmark runner: many systems, one query sweep, paired significance. + +Design constraints inherited from `tests/eval/harness.py`, deliberately, so that the +numbers here are comparable with the ones the product publishes: + + * one index per corpus, shared by every variant -- deltas measure retrieval, not + indexing variance; + * metrics come from `tests.eval.metrics` verbatim, never reimplemented; + * every variant sees the identical query in the identical order, so per-query + vectors stay aligned and the paired bootstrap / permutation tests are valid. + +One constraint is new here. The co-change relation must never see the query's own +commit, so queries are swept **oldest-first** and the history model is advanced +monotonically. Running all variants inside one sweep is therefore not just faster -- +it is what guarantees every variant sees exactly the same revealed history. +""" + +from __future__ import annotations + +import argparse +import json +import statistics +import sqlite3 +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable, Optional + +import yaml + +from codebase_index.retrieval.diversity import ( + normalize_code_tokens, simhash_distance, token_fingerprint, +) +from codebase_index.retrieval.tuning import RetrievalTuning +from tests.eval import metrics +from tests.eval.harness import EvalReport, format_significance, format_table, pool + +from . import baselines +from .relations import CoChangeModel, RelationGraph, RelationWeights, load_static_edges +from .search import NucleusParams, ObligationIndex, nucleus_search + + +@dataclass(frozen=True) +class Query: + query: str + category: str + expected_files: tuple[str, ...] + commit: str + position: int # git log position; larger == older + + +@dataclass +class Ctx: + """Everything a variant may need for one corpus.""" + conn: sqlite3.Connection + graph: RelationGraph + obligations: ObligationIndex + lsa: Optional[baselines.LSAIndex] = None + + +Variant = Callable[[Ctx, str], dict] + + +@dataclass +class Outcome: + ranked_files: list[str] + returned: list[tuple[str, int]] + latency_ms: float + total_tokens: int + duplicates: int + n_results: int + pool_files: list[str] + + +def _norm(p: str) -> str: + return p.replace("\\", "/") + + +def load_queries(path: Path, repo: Path) -> list[Query]: + import subprocess + + raw = subprocess.run(["git", "-C", str(repo), "log", "--format=%H"], + capture_output=True, text=True, encoding="utf-8", + errors="replace") + order = {s.strip()[:12]: i for i, s in enumerate(raw.stdout.splitlines()) if s.strip()} + out: list[Query] = [] + for e in yaml.safe_load(path.read_text(encoding="utf-8")) or []: + commit = e.get("commit", "") + pos = order.get(commit) + if pos is None: + continue + out.append(Query( + query=e["query"], category=e.get("category", "uncategorized"), + expected_files=tuple(_norm(f) for f in e.get("expected_files", ())), + commit=commit, position=pos, + )) + # Oldest first: the co-change model is append-only and must never see the future. + out.sort(key=lambda q: -q.position) + return out + + +def outcome_from_payload(payload: dict, latency_ms: float) -> Outcome: + """Mirrors tests/eval/harness.run_query's accounting exactly. + + Only snippets are billed (results past the budget carry a token_est for a chunk + the agent never receives); file-level ranking collapses several hits in one file + to its best rank; duplicates are measured with the pipeline's own fingerprint. + """ + pool_files: list[str] = [] + for entry in payload.get("diagnostics", {}).get("pool", ()): + p = _norm(entry["path"]) + if p not in pool_files: + pool_files.append(p) + + ranked_files: list[str] = [] + returned: list[tuple[str, int]] = [] + total_tokens = 0 + duplicates = 0 + fingerprints: list[int] = [] + results = payload.get("results", []) + for r in results: + p = _norm(r["path"]) + if p not in ranked_files: + ranked_files.append(p) + tokens = int(r.get("token_est") or 0) + returned.append((p, tokens)) + snippet = r.get("snippet") + if snippet: + total_tokens += tokens + chunk_tokens = normalize_code_tokens(snippet or "") + if chunk_tokens: + fp = token_fingerprint(chunk_tokens) + if any(simhash_distance(fp, seen) <= 3 for seen in fingerprints): + duplicates += 1 + fingerprints.append(fp) + return Outcome(ranked_files, returned, latency_ms, total_tokens, duplicates, + len(results), pool_files) + + +def aggregate(label: str, queries: list[Query], outcomes: list[Outcome], + *, token_budget: int) -> EvalReport: + scorers = { + "recall@5": lambda q, o: metrics.recall_at_k(o.ranked_files, q.expected_files, 5), + "recall@10": lambda q, o: metrics.recall_at_k(o.ranked_files, q.expected_files, 10), + "recall@15": lambda q, o: metrics.recall_at_k(o.ranked_files, q.expected_files, 15), + "MRR": lambda q, o: metrics.reciprocal_rank(o.ranked_files, q.expected_files), + "nDCG@10": lambda q, o: metrics.ndcg_at_k(o.ranked_files, q.expected_files, 10), + "hit@3": lambda q, o: metrics.hit_rate_at_k(o.ranked_files, q.expected_files, 3), + "P@5": lambda q, o: metrics.precision_at_k(o.ranked_files, q.expected_files, 5), + "MAP": lambda q, o: metrics.average_precision(o.ranked_files, q.expected_files), + "useful@budget": lambda q, o: metrics.useful_context_at_budget( + o.returned, q.expected_files, token_budget), + # Token-normalised recall at three budgets. This is the agent-centric view: + # not "is the answer ranked somewhere" but "is it inside the context the + # agent can afford", which is the only axis on which a system returning a + # bigger page can be compared fairly with one returning a smaller page. + "useful@500": lambda q, o: metrics.useful_context_at_budget( + o.returned, q.expected_files, 500), + "useful@1000": lambda q, o: metrics.useful_context_at_budget( + o.returned, q.expected_files, 1000), + "oracle": lambda q, o: metrics.oracle_reciprocal_rank(o.pool_files, q.expected_files), + "cand_recall": lambda q, o: metrics.recall_at_k( + o.pool_files, q.expected_files, len(o.pool_files)), + } + per_query = {n: [fn(q, o) for q, o in zip(queries, outcomes)] for n, fn in scorers.items()} + + def mean(n: str) -> float: + v = per_query[n] + return statistics.fmean(v) if v else 0.0 + + per_cat: dict[str, list[float]] = {} + for q, rr in zip(queries, per_query["MRR"]): + per_cat.setdefault(q.category, []).append(rr) + lat = [o.latency_ms for o in outcomes] + returned_total = sum(o.n_results for o in outcomes) + dup_total = sum(o.duplicates for o in outcomes) + return EvalReport( + label=label, n_queries=len(outcomes), + recall_at_5=mean("recall@5"), recall_at_10=mean("recall@10"), mrr=mean("MRR"), + ndcg_at_10=mean("nDCG@10"), hit_rate_at_3=mean("hit@3"), + precision_at_5=mean("P@5"), map_score=mean("MAP"), + useful_context=mean("useful@budget"), + p50_ms=metrics.percentile(lat, 50), p95_ms=metrics.percentile(lat, 95), + p99_ms=metrics.percentile(lat, 99), + mean_ms=statistics.fmean(lat) if lat else 0.0, + mean_tokens=statistics.fmean([o.total_tokens for o in outcomes]) if outcomes else 0.0, + duplicate_rate=(dup_total / returned_total) if returned_total else 0.0, + mean_candidates=(returned_total / len(outcomes)) if outcomes else 0.0, + oracle_mrr=mean("oracle"), candidate_recall=mean("cand_recall"), + mean_pool=statistics.fmean([len(o.pool_files) for o in outcomes]) if outcomes else 0.0, + per_category={c: statistics.fmean(v) for c, v in sorted(per_cat.items())}, + per_query=per_query, + ) + + +# --- variants ---------------------------------------------------------------- + + +LIMIT = 10 +BUDGET = 1500 + + +def make_variants(names: set[str], nucleus_params: dict[str, NucleusParams]) -> dict[str, Variant]: + v: dict[str, Variant] = {} + if "bm25" in names: + v["bm25"] = lambda ctx, q: baselines.finalize( + baselines.bm25_candidates(ctx.conn, q, limit=LIMIT), + query=q, token_budget=BUDGET, limit=LIMIT) + if "dense" in names: + v["dense"] = lambda ctx, q: baselines.finalize( + ctx.lsa.query(q, limit=LIMIT) if ctx.lsa else [], + query=q, token_budget=BUDGET, limit=LIMIT) + if "rag" in names: + v["rag"] = lambda ctx, q: baselines.finalize( + baselines.rrf_hybrid( + baselines.bm25_candidates(ctx.conn, q, limit=LIMIT), + ctx.lsa.query(q, limit=LIMIT) if ctx.lsa else []), + query=q, token_budget=BUDGET, limit=LIMIT) + if "hybrid13" in names: + def _hybrid13(ctx: Ctx, q: str) -> dict: + # The incumbent, allowed exactly the page growth NUCLEUS takes. Without + # this row a "bigger page wins" result would be indistinguishable from a + # "better page wins" result. + cands, pool_paths = baselines.product_candidates( + ctx.conn, q, limit=13, tuning=RetrievalTuning()) + return baselines.finalize(cands, query=q, token_budget=BUDGET, + limit=13, pool_paths=pool_paths) + v["hybrid13"] = _hybrid13 + if "hybrid" in names: + def _hybrid(ctx: Ctx, q: str) -> dict: + cands, pool_paths = baselines.product_candidates( + ctx.conn, q, limit=LIMIT, tuning=RetrievalTuning()) + return baselines.finalize(cands, query=q, token_budget=BUDGET, + limit=LIMIT, pool_paths=pool_paths) + v["hybrid"] = _hybrid + if "graph" in names: + def _graph(ctx: Ctx, q: str) -> dict: + cands, pool_paths = baselines.product_candidates( + ctx.conn, q, limit=LIMIT, tuning=baselines.GRAPH_TUNING, + force_graph=True) + return baselines.finalize(cands, query=q, token_budget=BUDGET, + limit=LIMIT, pool_paths=pool_paths) + v["graph"] = _graph + for label, params in nucleus_params.items(): + def mk(p: NucleusParams, lim: int) -> Variant: + return lambda ctx, q: nucleus_search( + ctx.conn, q, limit=lim, token_budget=BUDGET, graph=ctx.graph, + obligations=ctx.obligations, params=p, explain=True) + # `nucleus13` asks the sharpest question available: does accretion add + # anything ON TOP OF simply returning more results, which is the cheap + # alternative that `hybrid13` shows is already effective? + v[label] = mk(params, 13 if label.endswith("13") else LIMIT) + return v + + +def run_corpus( + name: str, repo: Path, queries_path: Path, index_path: Path, + variants_wanted: set[str], nucleus_params: dict[str, NucleusParams], + *, need_dense: bool, +) -> tuple[list[Query], dict[str, list[Outcome]]]: + queries = load_queries(queries_path, repo) + conn = sqlite3.connect(index_path) + conn.row_factory = sqlite3.Row + files = [r[0].replace("\\", "/") for r in conn.execute("SELECT path FROM files")] + cochange = CoChangeModel.from_repo(repo) + graph = RelationGraph(files, static=load_static_edges(index_path), cochange=cochange) + obligations = ObligationIndex(conn) + lsa = baselines.LSAIndex.build(conn) if need_dense else None + ctx = Ctx(conn=conn, graph=graph, obligations=obligations, lsa=lsa) + + variants = make_variants(variants_wanted, nucleus_params) + out: dict[str, list[Outcome]] = {k: [] for k in variants} + for q in queries: + cochange.advance_to(q.position) # reveal only strictly-older commits + for label, fn in variants.items(): + t0 = time.perf_counter() + payload = fn(ctx, q.query) + dt = (time.perf_counter() - t0) * 1000.0 + out[label].append(outcome_from_payload(payload, dt)) + conn.close() + return queries, out + + +CORPORA = [ + ("codebase-index", "."), + ("Civitas", "../Civitas"), + ("PoliternalSite", "../PoliternalSite"), + ("PoliternalParkour", "../PoliternalParkour"), + ("TerraForge", "../TerraForge"), + ("denfry.github.io", "../denfry.github.io"), + ("DevGraph", "../DevGraph"), + ("WinCleaner", "../Windows-Cleaner-and-Optimizer-main"), +] + +DATA = Path(__file__).parent.parent / "data" + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--systems", default="hybrid,nucleus", + help="comma list of bm25,dense,rag,hybrid,graph,nucleus") + ap.add_argument("--ablate", action="store_true") + ap.add_argument("--baseline-label", default="hybrid") + ap.add_argument("--corpus", action="append", default=None) + ap.add_argument("--json-out", default=None) + ap.add_argument("--per-corpus", action="store_true") + args = ap.parse_args() + + wanted = {s.strip() for s in args.systems.split(",") if s.strip()} + nucleus_params: dict[str, NucleusParams] = {} + if "nucleus" in wanted: + nucleus_params["nucleus"] = NucleusParams() + if "nucleus13" in wanted: + nucleus_params["nucleus13"] = NucleusParams() + if args.ablate: + base = NucleusParams() + nucleus_params.update({ + "-accretion13": NucleusParams(accrete=False), + "-cochange13": NucleusParams(weights=RelationWeights(cochange=0.0)), + "-edges13": NucleusParams(weights=RelationWeights(edge=0.0)), + "-testlink13": NucleusParams(weights=RelationWeights(testlink=0.0)), + "-stem13": NucleusParams(weights=RelationWeights(stem=0.0)), + "-dir13": NucleusParams(weights=RelationWeights(dirw=0.0)), + "-coverage_sel13": NucleusParams(coverage_select=False), + "-hub_penalty13": NucleusParams(hub_penalty=False), + "-compact13": NucleusParams(compact_completions=False), + "combine=max13": NucleusParams(combine="max"), + "gate=0.0__13": NucleusParams(min_score=0.0), + "slots=1__13": NucleusParams(max_completions=1), + }) + need_dense = bool({"dense", "rag"} & wanted) + + corpora = CORPORA + if args.corpus: + keep = set(args.corpus) + corpora = [c for c in CORPORA if c[0] in keep] + + all_reports: dict[str, list[EvalReport]] = {} + for name, repo in corpora: + qs = DATA / f"{name}.yml" + idx = DATA / "index" / f"{name}.sqlite" + if not qs.exists() or not idx.exists(): + print(f"[{name}] skipped (missing data)") + continue + t0 = time.perf_counter() + queries, outcomes = run_corpus( + name, Path(repo), qs, idx, wanted, nucleus_params, need_dense=need_dense) + for label, oc in outcomes.items(): + all_reports.setdefault(label, []).append( + aggregate(f"{label}", queries, oc, token_budget=BUDGET)) + print(f"[{name}] {len(queries)} queries, {len(outcomes)} systems, " + f"{time.perf_counter()-t0:.1f}s") + if args.per_corpus: + reps = [aggregate(label, queries, oc, token_budget=BUDGET) + for label, oc in outcomes.items()] + base = next((r for r in reps if r.label == args.baseline_label), None) + print(format_table(reps, baseline=base)) + print() + + pooled = {label: pool(reps, label=label) for label, reps in all_reports.items()} + order = [l for l in ("bm25", "dense", "rag", "graph", "hybrid", "hybrid13") + if l in pooled] + order += [l for l in pooled if l not in order] + reports = [pooled[l] for l in order] + base = pooled.get(args.baseline_label) + print("\n=== POOLED ===") + print(format_table(reports, baseline=base)) + if base is not None: + for label in order: + if label == args.baseline_label: + continue + print() + print(format_significance(base, pooled[label])) + + if args.json_out: + Path(args.json_out).write_text(json.dumps( + {l: pooled[l].as_row() for l in order}, indent=2), encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/research/nucleus/memory.py b/research/nucleus/memory.py new file mode 100644 index 0000000..abb3cd4 --- /dev/null +++ b/research/nucleus/memory.py @@ -0,0 +1,161 @@ +"""Evidence-keyed memory: sound reuse of agent work, and the granularity law. + +Hypotheses H3/H4. Unlike the retrieval plane, the claim here is about a *correctness* +property, so the experiment is not "does quality go up" but "how often does the +incumbent design return an answer that is silently wrong, and what does soundness +cost". + +The three cache designs compared: + + query-keyed the semantic cache. Key = the question. Reuse when the question + repeats (or is similar). Carries no information about whether the + code the answer was derived from still exists in that form. + + file-keyed evidence-keyed at file granularity. Key = content hashes of every + file the conclusion read. Sound: any edit to any of those files + invalidates. This is what a straightforward implementation does. + + span-keyed evidence-keyed at the granularity actually read -- the retrieved + line spans. Equally sound, but invalidated only by edits that + overlap the specific regions the conclusion depended on. + +Survival is measured retrospectively against real history: for a conclusion whose +evidence is a set of spans at HEAD, was that evidence disturbed over the last `h` +commits? Because `git diff HEAD~h HEAD` reports hunks on the new side in HEAD's +coordinates, the overlap test is exact -- no re-indexing of historical trees and no +line-number mapping heuristics. + +The semantic cache's unsound-hit rate is then exactly `1 - survival`: it would have +served every one of those conclusions, and each one whose evidence moved was stale. +""" + +from __future__ import annotations + +import hashlib +import re +import subprocess +from dataclasses import dataclass, field +from pathlib import Path + +_HUNK = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@") + + +@dataclass(frozen=True) +class Span: + path: str + start: int + end: int + tokens: int = 0 + + +@dataclass +class Conclusion: + """One unit of agent work, with the evidence it consumed recorded at write time.""" + + task: str + spans: tuple[Span, ...] + tokens: int = 0 + + @property + def files(self) -> frozenset[str]: + return frozenset(s.path for s in self.spans) + + def key(self, content_hash: dict[str, str], *, granularity: str) -> str: + """Content-addressed key. This is the whole mechanism. + + A conclusion is identified by what it *depended on*, not by what it was + asked. Two agents that read the same evidence share a key and therefore + share the work; an edit to that evidence changes the key, so the stale entry + is not evicted by a heuristic -- it is simply never looked up again. + """ + if granularity == "file": + parts = sorted(f"{p}:{content_hash.get(p, '')}" for p in self.files) + else: + parts = sorted( + f"{s.path}:{s.start}-{s.end}:{content_hash.get(s.path, '')}" + for s in self.spans + ) + return hashlib.sha256("\x00".join(parts).encode()).hexdigest()[:16] + + +def _git(repo: Path, *args: str) -> str: + p = subprocess.run(["git", "-C", str(repo), *args], capture_output=True, + text=True, encoding="utf-8", errors="replace") + if p.returncode != 0: + raise RuntimeError(f"git {' '.join(args)}: {p.stderr.strip()}") + return p.stdout + + +def changed_regions(repo: Path, horizon: int) -> tuple[set[str], dict[str, list[tuple[int, int]]]]: + """Files and new-side line ranges touched by the last `horizon` commits. + + Returns (changed_files, {path: [(start, end), ...]}) with ranges in HEAD + coordinates, which is what makes the overlap test against HEAD spans exact. + """ + try: + diff = _git(repo, "diff", "--unified=0", f"HEAD~{horizon}", "HEAD") + except RuntimeError: + return set(), {} + files: set[str] = set() + hunks: dict[str, list[tuple[int, int]]] = {} + current: str | None = None + for line in diff.splitlines(): + if line.startswith("+++ b/"): + current = line[6:].strip() + if current == "/dev/null": + current = None + else: + files.add(current) + hunks.setdefault(current, []) + elif line.startswith("@@") and current: + m = _HUNK.match(line) + if m: + start = int(m.group(1)) + length = int(m.group(2) or 1) + if length == 0: # pure deletion: mark the seam + hunks[current].append((start, start + 1)) + else: + hunks[current].append((start, start + length - 1)) + return files, hunks + + +def survives( + c: Conclusion, changed_files: set[str], hunks: dict[str, list[tuple[int, int]]], + *, granularity: str, +) -> bool: + if granularity == "file": + return not (c.files & changed_files) + for s in c.spans: + if s.path not in changed_files: + continue + for lo, hi in hunks.get(s.path, ()): + if s.start <= hi and lo <= s.end: + return False + # A file listed as changed with no parsable hunks (rename, mode change, + # binary) is treated as disturbing every span in it. Conservative in the + # direction that costs the mechanism hit-rate rather than soundness. + if not hunks.get(s.path): + return False + return True + + +@dataclass +class MemoStore: + """Content-addressed store of completed work, with exact invalidation.""" + + granularity: str = "span" + entries: dict[str, Conclusion] = field(default_factory=dict) + hits: int = 0 + misses: int = 0 + tokens_saved: int = 0 + + def get_or_compute(self, c: Conclusion, content_hash: dict[str, str]) -> bool: + """True on reuse. The caller does the work only on False.""" + k = c.key(content_hash, granularity=self.granularity) + if k in self.entries: + self.hits += 1 + self.tokens_saved += c.tokens + return True + self.entries[k] = c + self.misses += 1 + return False diff --git a/research/nucleus/relations.py b/research/nucleus/relations.py new file mode 100644 index 0000000..3aeda19 --- /dev/null +++ b/research/nucleus/relations.py @@ -0,0 +1,355 @@ +"""Typed relation union: the completion model P(c | anchor). + +The architecture's central bet is that the members of a required set are related to +*each other* by something computable, so that a high-precision anchor found by lexical +retrieval can be completed into a set. This module estimates those relations. + +Five relations, chosen because `research/diagnose_structure.py` measured each of them +carrying signal on 900 real anchor->target gold pairs, and because static structure and +history were shown to contribute *uniquely* (5.3% and 9.8% of pairs respectively) — +neither subsumes the other, so a single-relation completer leaves recall on the table. + +Temporal honesty +---------------- +`CoChange` is the one relation that can leak. Ground truth here is mined from commits, +so a co-change model that has seen the query's own commit would be reading the answer. +`CoChangeModel.advance_to()` therefore exposes counts from commits **strictly older** +than a given commit, and the benchmark sweeps queries oldest-first so the observable +history only ever grows. There is no code path that can see a commit at or after the +query's own; the filter is a property of the data structure, not a discipline the +caller has to remember. +""" + +from __future__ import annotations + +import math +import re +import sqlite3 +import subprocess +from collections import defaultdict +from dataclasses import dataclass, field +from pathlib import Path + +_WORD = re.compile(r"[A-Za-z][A-Za-z0-9]*") +_TEST_MARK = re.compile(r"(?:^|[._-])(?:test|tests|spec|specs)(?:$|[._-])", re.I) + +# A commit touching more than this many files couples everything to everything and is +# evidence about a release process, not about a design relation. +MAX_COMMIT_FILES = 40 + +# Additive smoothing on co-change confidence. Without it a file that appears in one +# prior commit alongside one other file scores confidence 1.0 on a single observation. +COCHANGE_PRIOR = 2.0 + + +def _norm(p: str) -> str: + return p.replace("\\", "/") + + +def stem_tokens(path: str) -> set[str]: + base = _norm(path).rsplit("/", 1)[-1].split(".", 1)[0] + parts = re.sub(r"([a-z0-9])([A-Z])", r"\1 \2", base).replace("_", " ").replace("-", " ") + return {w.lower() for w in _WORD.findall(parts) if len(w) > 2} + + +def core_stem(path: str) -> str: + base = _norm(path).rsplit("/", 1)[-1].split(".", 1)[0] + base = re.sub(r"^(?:test|spec)[._-]?", "", base, flags=re.I) + base = re.sub(r"[._-]?(?:test|tests|spec|specs)$", "", base, flags=re.I) + return base.lower() + + +def is_testish(path: str) -> bool: + p = _norm(path) + return bool(_TEST_MARK.search(p)) or "/test" in p.lower() + + +def directory(path: str) -> str: + p = _norm(path) + return p.rsplit("/", 1)[0] if "/" in p else "" + + +# --- static structure ------------------------------------------------------- + + +def load_static_edges(index_path: Path) -> dict[str, dict[str, float]]: + """file -> {file: weight} from resolved import/call/reference edges. + + Symbol targets are lifted to their defining file because the unit of the ground + truth, and of the agent's decision, is a file. Edge confidence is carried through + as the weight: an `inferred` import-path match is weaker evidence than an + `extracted` unique-symbol match, and the completer should know the difference. + """ + conf_w = {"extracted": 1.0, "inferred": 0.75, "ambiguous": 0.35} + adj: dict[str, dict[str, float]] = defaultdict(dict) + if not index_path.exists(): + return adj + conn = sqlite3.connect(index_path) + try: + rows = conn.execute( + """ + SELECT src.path AS a, + CASE WHEN e.dst_kind = 'file' THEN df.path ELSE sf.path END AS b, + e.confidence + FROM edges AS e + JOIN files AS src ON src.id = e.file_id + LEFT JOIN files AS df ON e.dst_kind = 'file' AND df.id = e.dst_id + LEFT JOIN symbols AS s ON e.dst_kind = 'symbol' AND s.id = e.dst_id + LEFT JOIN files AS sf ON sf.id = s.file_id + WHERE e.resolved = 1 + """ + ).fetchall() + except sqlite3.Error: + return adj + finally: + conn.close() + for a, b, conf in rows: + if not a or not b or a == b: + continue + a, b = _norm(a), _norm(b) + w = conf_w.get((conf or "extracted").lower(), 0.5) + # Undirected: "must I also look at this" is symmetric even though the edge + # is not. Direction is kept out of the weight because the diagnostic found + # no consistent asymmetry at file granularity. + adj[a][b] = max(adj[a].get(b, 0.0), w) + adj[b][a] = max(adj[b].get(a, 0.0), w) + return adj + + +# --- history ---------------------------------------------------------------- + + +def _git(repo: Path, *args: str) -> str: + p = subprocess.run( + ["git", "-C", str(repo), *args], + capture_output=True, text=True, encoding="utf-8", errors="replace", + ) + if p.returncode != 0: + raise RuntimeError(f"git {' '.join(args)}: {p.stderr.strip()}") + return p.stdout + + +@dataclass +class CoChangeModel: + """Incrementally-revealed co-change counts with a hard no-lookahead guarantee. + + Commits are held oldest-last (position 0 = newest, matching `git log` order). + `advance_to(pos)` folds in every commit strictly older than `pos`. Because the + benchmark visits queries oldest-first, `pos` decreases monotonically and the fold + is a single linear sweep over history per corpus rather than a rescan per query. + """ + + commits: list[tuple[int, list[str]]] = field(default_factory=list) # (pos, files) + _pair: dict[tuple[str, str], int] = field(default_factory=lambda: defaultdict(int)) + _single: dict[str, int] = field(default_factory=lambda: defaultdict(int)) + _folded: int = 0 # index into self.commits (which is sorted oldest-first) + _current_pos: int | None = None + + @classmethod + def from_repo(cls, repo: Path) -> "CoChangeModel": + order: dict[str, int] = {} + for i, sha in enumerate( + s.strip() for s in _git(repo, "log", "--format=%H").splitlines() if s.strip() + ): + order[sha[:12]] = i + raw = _git(repo, "log", "--no-merges", "--name-only", + "--pretty=format:%x01%H", "--diff-filter=ACMR") + commits: list[tuple[int, list[str]]] = [] + for rec in raw.split("\x01"): + rec = rec.strip("\n") + if not rec: + continue + head, _, body = rec.partition("\n") + pos = order.get(head.strip()[:12]) + if pos is None: + continue + files = [_norm(ln.strip()) for ln in body.splitlines() if ln.strip()] + if 0 < len(files) <= MAX_COMMIT_FILES: + commits.append((pos, files)) + # Oldest first, so advancing toward newer queries is an append-only fold. + commits.sort(key=lambda x: -x[0]) + return cls(commits=commits) + + def advance_to(self, pos: int) -> None: + """Reveal every commit strictly older than `pos` (i.e. position > pos).""" + if self._current_pos is not None and pos > self._current_pos: + raise ValueError( + "CoChangeModel is append-only; queries must be visited oldest-first " + f"(asked for pos={pos} after pos={self._current_pos})" + ) + self._current_pos = pos + while self._folded < len(self.commits) and self.commits[self._folded][0] > pos: + _, files = self.commits[self._folded] + uniq = sorted(set(files)) + for f in uniq: + self._single[f] += 1 + for i, a in enumerate(uniq): + for b in uniq[i + 1:]: + self._pair[(a, b)] += 1 + self._folded += 1 + + def confidence(self, a: str, b: str) -> float: + """P(b changes | a changes), smoothed. Asymmetric by construction.""" + if a == b: + return 0.0 + key = (a, b) if a < b else (b, a) + joint = self._pair.get(key, 0) + if not joint: + return 0.0 + return joint / (self._single.get(a, 0) + COCHANGE_PRIOR) + + def neighbours(self, a: str) -> list[str]: + """Files ever seen co-changing with `a` in the revealed history.""" + # Kept simple: the pair table is small enough (<2e5) that a per-anchor scan + # is cheaper than maintaining a second adjacency index. + out = [] + for (x, y), n in self._pair.items(): + if n and x == a: + out.append(y) + elif n and y == a: + out.append(x) + return out + + +class CoChangeIndex: + """Adjacency view over CoChangeModel, rebuilt lazily when history advances.""" + + def __init__(self, model: CoChangeModel) -> None: + self.model = model + self._adj: dict[str, list[str]] = {} + self._built_at = -1 + + def neighbours(self, a: str) -> list[str]: + if self._built_at != self.model._folded: + adj: dict[str, list[str]] = defaultdict(list) + for (x, y), n in self.model._pair.items(): + if n: + adj[x].append(y) + adj[y].append(x) + self._adj = adj + self._built_at = self.model._folded + return self._adj.get(a, []) + + +# --- the relation union ----------------------------------------------------- + + +@dataclass(frozen=True) +class RelationWeights: + """Linear weights over the relation union. + + Defaults are the leave-one-repository-out selection from `fit_weights.py`; they + are fitted parameters and are reported as such. + """ + + edge: float = 1.0 + cochange: float = 1.0 + testlink: float = 1.0 + stem: float = 1.0 + dirw: float = 0.3 + + def as_dict(self) -> dict[str, float]: + return {"edge": self.edge, "cochange": self.cochange, + "testlink": self.testlink, "stem": self.stem, "dir": self.dirw} + + +class RelationGraph: + """All five relations for one corpus, with candidate generation. + + Generation is *anchor-local*: candidates come from the anchors' relation + neighbourhoods, never from a scan of the corpus. That is what keeps completion at + O(|A0| * deg) instead of O(|corpus|) and is why the closure stage costs ~1ms. + """ + + def __init__( + self, + files: list[str], + *, + static: dict[str, dict[str, float]], + cochange: CoChangeModel, + ) -> None: + self.files = [_norm(f) for f in files] + self.static = static + self.cochange = cochange + self.cochange_index = CoChangeIndex(cochange) + + self._dir: dict[str, list[str]] = defaultdict(list) + self._by_stem_token: dict[str, list[str]] = defaultdict(list) + self._core_stem: dict[str, list[str]] = defaultdict(list) + self._tokens: dict[str, set[str]] = {} + for f in self.files: + self._dir[directory(f)].append(f) + toks = stem_tokens(f) + self._tokens[f] = toks + for t in toks: + self._by_stem_token[t].append(f) + self._core_stem[core_stem(f)].append(f) + + n = max(1, len(self.files)) + # IDF over file-name tokens: `index` in a repo full of `*_index.py` is not + # evidence of anything, while a token appearing in two names is strong. + self._idf = { + t: math.log(n / len(fs)) for t, fs in self._by_stem_token.items() + } + self._max_idf = max(self._idf.values(), default=1.0) or 1.0 + + # -- individual relation strengths, all normalised to [0, 1] -------------- + + def rel_edge(self, a: str, c: str) -> float: + return self.static.get(a, {}).get(c, 0.0) + + def rel_cochange(self, a: str, c: str) -> float: + return min(1.0, self.cochange.confidence(a, c)) + + def rel_testlink(self, a: str, c: str) -> float: + if is_testish(a) == is_testish(c): + return 0.0 + cs = core_stem(a) + return 1.0 if cs and cs == core_stem(c) else 0.0 + + def rel_stem(self, a: str, c: str) -> float: + shared = self._tokens.get(a, set()) & self._tokens.get(c, set()) + if not shared: + return 0.0 + # Best shared token by IDF, normalised. Sum would let three generic tokens + # outweigh one highly specific one, which is the wrong ordering. + return max(self._idf.get(t, 0.0) for t in shared) / self._max_idf + + def rel_dir(self, a: str, c: str) -> float: + d = directory(a) + if d != directory(c): + return 0.0 + # A 3-file package is a strong grouping; a 200-file dumping ground is not. + return 1.0 / math.log2(2 + len(self._dir.get(d, ()))) + + # -- candidate generation ------------------------------------------------- + + def candidates(self, anchor: str, *, dir_cap: int = 12) -> set[str]: + """Files related to `anchor` by at least one relation.""" + out: set[str] = set() + out.update(self.static.get(anchor, {}).keys()) + out.update(self.cochange_index.neighbours(anchor)) + out.update(self._core_stem.get(core_stem(anchor), ())) + for t in self._tokens.get(anchor, ()): + # Generic tokens fan out to hundreds of files and contribute ~0 IDF + # anyway; capping keeps generation bounded without changing the ranking. + fs = self._by_stem_token.get(t, ()) + if len(fs) <= 30: + out.update(fs) + siblings = self._dir.get(directory(anchor), ()) + if len(siblings) <= dir_cap: + out.update(siblings) + out.discard(anchor) + return out + + def score(self, anchor: str, c: str, w: RelationWeights) -> tuple[float, dict[str, float]]: + parts = { + "edge": self.rel_edge(anchor, c), + "cochange": self.rel_cochange(anchor, c), + "testlink": self.rel_testlink(anchor, c), + "stem": self.rel_stem(anchor, c), + "dir": self.rel_dir(anchor, c), + } + wd = w.as_dict() + total = sum(wd[k] * v for k, v in parts.items()) + return total, parts diff --git a/research/nucleus/search.py b/research/nucleus/search.py new file mode 100644 index 0000000..47b1849 --- /dev/null +++ b/research/nucleus/search.py @@ -0,0 +1,508 @@ +"""NUCLEUS retrieval: nucleate -> accrete -> select. + +The pipeline deliberately reuses the shipped retrieval stack for the *anchor* stage. +That is not laziness: the claim under test is about composition, not about lexical +matching, so the anchor stage must be the incumbent's best configuration or the +comparison is rigged. With `accrete=False` this module reproduces +`codebase_index.retrieval.pipeline.search` result-for-result — `test_nucleus.py` +asserts it — so every measured delta is attributable to accretion and selection alone. + +Three stages: + + NUCLEATE the incumbent hybrid retriever produces a ranked candidate list; the + head of it is treated as anchors (high precision, low recall). + + ACCRETE candidates related to the anchors are scored by the relation union + (`relations.RelationGraph`) *conditioned on the anchors, not the query*. + This is the stage that can retrieve a file sharing no vocabulary with + the question. + + SELECT accreted candidates compete for their own slot budget by coverage gain + per token, so completions do not duplicate each other's obligations. + +The budget separation is the point. Completions are appended behind a frozen anchor +head, so a completion can only ever take a slot a *lower-ranked baseline result* +would have held. The documented failure mode of graph retrieval in this codebase +(`tuning.graph_source = False`: "architectural neighbors displaced direct lexical +hits") is excluded structurally rather than by tuning. +""" + +from __future__ import annotations + +import re +import sqlite3 +from dataclasses import dataclass +from typing import Optional + +from codebase_index.retrieval.budget import apply_budget +from codebase_index.retrieval.diversity import deduplicate, mmr_select +from codebase_index.retrieval.fusion import fuse +from codebase_index.retrieval.intent import detect_intent +from codebase_index.retrieval.pipeline import _confidence, _diversify, _run_retrievers +from codebase_index.retrieval.rerank import rerank +from codebase_index.retrieval.skeleton import make_compactor +from codebase_index.retrieval.tuning import DEFAULT_TUNING, RetrievalTuning +from codebase_index.retrieval.types import Candidate + +from .relations import RelationGraph, RelationWeights, stem_tokens + +_TERM_RE = re.compile(r"[A-Za-z0-9_]+") + + +@dataclass(frozen=True) +class NucleusParams: + """Accretion configuration. Every field is an ablation switch.""" + + accrete: bool = True + anchor_head: int = 3 + """Baseline results whose rank is frozen. Completions are inserted after these.""" + + max_completions: int = 3 + """Completions to append. + + Under the eviction policy this had to be 1: `diagnose_slots.py` showed the + marginal precision of the second and third completion (0.0166 and 0.0072 + gold/query) fell below the cost of the rank-9 and rank-8 slots they evicted + (0.0339, 0.0247). Appending removes the eviction cost, so the binding constraint + becomes tokens and more completions become affordable. Ablated at 1/2/3.""" + + append_only: bool = True + """Add completions beyond the baseline page instead of evicting for them. + + The slot framing was wrong. `diagnose_slots.py` shows every eviction is a losing + trade, but that is an artifact of a fixed 10-slot page: the agent's real budget is + tokens, and the incumbent spends only ~1093 of 1500. Rendered as contract slices + rather than code chunks (`_completion_slice`), a completion costs a fraction of a + chunk and evicts nothing, so its expected value no longer has to beat a rank slot + -- only its own token cost. Compared honestly against a baseline allowed the same + page size and the same budget.""" + + compact_completions: bool = True + """Render completions as signatures (H8) rather than bodies.""" + + insert_at: str = "tail" + """Where completions go. + + 'after_head' inserts at `anchor_head + 1`, which shifts every baseline result + below it down by one. That is what made the first benchmark run lose: it left + recall unchanged while damaging MRR/nDCG/P@5, because a gold file at rank 4 + (density 0.096) was being pushed to rank 5+ to make room for a completion with + precision 0.073. 'tail' spends only the last slot, whose gold density is the + lowest on the page (0.035), and cannot reorder anything above it.""" + + min_score: float = 0.6 + """Gate, selected under leave-one-repository-out by `fit_gate.py`. + + Accretion proposes nothing at all on 58% of queries, and where it does propose, + its score is well calibrated: precision rises from 0.073 (fire always) to 0.353 + (fire on the top-scoring 10%). The gate converts a signal that is only 2.1x the + slot cost on average into one that is several times the slot cost when it spends. + + 0.6 was chosen in 7 of 8 leave-one-repository-out folds (the eighth chose 0.7) + and sits on a plateau: net gold/query is flat at 0.021-0.024 for every tau in + [0, 1.1]. It is a plateau interior, not a peak, which is the only kind of fitted + value worth shipping.""" + + weights: RelationWeights = RelationWeights() + + anchor_decay: float = 1.0 + """Anchor i gets weight 1/(1 + decay*i): rank-1 evidence outweighs rank-3.""" + + combine: str = "sum" + """'sum' rewards a candidate related to several anchors (evidence of belonging to + one change-set); 'max' treats relations as independent. Ablated.""" + + hub_penalty: bool = True + """Divide by sqrt(relation degree): a file related to everything is evidence for + nothing. The graph-retrieval god-node failure mode in miniature.""" + + coverage_select: bool = True + """H2: pick completions by coverage gain per token rather than by score alone.""" + + +# --- obligations ------------------------------------------------------------- + + +class ObligationIndex: + """path -> the set of symbol/identifier obligations that file participates in. + + Obligations are the latent objects the coverage objective is defined over. They + are approximated by identifiers, which is crude but has the two properties the + objective needs: shared obligations really do indicate "these files implement one + contract", and it costs one query per corpus to build. + """ + + def __init__(self, conn: sqlite3.Connection) -> None: + self.cov: dict[str, set[str]] = {} + for path, name in conn.execute( + "SELECT f.path, s.name FROM symbols s JOIN files f ON f.id = s.file_id" + ): + self.cov.setdefault(_norm(path), set()).add(name.lower()) + for path, name in conn.execute( + "SELECT f.path, e.dst_name FROM edges e JOIN files f ON f.id = e.file_id " + "WHERE e.dst_name IS NOT NULL" + ): + self.cov.setdefault(_norm(path), set()).add(str(name).lower()) + for path in list(self.cov): + self.cov[path] |= stem_tokens(path) + + def of(self, path: str) -> set[str]: + p = _norm(path) + if p not in self.cov: + self.cov[p] = set(stem_tokens(p)) + return self.cov[p] + + +def _norm(p: str) -> str: + return p.replace("\\", "/") + + +# --- completion snippets ----------------------------------------------------- + + +def _completion_slice( + conn: sqlite3.Connection, path: str +) -> Optional[tuple[int, int, str, int, Optional[str]]]: + """A file's *contract*: its declarations, not its body (hypothesis H8). + + An accreted file is proposed because something the agent is already looking at is + coupled to it -- not because it matched the question. What the agent needs is + enough to decide whether to open it: what lives here, and what it is called. A + signature digest answers that in a fraction of the tokens a code chunk costs, and + the ratio is what makes appending completions affordable at all. + """ + rows = conn.execute( + """ + SELECT s.name, s.kind, s.signature, s.line_start, s.line_end + FROM symbols s JOIN files f ON f.id = s.file_id + WHERE f.path = ? + ORDER BY (s.parent_id IS NOT NULL), s.line_start + LIMIT 8 + """, + (path,), + ).fetchall() + if not rows: + return None + lines = [f"# {path} (related to your anchors; contract only)"] + for name, kind, sig, ls, _le in rows: + text = (sig or name or "").strip().splitlines()[0][:140] if (sig or name) else "" + if text: + lines.append(f"{ls}: {kind or 'symbol'} {text}") + body = "\n".join(lines) + if len(lines) <= 1: + return None + first, last = int(rows[0][3]), int(rows[-1][4]) + # Matches the indexer's estimator (~4 chars/token) so the accounting stays + # symmetric with every other result on the page. + return (first, max(first, last), body, max(1, len(body) // 4), rows[0][0]) + + +def _completion_chunk( + conn: sqlite3.Connection, path: str, terms: set[str] +) -> Optional[tuple[int, int, str, int, Optional[str]]]: + """Best chunk of `path` to show for an accreted file. + + An accreted file frequently shares no vocabulary with the query — that is the + whole reason it had to be reached by relation rather than by search. When no + chunk matches a query term we fall back to the file's first symbol-bearing chunk, + i.e. its declaration/header region, which is the most informative fixed-size + window for orienting an agent that has never seen the file. + """ + rows = conn.execute( + """ + SELECT c.line_start, c.line_end, c.content, c.token_est, c.symbol_names + FROM chunks c JOIN files f ON f.id = c.file_id + WHERE f.path = ? + ORDER BY c.line_start + """, + (path,), + ).fetchall() + if not rows: + return None + best = None + best_hits = 0 + for r in rows: + content = (r[2] or "").lower() + hits = sum(1 for t in terms if t and t in content) + if hits > best_hits: + best_hits, best = hits, r + if best is None: + for r in rows: + if (r[4] or "").strip(): + best = r + break + best = best or rows[0] + return (int(best[0]), int(best[1]), best[2] or "", int(best[3] or 0), best[4]) + + +# --- accretion --------------------------------------------------------------- + + +def accrete( + conn: sqlite3.Connection, + *, + anchors: list[str], + graph: RelationGraph, + obligations: ObligationIndex, + params: NucleusParams, + exclude: set[str], + query_terms: set[str], +) -> list[tuple[str, float, dict[str, float]]]: + """Score files related to the anchor set. Returns [(path, score, parts)] desc.""" + if not anchors: + return [] + contrib = contributions(anchors=anchors, graph=graph, params=params, exclude=exclude) + ranked = rank_contributions(contrib, weights=params.weights, min_score=params.min_score) + return ranked + + +def contributions( + *, anchors: list[str], graph: RelationGraph, params: NucleusParams, + exclude: set[str], +) -> dict[str, dict[str, float]]: + """Per-candidate, per-relation evidence, aggregated over anchors. + + Deliberately kept **linear in the relation weights**: the score is + `sum_r theta_r * contrib[c][r]`, so refitting theta never requires re-walking the + graph. That is what makes the leave-one-repository-out fit in `fit_weights.py` + affordable, and it means the fitted weights are fitted against exactly the + quantities the retriever will use at query time. + + The hub normaliser is folded in here (not applied to the final score) so it scales + every relation identically and cannot be confused with a relation weight. + """ + contrib: dict[str, dict[str, float]] = {} + for i, a in enumerate(anchors): + alpha = 1.0 / (1.0 + params.anchor_decay * i) + for c in graph.candidates(a): + if c in exclude: + continue + _s, parts = graph.score(a, c, params.weights) + if not any(parts.values()): + continue + acc = contrib.setdefault(c, {}) + for k, v in parts.items(): + if v <= 0.0: + continue + if params.combine == "max": + acc[k] = max(acc.get(k, 0.0), alpha * v) + else: + acc[k] = acc.get(k, 0.0) + alpha * v + + if params.hub_penalty: + # A file reachable from several anchors is genuine evidence; a file reachable + # from *everything* is a hub. The normaliser is the candidate's total relation + # degree over the corpus, so multi-anchor support is still rewarded while a + # god-node is damped -- the graph-retrieval failure mode in miniature. + for c, acc in contrib.items(): + deg = len(graph.static.get(c, {})) + len(graph.cochange_index.neighbours(c)) + f = (1.0 + deg) ** 0.5 + for k in acc: + acc[k] /= f + return contrib + + +def rank_contributions( + contrib: dict[str, dict[str, float]], *, weights: RelationWeights, min_score: float, +) -> list[tuple[str, float, dict[str, float]]]: + wd = weights.as_dict() + scored = [ + (p, sum(wd.get(k, 0.0) * v for k, v in acc.items()), acc) + for p, acc in contrib.items() + ] + scored.sort(key=lambda t: (-t[1], t[0])) + return [t for t in scored if t[1] >= min_score] + + +def _select_completions( + ranked: list[tuple[str, float, dict[str, float]]], + *, + covered: set[str], + obligations: ObligationIndex, + params: NucleusParams, + conn: sqlite3.Connection, + query_terms: set[str], +) -> list[tuple[str, float, dict[str, float]]]: + """H2: budgeted greedy coverage over obligations, instead of plain top-C. + + Score alone will happily pick three files that all cover the same contract. The + marginal-gain rule prefers a slightly weaker candidate that covers something not + yet covered, which is what "minimal sufficient set" means operationally. + """ + if not params.coverage_select: + return ranked[: params.max_completions] + + chosen: list[tuple[str, float, dict[str, float]]] = [] + seen = set(covered) + pool = list(ranked[: params.max_completions * 6]) + while pool and len(chosen) < params.max_completions: + best = None + best_gain = -1.0 + for item in pool: + path, score, _ = item + new = obligations.of(path) - seen + # Marginal value per unit of score-normalised novelty. Token cost is + # nearly constant across chunks here (chunking is size-bounded), so + # novelty is the term that actually discriminates; keeping score as a + # multiplier stops a junk file with exotic identifiers from winning. + gain = score * (1.0 + len(new)) ** 0.5 + if gain > best_gain: + best_gain, best = gain, item + if best is None: + break + chosen.append(best) + seen |= obligations.of(best[0]) + pool.remove(best) + return chosen + + +# --- the pipeline ------------------------------------------------------------ + + +def nucleus_search( + conn: sqlite3.Connection, + query: str, + *, + mode: str = "hybrid", + limit: int = 10, + token_budget: int = 1500, + tuning: Optional[RetrievalTuning] = None, + graph: Optional[RelationGraph] = None, + obligations: Optional[ObligationIndex] = None, + params: NucleusParams = NucleusParams(), + explain: bool = False, +) -> dict: + tuning = tuning or DEFAULT_TUNING + plan = detect_intent(query) + base_limit = limit + if token_budget <= 0: + token_budget = plan.token_budget + + # --- NUCLEATE: the incumbent pipeline, unmodified ----------------------- + pool_mult = max(1, tuning.candidate_pool_multiplier) + pool_limit = limit if pool_mult == 1 else max(limit * pool_mult, 20) + lists, weights = _run_retrievers( + conn, query, mode=mode, limit=pool_limit, weights=plan.weights, + backend=None, tuning=tuning, graph_depth=tuning.graph_depth, + graph_node_cap=tuning.graph_node_cap, graph_strategy=plan.graph_strategy, + ) + fused = fuse(lists, weights=weights, k=tuning.rrf_k, + file_agreement=tuning.file_agreement_weight if tuning.file_agreement else 0.0) + pool_paths: list[str] = [] + for c in fused: + p = _norm(c.path) + if p not in pool_paths: + pool_paths.append(p) + ranked = rerank(fused, query=query, intent=plan.intent, tuning=tuning) + if tuning.dedup: + ranked = deduplicate(ranked, hamming_distance=tuning.dedup_hamming) + if tuning.mmr: + ranked = mmr_select(ranked, limit, tuning.mmr_lambda) + else: + ranked = _diversify(ranked, per_file=tuning.max_per_file) + baseline = ranked[:base_limit] + + accretion_log: list[dict] = [] + final = baseline + + # --- ACCRETE + SELECT --------------------------------------------------- + if params.accrete and graph is not None and obligations is not None and baseline: + terms = {t.lower() for t in _TERM_RE.findall(query) if len(t) > 2} + head = baseline[: params.anchor_head] + anchors: list[str] = [] + for c in head: + p = _norm(c.path) + if p not in anchors: + anchors.append(p) + # Only files already on the page are excluded from accretion; a file the + # baseline ranked below the page is a legitimate promotion target. + on_page = {_norm(c.path) for c in head} + if params.insert_at != "after_head": + # With tail placement a "promotion" from mid-page to the last slot is a + # demotion, not a gain, so only genuinely new files are worth a slot. + on_page = {_norm(c.path) for c in baseline} + covered: set[str] = set(terms) + for a in anchors: + covered |= obligations.of(a) + + cands = accrete( + conn, anchors=anchors, graph=graph, obligations=obligations, + params=params, exclude=on_page, query_terms=terms, + ) + picked = _select_completions( + cands, covered=covered, obligations=obligations, params=params, + conn=conn, query_terms=terms, + ) + + split = params.anchor_head if params.insert_at == "after_head" else len(baseline) + tail = baseline[split:] + tail_by_path: dict[str, Candidate] = {} + for c in baseline[params.anchor_head:]: + tail_by_path.setdefault(_norm(c.path), c) + + promoted: list[Candidate] = [] + consumed: set[int] = set() + for path, score, parts in picked: + existing = tail_by_path.get(path) + if existing is not None: + # Promotion: reuse the baseline candidate so its query-matched + # snippet (better than a synthesized one) is preserved. + promoted.append(existing) + consumed.add(id(existing)) + else: + chunk = None + if params.compact_completions: + chunk = _completion_slice(conn, path) + if chunk is None: + chunk = _completion_chunk(conn, path, terms) + if chunk is None: + continue + ls, le, content, tok, symnames = chunk + why = ",".join(f"{k}={v:.2f}" for k, v in sorted(parts.items()) if v > 0) + promoted.append( + Candidate( + path=path, line_start=ls, line_end=le, source="accretion", + score=score, kind=None, + symbol=(symnames or "").split()[0] if symnames else None, + content=content, token_est=tok, + reason=f"accreted from anchors ({why})", + ) + ) + accretion_log.append({"path": path, "score": round(score, 4), "parts": parts}) + + if params.append_only: + # Nothing is evicted; the page grows and the token budget arbitrates. + final = [c for c in baseline if id(c) not in consumed] + promoted + elif params.insert_at == "after_head": + rest = [c for c in tail if id(c) not in consumed] + final = (head + promoted + rest)[:limit] + else: + # Tail placement: completions claim the last `len(promoted)` slots of the + # page and nothing above them moves. When the baseline under-fills the + # page the slots are free and nothing is evicted at all. + kept = [c for c in baseline if id(c) not in consumed] + keep_n = max(0, limit - len(promoted)) + final = kept[:keep_n] + promoted + + confidence = _confidence(final) + compactor = make_compactor(intent=plan.intent, query=query, enabled=True, + min_reduction=0.25) + results, recommended = apply_budget(final, token_budget=token_budget, + compactor=compactor) + + payload: dict = { + "query": query, + "intent": plan.intent.value, + "mode": mode, + "confidence": confidence.value, + "results": results, + "recommended_reads": recommended, + "fallback_suggestions": {}, + } + if explain: + payload["diagnostics"] = { + "pool": [{"path": p} for p in pool_paths], + "pool_size": len(pool_paths), + "accreted": accretion_log, + } + return payload diff --git a/research/related-work.md b/research/related-work.md new file mode 100644 index 0000000..d181149 --- /dev/null +++ b/research/related-work.md @@ -0,0 +1,250 @@ +# Stage 1 — The existing frontier and where each approach structurally stops + +This is not a literature survey for its own sake. For each family the question is +narrow: **what does this method make impossible?** A limitation that a bigger model +or a better index would fix is an engineering gap, not a fundamental one. Only the +latter justifies a new architecture. + +Throughout, `q` is a query, `d` a document/chunk, `S` a retrieved set, `t` a task. + +--- + +## 1. Sparse lexical retrieval (BM25, FTS5, tf-idf) + +**What it computes.** `score(q,d) = Σ_{w∈q} IDF(w)·tf-saturation(w,d)`. A bag-of-words +independence assumption per term, summed. + +**Fundamental limitation — the vocabulary/necessity confusion.** BM25 can only rank a +document that *contains the query's words*. For code retrieval this is not a synonym +problem (fixable with expansion) but a **causality problem**: the file you must edit +alongside `auth/token.py` is `api/service.py`, which contains none of the query's +terms and never will. No re-weighting of a term-overlap function can produce a +document with zero term overlap. The ceiling is structural, not parametric. + +Measured in this repository (`docs/BENCHMARKS.md`): the shipped hybrid ranker reaches +`recall@10 = 0.693` while `oracle = 0.90`. The residual is dominated by gold files +that share no vocabulary with the query. + +## 2. Dense retrieval / embeddings (DPR, sentence encoders, code encoders) + +**What it computes.** `score(q,d) = ⟨E(q), E(d)⟩` — a single vector per document, +compared by inner product. + +**Fundamental limitation — the single-vector bottleneck and pointwise independence.** +Two distinct failures: + +1. *Capacity.* One vector must encode everything a document could ever be relevant + for. A file participating in 30 unrelated concerns gets one point in space. This + is a known information-theoretic bound: a `k`-dimensional vector cannot preserve + all pairwise relevance orderings of a corpus beyond a critical size (the + "embedding dimension vs. retrievable set" limit). +2. *Independence.* `score` factorises over documents. The model can express + "d is similar to q" but **cannot express "d is needed only if d' is also + retrieved"**. Every retrieval-as-ranking system inherits this: the top-k of a + pointwise scorer is not a set-optimal answer, it is `k` independently good answers. + +This second point is the one that matters here, and no amount of encoder quality +removes it. It is an artifact of the *objective*, not the representation. + +## 3. Hybrid + fusion (RRF, convex combination) + +**What it computes.** `RRF(d) = Σ_r w_r / (k + rank_r(d))`. + +**Fundamental limitation.** Fusion is still pointwise: it aggregates independent +opinions about the *same* document. It adds robustness, never set-awareness. This +repository's own data shows the ceiling precisely: 1.10.0 improved reranking +efficiency 0.639 → 0.660 while `oracle` and `cand_recall` moved by less than 1e-4. +Fusion redistributes the pool; it cannot create a candidate no retriever proposed. + +## 4. ColBERT / late interaction + +**What it computes.** `score(q,d) = Σ_i max_j ⟨E_i(q), E_j(d)⟩` — token-level +MaxSim, avoiding the single-vector bottleneck. + +**Fundamental limitation.** Fixes capacity, keeps independence, and multiplies index +size by ~`|d|`. Still `argmax` over documents scored in isolation, so it inherits §2.2 +in full. Late interaction makes similarity finer, not necessity computable. + +## 5. ANN indexes: HNSW, IVF, PQ, LSH + +**What they compute.** Approximate `argmax_d ⟨q,d⟩` in sublinear time. + +**Fundamental limitation — they optimise the wrong operation faster.** These are +accelerators for a metric-space nearest-neighbour query. They presuppose that +"relevant" = "near in a fixed metric". If the target relation is *asymmetric*, +*conditional* (`d` needed given `d'`), or *non-metric* (violates triangle +inequality — necessity plainly does: A needs B, B needs C, A may not need C), the +data structure has no way to represent it. PQ additionally trades recall for memory +in a way that is invisible to the caller — a silent quality knob. + +## 6. Graph retrieval / GraphRAG / knowledge graphs / PPR + +**What it computes.** Entry by similarity, then diffusion (personalised PageRank, +community summarisation, multi-hop traversal). + +**Fundamental limitation — undirected diffusion is not necessity, and it competes for +the same slots.** Two concrete failures, both visible in this repository: + +1. *Isotropy.* PPR spreads probability along every edge type at once. "Related to" + is not "required for". A hub file is reached from everywhere and therefore ranks + highly for everything — the god-node problem, documented in + `tests/benchmark_honest_RESULTS.md`. +2. *Slot competition.* When graph neighbours are injected as another retriever into + the fusion, they displace direct lexical hits. This repo measured exactly that and + **shipped `graph_source = False`** (`retrieval/tuning.py`): "the self-repository + ablation showed lower MRR when architectural neighbors displaced direct lexical + hits." + +This is the single most informative prior result available to me. The failure was not +"structure is useless" — my Stage-2 diagnostic shows resolved edges connect 30.7% of +anchor→target gold pairs. The failure was **architectural**: structural evidence was +forced to compete pointwise with lexical evidence in one ranked list, at one weight, +for one budget. That diagnosis is what the proposed architecture is built around. + +## 7. GraphRAG specifically + +Community detection + LLM-generated community summaries, queried by similarity. +**Limitation:** the summarisation is lossy and *query-independent* — it must guess in +advance which facts matter. It also costs an LLM pass over the whole corpus, which +makes incremental update expensive: one edited file can change community membership. +Good for global sensemaking questions, structurally poor at "which four files must I +edit". + +## 8. AST / symbol / code-structure indexing (LSP, tree-sitter, ctags, Aider repo-map) + +**What it computes.** Exact symbol definitions/references; repo-map ranks files by +PageRank over the symbol graph. + +**Fundamental limitation — static structure is *incomplete*, not merely noisy.** It +sees only relations the language makes explicit. It cannot see: a schema and the +migration that must accompany it; a config key and its consumer; a feature flag and +its test; a protocol and its two independent implementations. My diagnostic quantifies +this: static edges cover 30.7% of the pairs, while 9.8% are reachable *only* by +history. Structural indexing is a high-precision, low-recall relation. + +## 9. Evolutionary coupling / MSR (Zimmermann's ROSE, Ying et al.) + +**What it computes.** Association rules over co-changed files mined from VCS history. + +**Fundamental limitation.** Cold start (new files have no history), drift (couplings +decay as the design changes), and — decisively for benchmark honesty — **it is +trivially leaky if evaluated without a temporal split**. It is also a *recommender* +for an already-known seed file, not a retriever from a natural-language question. It +answers "what else changes with X", never "what is X". + +That is precisely why it composes with §1: lexical retrieval is good at finding X from +a question, and bad at finding what accompanies X. + +## 10. Semantic caching of LLM calls + +**What it computes.** Reuse a cached answer when `sim(q, q_cached) > τ`. + +**Fundamental limitation — it is unsound by construction.** Validity of a cached +answer depends on whether the *world it was computed from* has changed, and query +similarity carries no information about that. Two identical questions asked before and +after an edit must get different answers; a semantic cache returns the stale one with +high confidence. There is no threshold that fixes this, because the failure is not in +the similarity estimate — it is that the cache key omits the dependency. + +This is the gap the memory plane of the proposed architecture targets. + +## 11. Context compression (LLMLingua, RECOMP, selective context) + +**What it computes.** Drop low-information tokens from an already-retrieved context. + +**Fundamental limitation — it operates strictly downstream of a bad selection.** +Compression can shrink what you retrieved; it cannot retrieve what you missed. If the +required file was never in `S`, no compressor recovers it. Compression improves the +constant factor on `tokens`, and does nothing for `recall`. + +## 12. KV-cache reuse / prefix caching + +**Limitation.** Reuse is keyed on *literal token-prefix identity*. Any reordering or +one-token edit invalidates everything downstream. It is a systems optimisation with +zero semantic model of what the cached computation depended on — the same blind spot +as §10, at a different layer. + +## 13. Memory-augmented agents (episodic / semantic / procedural stores) + +**Limitation — write-time amnesia about provenance.** Nearly all implementations store +*what was concluded* and not *what it was concluded from*. Consequently they cannot +invalidate: a memory is retired by recency heuristics or LLM-judged staleness, never +by the fact that its evidence changed. They also store natural-language conclusions, +which cannot be checked for consistency automatically. + +## 14. Multi-agent routing / mixture-of-agents / task graphs + +**Limitation.** Routing is normally learned over *task descriptions*, so two agents +working on overlapping code are not detected as overlapping. Coordination cost grows +with agent count because there is no shared, addressable substrate — agents exchange +prose, so identical intermediate work is redone with no way to notice. + +## 15. Learned / neural indexes (DSI, generative retrieval) + +**Limitation.** The corpus is baked into model weights, so incremental update means +retraining or a fragile patching scheme. For a codebase changing hourly this is +disqualifying. Also inherits pointwise independence. + +--- + +## The common root + +Reading down the list, four distinct families collapse into **one** structural +assumption: + +> **Retrieval = rank documents independently by similarity to the query, then take the +> top k.** + +Everything above is an optimisation of some part of that sentence: better `sim` +(§2,§4), faster `argmax` (§5), more robust aggregation (§3), a wider candidate net +(§6,§8,§9), or post-hoc cleanup (§11). + +Three consequences follow, and none of them are fixable inside the assumption: + +1. **Sets are never optimised.** The objective factorises over documents, so + "necessary together" is inexpressible. The top-k of a pointwise scorer is `k` + individually-plausible documents, which is not the same object as a sufficient set. +2. **Necessity is conflated with similarity.** The system answers "what looks like the + question", where the agent needs "what must be true for the task to be completed". + These coincide only for lookup questions. +3. **Reuse is unsound.** Nothing records what a conclusion depended on, so nothing can + be safely reused after the corpus changes. + +Stage 2 attacks these three directly. + +--- + +## Prior art most adjacent to the proposal (stated up front, not buried) + +Honesty requires naming these before claiming anything. The proposed architecture is +**not** the first system to use co-change or structure for code retrieval: + +- **Zimmermann et al., "Mining Version Histories to Guide Software Changes" (ROSE, + ICSE'04 / TSE'05)** — association-rule co-change recommendation. Directly prior art + for the evolutionary relation. Differences: ROSE is seeded by a file the developer + already opened, not by a natural-language query; it recommends *changes*, not + retrieval context; and it optimises confidence/support, not a token budget. +- **Ying et al., "Predicting Source Code Changes by Mining Change History" (TSE'04)** — + frequent-pattern mining, same family, same seeding assumption. +- **Aider's repo-map / PageRank over the symbol graph** — structural ranking of files + for LLM context. Query-independent global ranking; no conditional completion, no + history, no budgeted coverage objective. +- **CoCoMIC, RepoHyper, RepoFusion, GraphCodeBERT** — cross-file code completion using + dependency context. Closest in spirit on the structural side; targeted at *code + completion at a cursor*, where the seed is given by construction, rather than at + retrieval from a task description. +- **Build systems (Bazel, Nix, `ccache`) and incremental computation (Adapton, + self-adjusting computation)** — content-addressed action caches with dependency-keyed + invalidation. This is the direct ancestor of the memory plane; the transplant to + *agent reasoning* is the part I claim as new, and §8 of `novelty.md` scopes that + claim carefully. +- **Submodular / facility-location summarisation (Lin & Bilmes 2011), MMR** — budgeted + set selection in IR. Prior art for the selection stage's mathematics. MMR maximises + *dissimilarity*; the objective proposed here maximises *coverage of implicated + obligations*, which is a different function with a different optimum. + +The claim under test is therefore not "co-change is new" or "greedy coverage is new". +It is that **conditional set-completion, budget-separated from anchor ranking, is the +composition that makes structural and historical evidence pay off where injecting it +into fusion measurably does not** — a claim this repository's own shipped +`graph_source = False` makes falsifiable rather than rhetorical. diff --git a/research/selection.md b/research/selection.md new file mode 100644 index 0000000..b00d888 --- /dev/null +++ b/research/selection.md @@ -0,0 +1,95 @@ +# Stage 4 — Scoring, TOP-3, and composition + +Scores are 1–5. `Feasible` means *testable with evidence available in this +environment* (eight real repositories, git history, no LLM API, no GPU) — a +hypothesis that can only be validated by machinery I do not have scores low no matter +how good it is, and is recorded as follow-up rather than silently dropped. + +| # | Hypothesis | Novelty | Theory | Feasible | Scale | Δtokens | Δlatency | Quality | Multi-agent | Risk | +|---|---|---|---|---|---|---|---|---|---|---| +| H1 | Conditional set completion | 4 | 5 | **5** | 5 | 3 | 4 | **5** | 3 | 2 | +| H2 | Budgeted coverage selection | 3 | 5 | **5** | 5 | **5** | 4 | 4 | 3 | 2 | +| H3 | Evidence-keyed memoisation | 4 | 4 | **5** | 5 | **5** | **5** | 3 | **5** | 2 | +| H4 | Granularity law of survival | 4 | 4 | **5** | 4 | 4 | 4 | 2 | 4 | 1 | +| H5 | Entropy-minimising retrieval | 4 | **5** | 1 | 2 | 4 | 1 | 4 | 2 | 4 | +| H6 | ddmin oracle for minimal context | **5** | **5** | 1 | 1 | 5 | 1 | 5 | 2 | 3 | +| H7 | Trace prefetching | 4 | 4 | 1 | 4 | 2 | 5 | 3 | 3 | 3 | +| H8 | Obligation-level slices | 2 | 3 | 4 | 5 | **5** | 4 | 2 | 2 | 1 | +| H9 | Index the derivative | 4 | 3 | 2 | 3 | 2 | 3 | 3 | 2 | **5** | +| H10 | Self-organising index | 3 | 4 | 1 | 4 | 2 | 2 | 4 | 4 | 4 | +| H11 | Obligation routing | 4 | 4 | 2 | 4 | 3 | 3 | 2 | **5** | 3 | +| H12 | Negative-result atoms | 4 | 2 | 3 | 4 | 4 | 4 | 2 | **5** | 2 | +| H13 | Speculative closure | 2 | 2 | 3 | 3 | 1 | 2 | 1 | 2 | 3 | + +## TOP-3 + +**H1 — conditional set completion.** Highest expected quality gain, premise already +measured at 71.8% reachability, and it has a *falsifiable prior*: this repository +shipped `graph_source = False` after structural evidence failed inside fusion. If H1's +composition claim is wrong, the experiment will reproduce that failure and say so. + +**H2 — budgeted coverage selection.** The only candidate that attacks token cost at the +objective level rather than by post-hoc compression, and it is the natural consumer of +H1's output: once completion produces a *set* of candidates with overlapping coverage, +top-k is provably the wrong selector. + +**H3 — evidence-keyed memoisation** (with **H4** as its measurement). Orthogonal to +H1/H2 — it addresses reuse, not retrieval — and it is the only one of the thirteen +that changes a *correctness* property rather than a quality metric: unsound reuse goes +to zero by construction. H4 makes it quantitative and is nearly free to measure once +H3 exists. + +## Rejected, with reasons + +- **H6** (ddmin oracle) is the single most valuable idea in the list and I cannot run + it: it needs `O(|S| log|S|)` LLM-agent executions per query plus a per-repo test + harness. Recorded as the top follow-up. Its absence is also the main threat to + validity of everything below, because it means I inherit the proxy label "files the + commit touched" rather than true minimal sufficiency — stated again in `experiments.md`. +- **H5, H7, H10** need a live agent workload or LLM scorer. Deferred, not disproven. +- **H13** rejected on measurement: closure latency turns out to be ~1 ms (§ results), + so there is nothing to amortise. +- **H9** carries an unacceptable circularity risk against a commit-derived benchmark; + its safe fragment (co-change under strict temporal split) is absorbed into H1. +- **H8** is largely already implemented here (`retrieval/skeleton.py`); it enters as a + cost model, not a claim. + +## Composition + +H1, H2, H3 compose without conflict because they act on different objects: + +``` + query + │ + ▼ + ┌─────────────┐ H1: anchors are found by the existing high-precision + │ NUCLEATE │ lexical/symbol retriever — unchanged, and deliberately so + └─────────────┘ + │ A₀ + ▼ + ┌─────────────┐ H1: completion conditioned on A₀, NOT on the query; + │ ACCRETE │ typed relation union {edge, cochange, testlink, stem, dir} + └─────────────┘ + │ candidates with coverage sets + ▼ + ┌─────────────┐ H2: greedy coverage-per-token under budget B, + │ SELECT │ replacing top-k + MMR + └─────────────┘ + │ S + ▼ + ┌─────────────┐ H3/H4: conclusions computed over S are stored keyed by + │ MEMOISE │ H(content hashes of S); invalidation is exact + └─────────────┘ +``` + +The composed system is specified in `architecture.md` under the working name +**NUCLEUS** (*Necessity-driven Unified Closure over Lexical, Evolutionary and +Structural relations*). + +One property of this composition is worth stating before any measurement, because it +is what makes the design defensible against the prior negative result: **accretion +never displaces an anchor.** Completions are allocated their own budget and appended +behind the anchor head, so a completion can only take a slot that a *lower-ranked +baseline result* would have taken. The 2015-vintage failure mode of graph retrieval — +architectural neighbours evicting direct hits — is excluded by construction rather +than by tuning. Whether that is enough is Stage 6's problem. diff --git a/research/test_nucleus.py b/research/test_nucleus.py new file mode 100644 index 0000000..5eeb087 --- /dev/null +++ b/research/test_nucleus.py @@ -0,0 +1,129 @@ +"""Correctness tests for the NUCLEUS prototype. + +Run explicitly (not part of the product suite, which is scoped to `tests/`): + + python -m pytest research/test_nucleus.py --no-cov -q + +The first test is the load-bearing one. Every headline delta in +`research/experiments.md` is "NUCLEUS vs the shipped pipeline", which is only a valid +attribution if NUCLEUS with accretion disabled *is* the shipped pipeline. If that +equivalence ever breaks, the benchmark is measuring an unrelated reimplementation and +the numbers mean nothing. +""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +import pytest + +from codebase_index.retrieval.pipeline import search +from research.nucleus.relations import ( + CoChangeModel, RelationGraph, core_stem, is_testish, load_static_edges, stem_tokens, +) +from research.nucleus.search import NucleusParams, ObligationIndex, nucleus_search + +INDEX = Path(__file__).parent / "data" / "index" / "codebase-index.sqlite" + +QUERIES = [ + "redact secrets before output", + "incremental index update", + "graph edge confidence", + "token budget for snippets", + "fuzzy symbol matching", +] + + +requires_index = pytest.mark.skipif( + not INDEX.exists(), reason="run research/build_indexes.py first" +) + + +@pytest.fixture() +def conn(): + c = sqlite3.connect(INDEX) + c.row_factory = sqlite3.Row + yield c + c.close() + + +@requires_index +@pytest.mark.parametrize("q", QUERIES) +def test_accretion_off_reproduces_shipped_pipeline(conn, q): + """NUCLEUS(accrete=False) must equal codebase_index's own search(), result by result.""" + theirs = search(conn, q, mode="hybrid", limit=10, token_budget=1500, + no_fallback=True, explain=False) + ours = nucleus_search(conn, q, limit=10, token_budget=1500, + params=NucleusParams(accrete=False)) + + def shape(payload): + return [ + (r["path"], r["line_start"], r["line_end"], r["token_est"], + (r.get("snippet") or "")[:200]) + for r in payload["results"] + ] + + assert shape(ours) == shape(theirs) + assert ours["intent"] == theirs["intent"] + assert ours["confidence"] == theirs["confidence"] + + +@requires_index +def test_accretion_changes_the_page(conn): + """Sanity: with a relation graph present, accretion actually does something.""" + files = [r[0].replace("\\", "/") for r in conn.execute("SELECT path FROM files")] + graph = RelationGraph(files, static=load_static_edges(INDEX), + cochange=CoChangeModel(commits=[])) + obl = ObligationIndex(conn) + changed = 0 + for q in QUERIES: + off = nucleus_search(conn, q, limit=10, token_budget=1500, + params=NucleusParams(accrete=False)) + on = nucleus_search(conn, q, limit=10, token_budget=1500, graph=graph, + obligations=obl, params=NucleusParams()) + if [r["path"] for r in off["results"]] != [r["path"] for r in on["results"]]: + changed += 1 + assert changed > 0, "accretion never altered any page — the graph is not wired in" + + +@requires_index +def test_anchor_head_is_never_displaced(conn): + """The design's central safety property, asserted rather than asserted-in-prose. + + A completion may only take a slot a lower-ranked baseline result would have held. + If this fails, NUCLEUS has reintroduced exactly the failure mode that made this + repository ship `graph_source = False`. + """ + files = [r[0].replace("\\", "/") for r in conn.execute("SELECT path FROM files")] + graph = RelationGraph(files, static=load_static_edges(INDEX), + cochange=CoChangeModel(commits=[])) + obl = ObligationIndex(conn) + params = NucleusParams() + for q in QUERIES: + off = nucleus_search(conn, q, limit=10, token_budget=1500, + params=NucleusParams(accrete=False)) + on = nucleus_search(conn, q, limit=10, token_budget=1500, graph=graph, + obligations=obl, params=params) + head = params.anchor_head + assert [r["path"] for r in on["results"][:head]] == \ + [r["path"] for r in off["results"][:head]] + + +def test_cochange_model_refuses_lookahead(): + """The no-lookahead guarantee is a property of the data structure, not a habit.""" + m = CoChangeModel(commits=[(5, ["a.py", "b.py"]), (3, ["a.py", "c.py"])]) + m.advance_to(4) + assert m.confidence("a.py", "b.py") > 0 # commit at position 5 is older + assert m.confidence("a.py", "c.py") == 0.0 # position 3 is newer, must be unseen + with pytest.raises(ValueError): + m.advance_to(9) # rewinding would reveal the future + + +def test_relation_primitives(): + assert core_stem("tests/test_service.py") == "service" + assert core_stem("src/Service.java") == "service" + assert is_testish("tests/test_service.py") + assert not is_testish("src/service.py") + assert "service" in stem_tokens("src/user_service.py") + assert "user" in stem_tokens("src/UserService.java") diff --git a/skill/SKILL.md b/skill/SKILL.md index 9404686..432005b 100644 --- a/skill/SKILL.md +++ b/skill/SKILL.md @@ -1,25 +1,26 @@ --- name: codebase-index -description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository. -allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob +description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository, and verify evidence gathered earlier before relying on it. +allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index verify *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob --- # Codebase Index Use the local index before reading repository files. -The operating principle is **Find → Trace → Predict**: +The operating principle is **Find → Trace → Verify → Predict**: - **Find** the implementation with ranked retrieval. - **Trace** behavior through definitions, callers, dependencies, and paths. +- **Verify** that evidence you already hold is still true before relying on it. - **Predict** change impact while preserving an explicit evidence trail. ## Route the question | Intent | Command | |---|---| -| Where is X implemented? | `codebase-index search "X" --json` | -| How does X work? | `codebase-index explain "X" --json` | +| Where is X implemented? | `codebase-index search "X" --session --json` | +| How does X work? | `codebase-index explain "X" --session --json` | | What is this codebase? | `codebase-index architecture --json` | | Find a named symbol | `codebase-index symbol "X" --json` | | Who calls or references X? | `codebase-index refs "X" --json` | @@ -27,6 +28,7 @@ The operating principle is **Find → Trace → Predict**: | What does my current diff affect? | `codebase-index diff-impact --json` | | How are X and Y connected? | `codebase-index path "X" "Y" --json` | | Describe X and its neighborhood | `codebase-index describe "X" --json` | +| Is what I read earlier still true? | `codebase-index verify --session --json` | | Produce a human graph | `codebase-index graph "X" --output ` | Use `search --mode symbol` for exact symbol work, `--mode fts` for text and @@ -38,21 +40,41 @@ or routing remain unclear. ## Evidence protocol -1. Run the best-matching command with `--json`. -2. Check `index` before trusting the payload: +1. Pick one session tag for this conversation (for example `auth-fix-1`) and + pass `--session ` to every `search` and `explain`. +2. Run the best-matching command with `--json`. +3. Check `index` before trusting the payload: - missing → run `codebase-index index`, then repeat; - stale with fewer than 20 changed files → run `codebase-index update`; - stale with 20 or more changed files → run `codebase-index index`; - fresh → continue. -3. Start with ranks 1–3. Read only `recommended_reads` line ranges. -4. Trace one additional hop only when the question requires behavior, +4. Start with ranks 1–3. Read only `recommended_reads` line ranges. +5. Trace one additional hop only when the question requires behavior, ownership, or impact. -5. Answer with `file:line` evidence and state uncertainty explicitly. +6. Before answering or editing from evidence gathered earlier in the task, run + `codebase-index verify --session --json` and reread anything whose + state is not `valid` or `relocated`. +7. Answer with `file:line` evidence and state uncertainty explicitly. Do not open whole files when a line range is available. A snippet may already be sufficient. `skeletonized: true` means the response intentionally folded unrelated body lines; read the supplied range when the missing body matters. +## Evidence memory + +- `reused: true` with `snippet: null` — this session already received that + exact text and its source is unchanged. Use your earlier copy; if you can no + longer see it, Read the range. +- `memory.invalidated` — evidence this session received has changed since. + Treat your earlier copy as wrong and reread before relying on it. +- `stale: true` — the index is older than the file. Run `codebase-index update` + or Read the range. +- A tag belongs to one context. Never give it to a subagent or another + conversation. Start a new tag after the context is cleared or compacted, or + whenever earlier snippets are no longer visible to you. + +Verdict states and citing evidence in notes: [references/memory.md](references/memory.md). + ## Confidence contract - **high** — answer from the indexed evidence. diff --git a/skill/references/commands.md b/skill/references/commands.md index 0efb464..79c91a6 100644 --- a/skill/references/commands.md +++ b/skill/references/commands.md @@ -17,6 +17,9 @@ Useful search options: - `--offset ` - `--raw` to disable snippet skeletonization - `--no-fallback` to suppress fallback suggestions +- `--session ` to name this conversation's context: unchanged evidence it + already received comes back as `reused: true` without the snippet, and + evidence that changed is listed under `memory.invalidated` `explain` uses the HOW_IT_WORKS intent and a larger default token budget. Prefer it over repeatedly rewording a broad search. @@ -49,6 +52,18 @@ codebase-index graph "" --direction both --depth 2 --output graph.html For headless work, use `--output`; do not use `--open`. Exports also support `--format graphml|dot|neo4j`. +## Evidence + +```bash +codebase-index verify --session --json +codebase-index verify "" ... --json +``` + +- `verify` is read-only and needs no index: it checks evidence against the + working tree. `all_valid` is true only when every checked span still holds. +- `--strict` exits 1 when anything is invalid (useful in scripts). +- See [memory.md](memory.md) for verdict states and when to reread. + ## Index health ```bash diff --git a/skill/references/memory.md b/skill/references/memory.md new file mode 100644 index 0000000..fa533ad --- /dev/null +++ b/skill/references/memory.md @@ -0,0 +1,49 @@ +# Evidence Memory + +Load this when you use `--session`, when a packet contains `reused`, `stale` or +`memory`, or when you want to cite evidence for later. + +## What is guaranteed + +Evidence is identified by the exact bytes of a file span. `reused: true` is set +only when this session already received the same text — or the whole span — +and those bytes are unchanged now. Nothing is decided by query similarity, and +memory stores no source text. + +## Verdict states + +| state | still true? | do this | +|---|---|---| +| `valid` | yes | use it | +| `relocated` | yes — identical content moved inside the file | use it; cite the new lines | +| `changed` | no | reread the range before relying on it | +| `ambiguous` | no — identical content now occurs more than once | reread | +| `deleted` | no — file gone; a rename or move counts as deleted | search again | +| `excluded` | no — now ignored or secret-like | do not read it | +| `unreadable` | no | search again | + +## Reread even when evidence is valid + +- You are about to edit and need the exact current text. +- The earlier snippet was a skeleton or a signature and you need the body. +- Your context was cleared or compacted, or you cannot see the earlier snippet. +- A tool output was truncated. + +## Citing evidence across conversations + +`codebase-index verify --session --json` lists everything the session +received as `path:start-end@hash` references. Keep the references next to the +conclusions they support in notes or handoffs: + +```text +Refunds are capped at the invoice total [billing/refund.py:3-4@3f9a2c1b7d4e8a90] +``` + +Later, from any agent, check them before trusting the note: + +```bash +codebase-index verify "billing/refund.py:3-4@3f9a2c1b7d4e8a90" --json +``` + +`all_valid: true` means every cited span still holds exactly. Otherwise reread +the invalid spans and re-derive the conclusion; do not patch the old one. diff --git a/skill/references/response-contract.md b/skill/references/response-contract.md index 798f152..98af67f 100644 --- a/skill/references/response-contract.md +++ b/skill/references/response-contract.md @@ -16,6 +16,9 @@ Each result can contain: - `snippet` - `skeletonized` - `elided_lines` +- `stale` — only when the index text no longer matches the file +- `reused` — only with `--session`, when `snippet` was withheld because this + session already received that exact text from unchanged source `recommended_reads` is the read plan. Start with its first one to three entries and use exact line ranges. An entry with `truncated: true` was capped at the @@ -23,6 +26,11 @@ definition head (`max_read_lines`, default 120); `line_end_full` gives the real extent. Read the capped range first and continue only when the head is not enough. +With `--session`, the packet also carries `memory`: `session`, `reused`, +`tokens_saved`, and `invalidated` — references to evidence the session received +that has since changed. `available: false` means nothing was withheld. See +[memory.md](memory.md). + `pagination.has_more` and `pagination.next_offset` indicate additional results. Prefer a more specific command or a larger token budget before paging. diff --git a/skill/scripts/cbx b/skill/scripts/cbx index 94a9114..18c3ed0 100644 --- a/skill/scripts/cbx +++ b/skill/scripts/cbx @@ -4,7 +4,7 @@ # - Whitelists subcommands so the skill can never invoke destructive ones (clean/init/watch). set -euo pipefail -ALLOWED="search explain architecture symbol refs impact diff-impact path describe graph stats doctor update index" +ALLOWED="search explain architecture symbol refs impact diff-impact path describe verify graph stats doctor update index" sub="${1:-}" case " $ALLOWED " in diff --git a/skill/scripts/cbx.ps1 b/skill/scripts/cbx.ps1 index eee56ae..6419e37 100644 --- a/skill/scripts/cbx.ps1 +++ b/skill/scripts/cbx.ps1 @@ -10,7 +10,7 @@ param( $ErrorActionPreference = "Stop" $allowed = @( "search", "explain", "architecture", "symbol", "refs", "impact", "diff-impact", - "path", "describe", "graph", "stats", "doctor", "update", "index" + "path", "describe", "verify", "graph", "stats", "doctor", "update", "index" ) if ($allowed -notcontains $Subcommand) { diff --git a/skill/scripts/doctor.py b/skill/scripts/doctor.py index 25aeac4..957dbf7 100644 --- a/skill/scripts/doctor.py +++ b/skill/scripts/doctor.py @@ -69,7 +69,7 @@ def check_skill_installed(project_root: Path) -> bool: print(f"[OK] Skill installed at: {skill_dir}") return True else: - print(f"[INFO] Skill not installed in .claude/skills/") + print("[INFO] Skill not installed in .claude/skills/") print(" Run: python skill/scripts/install.py") return True # Not a hard failure @@ -86,7 +86,7 @@ def check_config(project_root: Path) -> bool: print(f"[FAIL] Invalid JSON in {config_path}") return False else: - print(f"[INFO] No config file (using defaults)") + print("[INFO] No config file (using defaults)") return True diff --git a/skill/scripts/smoke_test.py b/skill/scripts/smoke_test.py index 7e4f0fa..a92b866 100644 --- a/skill/scripts/smoke_test.py +++ b/skill/scripts/smoke_test.py @@ -11,7 +11,6 @@ import json import os -import shutil import subprocess import sys import tempfile diff --git a/skills/codebase-index/SKILL.md b/skills/codebase-index/SKILL.md index 9404686..432005b 100644 --- a/skills/codebase-index/SKILL.md +++ b/skills/codebase-index/SKILL.md @@ -1,25 +1,26 @@ --- name: codebase-index -description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository. -allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob +description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository, and verify evidence gathered earlier before relying on it. +allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index verify *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob --- # Codebase Index Use the local index before reading repository files. -The operating principle is **Find → Trace → Predict**: +The operating principle is **Find → Trace → Verify → Predict**: - **Find** the implementation with ranked retrieval. - **Trace** behavior through definitions, callers, dependencies, and paths. +- **Verify** that evidence you already hold is still true before relying on it. - **Predict** change impact while preserving an explicit evidence trail. ## Route the question | Intent | Command | |---|---| -| Where is X implemented? | `codebase-index search "X" --json` | -| How does X work? | `codebase-index explain "X" --json` | +| Where is X implemented? | `codebase-index search "X" --session --json` | +| How does X work? | `codebase-index explain "X" --session --json` | | What is this codebase? | `codebase-index architecture --json` | | Find a named symbol | `codebase-index symbol "X" --json` | | Who calls or references X? | `codebase-index refs "X" --json` | @@ -27,6 +28,7 @@ The operating principle is **Find → Trace → Predict**: | What does my current diff affect? | `codebase-index diff-impact --json` | | How are X and Y connected? | `codebase-index path "X" "Y" --json` | | Describe X and its neighborhood | `codebase-index describe "X" --json` | +| Is what I read earlier still true? | `codebase-index verify --session --json` | | Produce a human graph | `codebase-index graph "X" --output ` | Use `search --mode symbol` for exact symbol work, `--mode fts` for text and @@ -38,21 +40,41 @@ or routing remain unclear. ## Evidence protocol -1. Run the best-matching command with `--json`. -2. Check `index` before trusting the payload: +1. Pick one session tag for this conversation (for example `auth-fix-1`) and + pass `--session ` to every `search` and `explain`. +2. Run the best-matching command with `--json`. +3. Check `index` before trusting the payload: - missing → run `codebase-index index`, then repeat; - stale with fewer than 20 changed files → run `codebase-index update`; - stale with 20 or more changed files → run `codebase-index index`; - fresh → continue. -3. Start with ranks 1–3. Read only `recommended_reads` line ranges. -4. Trace one additional hop only when the question requires behavior, +4. Start with ranks 1–3. Read only `recommended_reads` line ranges. +5. Trace one additional hop only when the question requires behavior, ownership, or impact. -5. Answer with `file:line` evidence and state uncertainty explicitly. +6. Before answering or editing from evidence gathered earlier in the task, run + `codebase-index verify --session --json` and reread anything whose + state is not `valid` or `relocated`. +7. Answer with `file:line` evidence and state uncertainty explicitly. Do not open whole files when a line range is available. A snippet may already be sufficient. `skeletonized: true` means the response intentionally folded unrelated body lines; read the supplied range when the missing body matters. +## Evidence memory + +- `reused: true` with `snippet: null` — this session already received that + exact text and its source is unchanged. Use your earlier copy; if you can no + longer see it, Read the range. +- `memory.invalidated` — evidence this session received has changed since. + Treat your earlier copy as wrong and reread before relying on it. +- `stale: true` — the index is older than the file. Run `codebase-index update` + or Read the range. +- A tag belongs to one context. Never give it to a subagent or another + conversation. Start a new tag after the context is cleared or compacted, or + whenever earlier snippets are no longer visible to you. + +Verdict states and citing evidence in notes: [references/memory.md](references/memory.md). + ## Confidence contract - **high** — answer from the indexed evidence. diff --git a/skills/codebase-index/references/commands.md b/skills/codebase-index/references/commands.md index 0efb464..79c91a6 100644 --- a/skills/codebase-index/references/commands.md +++ b/skills/codebase-index/references/commands.md @@ -17,6 +17,9 @@ Useful search options: - `--offset ` - `--raw` to disable snippet skeletonization - `--no-fallback` to suppress fallback suggestions +- `--session ` to name this conversation's context: unchanged evidence it + already received comes back as `reused: true` without the snippet, and + evidence that changed is listed under `memory.invalidated` `explain` uses the HOW_IT_WORKS intent and a larger default token budget. Prefer it over repeatedly rewording a broad search. @@ -49,6 +52,18 @@ codebase-index graph "" --direction both --depth 2 --output graph.html For headless work, use `--output`; do not use `--open`. Exports also support `--format graphml|dot|neo4j`. +## Evidence + +```bash +codebase-index verify --session --json +codebase-index verify "" ... --json +``` + +- `verify` is read-only and needs no index: it checks evidence against the + working tree. `all_valid` is true only when every checked span still holds. +- `--strict` exits 1 when anything is invalid (useful in scripts). +- See [memory.md](memory.md) for verdict states and when to reread. + ## Index health ```bash diff --git a/skills/codebase-index/references/memory.md b/skills/codebase-index/references/memory.md new file mode 100644 index 0000000..fa533ad --- /dev/null +++ b/skills/codebase-index/references/memory.md @@ -0,0 +1,49 @@ +# Evidence Memory + +Load this when you use `--session`, when a packet contains `reused`, `stale` or +`memory`, or when you want to cite evidence for later. + +## What is guaranteed + +Evidence is identified by the exact bytes of a file span. `reused: true` is set +only when this session already received the same text — or the whole span — +and those bytes are unchanged now. Nothing is decided by query similarity, and +memory stores no source text. + +## Verdict states + +| state | still true? | do this | +|---|---|---| +| `valid` | yes | use it | +| `relocated` | yes — identical content moved inside the file | use it; cite the new lines | +| `changed` | no | reread the range before relying on it | +| `ambiguous` | no — identical content now occurs more than once | reread | +| `deleted` | no — file gone; a rename or move counts as deleted | search again | +| `excluded` | no — now ignored or secret-like | do not read it | +| `unreadable` | no | search again | + +## Reread even when evidence is valid + +- You are about to edit and need the exact current text. +- The earlier snippet was a skeleton or a signature and you need the body. +- Your context was cleared or compacted, or you cannot see the earlier snippet. +- A tool output was truncated. + +## Citing evidence across conversations + +`codebase-index verify --session --json` lists everything the session +received as `path:start-end@hash` references. Keep the references next to the +conclusions they support in notes or handoffs: + +```text +Refunds are capped at the invoice total [billing/refund.py:3-4@3f9a2c1b7d4e8a90] +``` + +Later, from any agent, check them before trusting the note: + +```bash +codebase-index verify "billing/refund.py:3-4@3f9a2c1b7d4e8a90" --json +``` + +`all_valid: true` means every cited span still holds exactly. Otherwise reread +the invalid spans and re-derive the conclusion; do not patch the old one. diff --git a/skills/codebase-index/references/response-contract.md b/skills/codebase-index/references/response-contract.md index 798f152..98af67f 100644 --- a/skills/codebase-index/references/response-contract.md +++ b/skills/codebase-index/references/response-contract.md @@ -16,6 +16,9 @@ Each result can contain: - `snippet` - `skeletonized` - `elided_lines` +- `stale` — only when the index text no longer matches the file +- `reused` — only with `--session`, when `snippet` was withheld because this + session already received that exact text from unchanged source `recommended_reads` is the read plan. Start with its first one to three entries and use exact line ranges. An entry with `truncated: true` was capped at the @@ -23,6 +26,11 @@ definition head (`max_read_lines`, default 120); `line_end_full` gives the real extent. Read the capped range first and continue only when the head is not enough. +With `--session`, the packet also carries `memory`: `session`, `reused`, +`tokens_saved`, and `invalidated` — references to evidence the session received +that has since changed. `available: false` means nothing was withheld. See +[memory.md](memory.md). + `pagination.has_more` and `pagination.next_offset` indicate additional results. Prefer a more specific command or a larger token budget before paging. diff --git a/src/codebase_index/__init__.py b/src/codebase_index/__init__.py index af8c17b..61ed905 100644 --- a/src/codebase_index/__init__.py +++ b/src/codebase_index/__init__.py @@ -4,4 +4,4 @@ See docs/ARCHITECTURE.md for the module map. """ -__version__ = "1.9.0" +__version__ = "2.0.0" diff --git a/src/codebase_index/cli.py b/src/codebase_index/cli.py index 82c8f79..71e6f99 100644 --- a/src/codebase_index/cli.py +++ b/src/codebase_index/cli.py @@ -75,6 +75,32 @@ def _open_in_browser(path: Path) -> None: subprocess.Popen(["xdg-open", uri]) +def _checked_session(session: Optional[str]) -> Optional[str]: + """Validate a --session tag before any work is done; exit 2 when malformed.""" + from .service import session_tag + + try: + return session_tag(session) + except ValueError as exc: + typer.echo(f"[codebase-index] {exc}", err=True) + raise typer.Exit(code=2) + + +def _memory_line(memory: dict) -> str: + if not memory.get("enabled", True): + return "memory: disabled" + if not memory.get("exists"): + return "memory: empty (starts with the first --session search)" + if memory.get("available") is False: + return f"memory: unavailable — {memory.get('reason')}" + return ( + f"memory: {memory.get('atoms', 0)} evidence atoms · {memory.get('sessions', 0)} " + f"sessions · {memory.get('tokens_saved', 0)} tokens not resent · " + f"{memory.get('invalidations', 0)} invalidations · " + f"{memory.get('bytes', 0) // 1024} KB" + ) + + def _resolve_backend_for_search(ctx: "typer.Context"): """Embedding backend for query-time vector search (see service.search_backend).""" from .config import load @@ -382,6 +408,11 @@ def search( False, "--raw", help="Disable snippet skeletonization; return full raw snippets.", ), + session: Optional[str] = typer.Option( + None, "--session", + help="Tag naming ONE agent context: evidence it already received and that is " + "unchanged is not resent; changes to it are reported (docs/MEMORY.md).", + ), json_out: bool = typer.Option(False, "--json", help="Emit machine-readable JSON."), ) -> None: """Hybrid ranked search; returns compact results + recommended_reads.""" @@ -389,6 +420,7 @@ def search( from .output import markdown as md_renderer from .service import search_payload + session = _checked_session(session) if offset < 0: typer.echo("[codebase-index] --offset must be >= 0.") raise typer.Exit(code=2) @@ -407,6 +439,7 @@ def search( payload = search_payload( db_path, cfg, query, mode=mode, limit=limit, offset=offset, token_budget=token_budget, no_fallback=no_fallback, backend=backend, raw=raw, + session=session, ) want_json = json_out or (ctx.obj and ctx.obj.get("json")) @@ -522,6 +555,11 @@ def explain( False, "--raw", help="Disable snippet skeletonization; return full raw snippets.", ), + session: Optional[str] = typer.Option( + None, "--session", + help="Tag naming ONE agent context: evidence it already received and that is " + "unchanged is not resent; changes to it are reported (docs/MEMORY.md).", + ), json_out: bool = typer.Option(False, "--json", help="Emit machine-readable JSON."), ) -> None: """Intent-aware bundle for 'how does X work' / overview questions.""" @@ -529,12 +567,14 @@ def explain( from .output import markdown as md_renderer from .service import normalize_explain_query, search_payload + session = _checked_session(session) backend = _resolve_backend_for_search(ctx) db_path, cfg = _ensure_index(ctx) payload = search_payload( db_path, cfg, normalize_explain_query(query), mode="hybrid", limit=10, token_budget=token_budget, no_fallback=False, backend=backend, raw=raw, + session=session, ) want_json = json_out or (ctx.obj and ctx.obj.get("json")) @@ -602,6 +642,104 @@ def describe( typer.echo(json_renderer.render(payload) if is_json else md_renderer.render_describe(payload)) +@app.command("verify") +def verify( + ctx: typer.Context, + refs: Optional[list[str]] = typer.Argument( + None, help="Evidence references: path:start-end@hash (as printed by memory)." + ), + session: Optional[str] = typer.Option( + None, "--session", help="Also verify everything this session was given." + ), + strict: bool = typer.Option( + False, "--strict", help="Exit 1 unless every piece of evidence is still valid." + ), + json_flag: bool = typer.Option(False, "--json", help="Emit machine-readable JSON."), +) -> None: + """Is this evidence still true? Checks references against the working tree (read-only).""" + import json as _json + + from .output import markdown as md_renderer + from .service import resolve_db, verify_payload + + session = _checked_session(session) + if not refs and session is None: + typer.echo("[codebase-index] pass evidence references, --session, or both.", err=True) + raise typer.Exit(code=2) + _db_path, cfg = resolve_db(ctx.obj.get("root") if ctx.obj else None) + payload = verify_payload(cfg, refs or [], session) + is_json = json_flag or bool(ctx.obj and ctx.obj.get("json")) + typer.echo(_json.dumps(payload) if is_json else md_renderer.render_verify(payload)) + if strict and not payload["all_valid"]: + raise typer.Exit(code=1) + + +memory_app = typer.Typer( + help="Evidence memory maintenance. Deliberately not exposed to agents or MCP.", + no_args_is_help=True, +) +app.add_typer(memory_app, name="memory") + + +@memory_app.command("gc") +def memory_gc( + ctx: typer.Context, + json_flag: bool = typer.Option(False, "--json", help="Emit machine-readable JSON."), +) -> None: + """Drop expired sessions and orphaned evidence, then compact memory.sqlite.""" + import json as _json + + from .memory.store import MemoryUnavailable + from .service import memory_gc_payload, resolve_db + + _db_path, cfg = resolve_db(ctx.obj.get("root") if ctx.obj else None) + try: + payload = memory_gc_payload(cfg) + except MemoryUnavailable as exc: + typer.echo(f"[codebase-index] {exc}", err=True) + raise typer.Exit(code=1) + if json_flag or bool(ctx.obj and ctx.obj.get("json")): + typer.echo(_json.dumps(payload)) + elif not payload.get("exists"): + typer.echo("No evidence memory to collect.") + else: + typer.echo( + f"Removed {payload['expired_sessions']} expired and {payload['capped_sessions']} " + f"over-limit session(s), {payload['orphan_atoms']} orphaned atom(s). " + f"{payload['sessions']} session(s) and {payload['atoms']} atom(s) remain." + ) + + +@memory_app.command("clear") +def memory_clear( + ctx: typer.Context, + session: Optional[str] = typer.Option(None, "--session", help="Forget only this session."), + yes: bool = typer.Option(False, "--yes", help="Skip the confirmation prompt."), + json_flag: bool = typer.Option(False, "--json", help="Emit machine-readable JSON."), +) -> None: + """Forget one session, or all evidence memory for this repository. Source is untouched.""" + import json as _json + + from .memory.store import MemoryUnavailable + from .service import memory_clear_payload, resolve_db + + session = _checked_session(session) + is_json = json_flag or bool(ctx.obj and ctx.obj.get("json")) + _db_path, cfg = resolve_db(ctx.obj.get("root") if ctx.obj else None) + if not yes and not is_json and sys.stdin.isatty(): + what = f"session '{session}'" if session else "all evidence memory for this repository" + typer.confirm(f"Forget {what}?", abort=True) + try: + payload = memory_clear_payload(cfg, session) + except MemoryUnavailable as exc: + typer.echo(f"[codebase-index] {exc}", err=True) + raise typer.Exit(code=1) + if is_json: + typer.echo(_json.dumps(payload)) + else: + typer.echo(f"Forgot {payload['removed_sessions']} session(s).") + + @app.command("graph") def graph_view( ctx: typer.Context, @@ -686,7 +824,7 @@ def stats( raise typer.Exit(code=0) with Database(db_path) as db: - payload = stats_payload(db.conn) + payload = stats_payload(db.conn, cfg=_cfg) if is_json: typer.echo(_json.dumps(payload)) @@ -699,6 +837,7 @@ def stats( flag = " ⚠ 0 symbols" if (r["symbols"] or 0) == 0 and r["files"] >= 3 else "" tier = " · partial graph (Tier-B)" if r["graph"] == "partial" else "" typer.echo(f" {r['lang']}: {r['files']} files, {r['symbols']} symbols{flag}{tier}") + typer.echo(_memory_line(payload.get("memory") or {})) @app.command() diff --git a/src/codebase_index/config.py b/src/codebase_index/config.py index eac4cb8..dae47bc 100644 --- a/src/codebase_index/config.py +++ b/src/codebase_index/config.py @@ -49,6 +49,14 @@ class GraphConfig(BaseModel): node_cap: int = 40 +class MemoryConfig(BaseModel): + """Evidence memory (docs/MEMORY.md). Not indexing-relevant, so not in config_hash.""" + + enabled: bool = True + retention_days: int = 14 + max_deliveries: int = 50_000 + + class Config(BaseModel): root: str = "." languages: Union[Literal["auto"], list[str]] = "auto" @@ -59,6 +67,7 @@ class Config(BaseModel): retrieval: RetrievalConfig = RetrievalConfig() embeddings: EmbeddingsConfig = EmbeddingsConfig() graph: GraphConfig = GraphConfig() + memory: MemoryConfig = MemoryConfig() redaction: dict = {"enabled": True} def config_hash(self) -> str: diff --git a/src/codebase_index/discovery/gates.py b/src/codebase_index/discovery/gates.py new file mode 100644 index 0000000..878f88f --- /dev/null +++ b/src/codebase_index/discovery/gates.py @@ -0,0 +1,117 @@ +"""Discovery gates shared by indexing and by every later read of the working tree. + +A path the indexer refuses (ignored, dependency/build directory, secret filename, +oversized, binary) must be refused identically wherever else the tool reads files. +Keeping a single implementation is how that stays true: the walker and evidence +validation both go through `PathGate`. +""" + +from __future__ import annotations + +import stat as stat_mod +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +from ..config import Config +from . import classify +from .ignore import IgnoreMatcher + +BINARY_SNIFF_BYTES = 4096 + + +@dataclass(frozen=True) +class GateRead: + """Outcome of reading one repo-relative file through the gates. + + ``state`` is ``ok`` (``data`` holds the bytes), ``excluded`` (a gate refused it and the + file was not opened unless the refusal is content-based), ``deleted`` or ``unreadable``. + """ + + state: str + data: bytes = b"" + reason: str = "" + + +class PathGate: + def __init__(self, root: Path | str, config: Config) -> None: + self.root = Path(root).resolve() + self.max_file_bytes = config.max_file_bytes + self.matcher = IgnoreMatcher.from_root( + self.root, + ignore_files=config.ignore_files, + extra_ignore=config.extra_ignore, + ) + + # -- the walker's gates, in the walker's order ----------------------------------- + def dir_allowed(self, name: str, rel_dir: str) -> bool: + return not self.matcher.is_ignored_dir(name) and not self.matcher.is_ignored( + rel_dir + "/" + ) + + def name_rejection(self, rel: str) -> Optional[str]: + if self.matcher.is_ignored(rel): + return "ignored by an ignore file or the built-in denylist" + if classify.is_secret_filename(rel): + return "secret-like filename" + return None + + def content_rejection(self, size: int, head: bytes) -> Optional[str]: + if size > self.max_file_bytes: + return "larger than max_file_bytes" + if classify.looks_binary(head): + return "binary content" + return None + + # -- whole-path checks for reads that did not come from a walk ------------------- + def path_rejection(self, rel: str) -> Optional[str]: + """Every name-based gate for a repo-relative POSIX path, including its directories.""" + parts = rel.split("/") + for depth in range(1, len(parts)): + if not self.dir_allowed(parts[depth - 1], "/".join(parts[:depth])): + return "inside an ignored directory" + return self.name_rejection(rel) + + def read(self, rel: str) -> GateRead: + """Read an admitted file, or explain why it is not admitted. + + Name-based gates run before the file is touched. The resolved on-disk path is gated + again, so a symlink or (on case-insensitive filesystems) a differently-cased path + cannot reach a file the walker would never have indexed. + """ + reason = self.path_rejection(rel) + if reason: + return GateRead("excluded", reason=reason) + path = self.root / rel + try: + resolved = path.resolve() + except OSError as exc: + return GateRead("unreadable", reason=str(exc)) + try: + actual_rel = resolved.relative_to(self.root).as_posix() + except ValueError: + return GateRead("excluded", reason="resolves outside the repository") + if actual_rel != rel: + reason = self.path_rejection(actual_rel) + if reason: + return GateRead("excluded", reason=reason) + try: + st = path.stat() + except FileNotFoundError: + return GateRead("deleted", reason="file no longer exists") + except OSError as exc: + return GateRead("unreadable", reason=str(exc)) + if not stat_mod.S_ISREG(st.st_mode): + return GateRead("deleted", reason="no longer a regular file") + if st.st_size > self.max_file_bytes: + return GateRead("excluded", reason="larger than max_file_bytes") + try: + data = path.read_bytes() + except FileNotFoundError: + return GateRead("deleted", reason="file no longer exists") + except OSError as exc: + return GateRead("unreadable", reason=str(exc)) + reason = self.content_rejection(len(data), data[:BINARY_SNIFF_BYTES]) + if reason: + return GateRead("excluded", reason=reason) + return GateRead("ok", data=data) diff --git a/src/codebase_index/discovery/walker.py b/src/codebase_index/discovery/walker.py index 9c707a0..e3199b8 100644 --- a/src/codebase_index/discovery/walker.py +++ b/src/codebase_index/discovery/walker.py @@ -9,9 +9,7 @@ from ..config import Config from . import classify -from .ignore import IgnoreMatcher - -_BINARY_SNIFF_BYTES = 4096 +from .gates import BINARY_SNIFF_BYTES, PathGate @dataclass @@ -26,38 +24,36 @@ class Candidate: def walk(root: Path, config: Config) -> Iterator[Candidate]: root = Path(root).resolve() - matcher = IgnoreMatcher.from_root( - root, - ignore_files=config.ignore_files, - extra_ignore=config.extra_ignore, - ) + gate = PathGate(root, config) for dirpath, dirnames, filenames in os.walk(root): - dirnames[:] = [ - d - for d in dirnames - if not matcher.is_ignored_dir(d) - and not matcher.is_ignored(_rel(root, Path(dirpath) / d) + "/") - ] + kept: list[str] = [] + for d in dirnames: + rel_dir = _rel(root, Path(dirpath) / d) + if rel_dir is not None and gate.dir_allowed(d, rel_dir): + kept.append(d) + dirnames[:] = kept for fname in filenames: abs_path = Path(dirpath) / fname rel = _rel(root, abs_path) + if rel is None: + continue - if matcher.is_ignored(rel) or classify.is_secret_filename(rel): + if gate.name_rejection(rel): continue try: size = abs_path.stat().st_size except OSError: continue - if size > config.max_file_bytes: + if size > gate.max_file_bytes: continue try: with abs_path.open("rb") as fh: - head = fh.read(_BINARY_SNIFF_BYTES) + head = fh.read(BINARY_SNIFF_BYTES) except OSError: continue - if classify.looks_binary(head): + if gate.content_rejection(size, head): continue lang = classify.detect_language(rel) @@ -71,5 +67,15 @@ def walk(root: Path, config: Config) -> Iterator[Candidate]: ) -def _rel(root: Path, path: Path) -> str: - return path.resolve().relative_to(root).as_posix() +def _rel(root: Path, path: Path) -> Optional[str]: + """Root-relative POSIX path of ``path``, or ``None`` if it resolves outside ``root``. + + ``os.walk`` lists symlinks lexically under ``root``, but a link may point elsewhere + (Bazel's ``bazel-out`` convenience symlinks are the common case). Such entries are + skipped rather than raising, matching the gate's "resolves outside the repository" + exclusion. + """ + try: + return path.resolve().relative_to(root).as_posix() + except (ValueError, OSError): + return None diff --git a/src/codebase_index/doctor.py b/src/codebase_index/doctor.py index 0770cae..c4fcadd 100644 --- a/src/codebase_index/doctor.py +++ b/src/codebase_index/doctor.py @@ -7,6 +7,7 @@ from __future__ import annotations +import sqlite3 from dataclasses import dataclass from pathlib import Path from typing import Literal @@ -126,9 +127,46 @@ def run_doctor(root: Path, config: Config) -> list[Finding]: ) ) + findings.append(_memory_finding(config)) return findings +def _memory_finding(config: Config) -> Finding: + """Read-only probe of memory.sqlite. Doctor reports; it never repairs or migrates.""" + from .memory.store import SCHEMA_VERSION + from .service import memory_enabled, memory_path_for + + if not memory_enabled(config): + return Finding("memory_store", True, "info", "evidence memory is disabled") + path = memory_path_for(config) + quarantined = [p for p in path.parent.glob(f"{path.name}.corrupt-*") + if not p.name.endswith(("-wal", "-shm"))] + kept = f"; {len(quarantined)} quarantined corrupt store(s) kept beside it" if quarantined else "" + if not path.exists(): + return Finding("memory_store", True, "info", + f"no evidence memory yet (created on first --session use){kept}") + try: + conn = sqlite3.connect(f"file:{path.as_posix()}?mode=ro", uri=True) + try: + check = conn.execute("PRAGMA quick_check").fetchone()[0] + row = conn.execute("SELECT value FROM meta WHERE key = 'schema_version'").fetchone() + finally: + conn.close() + except sqlite3.Error as exc: + return Finding("memory_store", False, "medium", + f"memory store unreadable ({exc}); it is moved aside and recreated " + f"on next use{kept}") + version = int(row[0]) if row else 0 + if check != "ok": + return Finding("memory_store", False, "medium", + f"memory store integrity check failed: {check}{kept}") + if version > SCHEMA_VERSION: + return Finding("memory_store", False, "medium", + f"memory store schema {version} is newer than supported " + f"{SCHEMA_VERSION}; memory is off until codebase-index is upgraded{kept}") + return Finding("memory_store", True, "info", f"memory store healthy (schema {version}){kept}") + + # Threshold above which a tree-sitter language with zero symbols is treated as broken rather # than just a tiny/empty repo. _ZERO_SYMBOL_FILE_THRESHOLD = 3 diff --git a/src/codebase_index/mcp/server.py b/src/codebase_index/mcp/server.py index fae85f5..fe8866d 100644 --- a/src/codebase_index/mcp/server.py +++ b/src/codebase_index/mcp/server.py @@ -48,7 +48,11 @@ "Local codebase index. Use search_code for general queries, find_symbol for exact " "symbol lookups, find_refs to find callers/usages, impact_of for blast-radius analysis, " "explain_code for architecture/how-it-works questions, and architecture_overview to map " - "the codebase's modules, god nodes, and surprising connections before diving in." + "the codebase's modules, god nodes, and surprising connections before diving in. " + "Pass the same `session` tag on search_code/explain_code calls made from one context " + "so unchanged evidence you already received is not resent; use a new tag after your " + "context is cleared or compacted. verify_evidence checks whether evidence you hold or " + "cite is still true in the working tree." ), ) @@ -120,6 +124,9 @@ def healthcheck() -> str: "path": str(db_path), **compute_freshness(db.conn, Path(cfg.root), cfg).model_dump(), } + from ..service import memory_status_payload + + payload["memory"] = memory_status_payload(cfg) return _emit("healthcheck", payload) @@ -131,6 +138,7 @@ def search_code( token_budget: int = 1500, offset: int = 0, raw: bool = False, + session: Optional[str] = None, ) -> str: """Hybrid search over the codebase index. @@ -148,6 +156,11 @@ def search_code( offset: Result offset for pagination. Pass ``next_offset`` from a previous response to fetch the next page. raw: If true, return full raw snippets instead of skeletons. + session: Optional tag naming this one agent context. A result whose snippet this + session already received from unchanged source comes back with + ``snippet: null, reused: true``; evidence it received that has since + changed is listed under ``memory.invalidated``. Never share a tag with + another context; use a new tag after the context is cleared or compacted. """ db_path, cfg = _resolve_db() if not db_path.exists(): @@ -155,11 +168,14 @@ def search_code( from ..service import search_payload - payload = search_payload( - db_path, cfg, query, mode=mode, limit=limit, offset=offset, - token_budget=token_budget, no_fallback=False, backend=_search_backend(cfg), - raw=raw, - ) + try: + payload = search_payload( + db_path, cfg, query, mode=mode, limit=limit, offset=offset, + token_budget=token_budget, no_fallback=False, backend=_search_backend(cfg), + raw=raw, session=session, + ) + except ValueError as exc: + payload = {"error": str(exc)} return _emit("search_code", payload) @@ -279,6 +295,7 @@ def explain_code( token_budget: int = 2200, offset: int = 0, raw: bool = False, + session: Optional[str] = None, ) -> str: """Intent-aware retrieval for architecture / how-does-X-work questions. @@ -291,6 +308,7 @@ def explain_code( offset: Result offset for pagination. Pass ``next_offset`` from a previous response to fetch the next page. raw: If true, return full raw snippets instead of skeletons. + session: Optional tag naming this one agent context (see search_code). """ db_path, cfg = _resolve_db() if not db_path.exists(): @@ -298,14 +316,45 @@ def explain_code( from ..service import normalize_explain_query, search_payload - payload = search_payload( - db_path, cfg, normalize_explain_query(query), mode="hybrid", limit=10, - offset=offset, token_budget=token_budget, no_fallback=False, - backend=_search_backend(cfg), raw=raw, - ) + try: + payload = search_payload( + db_path, cfg, normalize_explain_query(query), mode="hybrid", limit=10, + offset=offset, token_budget=token_budget, no_fallback=False, + backend=_search_backend(cfg), raw=raw, session=session, + ) + except ValueError as exc: + payload = {"error": str(exc)} return _emit("explain_code", payload) +@_tool() +def verify_evidence(refs: Optional[list[str]] = None, session: Optional[str] = None) -> str: + """Check whether evidence is still true in the current working tree (read-only). + + Use before relying on something read earlier — especially after edits, a branch + switch, or when resuming from notes that cite evidence. Works without an index. + + Args: + refs: Evidence references ``path:start-end@hash``, e.g. from ``memory.invalidated`` + or from notes that cite evidence. + session: Also verify everything this session tag was given. + + Each verdict has ``state``: valid, relocated (identical content moved within the + file) — both still true — or changed, ambiguous, deleted, excluded, unreadable. + ``all_valid`` is true only when every checked piece of evidence holds. + """ + _, cfg = _resolve_db() + from ..service import verify_payload + + if not refs and not session: + return _emit("verify_evidence", {"error": "pass refs, session, or both"}) + try: + payload = verify_payload(cfg, refs or [], session) + except ValueError as exc: + payload = {"error": str(exc)} + return _emit("verify_evidence", payload) + + @_tool() def architecture_overview() -> str: """High-level map of the codebase from the cached graph analytics. @@ -373,8 +422,8 @@ def describe_symbol(symbol: str) -> str: @_tool() def index_stats() -> str: - """Return index freshness, file count, symbol count, and per-language coverage.""" - db_path, _ = _resolve_db() + """Return index freshness, file count, symbol count, per-language coverage, and memory.""" + db_path, cfg = _resolve_db() if not db_path.exists(): return _emit("index_stats", {"exists": False, "error": "No index found."}) @@ -382,7 +431,7 @@ def index_stats() -> str: from ..storage.db import Database with Database(db_path) as db: - payload = stats_payload(db.conn) + payload = stats_payload(db.conn, cfg=cfg) return _emit("index_stats", payload) diff --git a/src/codebase_index/memory/__init__.py b/src/codebase_index/memory/__init__.py new file mode 100644 index 0000000..b1400ce --- /dev/null +++ b/src/codebase_index/memory/__init__.py @@ -0,0 +1,5 @@ +"""Evidence memory: verifiable references to repository evidence and session reuse. + +See docs/MEMORY.md. Nothing in this package stores source text; identity is a hash of +the exact bytes a span held, and validity is always re-checked against the working tree. +""" diff --git a/src/codebase_index/memory/identity.py b/src/codebase_index/memory/identity.py new file mode 100644 index 0000000..ba97633 --- /dev/null +++ b/src/codebase_index/memory/identity.py @@ -0,0 +1,181 @@ +"""Evidence identity: the line model, span hashing, and verifiable references. + +Pure functions only — no filesystem, no database — so every identity property can be +tested exhaustively and cannot depend on I/O timing. + +The line model deliberately mirrors the indexer. It reads files with universal newlines +and slices chunks with ``str.splitlines()`` (parsers/line_chunker.py, +parsers/symbol_chunks.py), so a snippet and the span hashed for it are the same lines. +Two things differ on purpose: + +* bytes are decoded with ``surrogateescape`` instead of ``ignore``, so editing an + undecodable byte still changes the hash; +* only line terminators are normalised. Whitespace, comments and case are part of the + identity: a strict hash can never equate two different programs. +""" + +from __future__ import annotations + +import hashlib +import re +import sys +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Optional, Sequence + +REF_HASH_CHARS = 16 +"""Hex characters of the span hash shown in a reference (64 bits).""" + +MIN_REF_HASH_CHARS = 12 + +_REF_RE = re.compile(r"^(?P.+):(?P\d+)-(?P\d+)@(?P[0-9a-fA-F]+)$") +_DRIVE_RE = re.compile(r"^[A-Za-z]:") + + +def split_lines(raw: bytes) -> list[str]: + """Split file bytes into lines exactly as the chunkers number them.""" + text = raw.decode("utf-8", "surrogateescape") + if "\r" in text: + text = text.replace("\r\n", "\n").replace("\r", "\n") + return text.splitlines() + + +def _encode(text: str) -> bytes: + return text.encode("utf-8", "surrogateescape") + + +def sha_hex(text: str) -> str: + return hashlib.sha256(_encode(text)).hexdigest() + + +def span_text(lines: Sequence[str], line_start: int, line_end: int) -> Optional[str]: + """Text of 1-based inclusive lines, or None when the range is not inside the file.""" + if line_start < 1 or line_end < line_start or line_end > len(lines): + return None + return "\n".join(lines[line_start - 1 : line_end]) + + +def span_sha(lines: Sequence[str], line_start: int, line_end: int) -> Optional[str]: + text = span_text(lines, line_start, line_end) + return None if text is None else sha_hex(text) + + +def line_sha(line: str) -> str: + """Short hash of one line; used only to find relocation candidates, never as proof.""" + return sha_hex(line)[:REF_HASH_CHARS] + + +def visible_text(text: str) -> str: + """The text the indexer stored for these lines (undecodable bytes dropped).""" + return _encode(text).decode("utf-8", "ignore") + + +def content_matches(index_content: Optional[str], span: str) -> bool: + """Does text taken from the index still describe these working-tree lines? + + Chunk results store the whole span, so the common case is equality. Symbol and doc + results store an excerpt (a signature, a heading), which must occur inside the span. + Either way the snippet delivered to the agent is text that exists in the current + file at that locator, which is all an evidence atom asserts. + """ + if not index_content: + return False + visible = visible_text(span) + content = index_content.replace("\r\n", "\n") + return content == visible or content in visible + + +def is_full_span(index_content: Optional[str], span: str) -> bool: + """True when the index text is the entire span rather than an excerpt of it.""" + if not index_content: + return False + return index_content.replace("\r\n", "\n") == visible_text(span) + + +def normalize_rel_path(path: str) -> str: + """Canonical repo-relative POSIX path, or ValueError for anything that could escape. + + References arrive from agents, notes and other tools, so they are untrusted input: an + absolute path, a drive letter, a ``..`` segment or a NUL byte is rejected before any + filesystem access is attempted. + """ + if not path or "\x00" in path: + raise ValueError("evidence path is empty or contains NUL") + posix = path.replace("\\", "/") + if posix.startswith("/") or _DRIVE_RE.match(posix): + raise ValueError(f"evidence path must be repository-relative: {path!r}") + parts = [p for p in PurePosixPath(posix).parts if p not in ("", ".")] + if not parts or any(p == ".." for p in parts): + raise ValueError(f"evidence path must stay inside the repository: {path!r}") + return "/".join(parts) + + +@dataclass(frozen=True) +class EvidenceRef: + """``:-@`` — a citable, re-verifiable piece of evidence. + + The line range is the locator at the time the evidence was observed; identity is the + path plus the span hash. ``sha`` may be a prefix (from a printed reference) or a full + 256-bit digest (from the store). + """ + + path: str + line_start: int + line_end: int + sha: str + + @property + def line_count(self) -> int: + return self.line_end - self.line_start + 1 + + def matches(self, full_sha: Optional[str]) -> bool: + return bool(full_sha) and str(full_sha).startswith(self.sha) + + def __str__(self) -> str: + return f"{self.path}:{self.line_start}-{self.line_end}@{self.sha[:REF_HASH_CHARS]}" + + +def make_ref(path: str, line_start: int, line_end: int, full_sha: str) -> EvidenceRef: + return EvidenceRef(normalize_rel_path(path), int(line_start), int(line_end), full_sha.lower()) + + +def parse_ref(text: str) -> EvidenceRef: + """Parse a printed reference. Paths may contain ``:`` or ``@``; parsing is right-anchored.""" + match = _REF_RE.match(text.strip()) + if match is None: + raise ValueError(f"not an evidence reference (expected path:start-end@hash): {text!r}") + start, end = int(match["start"]), int(match["end"]) + if start < 1 or end < start: + raise ValueError(f"invalid line range in evidence reference: {text!r}") + sha = match["sha"].lower() + if not MIN_REF_HASH_CHARS <= len(sha) <= 64: + raise ValueError( + f"evidence hash must be {MIN_REF_HASH_CHARS}-64 hex characters: {text!r}" + ) + return EvidenceRef(normalize_rel_path(match["path"]), start, end, sha) + + +def repo_id_for(root: Path | str) -> str: + """Scope key for one checkout. Case-folded on Windows, where paths are case-insensitive.""" + canonical = Path(root).resolve().as_posix() + if sys.platform == "win32": + canonical = canonical.casefold() + return hashlib.sha256(f"repo:{canonical}".encode("utf-8")).hexdigest() + + +def session_key(repo_id: str, tag: str) -> str: + """Stored form of a caller's session tag; the tag itself is never persisted.""" + return hashlib.sha256(f"session:{repo_id}:{tag}".encode("utf-8")).hexdigest() + + +_SESSION_TAG_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$") + + +def validate_session_tag(tag: str) -> str: + tag = tag.strip() + if not _SESSION_TAG_RE.match(tag): + raise ValueError( + "session tag must be 1-128 characters of letters, digits, '.', '_', ':' or '-', " + "starting with a letter or digit" + ) + return tag diff --git a/src/codebase_index/memory/session.py b/src/codebase_index/memory/session.py new file mode 100644 index 0000000..7691d2a --- /dev/null +++ b/src/codebase_index/memory/session.py @@ -0,0 +1,230 @@ +"""Evidence at retrieval time: verify each delivered snippet, reuse it within a session. + +`retrieval.pipeline.search` calls the processor after ranking, budgeting and pagination +are final (docs/MEMORY.md). It never adds, removes or reorders a result, and it never +changes which results carry a snippet. It can only: + +* mark a result ``stale`` when the index text it came from no longer matches the file; +* in a session, replace a snippet that session already received from byte-identical + source with ``snippet: null, reused: true``; +* in a session, list evidence the session received earlier that has since changed. + +A session tag names exactly one agent context. It is always supplied by the caller — +never inferred from a process, an environment variable or a time window — because none +of those identify what an agent still holds in its context window. +""" + +from __future__ import annotations + +import sqlite3 +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Any, Callable, Iterator, Optional, Sequence, Union + +from ..config import Config +from ..discovery.gates import PathGate +from . import identity as ident +from .store import MemoryStore, MemoryUnavailable, NewDelivery, utc_now +from .validate import WorkingTree, validate + + +@dataclass +class Session: + tag: str + repo_id: str + store: Optional[MemoryStore] = None + session_id: Optional[int] = None + unavailable: Optional[str] = None + + @property + def usable(self) -> bool: + return self.store is not None and self.session_id is not None and not self.unavailable + + +IndexSha = Callable[[str], Optional[str]] +_STALE = "stale" + + +def index_sha_lookup(conn: sqlite3.Connection) -> IndexSha: + """``files.sha256`` by path, cached for one call: the fingerprint the index was built from.""" + from ..storage import repo + + cache: dict[str, Optional[str]] = {} + + def lookup(rel: str) -> Optional[str]: + if rel not in cache: + row = repo.get_file(conn, rel) + cache[rel] = row["sha256"] if row else None + return cache[rel] + + return lookup + + +class EvidenceProcessor: + def __init__(self, *, root: Path, config: Config, now: datetime, + session: Optional[Session] = None, + index_sha: Optional[IndexSha] = None) -> None: + self.tree = WorkingTree(PathGate(root, config)) + self.config = config + self.now = now + self.session = session + self.index_sha = index_sha + + def __call__(self, payload: dict, candidates: Sequence[Any]) -> None: + session = self.session + notices = self._notices(session) if session is not None and session.usable else [] + reused = tokens_saved = tokens_delivered = 0 + fresh: list[NewDelivery] = [] + + for result, candidate in zip(payload.get("results", []), candidates): + snippet = result.get("snippet") + if not snippet: + continue # nothing was delivered for this result + observed = self._observe(result, candidate) + if isinstance(observed, str): # _STALE + result["stale"] = True + continue + if observed is None or session is None or not session.usable: + continue # derived index text is accurate but not byte-verifiable: never withheld + rel, span, span_sha = observed + tokens = int(result.get("token_est") or 0) + snippet_sha = ident.sha_hex(snippet) + if self._known(session, rel, span_sha, snippet_sha): + result["snippet"] = None + result["reused"] = True + reused += 1 + tokens_saved += tokens + continue + fresh.append( + NewDelivery( + path=rel, + span_sha=span_sha, + line_count=int(result["line_end"]) - int(result["line_start"]) + 1, + first_line_sha=ident.line_sha(span.split("\n", 1)[0]), + snippet_sha=snippet_sha, + full=not result.get("skeletonized") + and ident.is_full_span(getattr(candidate, "content", None), span), + line_start=int(result["line_start"]), + line_end=int(result["line_end"]), + token_est=tokens, + ) + ) + tokens_delivered += tokens + + if session is not None: + payload["memory"] = self._finish( + session, fresh, notices, + reused=reused, tokens_saved=tokens_saved, tokens_delivered=tokens_delivered, + ) + + def _observe(self, result: dict, candidate: Any) -> Union[tuple[str, str, str], str, None]: + """Classify one delivered snippet against the working tree. + + * ``(path, span text, span sha)`` — the snippet is text these exact lines hold now; + * ``"stale"`` — the file is gone, excluded, or its bytes differ from what was indexed; + * ``None`` — the index is current for this file but its text was derived (config-key + and section summaries), so it cannot be checked byte-for-byte. + """ + try: + rel = ident.normalize_rel_path(str(result["path"])) + except ValueError: + return _STALE + view, _state, _reason = self.tree.view(rel) + if view is None: + return _STALE + span = ident.span_text(view.lines, int(result["line_start"]), int(result["line_end"])) + if span is not None and ident.content_matches(getattr(candidate, "content", None), span): + return rel, span, ident.sha_hex(span) + if self.index_sha is None: + return None + return None if self.index_sha(rel) == view.sha256 else _STALE + + def _known(self, session: Session, rel: str, span_sha: str, snippet_sha: str) -> bool: + assert session.store is not None and session.session_id is not None + try: + return session.store.is_known(session.session_id, rel, span_sha, snippet_sha) + except sqlite3.Error as exc: + session.unavailable = f"memory store unavailable: {exc}" + return False + + def _notices(self, session: Session) -> list[dict]: + assert session.store is not None and session.session_id is not None + try: + pending = session.store.pending(session.session_id) + except sqlite3.Error as exc: + session.unavailable = f"memory store unavailable: {exc}" + return [] + notices: list[dict] = [] + states: list[tuple[int, str]] = [] + for delivery in pending: + ref = ident.EvidenceRef(delivery.path, delivery.line_start, delivery.line_end, + delivery.span_sha) + verdict = validate(ref, self.tree, first_line_sha=delivery.first_line_sha) + if not verdict.valid: + states.append((delivery.atom_id, verdict.state)) + notices.append({"ref": str(ref), "state": verdict.state}) + if states: + try: + session.store.mark_invalid(session.session_id, states) + except (MemoryUnavailable, sqlite3.Error) as exc: + session.unavailable = str(exc) + return notices + + def _finish(self, session: Session, fresh: list[NewDelivery], notices: list[dict], *, + reused: int, tokens_saved: int, tokens_delivered: int) -> dict: + block: dict = { + "session": session.tag, + "reused": reused, + "tokens_saved": tokens_saved, + "invalidated": notices, + } + if not session.usable: + block["available"] = False + block["reason"] = session.unavailable or "memory store unavailable" + return block + store = session.store + assert store is not None and session.session_id is not None + try: + store.record(session.repo_id, session.session_id, fresh, now=self.now) + store.add_counters( + session.session_id, tokens_delivered=tokens_delivered, + tokens_saved=tokens_saved, reused=reused, invalidations=len(notices), + ) + if store.gc_due(now=self.now): + store.gc(now=self.now, retention_days=self.config.memory.retention_days, + max_deliveries=self.config.memory.max_deliveries) + except (MemoryUnavailable, sqlite3.Error) as exc: + # The packet is already correct; only future withholding is affected. + block["degraded"] = str(exc) + return block + + +@contextmanager +def open_evidence(*, root: Path, config: Config, memory_path: Path, tag: Optional[str], + now: Optional[datetime] = None, + index_conn: Optional[sqlite3.Connection] = None) -> Iterator[EvidenceProcessor]: + """Processor for one retrieval call; opens the store only when a session is named.""" + now = now or utc_now() + session: Optional[Session] = None + store: Optional[MemoryStore] = None + if tag is not None: + repo_id = ident.repo_id_for(root) + session = Session(tag=tag, repo_id=repo_id) + try: + store = MemoryStore.open(memory_path) + session.store = store + session.session_id = store.touch_session( + repo_id, ident.session_key(repo_id, tag), now=now + ) + except (MemoryUnavailable, sqlite3.Error) as exc: + session.unavailable = str(exc) + try: + yield EvidenceProcessor( + root=Path(root), config=config, now=now, session=session, + index_sha=index_sha_lookup(index_conn) if index_conn is not None else None, + ) + finally: + if store is not None: + store.close() diff --git a/src/codebase_index/memory/store.py b/src/codebase_index/memory/store.py new file mode 100644 index 0000000..aa81aa7 --- /dev/null +++ b/src/codebase_index/memory/store.py @@ -0,0 +1,435 @@ +"""Persistent evidence memory: ``memory.sqlite``, separate from the index. + +Separate because ``index --rebuild``, schema-triggered rebuilds and ``clean`` delete +``index.sqlite``, and because ledger writes at search time must not queue behind a long +``update`` transaction on the same WAL. + +The store is content-free: paths, line numbers, hashes, token counts and timestamps. +No source text, query, prompt or session tag is ever written. +""" + +from __future__ import annotations + +import sqlite3 +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Callable, Iterator, Optional, Sequence + +SCHEMA_VERSION = 1 +BUSY_TIMEOUT_MS = 2000 + + +class MemoryUnavailable(RuntimeError): + """Memory cannot serve this call. Retrieval continues without withholding anything.""" + + +def utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def iso(moment: datetime) -> str: + return moment.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +_V1 = ( + "CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)", + """CREATE TABLE IF NOT EXISTS atoms ( + id INTEGER PRIMARY KEY, + repo_id TEXT NOT NULL, + path TEXT NOT NULL, + span_sha TEXT NOT NULL, + line_count INTEGER NOT NULL, + first_line_sha TEXT NOT NULL, + first_seen_at TEXT NOT NULL, + UNIQUE (repo_id, path, span_sha) + )""", + """CREATE TABLE IF NOT EXISTS sessions ( + id INTEGER PRIMARY KEY, + repo_id TEXT NOT NULL, + tag_sha TEXT NOT NULL, + created_at TEXT NOT NULL, + last_used_at TEXT NOT NULL, + calls INTEGER NOT NULL DEFAULT 0, + tokens_delivered INTEGER NOT NULL DEFAULT 0, + tokens_saved INTEGER NOT NULL DEFAULT 0, + reused INTEGER NOT NULL DEFAULT 0, + invalidations INTEGER NOT NULL DEFAULT 0, + UNIQUE (repo_id, tag_sha) + )""", + """CREATE TABLE IF NOT EXISTS deliveries ( + session_id INTEGER NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, + atom_id INTEGER NOT NULL REFERENCES atoms(id) ON DELETE CASCADE, + snippet_sha TEXT NOT NULL, + full INTEGER NOT NULL, + line_start INTEGER NOT NULL, + line_end INTEGER NOT NULL, + token_est INTEGER NOT NULL, + delivered_at TEXT NOT NULL, + invalid_state TEXT, + PRIMARY KEY (session_id, atom_id, snippet_sha) + )""", + "CREATE INDEX IF NOT EXISTS idx_deliveries_atom ON deliveries(atom_id)", + "CREATE INDEX IF NOT EXISTS idx_sessions_used ON sessions(last_used_at)", +) + +# version -> statements that bring a store from version-1 to version. Applied in order, +# each inside one transaction together with the version bump. +MIGRATIONS: dict[int, Sequence[str]] = {1: _V1} + + +@dataclass(frozen=True) +class NewDelivery: + path: str + span_sha: str + line_count: int + first_line_sha: str + snippet_sha: str + full: bool + line_start: int + line_end: int + token_est: int + + +@dataclass(frozen=True) +class Delivery: + atom_id: int + path: str + span_sha: str + line_count: int + first_line_sha: str + line_start: int + line_end: int + + +class MemoryStore: + def __init__(self, conn: sqlite3.Connection, path: Path, recovered_from: Optional[str]): + self.conn = conn + self.path = path + self.recovered_from = recovered_from + + # -- lifecycle ------------------------------------------------------------------- + @classmethod + def open(cls, path: Path | str, *, migrations: Optional[dict[int, Sequence[str]]] = None, + schema_version: int = SCHEMA_VERSION, + busy_timeout_ms: Optional[int] = None) -> "MemoryStore": + """Open (creating or migrating) a store, or raise ``MemoryUnavailable``. + + A file SQLite cannot read as a database is moved aside to + ``memory.sqlite.corrupt-`` — preserved, never deleted — and a fresh + store is created. A lock or I/O failure is *not* corruption and is reported as + unavailable without touching the file. + """ + path = Path(path) + migrations = MIGRATIONS if migrations is None else migrations + if busy_timeout_ms is None: + busy_timeout_ms = BUSY_TIMEOUT_MS + try: + path.parent.mkdir(parents=True, exist_ok=True) + except OSError as exc: + raise MemoryUnavailable(f"cannot create memory directory: {exc}") from exc + recovered: Optional[str] = None + try: + conn = cls._connect_and_migrate(path, migrations, schema_version, busy_timeout_ms) + except sqlite3.OperationalError as exc: + raise MemoryUnavailable(f"memory store unavailable: {exc}") from exc + except sqlite3.DatabaseError: + recovered = _quarantine(path) + try: + conn = cls._connect_and_migrate(path, migrations, schema_version, busy_timeout_ms) + except sqlite3.Error as exc: + raise MemoryUnavailable(f"memory store unavailable after recovery: {exc}") from exc + return cls(conn, path, recovered) + + @staticmethod + def _connect_and_migrate(path: Path, migrations: dict[int, Sequence[str]], + schema_version: int, busy_timeout_ms: int) -> sqlite3.Connection: + conn = sqlite3.connect(path, isolation_level=None, timeout=busy_timeout_ms / 1000) + try: + conn.execute(f"PRAGMA busy_timeout = {int(busy_timeout_ms)}") + conn.execute("PRAGMA journal_mode = WAL") + conn.execute("PRAGMA synchronous = NORMAL") + conn.execute("PRAGMA foreign_keys = ON") + current = _stored_version(conn) + if current > schema_version: + raise MemoryUnavailable( + f"memory store schema {current} is newer than this version supports " + f"({schema_version}); upgrade codebase-index or run `memory clear`" + ) + for version in range(current + 1, schema_version + 1): + with _transaction(conn): + for statement in migrations[version]: + conn.execute(statement) + conn.execute( + "INSERT INTO meta(key, value) VALUES ('schema_version', ?) " + "ON CONFLICT(key) DO UPDATE SET value = excluded.value", + (str(version),), + ) + except BaseException: + conn.close() + raise + return conn + + def close(self) -> None: + self.conn.close() + + def __enter__(self) -> "MemoryStore": + return self + + def __exit__(self, *exc: object) -> None: + self.close() + + @property + def schema_version(self) -> int: + return _stored_version(self.conn) + + @contextmanager + def _write(self) -> Iterator[sqlite3.Connection]: + try: + with _transaction(self.conn): + yield self.conn + except sqlite3.OperationalError as exc: + raise MemoryUnavailable(f"memory store busy: {exc}") from exc + + # -- sessions -------------------------------------------------------------------- + def touch_session(self, repo_id: str, tag_sha: str, *, now: datetime) -> int: + stamp = iso(now) + with self._write() as conn: + conn.execute( + "INSERT INTO sessions(repo_id, tag_sha, created_at, last_used_at, calls) " + "VALUES (?, ?, ?, ?, 1) ON CONFLICT(repo_id, tag_sha) DO UPDATE SET " + "last_used_at = excluded.last_used_at, calls = calls + 1", + (repo_id, tag_sha, stamp, stamp), + ) + row = conn.execute( + "SELECT id FROM sessions WHERE repo_id = ? AND tag_sha = ?", (repo_id, tag_sha) + ).fetchone() + return int(row[0]) + + def find_session(self, repo_id: str, tag_sha: str) -> Optional[int]: + row = self.conn.execute( + "SELECT id FROM sessions WHERE repo_id = ? AND tag_sha = ?", (repo_id, tag_sha) + ).fetchone() + return int(row[0]) if row else None + + def is_known(self, session_id: int, path: str, span_sha: str, snippet_sha: str) -> bool: + """Was exactly this text — or the whole unchanged span — delivered to this session? + + Deliveries later found invalid never count, even if the bytes come back: the + agent was told that evidence changed, so it gets the text again. + """ + row = self.conn.execute( + "SELECT 1 FROM deliveries d JOIN atoms a ON a.id = d.atom_id " + "WHERE d.session_id = ? AND a.path = ? AND a.span_sha = ? " + "AND d.invalid_state IS NULL AND (d.full = 1 OR d.snippet_sha = ?) LIMIT 1", + (session_id, path, span_sha, snippet_sha), + ).fetchone() + return row is not None + + def pending(self, session_id: int) -> list[Delivery]: + """Distinct evidence delivered to this session and not yet reported invalid.""" + return self._deliveries(session_id, "AND d.invalid_state IS NULL") + + def delivered(self, session_id: int) -> list[Delivery]: + """Every distinct piece of evidence delivered to this session, valid or not.""" + return self._deliveries(session_id, "") + + def _deliveries(self, session_id: int, condition: str) -> list[Delivery]: + rows = self.conn.execute( + "SELECT a.id, a.path, a.span_sha, a.line_count, a.first_line_sha, " + " MIN(d.line_start), MIN(d.line_end) " + "FROM deliveries d JOIN atoms a ON a.id = d.atom_id " + f"WHERE d.session_id = ? {condition} " + "GROUP BY a.id ORDER BY a.path, MIN(d.line_start)", + (session_id,), + ).fetchall() + return [Delivery(int(r[0]), r[1], r[2], int(r[3]), r[4], int(r[5]), int(r[6])) + for r in rows] + + def record(self, repo_id: str, session_id: int, items: Sequence[NewDelivery], *, + now: datetime) -> None: + if not items: + return + stamp = iso(now) + with self._write() as conn: + for item in items: + conn.execute( + "INSERT INTO atoms(repo_id, path, span_sha, line_count, first_line_sha, " + "first_seen_at) VALUES (?, ?, ?, ?, ?, ?) " + "ON CONFLICT(repo_id, path, span_sha) DO NOTHING", + (repo_id, item.path, item.span_sha, item.line_count, + item.first_line_sha, stamp), + ) + atom_id = conn.execute( + "SELECT id FROM atoms WHERE repo_id = ? AND path = ? AND span_sha = ?", + (repo_id, item.path, item.span_sha), + ).fetchone()[0] + conn.execute( + "INSERT INTO deliveries(session_id, atom_id, snippet_sha, full, line_start, " + "line_end, token_est, delivered_at, invalid_state) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL) " + "ON CONFLICT(session_id, atom_id, snippet_sha) DO UPDATE SET " + "full = MAX(full, excluded.full), line_start = excluded.line_start, " + "line_end = excluded.line_end, token_est = excluded.token_est, " + "delivered_at = excluded.delivered_at, invalid_state = NULL", + (session_id, atom_id, item.snippet_sha, int(item.full), item.line_start, + item.line_end, item.token_est, stamp), + ) + + def mark_invalid(self, session_id: int, states: Sequence[tuple[int, str]]) -> None: + if not states: + return + with self._write() as conn: + conn.executemany( + "UPDATE deliveries SET invalid_state = ? WHERE session_id = ? AND atom_id = ?", + [(state, session_id, atom_id) for atom_id, state in states], + ) + + def add_counters(self, session_id: int, *, tokens_delivered: int, tokens_saved: int, + reused: int, invalidations: int) -> None: + with self._write() as conn: + conn.execute( + "UPDATE sessions SET tokens_delivered = tokens_delivered + ?, " + "tokens_saved = tokens_saved + ?, reused = reused + ?, " + "invalidations = invalidations + ? WHERE id = ?", + (tokens_delivered, tokens_saved, reused, invalidations, session_id), + ) + + def first_line_hint(self, repo_id: str, path: str, sha_prefix: str) -> Optional[str]: + """Relocation hint for a printed reference, when this store has seen the evidence.""" + rows = self.conn.execute( + "SELECT first_line_sha FROM atoms WHERE repo_id = ? AND path = ? " + "AND span_sha >= ? AND span_sha < ? LIMIT 2", + (repo_id, path, sha_prefix, sha_prefix + "g"), + ).fetchall() + return rows[0][0] if len(rows) == 1 else None + + # -- observability and maintenance ----------------------------------------------- + def stats(self, repo_id: str) -> dict: + one = self.conn.execute + sessions = one( + "SELECT COUNT(*), COALESCE(SUM(tokens_delivered),0), COALESCE(SUM(tokens_saved),0), " + "COALESCE(SUM(reused),0), COALESCE(SUM(invalidations),0), MAX(last_used_at) " + "FROM sessions WHERE repo_id = ?", (repo_id,), + ).fetchone() + atoms = one("SELECT COUNT(*) FROM atoms WHERE repo_id = ?", (repo_id,)).fetchone()[0] + deliveries = one( + "SELECT COUNT(*) FROM deliveries d JOIN sessions s ON s.id = d.session_id " + "WHERE s.repo_id = ?", (repo_id,), + ).fetchone()[0] + last_gc = one("SELECT value FROM meta WHERE key = 'last_gc_at'").fetchone() + return { + "schema_version": self.schema_version, + "sessions": int(sessions[0]), + "atoms": int(atoms), + "deliveries": int(deliveries), + "tokens_delivered": int(sessions[1]), + "tokens_saved": int(sessions[2]), + "reused": int(sessions[3]), + "invalidations": int(sessions[4]), + "last_used_at": sessions[5], + "last_gc_at": last_gc[0] if last_gc else None, + "bytes": _file_bytes(self.path), + } + + def gc_due(self, *, now: datetime, every: timedelta = timedelta(days=1)) -> bool: + row = self.conn.execute("SELECT value FROM meta WHERE key = 'last_gc_at'").fetchone() + return row is None or row[0] < iso(now - every) + + def gc(self, *, now: datetime, retention_days: int, max_deliveries: int) -> dict: + """Delete expired sessions, sessions beyond the size cap, and orphaned atoms. + + Only rows are removed. Validity is recomputed from the working tree on every + check, so GC can make memory withhold less, never make it withhold wrongly. + """ + cutoff = iso(now - timedelta(days=max(0, retention_days))) + with self._write() as conn: + expired = conn.execute( + "DELETE FROM sessions WHERE last_used_at < ?", (cutoff,) + ).rowcount + total = conn.execute("SELECT COUNT(*) FROM deliveries").fetchone()[0] + capped = 0 + if total > max_deliveries: + for session_id, count in conn.execute( + "SELECT s.id, COUNT(d.atom_id) FROM sessions s " + "LEFT JOIN deliveries d ON d.session_id = s.id " + "GROUP BY s.id ORDER BY s.last_used_at, s.id" + ).fetchall(): + if total <= max_deliveries: + break + conn.execute("DELETE FROM sessions WHERE id = ?", (session_id,)) + total -= count + capped += 1 + orphans = conn.execute( + "DELETE FROM atoms WHERE id NOT IN (SELECT DISTINCT atom_id FROM deliveries)" + ).rowcount + conn.execute( + "INSERT INTO meta(key, value) VALUES ('last_gc_at', ?) " + "ON CONFLICT(key) DO UPDATE SET value = excluded.value", + (iso(now),), + ) + return {"expired_sessions": max(0, expired), "capped_sessions": capped, + "orphan_atoms": max(0, orphans)} + + def clear(self, repo_id: str, tag_sha: Optional[str] = None) -> int: + with self._write() as conn: + if tag_sha is None: + removed = conn.execute( + "DELETE FROM sessions WHERE repo_id = ?", (repo_id,) + ).rowcount + conn.execute("DELETE FROM atoms WHERE repo_id = ?", (repo_id,)) + else: + removed = conn.execute( + "DELETE FROM sessions WHERE repo_id = ? AND tag_sha = ?", (repo_id, tag_sha) + ).rowcount + conn.execute( + "DELETE FROM atoms WHERE id NOT IN (SELECT DISTINCT atom_id FROM deliveries)" + ) + return max(0, removed) + + def vacuum(self) -> None: + try: + self.conn.execute("VACUUM") + except sqlite3.OperationalError as exc: + raise MemoryUnavailable(f"memory store busy: {exc}") from exc + + +@contextmanager +def _transaction(conn: sqlite3.Connection) -> Iterator[None]: + conn.execute("BEGIN IMMEDIATE") + try: + yield + except BaseException: + conn.execute("ROLLBACK") + raise + conn.execute("COMMIT") + + +def _stored_version(conn: sqlite3.Connection) -> int: + has_meta = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'meta'" + ).fetchone() + if not has_meta: + return 0 + row = conn.execute("SELECT value FROM meta WHERE key = 'schema_version'").fetchone() + return int(row[0]) if row else 0 + + +def _file_bytes(path: Path) -> int: + return sum( + p.stat().st_size + for p in (path, path.with_name(path.name + "-wal"), path.with_name(path.name + "-shm")) + if p.exists() + ) + + +def _quarantine(path: Path, clock: Callable[[], datetime] = utc_now) -> str: + """Move an unreadable store (and its WAL sidecars) aside; return the new name.""" + suffix = clock().strftime("%Y%m%dT%H%M%SZ") + target = path.with_name(f"{path.name}.corrupt-{suffix}") + for extra in ("", "-wal", "-shm"): + src = path.with_name(path.name + extra) + if src.exists(): + src.replace(target.with_name(target.name + extra)) + return target.name diff --git a/src/codebase_index/memory/validate.py b/src/codebase_index/memory/validate.py new file mode 100644 index 0000000..9b5a10f --- /dev/null +++ b/src/codebase_index/memory/validate.py @@ -0,0 +1,150 @@ +"""Validate evidence references against the current working tree. + +Validity is byte identity with the working tree — never a commit, a timestamp, or a +similarity score. Every uncertain outcome is reported as invalid: a false invalidation +costs the agent one re-read, a false validation costs it a wrong answer. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from typing import Optional + +from ..discovery.gates import PathGate +from . import identity as ident + +VALID_STATES = frozenset({"valid", "relocated"}) + +MAX_SCAN_STEPS = 200_000 +"""Upper bound on line-slices hashed while searching a file for moved content.""" + + +@dataclass(frozen=True) +class Verdict: + ref: ident.EvidenceRef + state: str + line_start: Optional[int] = None + line_end: Optional[int] = None + reason: str = "" + + @property + def valid(self) -> bool: + return self.state in VALID_STATES + + def as_dict(self) -> dict: + return { + "ref": str(self.ref), + "state": self.state, + "valid": self.valid, + "line_start": self.line_start, + "line_end": self.line_end, + "reason": self.reason, + } + + +class FileView: + """Lines of one working-tree file; per-line hashes are computed only if needed. + + ``sha256`` is the digest of the raw bytes — the same fingerprint the indexer stores in + ``files.sha256`` — so a caller can tell "the index is older than this file" apart from + "the index text was derived rather than copied". + """ + + def __init__(self, lines: list[str], sha256: Optional[str] = None) -> None: + self.lines = lines + self.sha256 = sha256 + self._line_shas: Optional[list[str]] = None + + def line_shas(self) -> list[str]: + if self._line_shas is None: + self._line_shas = [ident.line_sha(line) for line in self.lines] + return self._line_shas + + def sha(self, line_start: int, line_end: int) -> Optional[str]: + return ident.span_sha(self.lines, line_start, line_end) + + +def locate( + view: FileView, + sha: str, + line_count: int, + *, + first_line_sha: Optional[str] = None, + limit: int = 2, + max_steps: Optional[int] = None, +) -> Optional[list[int]]: + """1-based start lines where a span with this hash occurs (at most ``limit``). + + Returns ``None`` when the bounded search gave up, which callers must treat as "not + found". With ``first_line_sha`` only lines that hash like the span's first line are + tried, which makes relocation cheap for stored evidence. + """ + if max_steps is None: + max_steps = MAX_SCAN_STEPS + total = len(view.lines) + if line_count < 1 or line_count > total: + return [] + starts: list[int] | range = range(total - line_count + 1) + if first_line_sha is not None: + shas = view.line_shas() + starts = [i for i in starts if shas[i] == first_line_sha] + if len(starts) * line_count > max_steps: + return None + found: list[int] = [] + for i in starts: + if ident.sha_hex("\n".join(view.lines[i : i + line_count])).startswith(sha): + found.append(i + 1) + if len(found) >= limit: + break + return found + + +class WorkingTree: + """Gated, cached file access for one validation batch (one CLI/MCP call).""" + + def __init__(self, gate: PathGate) -> None: + self.gate = gate + self._views: dict[str, tuple[Optional[FileView], str, str]] = {} + + def view(self, rel: str) -> tuple[Optional[FileView], str, str]: + cached = self._views.get(rel) + if cached is None: + result = self.gate.read(rel) + if result.state == "ok": + view = FileView(ident.split_lines(result.data), + hashlib.sha256(result.data).hexdigest()) + cached = (view, "ok", "") + else: + cached = (None, result.state, result.reason) + self._views[rel] = cached + return cached + + +def validate( + ref: ident.EvidenceRef, + tree: WorkingTree, + *, + first_line_sha: Optional[str] = None, +) -> Verdict: + view, state, reason = tree.view(ref.path) + if view is None: + return Verdict(ref, state, reason=reason) + if ref.matches(view.sha(ref.line_start, ref.line_end)): + return Verdict(ref, "valid", ref.line_start, ref.line_end, "unchanged") + starts = locate(view, ref.sha, ref.line_count, first_line_sha=first_line_sha) + if starts is None: + return Verdict( + ref, "changed", + reason="differs at the recorded lines; file too large to search for moved content", + ) + if len(starts) == 1: + start = starts[0] + end = start + ref.line_count - 1 + return Verdict(ref, "relocated", start, end, f"identical content now at lines {start}-{end}") + if len(starts) > 1: + return Verdict( + ref, "ambiguous", + reason="identical content occurs more than once in the file; cannot identify it", + ) + return Verdict(ref, "changed", reason="content no longer occurs in the file") diff --git a/src/codebase_index/output/markdown.py b/src/codebase_index/output/markdown.py index 2994e1e..6914e34 100644 --- a/src/codebase_index/output/markdown.py +++ b/src/codebase_index/output/markdown.py @@ -30,8 +30,12 @@ def _render_dict(payload: dict) -> str: ) lines.append("") for r in payload["results"]: - if r.get("snippet"): - lines.append(f"`{r['path']}:{r['line_start']}-{r['line_end']}`") + location = f"`{r['path']}:{r['line_start']}-{r['line_end']}`" + if r.get("reused"): + lines.append(f"{location} — unchanged; already delivered in this session") + elif r.get("snippet"): + stale = " — index is older than the file; read the range" if r.get("stale") else "" + lines.append(f"{location}{stale}") lines.append("```") lines.append(r["snippet"]) lines.append("```") @@ -41,6 +45,9 @@ def _render_dict(payload: dict) -> str: for rr in payload["recommended_reads"]: lines.append(f"- `{rr['path']}:{rr['line_start']}-{rr['line_end']}`") + if payload.get("memory"): + lines.extend(_render_memory(payload["memory"])) + fb = payload.get("fallback_suggestions", {}).get("ripgrep") if fb: lines.append("\n**Fallback (low confidence) — try:**") @@ -58,6 +65,49 @@ def _render_dict(payload: dict) -> str: return "\n".join(lines) +def _render_memory(memory: dict) -> list[str]: + session = memory.get("session") + if memory.get("available") is False: + return [f"\n**Memory** (session `{session}`): unavailable — {memory.get('reason')}; " + "nothing was withheld."] + out = [f"\n**Memory** (session `{session}`): {memory.get('reused', 0)} snippet(s) not " + f"resent ({memory.get('tokens_saved', 0)} tokens)."] + invalidated = memory.get("invalidated") or [] + if invalidated: + out.append("**Changed since this session received it — do not rely on the earlier text:**") + out.extend(f"- `{n['ref']}` — {n['state']}" for n in invalidated) + return out + + +def render_verify(payload: dict) -> str: + """Render evidence verdicts: one row per reference, state first.""" + if payload.get("error"): + return f"_{payload['error']}_\n" + verdict = ("all evidence is still valid" if payload.get("all_valid") + else "some evidence is not valid") + lines = [f"**verify:** {verdict}"] + session = payload.get("session") + if session: + name = f"session `{session['session']}`" + if session.get("available") is False: + lines.append(f"{name}: unavailable — {session.get('reason')}") + elif not session.get("found"): + lines.append(f"{name}: no evidence recorded") + else: + lines.append(f"{name}: {session.get('evidence', 0)} piece(s) of evidence") + lines.append("") + rows = payload.get("evidence", []) + if rows: + lines.append("| state | evidence | lines now | reason |") + lines.append("|---|---|---|---|") + for v in rows: + now = f"{v['line_start']}-{v['line_end']}" if v.get("line_start") else "—" + lines.append(f"| {v['state'].upper()} | `{v['ref']}` | {now} | {v.get('reason', '')} |") + for err in payload.get("errors", []): + lines.append(f"- not a valid reference `{err['ref']}`: {err['error']}") + return "\n".join(lines).rstrip() + "\n" + + def _render_search_response(resp: SearchResponse) -> str: lines: list[str] = [] freshness = "fresh" if not resp.index.stale else "STALE" diff --git a/src/codebase_index/retrieval/diversity.py b/src/codebase_index/retrieval/diversity.py index d94be5d..0619fae 100644 --- a/src/codebase_index/retrieval/diversity.py +++ b/src/codebase_index/retrieval/diversity.py @@ -18,6 +18,26 @@ _MASK64 = (1 << 64) - 1 +# Leading characters of a candidate body that decide near-duplication. See +# `deduplicate` for why bounding this is safe and what it buys. +_FINGERPRINT_CHARS = 2000 + +# Multi-character operators kept as one token, grouped by length so a match can be +# found with a slice and a set lookup instead of a linear scan. Probed longest-first. +_OPERATORS_BY_LEN: tuple[tuple[int, frozenset[str]], ...] = ( + (4, frozenset({">>>="})), + (3, frozenset({"===", "!==", "**=", "..."})), + ( + 2, + frozenset( + { + "=>", "->", "::", "==", "!=", "<=", ">=", "&&", "||", "++", + "--", "+=", "-=", "*=", "/=", "%=", "<<", ">>", "**", "??", + } + ), + ), +) + def normalize_code_tokens(content: str | None) -> tuple[str, ...]: """Return normalized code tokens, omitting whitespace and comments.""" @@ -27,10 +47,15 @@ def normalize_code_tokens(content: str | None) -> tuple[str, ...]: tokens: list[str] = [] length = len(content) index = 0 - operators = ( - "===", "!==", ">>>=", "**=", "...", "=>", "->", "::", "==", "!=", "<=", ">=", - "&&", "||", "++", "--", "+=", "-=", "*=", "/=", "%=", "<<", ">>", "**", "??", - ) + # `operators` used to be scanned with `next(c for c in operators if + # content.startswith(c, index))`, i.e. up to 24 `str.startswith` calls for every + # punctuation character in every candidate chunk. On the Java corpus that single + # generator was 2.1s of the 5.9s spent tokenising. `_OPERATORS_BY_LEN` replaces it + # with at most three slice-and-set-lookup probes, longest first. + # + # This is bit-identical, not merely equivalent-looking: every 3- and 4-character + # operator appeared before every 2-character one in the old tuple, so first-match + # and longest-match already agreed, and the emitted token stream is unchanged. while index < length: char = content[index] @@ -81,10 +106,11 @@ def normalize_code_tokens(content: str | None) -> tuple[str, ...]: index = cursor continue - operator = next((candidate for candidate in operators if content.startswith(candidate, index)), None) - if operator is not None: - tokens.append(operator) - index += len(operator) + for width, group in _OPERATORS_BY_LEN: + if content[index : index + width] in group: + tokens.append(content[index : index + width]) + index += width + break else: tokens.append(char) index += 1 @@ -171,7 +197,17 @@ def deduplicate(candidates: Sequence[Candidate], hamming_distance: int = 3) -> l # Only the fingerprint is needed here; building the token set as well # doubled the per-candidate cost of the pipeline's hottest stage. "No # fingerprint" means "no tokens", never "fingerprint happened to be 0". - tokens = normalize_code_tokens(candidate.content) + # + # Only the leading _FINGERPRINT_CHARS decide duplication. Tokenising whole + # chunk bodies made this the single most expensive stage of the query path + # (42% of it on the Java corpus) to answer a question the first ~50 lines + # already answer: two chunks that agree for 2000 characters are the same + # snippet. Measured over 420 queries on eight repositories the bound leaves + # recall, duplicate rate and useful-context identical and MRR within + # -0.0003 (p=0.51), for -3ms p50. The retained candidate keeps its full + # content; only the comparison window is bounded. + content = candidate.content + tokens = normalize_code_tokens(content[:_FINGERPRINT_CHARS] if content else None) fingerprint = token_fingerprint(tokens) if tokens else None if fingerprint is None: representatives.append(candidate) diff --git a/src/codebase_index/retrieval/features.py b/src/codebase_index/retrieval/features.py new file mode 100644 index 0000000..6aa9a7b --- /dev/null +++ b/src/codebase_index/retrieval/features.py @@ -0,0 +1,119 @@ +"""Deterministic query<->candidate ranking features. + +The fused RRF score answers "how many retrievers liked this candidate, and how +much". It cannot answer "does this candidate's *name* actually talk about the +thing the query asked for", because every retriever scores each query term +independently and RRF only ever sums those independent verdicts. + +That gap is a measured failure mode, not a hypothetical one. Two examples from +the benchmark, both ranked wrong by 1.9.0: + + "graph resolution + traversal accessors" + won by graph/retrieval.py (`graph` — one term) + beating test_graph_accessors_... (`graph` + `accessors` + `resolve`) + + "greedy token budgeting with redaction" + won by output/redact.py (`redact` — one term) + beating retrieval/budget.py (`budget` + `token`) + +The winner in each pair matched *one* query term very well. The correct answer +matched *several* terms in a single name. Summing per-term evidence cannot tell +those apart, so the signal has to be an interaction: credit terms for occurring +*together* in one name, over and above their individual matches. + +Design constraints, all of which the eval harness enforces: + + * Bounded. Cost is O(len(path) + len(symbol) + len(terms)) per candidate, with + no corpus statistics, no posting-list scan, and no second query. + * Deterministic. Set membership over identifier components; no floating-point + accumulation order dependence, no randomness, no I/O. + * Explainable. The score reduces to a count the reranker can print verbatim. + +Feature variants that were measured and rejected are recorded in +`docs/RETRIEVAL.md`; the short version is that idf weighting (pool-local or +corpus-wide), substring matching, proximity, ordered-subsequence matching, and +body-text coverage all failed to beat this one feature on held-out repositories. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from .lexical import salient_terms, split_identifier + + +def _components(text: str) -> set[str]: + """Lowercase identifier components of *text* (camelCase and snake_case aware). + + `split_identifier` is applied to the whole string rather than to regex-matched + words first: it already treats `_`, `-` and every non-alphanumeric character as + a separator, and it decides "alphanumeric" with `str.isalnum`. Pre-splitting on + an ASCII word pattern would silently reduce `расчёт_налога` to nothing, so + non-ASCII identifiers could never earn this bonus while the query side — which + parses Unicode correctly — happily produced the matching terms. + """ + return set(split_identifier(text)) + + +def name_zone(path: str, symbol: str | None) -> set[str]: + """Identifier components of a candidate's *name*: file basename + symbol. + + Directories are excluded deliberately. A path prefix such as + ``src/main/java/net/denfry/newtowny/`` is shared by hundreds of files, so its + components carry no evidence about which of them answers the query while + adding co-occurrence noise to all of them. Measured on eight repositories, + including the parent directory was worth -0.001 MRR: indistinguishable from + noise, for strictly more code. + """ + basename = path.replace("\\", "/").rsplit("/", 1)[-1] + stem = basename[: basename.rfind(".")] if "." in basename else basename + zone = _components(stem) + if symbol: + zone |= _components(symbol) + return zone + + +@dataclass(frozen=True, slots=True) +class QueryProfile: + """Query terms worth matching against a candidate name, computed once per query. + + Built from :func:`lexical.salient_terms`, so stopword policy stays in one + place and the reranker cannot drift from the retrievers' idea of a term. + """ + + terms: tuple[str, ...] + + @property + def n_terms(self) -> int: + return len(self.terms) + + +def query_profile(query: str) -> QueryProfile: + return QueryProfile(terms=salient_terms(query)) + + +def name_cooccurrence(profile: QueryProfile, zone: set[str]) -> tuple[int, float]: + """Return (terms matched in *zone*, co-occurrence score in [0, 1]). + + The score credits only the query terms *beyond the first*: + + score = max(0, matched - 1) / (n_terms - 1) + + A candidate whose name matches one query term scores 0 — it has shown no + interaction, and its single match is already fully paid for by the retriever + that surfaced it. The second matched term is what distinguishes + ``test_graph_accessors_resolve_and_walk`` from ``graph_candidates`` on the + query "graph resolution + traversal accessors", so that is where the credit + starts. Normalising by ``n_terms - 1`` keeps the feature comparable between a + two-term and a nine-term query. + + Single-term queries return 0.0: there is no co-occurrence to observe, and the + exact-name machinery in the symbol retriever already handles them. + """ + n = profile.n_terms + if n < 2 or not zone: + return 0, 0.0 + matched = sum(1 for term in profile.terms if term in zone) + if matched < 2: + return matched, 0.0 + return matched, (matched - 1) / (n - 1) diff --git a/src/codebase_index/retrieval/pipeline.py b/src/codebase_index/retrieval/pipeline.py index 56c9e70..11f7bde 100644 --- a/src/codebase_index/retrieval/pipeline.py +++ b/src/codebase_index/retrieval/pipeline.py @@ -9,7 +9,7 @@ import re import sqlite3 from pathlib import Path -from typing import Optional +from typing import Callable, Optional from ..config import Config from ..indexer.freshness import compute_freshness @@ -144,6 +144,27 @@ def _fallback_suggestions(query, ranked) -> dict: return {"ripgrep": rg} +def _pool_entry(rank: int, c) -> dict: + """One pre-rerank candidate, as the ranking-diagnostics view of it. + + Recorded before rerank mutates `score`/`reason` in place, so the fused order + stays comparable with the final order. Only built when `explain` is set: the + default query path allocates nothing. + """ + return { + "rank": rank, + "path": c.path, + "line_start": c.line_start, + "line_end": c.line_end, + "source": c.source, + "symbol": c.symbol, + "kind": c.kind, + "fused_score": round(c.score, 4), + "agreeing_sources": c.agreeing_sources, + "exact_symbol": c.exact_symbol, + } + + def _bounded_read(entry: dict, max_lines: int) -> dict: """Cap one read-plan entry to `max_lines`, keeping the original span visible. @@ -179,6 +200,8 @@ def search( offset: int = 0, compact: bool = True, compact_min_reduction: float = 0.25, + explain: bool = False, + evidence: Optional[Callable[[dict, list], None]] = None, max_read_lines: int = 120, ) -> dict: tuning = tuning or DEFAULT_TUNING @@ -211,6 +234,7 @@ def search( k=tuning.rrf_k, file_agreement=tuning.file_agreement_weight if tuning.file_agreement else 0.0, ) + pool = [_pool_entry(rank, c) for rank, c in enumerate(fused, start=1)] if explain else [] ranked = rerank(fused, query=query, intent=plan.intent, tuning=tuning) if tuning.dedup: ranked = deduplicate(ranked, hamming_distance=tuning.dedup_hamming) @@ -276,4 +300,30 @@ def search( "has_more": has_more, "next_offset": offset + limit if has_more else None, } + if explain: + payload["diagnostics"] = { + "weights": plan.weights, + "pool_size": len(pool), + "pool": pool, + "ranked": [ + { + "rank": rank, + "path": c.path, + "line_start": c.line_start, + "line_end": c.line_end, + "source": c.source, + "symbol": c.symbol, + "score": round(c.score, 4), + "reason": c.reason, + "agreeing_sources": c.agreeing_sources, + "exact_symbol": c.exact_symbol, + } + for rank, c in enumerate(ranked, start=1) + ], + } + if evidence is not None: + # Evidence memory (memory/session.py) sees the delivered page and the candidates + # behind it only after ranking, budgeting and pagination are final, so it cannot + # change which results are returned or which carry snippets. + evidence(payload, ranked[offset:offset + limit]) return payload diff --git a/src/codebase_index/retrieval/rerank.py b/src/codebase_index/retrieval/rerank.py index 4b976d8..464bb8c 100644 --- a/src/codebase_index/retrieval/rerank.py +++ b/src/codebase_index/retrieval/rerank.py @@ -11,6 +11,7 @@ import re from ..discovery.classify import is_test_path +from .features import name_cooccurrence, name_zone, query_profile from .priors import source_role_prior from .tuning import DEFAULT_TUNING, RetrievalTuning from .types import Candidate, Intent @@ -35,9 +36,33 @@ def rerank( tuning: RetrievalTuning = DEFAULT_TUNING, ) -> list[Candidate]: terms = {t.lower() for t in _TERM_RE.findall(query)} + wants_tests = "test" in terms or "tests" in terms + profile = query_profile(query) if tuning.name_cooccurrence else None for c in candidates: bonus = 0.0 reasons: list[str] = [] + # Computed before the bonuses because the name-co-occurrence signal is + # conditioned on it, not merely penalised after the fact. + demoted = c.is_generated or (is_test_path(c.path) and not wants_tests) + + # Name co-occurrence is *discounted*, not withheld, for sources the ranker + # is unwilling to promote. Test symbol names are descriptive sentences + # (`test_compactor_output_is_redacted`), so they harvest query-term + # co-occurrences that real identifiers never do, and a bonus reaching + # +`name_cooccurrence_weight` is not counterbalanced by a flat -0.15 + # demotion calibrated when the largest name bonus was +0.05. Withholding it + # outright over-corrects: on git-derived ground truth the changed file often + # *is* the test, and zeroing the bonus cost -0.017 MRR across eight + # repositories. The discount keeps a strongly-matching test ahead of a + # barely-matching implementation while restoring the intended role ordering + # when both match comparably. + if profile is not None: + matched, cooccurrence = name_cooccurrence(profile, name_zone(c.path, c.symbol)) + if cooccurrence: + if demoted: + cooccurrence *= tuning.name_cooccurrence_demoted_scale + bonus += tuning.name_cooccurrence_weight * cooccurrence + reasons.append(f"{matched}/{profile.n_terms} query terms co-occur in name") if c.source == "symbol" and c.kind in {"function", "method", "class", "interface", "type"}: bonus += 0.05 @@ -71,8 +96,7 @@ def rerank( bonus += prior reasons.append(f"source prior {prior:+.2f}") - wants_tests = "test" in terms or "tests" in terms - if c.is_generated or (is_test_path(c.path) and not wants_tests): + if demoted: bonus -= 0.15 reasons.append("generated/test demoted") diff --git a/src/codebase_index/retrieval/tuning.py b/src/codebase_index/retrieval/tuning.py index 9898d6e..6d47543 100644 --- a/src/codebase_index/retrieval/tuning.py +++ b/src/codebase_index/retrieval/tuning.py @@ -44,7 +44,21 @@ class RetrievalTuning: query_expansion: bool = True """Down-weighted code-synonym expansion (auth->authentication, ...). Original - terms always keep a strictly higher weight so precision is preserved.""" + terms always keep a strictly higher weight so precision is preserved. + + 1.10.0 tried to delete this. On the 420-query git-derived benchmark the + vocabulary is worth nothing measurable — MRR -0.0022 (p=0.40) when removed — + and the flag's apparent +0.006 turned out to come from it also swapping the + symbol retriever's tokenizer, not from the synonyms. + + It survives because the git benchmark cannot see what it does. A commit + subject is written by someone looking at the identifiers they just changed, + so it reuses the codebase's own spelling; a user asking a question does not. + On the 36 hand-written natural-language queries — the only set that phrases + things the way a person would — removing the vocabulary cost -0.060 MRR, + because "where are secrets redacted" has to reach `redact_snippet` and + "how does authentication work" has to reach `auth/`. Both benchmark families + are needed to make this call, and only one of them can measure this.""" graph_source: bool = False """Personalized-PageRank candidate source seeded from lexical/symbol hits. @@ -106,9 +120,62 @@ class RetrievalTuning: """Discount applied to same-file, different-locator evidence. Tuned on 305 queries over three repositories; the 0.3-0.6 plateau peaks here.""" + name_cooccurrence: bool = True + """Reward query terms that co-occur in one candidate *name* (file basename + + symbol), superlinearly, over and above their independent per-term matches. + + RRF sums independent per-term verdicts, so it cannot distinguish a candidate + that matched one query term very well from one that matched three terms in a + single name. Measured over eight repositories and 420 queries, that confusion + was the largest single reranking loss: the correct answer sat in the candidate + pool with a better name-level match and lost to a one-term winner. + + See `retrieval/features.py` for the feature and the variants it beat.""" + + name_cooccurrence_weight: float = 1.8 + """Bonus at full co-occurrence (every query term present in one name). + + Chosen on a plateau, not at a peak. Pooled MRR rises to w≈1.8 and then flattens + (0.599 at 1.8, 0.603 at 4.0, saturating at 0.603 beyond); past 1.8 the extra + movement is churn, with per-query wins flat and losses nearly doubling + (37W/19L at 1.8 against 39W/32L at 4.0), because a larger bonus turns the + feature into the primary sort key and reduces fusion to a tiebreak. Every + value in 1.0-4.0 leaves all eight corpora at or above 1.9.0; 1.8 is the + interior of that region with the strongest significance (p=0.004).""" + + name_cooccurrence_demoted_scale: float = 0.5 + """Fraction of the co-occurrence bonus granted to test/generated sources. + + 1.0 lets descriptive test function names outrank implementations on questions + like "where are secrets redacted before output"; 0.0 over-corrects, since on + git-derived ground truth the changed file often *is* the test. + + Decided by splitting the benchmark on whether its own ground truth is a test: + across the 261 queries whose answer is *not* a test, the gain is flat at + +0.020 MRR for every scale, so the whole aggregate difference between 0.5 and + 1.0 comes from the 159 test-answer queries — an artifact of mining ground + truth from commits, which touch tests. 0.5 is the only setting that improves + both partitions (+0.015 test-answer, +0.020 implementation-answer).""" + + # --- selection, continued ----------------------------------------------- + max_per_file: int = 1 + """Hits from one file kept in place before the rest are pushed to the tail. + + The agent's unit of decision is "which file do I open", so a page of 10 + results that spends three slots on three regions of one file offers seven + choices, not ten. Measured across eight repositories the page held 7.1 + distinct files on average, and 45 of 57 queries whose answer was in the + candidate pool but absent from the page had it sitting past rank 10. + + Monotone over 1-5 (1 > 2 > 3 > 4 > 5), so this is a plateau boundary rather + than a fitted peak: recall@10 +0.045 and nDCG@10 +0.015 against 3, at + unchanged token cost and with no metric or corpus regressing. Nothing is + dropped — overflow hits keep their relative order at the tail — so a file + with several relevant regions still surfaces them below the first page. + """ + # --- fixed parameters ---------------------------------------------------- rrf_k: int = 60 - max_per_file: int = 3 graph_damping: float = 0.85 graph_iterations: int = 12 graph_weight: float = 0.1 @@ -127,10 +194,25 @@ def baseline(cls) -> RetrievalTuning: dedup=False, source_priors=False, file_agreement=False, - # 1.7.0 had no over-fetch: the pool was exactly the requested page. + name_cooccurrence=False, + # 1.7.0 had no over-fetch: the pool was exactly the requested page, + # and it kept up to three hits from any one file. candidate_pool_multiplier=1, + max_per_file=3, ) + @classmethod + def v190(cls) -> RetrievalTuning: + """The 1.9.0 shipped configuration, as the immutable "before" column. + + `baseline()` reaches back to 1.7.0 and answers "was any of this worth it"; + this answers the narrower question every 1.10.0 change is judged on: is it + better than the release it replaces. Both must keep working, so a later + default change can never silently redefine its own comparison point. + """ + return cls(name_cooccurrence=False, max_per_file=3) + + def without(self, flag: str) -> RetrievalTuning: """Return a copy with one boolean signal disabled (single-signal ablation).""" field_names = {f.name for f in fields(self)} diff --git a/src/codebase_index/scaffold.py b/src/codebase_index/scaffold.py index e49ea0b..72a65c4 100644 --- a/src/codebase_index/scaffold.py +++ b/src/codebase_index/scaffold.py @@ -166,10 +166,15 @@ def write_codex_agents(root: Path) -> Path: Skill resources: `{rel.as_posix()}` -Run `codebase-index search "" --json` for general questions, or use -`symbol`, `refs`, `impact`, and `graph` for symbol lookup, references, change -impact, and HTML graph export. Search/read commands auto-build the index when -it is missing; run `codebase-index update` when responses report stale data. +Run `codebase-index search "" --session --json` for general +questions, or use `symbol`, `refs`, `impact`, and `graph` for symbol lookup, +references, change impact, and HTML graph export. Use one session tag per +conversation: unchanged evidence you already received comes back as +`reused: true`, and evidence that changed is listed under `memory.invalidated`. +Before relying on something read earlier, run +`codebase-index verify --session --json`. Search/read commands auto-build +the index when it is missing; run `codebase-index update` when responses report +stale data. """ return _upsert_managed_block(root / "AGENTS.md", content) @@ -192,6 +197,9 @@ def write_opencode_files(root: Path) -> list[Path]: Use `symbol `, `refs `, or `impact ` when those match the request. If the index is missing, run `codebase-index index` first. + +In a multi-step task pass `--session ` (one tag per conversation) and check +evidence read earlier with `codebase-index verify --session --json`. """, encoding="utf-8", ) diff --git a/src/codebase_index/service.py b/src/codebase_index/service.py index 7dcfcb5..33f85d5 100644 --- a/src/codebase_index/service.py +++ b/src/codebase_index/service.py @@ -12,8 +12,9 @@ import os import sqlite3 import subprocess +from contextlib import contextmanager from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Optional, Union +from typing import TYPE_CHECKING, Any, Callable, Iterator, Optional, Sequence, Union if TYPE_CHECKING: from .config import Config @@ -77,34 +78,89 @@ def search_payload( no_fallback: bool = False, backend: Any = None, raw: bool = False, + session: Optional[str] = None, ) -> dict: """One search session: open the DB (vector-enabled when the backend is live), run retrieval, return the payload dict both surfaces serialize. ``raw`` forces full snippets; otherwise snippets are skeletonized when - ``cfg.retrieval.compact_snippets`` is on (the default).""" + ``cfg.retrieval.compact_snippets`` is on (the default). + + ``session`` names one agent context for evidence reuse (docs/MEMORY.md); + ``ValueError`` for a malformed tag. Without it the packet is the plain + retrieval packet, plus ``stale`` on any result whose index text no longer + matches the working tree.""" from .retrieval.pipeline import search as run_search from .storage.db import Database compact = cfg.retrieval.compact_snippets and not raw + tag = session_tag(session) + enabled = memory_enabled(cfg) with Database(db_path) as db: if backend is not None and getattr(backend, "enabled", False): db.enable_vectors() - return run_search( - db.conn, - query, - mode=mode, - limit=limit, - offset=offset, - token_budget=token_budget, - no_fallback=no_fallback, - backend=backend, - root=Path(cfg.root), - config=cfg, - compact=compact, - compact_min_reduction=cfg.retrieval.compact_min_reduction, - max_read_lines=cfg.retrieval.max_read_lines, - ) + with _evidence(cfg, tag, enabled, db.conn) as evidence: + payload = run_search( + db.conn, + query, + mode=mode, + limit=limit, + offset=offset, + token_budget=token_budget, + no_fallback=no_fallback, + backend=backend, + root=Path(cfg.root), + config=cfg, + compact=compact, + compact_min_reduction=cfg.retrieval.compact_min_reduction, + evidence=evidence, + max_read_lines=cfg.retrieval.max_read_lines, + ) + if tag is not None and not enabled: + payload["memory"] = {"session": tag, "available": False, "reason": "memory is disabled"} + return payload + + +def memory_enabled(cfg: "Config") -> bool: + """Evidence memory is on unless the config disables it or ``CBX_MEMORY=0``.""" + if os.environ.get("CBX_MEMORY", "").strip() == "0": + return False + return bool(cfg.memory.enabled) + + +def memory_path_for(cfg: "Config") -> Path: + """``CBX_MEMORY_PATH``, else next to a ``CBX_DB_PATH`` override, else the cache dir.""" + override = os.environ.get("CBX_MEMORY_PATH") + if override: + return Path(override) + db_override = os.environ.get("CBX_DB_PATH") + if db_override: + return Path(db_override).with_name("memory.sqlite") + return cache_dir_for(cfg) / "memory.sqlite" + + +def session_tag(session: Optional[str]) -> Optional[str]: + """Validated session tag, or None. Sessions are only ever named explicitly.""" + if session is None or not session.strip(): + return None + from .memory.identity import validate_session_tag + + return validate_session_tag(session) + + +@contextmanager +def _evidence(cfg: "Config", tag: Optional[str], enabled: bool, + conn: sqlite3.Connection) -> Iterator[Any]: + if not enabled: + yield None + return + from .memory.session import open_evidence + + with open_evidence( + root=Path(cfg.root), config=cfg, memory_path=memory_path_for(cfg), tag=tag, + index_conn=conn, + ) as processor: + yield processor def diff_impact_payload( @@ -269,7 +325,133 @@ def architecture_payload(db_path: Path, cfg: "Config") -> dict[str, Any]: return {"exists": True, "available": True, "index": fresh.model_dump(), **summary} -def stats_payload(conn: sqlite3.Connection) -> dict[str, Any]: +def verify_payload( + cfg: "Config", refs: Sequence[str], session: Optional[str] = None +) -> dict[str, Any]: + """Re-check evidence references and/or everything one session was given. + + Read-only: nothing is recorded, so it is safe to run at any time and from any agent. + References are untrusted input and are validated before any file is read. + """ + from .discovery.gates import PathGate + from .memory import identity as ident + from .memory.store import MemoryStore, MemoryUnavailable + from .memory.validate import WorkingTree, validate + + tag = session_tag(session) + root = Path(cfg.root) + repo_id = ident.repo_id_for(root) + tree = WorkingTree(PathGate(root, cfg)) + enabled = memory_enabled(cfg) + path = memory_path_for(cfg) + store: Optional[MemoryStore] = None + store_problem: Optional[str] = None + if enabled and path.exists(): + try: + store = MemoryStore.open(path) + except MemoryUnavailable as exc: + store_problem = str(exc) + + evidence: list[dict] = [] + errors: list[dict] = [] + session_block: Optional[dict[str, Any]] = None + try: + for text in refs: + try: + ref = ident.parse_ref(text) + except ValueError as exc: + errors.append({"ref": text, "error": str(exc)}) + continue + hint = store.first_line_hint(repo_id, ref.path, ref.sha) if store else None + evidence.append(validate(ref, tree, first_line_sha=hint).as_dict()) + if tag is not None: + session_block = {"session": tag} + session_id = None + if not enabled: + session_block.update(available=False, reason="memory is disabled") + elif store_problem is not None: + session_block.update(available=False, reason=store_problem) + elif store is not None: + session_id = store.find_session(repo_id, ident.session_key(repo_id, tag)) + session_block["found"] = session_id is not None + if store is not None and session_id is not None: + delivered = store.delivered(session_id) + session_block["evidence"] = len(delivered) + for item in delivered: + ref = ident.EvidenceRef(item.path, item.line_start, item.line_end, + item.span_sha) + evidence.append( + validate(ref, tree, first_line_sha=item.first_line_sha).as_dict()) + finally: + if store is not None: + store.close() + + summary: dict[str, int] = {} + for verdict in evidence: + summary[verdict["state"]] = summary.get(verdict["state"], 0) + 1 + payload: dict[str, Any] = { + "all_valid": bool(evidence) and not errors and all(v["valid"] for v in evidence), + "summary": summary, + } + if session_block is not None: + payload["session"] = session_block + payload["evidence"] = evidence + payload["errors"] = errors + return payload + + +def memory_status_payload(cfg: "Config") -> dict[str, Any]: + """Evidence-memory health and counters for this repository (no paths, no content).""" + from .memory.identity import repo_id_for + from .memory.store import MemoryStore, MemoryUnavailable + + path = memory_path_for(cfg) + block: dict[str, Any] = {"enabled": memory_enabled(cfg), "exists": path.exists()} + if not block["exists"]: + return block + try: + with MemoryStore.open(path) as store: + block.update(store.stats(repo_id_for(Path(cfg.root)))) + if store.recovered_from: + block["recovered_from"] = store.recovered_from + except MemoryUnavailable as exc: + block.update(available=False, reason=str(exc)) + return block + + +def memory_gc_payload(cfg: "Config") -> dict[str, Any]: + """Apply retention and size limits, drop orphan atoms, and compact the store.""" + from .memory.identity import repo_id_for + from .memory.store import MemoryStore, utc_now + + path = memory_path_for(cfg) + if not path.exists(): + return {"exists": False} + with MemoryStore.open(path) as store: + removed = store.gc(now=utc_now(), retention_days=cfg.memory.retention_days, + max_deliveries=cfg.memory.max_deliveries) + store.vacuum() + return {"exists": True, **removed, **store.stats(repo_id_for(Path(cfg.root)))} + + +def memory_clear_payload(cfg: "Config", session: Optional[str] = None) -> dict[str, Any]: + """Forget one session, or all evidence memory for this repository.""" + from .memory import identity as ident + from .memory.store import MemoryStore + + tag = session_tag(session) + path = memory_path_for(cfg) + if not path.exists(): + return {"exists": False, "removed_sessions": 0, "session": tag} + repo_id = ident.repo_id_for(Path(cfg.root)) + with MemoryStore.open(path) as store: + removed = store.clear(repo_id, ident.session_key(repo_id, tag) if tag else None) + if tag is None: + store.vacuum() + return {"exists": True, "removed_sessions": removed, "session": tag} + + +def stats_payload(conn: sqlite3.Connection, cfg: Optional["Config"] = None) -> dict[str, Any]: """Index size, freshness, and per-language coverage with the graph tier.""" from .parsers.languages import has_full_graph from .storage import repo @@ -285,7 +467,7 @@ def stats_payload(conn: sqlite3.Connection) -> dict[str, Any]: } for r in repo.treesitter_coverage(conn) ] - return { + payload: dict[str, Any] = { "files": repo.count_files(conn), "symbols": repo.count_symbols(conn), "built_at": repo.get_meta(conn, "built_at"), @@ -293,3 +475,6 @@ def stats_payload(conn: sqlite3.Connection) -> dict[str, Any]: "treesitter_coverage": coverage, "exists": True, } + if cfg is not None: + payload["memory"] = memory_status_payload(cfg) + return payload diff --git a/src/codebase_index/skill_template/SKILL.md b/src/codebase_index/skill_template/SKILL.md index 9404686..432005b 100644 --- a/src/codebase_index/skill_template/SKILL.md +++ b/src/codebase_index/skill_template/SKILL.md @@ -1,25 +1,26 @@ --- name: codebase-index -description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository. -allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob +description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository, and verify evidence gathered earlier before relying on it. +allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index verify *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob --- # Codebase Index Use the local index before reading repository files. -The operating principle is **Find → Trace → Predict**: +The operating principle is **Find → Trace → Verify → Predict**: - **Find** the implementation with ranked retrieval. - **Trace** behavior through definitions, callers, dependencies, and paths. +- **Verify** that evidence you already hold is still true before relying on it. - **Predict** change impact while preserving an explicit evidence trail. ## Route the question | Intent | Command | |---|---| -| Where is X implemented? | `codebase-index search "X" --json` | -| How does X work? | `codebase-index explain "X" --json` | +| Where is X implemented? | `codebase-index search "X" --session --json` | +| How does X work? | `codebase-index explain "X" --session --json` | | What is this codebase? | `codebase-index architecture --json` | | Find a named symbol | `codebase-index symbol "X" --json` | | Who calls or references X? | `codebase-index refs "X" --json` | @@ -27,6 +28,7 @@ The operating principle is **Find → Trace → Predict**: | What does my current diff affect? | `codebase-index diff-impact --json` | | How are X and Y connected? | `codebase-index path "X" "Y" --json` | | Describe X and its neighborhood | `codebase-index describe "X" --json` | +| Is what I read earlier still true? | `codebase-index verify --session --json` | | Produce a human graph | `codebase-index graph "X" --output ` | Use `search --mode symbol` for exact symbol work, `--mode fts` for text and @@ -38,21 +40,41 @@ or routing remain unclear. ## Evidence protocol -1. Run the best-matching command with `--json`. -2. Check `index` before trusting the payload: +1. Pick one session tag for this conversation (for example `auth-fix-1`) and + pass `--session ` to every `search` and `explain`. +2. Run the best-matching command with `--json`. +3. Check `index` before trusting the payload: - missing → run `codebase-index index`, then repeat; - stale with fewer than 20 changed files → run `codebase-index update`; - stale with 20 or more changed files → run `codebase-index index`; - fresh → continue. -3. Start with ranks 1–3. Read only `recommended_reads` line ranges. -4. Trace one additional hop only when the question requires behavior, +4. Start with ranks 1–3. Read only `recommended_reads` line ranges. +5. Trace one additional hop only when the question requires behavior, ownership, or impact. -5. Answer with `file:line` evidence and state uncertainty explicitly. +6. Before answering or editing from evidence gathered earlier in the task, run + `codebase-index verify --session --json` and reread anything whose + state is not `valid` or `relocated`. +7. Answer with `file:line` evidence and state uncertainty explicitly. Do not open whole files when a line range is available. A snippet may already be sufficient. `skeletonized: true` means the response intentionally folded unrelated body lines; read the supplied range when the missing body matters. +## Evidence memory + +- `reused: true` with `snippet: null` — this session already received that + exact text and its source is unchanged. Use your earlier copy; if you can no + longer see it, Read the range. +- `memory.invalidated` — evidence this session received has changed since. + Treat your earlier copy as wrong and reread before relying on it. +- `stale: true` — the index is older than the file. Run `codebase-index update` + or Read the range. +- A tag belongs to one context. Never give it to a subagent or another + conversation. Start a new tag after the context is cleared or compacted, or + whenever earlier snippets are no longer visible to you. + +Verdict states and citing evidence in notes: [references/memory.md](references/memory.md). + ## Confidence contract - **high** — answer from the indexed evidence. diff --git a/src/codebase_index/skill_template/references/commands.md b/src/codebase_index/skill_template/references/commands.md index 0efb464..79c91a6 100644 --- a/src/codebase_index/skill_template/references/commands.md +++ b/src/codebase_index/skill_template/references/commands.md @@ -17,6 +17,9 @@ Useful search options: - `--offset ` - `--raw` to disable snippet skeletonization - `--no-fallback` to suppress fallback suggestions +- `--session ` to name this conversation's context: unchanged evidence it + already received comes back as `reused: true` without the snippet, and + evidence that changed is listed under `memory.invalidated` `explain` uses the HOW_IT_WORKS intent and a larger default token budget. Prefer it over repeatedly rewording a broad search. @@ -49,6 +52,18 @@ codebase-index graph "" --direction both --depth 2 --output graph.html For headless work, use `--output`; do not use `--open`. Exports also support `--format graphml|dot|neo4j`. +## Evidence + +```bash +codebase-index verify --session --json +codebase-index verify "" ... --json +``` + +- `verify` is read-only and needs no index: it checks evidence against the + working tree. `all_valid` is true only when every checked span still holds. +- `--strict` exits 1 when anything is invalid (useful in scripts). +- See [memory.md](memory.md) for verdict states and when to reread. + ## Index health ```bash diff --git a/src/codebase_index/skill_template/references/memory.md b/src/codebase_index/skill_template/references/memory.md new file mode 100644 index 0000000..fa533ad --- /dev/null +++ b/src/codebase_index/skill_template/references/memory.md @@ -0,0 +1,49 @@ +# Evidence Memory + +Load this when you use `--session`, when a packet contains `reused`, `stale` or +`memory`, or when you want to cite evidence for later. + +## What is guaranteed + +Evidence is identified by the exact bytes of a file span. `reused: true` is set +only when this session already received the same text — or the whole span — +and those bytes are unchanged now. Nothing is decided by query similarity, and +memory stores no source text. + +## Verdict states + +| state | still true? | do this | +|---|---|---| +| `valid` | yes | use it | +| `relocated` | yes — identical content moved inside the file | use it; cite the new lines | +| `changed` | no | reread the range before relying on it | +| `ambiguous` | no — identical content now occurs more than once | reread | +| `deleted` | no — file gone; a rename or move counts as deleted | search again | +| `excluded` | no — now ignored or secret-like | do not read it | +| `unreadable` | no | search again | + +## Reread even when evidence is valid + +- You are about to edit and need the exact current text. +- The earlier snippet was a skeleton or a signature and you need the body. +- Your context was cleared or compacted, or you cannot see the earlier snippet. +- A tool output was truncated. + +## Citing evidence across conversations + +`codebase-index verify --session --json` lists everything the session +received as `path:start-end@hash` references. Keep the references next to the +conclusions they support in notes or handoffs: + +```text +Refunds are capped at the invoice total [billing/refund.py:3-4@3f9a2c1b7d4e8a90] +``` + +Later, from any agent, check them before trusting the note: + +```bash +codebase-index verify "billing/refund.py:3-4@3f9a2c1b7d4e8a90" --json +``` + +`all_valid: true` means every cited span still holds exactly. Otherwise reread +the invalid spans and re-derive the conclusion; do not patch the old one. diff --git a/src/codebase_index/skill_template/references/response-contract.md b/src/codebase_index/skill_template/references/response-contract.md index 798f152..98af67f 100644 --- a/src/codebase_index/skill_template/references/response-contract.md +++ b/src/codebase_index/skill_template/references/response-contract.md @@ -16,6 +16,9 @@ Each result can contain: - `snippet` - `skeletonized` - `elided_lines` +- `stale` — only when the index text no longer matches the file +- `reused` — only with `--session`, when `snippet` was withheld because this + session already received that exact text from unchanged source `recommended_reads` is the read plan. Start with its first one to three entries and use exact line ranges. An entry with `truncated: true` was capped at the @@ -23,6 +26,11 @@ definition head (`max_read_lines`, default 120); `line_end_full` gives the real extent. Read the capped range first and continue only when the head is not enough. +With `--session`, the packet also carries `memory`: `session`, `reused`, +`tokens_saved`, and `invalidated` — references to evidence the session received +that has since changed. `available: false` means nothing was withheld. See +[memory.md](memory.md). + `pagination.has_more` and `pagination.next_offset` indicate additional results. Prefer a more specific command or a larger token budget before paging. diff --git a/src/codebase_index/skill_template/scripts/cbx b/src/codebase_index/skill_template/scripts/cbx index 94a9114..18c3ed0 100644 --- a/src/codebase_index/skill_template/scripts/cbx +++ b/src/codebase_index/skill_template/scripts/cbx @@ -4,7 +4,7 @@ # - Whitelists subcommands so the skill can never invoke destructive ones (clean/init/watch). set -euo pipefail -ALLOWED="search explain architecture symbol refs impact diff-impact path describe graph stats doctor update index" +ALLOWED="search explain architecture symbol refs impact diff-impact path describe verify graph stats doctor update index" sub="${1:-}" case " $ALLOWED " in diff --git a/src/codebase_index/skill_template/scripts/cbx.ps1 b/src/codebase_index/skill_template/scripts/cbx.ps1 index eee56ae..6419e37 100644 --- a/src/codebase_index/skill_template/scripts/cbx.ps1 +++ b/src/codebase_index/skill_template/scripts/cbx.ps1 @@ -10,7 +10,7 @@ param( $ErrorActionPreference = "Stop" $allowed = @( "search", "explain", "architecture", "symbol", "refs", "impact", "diff-impact", - "path", "describe", "graph", "stats", "doctor", "update", "index" + "path", "describe", "verify", "graph", "stats", "doctor", "update", "index" ) if ($allowed -notcontains $Subcommand) { diff --git a/tests/eval/README.md b/tests/eval/README.md index 878d28c..923785c 100644 --- a/tests/eval/README.md +++ b/tests/eval/README.md @@ -62,17 +62,50 @@ Conventional Commits prefix is stripped; duplicate subjects collapse. Changelog- files are never accepted as answers because they paraphrase commit subjects, and any commit touching `tests/eval/` is dropped so the benchmark cannot grade itself. -Corpora used to validate the 1.9.0 ranking changes, beyond this repository: +Corpora used to validate the 1.10.0 ranking changes, beyond this repository: | Corpus | Language | Files | Queries | |---|---|---|---| | Civitas | Java | 944 | 64 | -| PoliternalSite | TypeScript / TSX | 443 | 118 | +| PoliternalSite | TypeScript / TSX | 450 | 120 | +| PoliternalParkour | Java | 96 | 64 | +| TerraForge | Java | 325 | 17 | +| denfry.github.io | TypeScript / TSX | 48 | 39 | +| DevGraph | Python | 59 | 12 | +| Windows-Cleaner-and-Optimizer | PowerShell | 39 | 17 | Those repositories are not vendored here — shipping someone else's source to run a benchmark is not reproducible either. The generator is the reproducible part: point it at any git repository and the protocol is identical. +## Two query families, and why both are load-bearing + +The git-derived sets are large and objective, but they are not a substitute for +human phrasing, and 1.10.0 turned up a case where the difference decided a release. + +A commit subject is written by someone looking at the identifiers they just +changed, so it reuses the codebase's own spelling. A user asking a question does +not: they type "where are secrets redacted", not "redact". So the git sets are +structurally blind to morphology. Removing the synonym vocabulary looked free on 420 +git-derived queries (MRR −0.0022, p=0.40) and cost −0.060 MRR on the 36 hand-written +ones. The vocabulary stayed. + +The reverse also holds. Ground truth mined from commits counts a *test* file as the +answer whenever the commit touched tests (159 of 420 queries here), which the +hand-written set never does. Any signal that rewards test files therefore looks +better than it is. `run_eval.py` reports both families; a change needs to survive +the large set and not visibly break the small one. + +## Held-out validation + +Hand-tuned coefficients are fitted parameters, so tuning them on the same corpora +that report the result is how a benchmark gets gamed by accident. Ranking parameters +are selected under **leave-one-repository-out**: choose on seven corpora, score on +the eighth, pool the eight held-out scores. A parameter is only shipped from a +*plateau* — a region where neighbouring values are statistically indistinguishable — +never from a single peak, and per-corpus scores are inspected so that no fold is +allowed to regress in exchange for a better aggregate. + ## What is measured Ranking quality is scored at **file** granularity, because the agent's unit of @@ -88,6 +121,24 @@ best rank. - `dup%` — fraction of returned results that near-duplicate an earlier result - `p50/p95/p99` latency, in-process, excluding interpreter start-up +### Oracle metrics: is this a recall problem or a ranking problem? + +MRR alone cannot separate "the retrievers never found it" from "the ranker had it +and buried it", yet those two failures share no fix. Each variant therefore also +reports, against the *pre-rerank candidate pool* it actually generated: + +- `oracle` — the MRR a perfect reranker would score over that pool. `1 - oracle` is + the share of the query set that only better recall can ever fix. +- `cand_recall` — mean fraction of expected files present anywhere in the pool. +- `eff` = `MRR / oracle` — the share of achievable ranking quality delivered. + +At 1.9.0 this read `oracle = 0.902`, `MRR = 0.577`, `eff = 0.639`: a third of the +answers already in the pool were ranked below something else, an error mode about +three times larger than the 0.098 of remaining recall headroom. That is why 1.10.0 +worked on the reranker rather than adding retrievers, and it is also the cheapest +available check that a "ranking" change did not quietly change recall instead — +`-name_cooccurrence` moves eight quality metrics and leaves `oracle` at ±0.0000. + ## Baselines on public repositories `run_eval.py` compares the ranker with itself. `run_baselines.py` compares it with diff --git a/tests/eval/harness.py b/tests/eval/harness.py index 7802301..98ad725 100644 --- a/tests/eval/harness.py +++ b/tests/eval/harness.py @@ -32,7 +32,7 @@ from codebase_index.retrieval.tuning import RetrievalTuning from codebase_index.storage.db import Database -from . import metrics +from . import gen_queries, metrics from .gen_queries import CHANGELOG_EXCLUDES QUERY_DIR = Path(__file__).parent / "queries" @@ -59,6 +59,12 @@ class QueryOutcome: duplicates: int = 0 """Results whose snippet near-duplicates an earlier result in the same page.""" n_results: int = 0 + pool_files: list[str] = field(default_factory=list) + """Distinct files in the pre-rerank candidate pool, in fused order. + + The oracle metrics are computed against this, so they measure exactly what the + ranker was handed — not what the retrievers could have found with other + settings.""" @dataclass @@ -85,6 +91,14 @@ class EvalReport: """Fraction of returned results that near-duplicate an earlier result.""" mean_candidates: float = 0.0 """Mean results returned per query, before the agent reads anything.""" + oracle_mrr: float = 0.0 + """Ceiling MRR a perfect reranker could reach over the pool actually generated. + + `1 - oracle_mrr` is the share of the query set no reranking can ever fix.""" + candidate_recall: float = 0.0 + """Mean fraction of expected files present anywhere in the candidate pool.""" + mean_pool: float = 0.0 + """Mean distinct files in the candidate pool, i.e. what the ranker chose from.""" per_category: dict[str, float] = field(default_factory=dict) per_query: dict[str, list[float]] = field(default_factory=dict) """Per-query metric vectors, in query order. Required for paired significance @@ -108,6 +122,10 @@ def as_row(self) -> dict[str, float | str | int]: "p50_ms": self.p50_ms, "p95_ms": self.p95_ms, "p99_ms": self.p99_ms, + "oracle": self.oracle_mrr, + "eff": metrics.rerank_efficiency(self.mrr, self.oracle_mrr), + "cand_recall": self.candidate_recall, + "pool": self.mean_pool, } @@ -146,7 +164,20 @@ def validate_queries(queries: Iterable[EvalQuery], root: Path) -> list[str]: # the corpus makes it the top lexical hit for almost every query — a measurement # artifact that depresses scores and hides real ranking behaviour. Benchmark # scaffolding is excluded from the corpus it grades. -CORPUS_EXCLUDES = ("tests/eval/**", "tests/benchmark_*", "tests/fixtures/expected_answers.yml") +# +# Changelog-like files are excluded for the mirror-image reason: a git-derived +# query *is* a commit subject, and a changelog entry paraphrases that subject +# verbatim while never being an accepted answer (`gen_queries._ANSWER_DENY_RE`). +# Indexing them plants an unbeatable distractor at rank 1 for a large share of +# queries, which compresses every variant's score toward the same floor and hides +# ranking differences. `gen_queries.CHANGELOG_EXCLUDES` documented this exclusion; +# it was never applied to the corpus. +CORPUS_EXCLUDES = ( + "tests/eval/**", + "tests/benchmark_*", + "tests/fixtures/expected_answers.yml", + *gen_queries.CHANGELOG_EXCLUDES, +) # Changelog-style files paraphrase commit subjects, so for a git-derived query set # they are the answer key in prose. They are never accepted as *answers* @@ -188,8 +219,18 @@ def run_query( token_budget=token_budget, no_fallback=True, tuning=tuning, + # The pre-rerank pool is what the oracle metrics are measured against. + # Building it costs ~30 dict literals per query, which is inside the noise + # of a 38ms query (measured: -0.8ms p50, i.e. unmeasurable), so it stays on + # for the timed call rather than forcing a second untimed pass. + explain=True, ) latency_ms = (time.perf_counter() - start) * 1000.0 + pool_files: list[str] = [] + for entry in payload.get("diagnostics", {}).get("pool", ()): + p = _normalise(entry["path"]) + if p not in pool_files: + pool_files.append(p) ranked_files: list[str] = [] returned: list[tuple[str, int]] = [] @@ -228,6 +269,7 @@ def run_query( total_tokens=total_tokens, duplicates=duplicates, n_results=len(results), + pool_files=pool_files, ) @@ -267,6 +309,15 @@ def evaluate( "useful@budget": lambda o: metrics.useful_context_at_budget( o.returned, o.query.expected_files, token_budget ), + # Scored per query so the oracle ceiling gets the same paired significance + # treatment as everything else: a ranking change that only moved the + # ceiling has not improved ranking. + "oracle": lambda o: metrics.oracle_reciprocal_rank( + o.pool_files, o.query.expected_files + ), + "cand_recall": lambda o: metrics.recall_at_k( + o.pool_files, o.query.expected_files, len(o.pool_files) + ), } per_query = {name: [fn(o) for o in outcomes] for name, fn in scorers.items()} @@ -301,6 +352,11 @@ def mean(name: str) -> float: ), duplicate_rate=(duplicate_total / returned_total) if returned_total else 0.0, mean_candidates=(returned_total / len(outcomes)) if outcomes else 0.0, + oracle_mrr=mean("oracle"), + candidate_recall=mean("cand_recall"), + mean_pool=( + statistics.fmean([len(o.pool_files) for o in outcomes]) if outcomes else 0.0 + ), per_category={ cat: statistics.fmean(vals) for cat, vals in sorted(per_category.items()) }, @@ -319,8 +375,8 @@ def format_table( ) -> str: """Render reports as a Markdown table, with deltas against `baseline`.""" cols = list(columns) if columns else [ - "label", "recall@5", "recall@10", "MRR", "nDCG@10", "hit@3", "P@5", - "MAP", "useful@budget", "tokens", "dup%", "p50_ms", "p95_ms", + "label", "recall@5", "recall@10", "MRR", "oracle", "eff", "nDCG@10", + "hit@3", "P@5", "MAP", "useful@budget", "tokens", "dup%", "p50_ms", "p95_ms", ] lines = ["| " + " | ".join(cols) + " |", "|" + "|".join("---" for _ in cols) + "|"] @@ -394,6 +450,9 @@ def weighted(attr: str) -> float: mean_tokens=weighted("mean_tokens"), duplicate_rate=weighted("duplicate_rate"), mean_candidates=weighted("mean_candidates"), + oracle_mrr=mean("oracle"), + candidate_recall=mean("cand_recall"), + mean_pool=weighted("mean_pool"), per_category={c: statistics.fmean(v) for c, v in sorted(per_category.items())}, per_query=per_query, ) diff --git a/tests/eval/memory_eval.py b/tests/eval/memory_eval.py new file mode 100644 index 0000000..b6d15e7 --- /dev/null +++ b/tests/eval/memory_eval.py @@ -0,0 +1,660 @@ +#!/usr/bin/env python3 +"""Sequential real-history benchmark for evidence memory. + +Replays a repository's own history. For every git-derived query (a commit subject whose +answer is the files that commit changed) the working tree is checked out at the commit's +*parent* — what an agent sees before making the change — the index is updated +incrementally, and one retrieval call is made. Consecutive tasks form agent sessions of K +tasks, so the repository genuinely evolves between the calls of one session: unrelated +commits, edits to evidence already delivered, moves, deletions and refactors. + +Each task produces one retrieval packet. Every arm is computed from that same packet and +the same candidates, so arms differ only in what they do with evidence: + + A read the full file behind every delivered result ("just reread the file") + A-mem A, skipping a file this session already read whose bytes are unchanged + B the 1.10.0 packet: every snippet, every time + S unsafe dedup: withhold any result whose (path, lines) the session saw before, + without looking at the source ("trust what you already read") + C 2.0 evidence memory: withhold only byte-identical evidence; report changes + +The oracle is independent of memory's hashing and relocation code. It keeps the text each +session was actually handed; a withheld snippet is **stale** unless the text B would +deliver now is text that session already holds. Invalidation notices are scored against a +list-comparison re-implementation of span presence. + +Sources are never modified: each corpus is replayed in a `git clone --shared` inside a +temporary directory. + + python tests/eval/memory_eval.py + python tests/eval/memory_eval.py --corpus ../svc:/tmp/svc.yml --sessions 1,5,10,all +""" + +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import statistics +import subprocess +import sys +import tempfile +import time +from collections.abc import Callable, Sequence +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Optional + +REPO_ROOT = Path(__file__).resolve().parents[2] +if __package__ in (None, ""): + sys.path.insert(0, str(REPO_ROOT / "tests")) + sys.path.insert(0, str(REPO_ROOT / "src")) + from eval import harness, metrics # type: ignore[no-redef] +else: + from . import harness, metrics + +import yaml # noqa: E402 + +from codebase_index.config import Config # noqa: E402 +from codebase_index.indexer.pipeline import build_index, update_index # noqa: E402 +from codebase_index.memory import identity as ident # noqa: E402 +from codebase_index.memory.session import ( # noqa: E402 + EvidenceProcessor, + Session, + index_sha_lookup, +) +from codebase_index.memory.store import MemoryStore # noqa: E402 +from codebase_index.output.redact import redact_snippet # noqa: E402 +from codebase_index.parsers.line_chunker import estimate_tokens # noqa: E402 +from codebase_index.retrieval.pipeline import search # noqa: E402 +from codebase_index.storage.db import Database # noqa: E402 + +LIMIT = 10 +BUDGET = 1500 +SURVIVAL_HORIZONS = (1, 5, 10, 20) +_EPOCH = datetime(2026, 1, 1, tzinfo=timezone.utc) + + +# --- replay setup ------------------------------------------------------------------ + + +@dataclass(frozen=True) +class Task: + query: str + expected_files: tuple[str, ...] + commit: str + parent: str + + +def git(repo: Path, *args: str) -> str: + proc = subprocess.run(["git", "-C", str(repo), *args], capture_output=True, text=True, + encoding="utf-8", errors="replace") + if proc.returncode != 0: + raise RuntimeError(f"git {' '.join(args)}: {proc.stderr.strip()}") + return proc.stdout + + +def load_tasks(queries_path: Path, work: Path) -> tuple[list[Task], dict[str, int]]: + """Tasks oldest-first, plus each commit's `git log` position (0 = newest).""" + order: dict[str, int] = {} + parents: dict[str, Optional[str]] = {} + for i, line in enumerate(git(work, "log", "--format=%H %P", "HEAD").splitlines()): + parts = line.split() + order[parts[0]] = i + parents[parts[0]] = parts[1] if len(parts) > 1 else None + by_prefix = {sha[:12]: sha for sha in order} + tasks: list[Task] = [] + for entry in yaml.safe_load(queries_path.read_text(encoding="utf-8")) or []: + full = by_prefix.get(str(entry.get("commit", ""))[:12]) + if not full or not parents.get(full): + continue # unknown commit or a root commit: no "before" state to replay + tasks.append(Task( + query=entry["query"], + expected_files=tuple(f.replace("\\", "/") for f in entry.get("expected_files", ())), + commit=full, + parent=str(parents[full]), + )) + tasks.sort(key=lambda t: -order[t.commit]) + return tasks, order + + +def corpus_config(work: Path) -> Config: + cfg = Config() + cfg.root = str(work) + cfg.embeddings.enabled = False + cfg.extra_ignore = [*cfg.extra_ignore, *harness.CORPUS_EXCLUDES] + cfg.memory.retention_days = 36500 # the replay's synthetic clock must never expire a session + return cfg + + +# --- oracle ------------------------------------------------------------------------ + + +class FileCache: + """Working-tree reads for one task, shared by every arm's oracle.""" + + def __init__(self, work: Path) -> None: + self.work = work + self._bytes: dict[str, Optional[bytes]] = {} + + def raw(self, rel: str) -> Optional[bytes]: + if rel not in self._bytes: + path = self.work / rel + self._bytes[rel] = path.read_bytes() if path.is_file() else None + return self._bytes[rel] + + def lines(self, rel: str) -> Optional[list[str]]: + raw = self.raw(rel) + if raw is None: + return None + text = raw.decode("utf-8", "surrogateescape") + return text.replace("\r\n", "\n").replace("\r", "\n").splitlines() + + +def _sha16(lines: list[str]) -> str: + return hashlib.sha256("\n".join(lines).encode("utf-8", "surrogateescape")).hexdigest()[:16] + + +def span_present(lines: Optional[list[str]], span: list[str], line_start: int) -> bool: + """List comparison: the span sits at its recorded lines, or occurs exactly once.""" + if lines is None or not span: + return False + n = len(span) + if lines[line_start - 1:line_start - 1 + n] == span: + return True + return sum(1 for i in range(len(lines) - n + 1) if lines[i:i + n] == span) == 1 + + +def _occurrences(lines: Optional[list[str]], span: list[str]) -> Optional[int]: + if lines is None or not span: + return None + n = len(span) + return sum(1 for i in range(len(lines) - n + 1) if lines[i:i + n] == span) + + +def session_holds(truth: "SessionTruth", path: str, snippet: str, span_text: str) -> bool: + """Does the session hold what it would be sent now? + + Yes if it was handed this exact snippet before, or the whole current span (the + snippet — skeleton, signature or full text — is derived from those very bytes). + Both sides are plain strings read from packets and files, never memory's hashes. + """ + return (snippet in truth.held.get(path, set()) + or span_text in truth.full_spans.get(path, set())) + + +@dataclass +class Atom: + path: str + line_start: int + span: list[str] + delivered_position: int + last_valid: int = 0 + invalid_at: Optional[int] = None + + +@dataclass +class SessionTruth: + held: dict[str, set[str]] = field(default_factory=dict) + full_spans: dict[str, set[str]] = field(default_factory=dict) + seen_at: dict[tuple[str, int, int], str] = field(default_factory=dict) + files_read: dict[str, str] = field(default_factory=dict) + atoms: dict[tuple[str, str], Atom] = field(default_factory=dict) + + +@dataclass +class Totals: + tasks: int = 0 + sessions: int = 0 + deliveries: int = 0 + b_tokens: int = 0 + c_tokens: int = 0 + s_tokens: int = 0 + a_tokens: int = 0 + amem_tokens: int = 0 + b_packet_tokens: int = 0 + c_packet_tokens: int = 0 + c_reused: int = 0 + s_reused: int = 0 + c_stale_withheld: int = 0 + s_stale_withheld: int = 0 + s_stale_gold: int = 0 + c_stale_flags: int = 0 + notices: int = 0 + notice_tp: int = 0 + notice_fp: int = 0 + notice_fn: int = 0 + page_mismatches: int = 0 + distinct_atoms: int = 0 + store_bytes: int = 0 + survival: dict[int, list[int]] = field( + default_factory=lambda: {h: [0, 0] for h in SURVIVAL_HORIZONS}) + useful_b: list[float] = field(default_factory=list) + useful_s: list[float] = field(default_factory=list) + saved_per_task: list[float] = field(default_factory=list) + memory_ms: list[float] = field(default_factory=list) + examples: list[dict] = field(default_factory=list) + """Bounded diagnostics for any event that should not happen (never committed: they + can quote private source).""" + + def example(self, kind: str, **info: Any) -> None: + if sum(1 for e in self.examples if e["kind"] == kind) < 10: + self.examples.append({"kind": kind, **info}) + + def merge(self, other: "Totals") -> None: + for name, value in vars(other).items(): + mine = getattr(self, name) + if isinstance(value, int): + setattr(self, name, mine + value) + elif isinstance(value, list): + mine.extend(value) + elif isinstance(value, dict): + for h, (survived, observed) in value.items(): + mine[h][0] += survived + mine[h][1] += observed + + +def close_session(truth: SessionTruth, stat: Totals) -> None: + """Fold one session's evidence into survival counts (interval-censored).""" + stat.distinct_atoms += len(truth.atoms) + for atom in truth.atoms.values(): + for h in SURVIVAL_HORIZONS: + if atom.last_valid >= h: + stat.survival[h][0] += 1 + stat.survival[h][1] += 1 + elif atom.invalid_at is not None and atom.invalid_at <= h: + stat.survival[h][1] += 1 + + +# --- scoring one task -------------------------------------------------------------- + + +def score_task(*, files: FileCache, processor: EvidenceProcessor, task: Task, packet: dict, + candidates: list, truth: SessionTruth, stat: Totals, position: int, + token_budget: int) -> None: + stat.tasks += 1 + + # Ground truth for evidence this session holds, established before memory runs. + truth_invalid: set[tuple[str, str]] = set() + for key, atom in truth.atoms.items(): + if atom.invalid_at is not None: + continue + distance = atom.delivered_position - position + if span_present(files.lines(atom.path), atom.span, atom.line_start): + atom.last_valid = max(atom.last_valid, distance) + else: + atom.invalid_at = distance + truth_invalid.add(key) + + memory_packet = copy.deepcopy(packet) + started = time.perf_counter() + processor(memory_packet, candidates) + stat.memory_ms.append((time.perf_counter() - started) * 1000) + + notices = {(r.path, r.sha[:16]) for r in + (ident.parse_ref(n["ref"]) for n in memory_packet["memory"]["invalidated"])} + stat.notices += len(notices) + stat.notice_tp += len(notices & truth_invalid) + stat.notice_fp += len(notices - truth_invalid) + stat.notice_fn += len(truth_invalid - notices) + states = {(r.path, r.sha[:16]): n["state"] for n in memory_packet["memory"]["invalidated"] + for r in [ident.parse_ref(n["ref"])]} + for key in sorted(notices ^ truth_invalid): + atom = truth.atoms.get(key) + stat.example("notice_fp" if key in notices else "notice_fn", path=key[0], sha=key[1], + memory_state=states.get(key), oracle_known=atom is not None, + oracle_line_start=atom.line_start if atom else None, + oracle_occurrences=_occurrences(files.lines(key[0]), atom.span) + if atom else None) + + # Filling withheld snippets back in must reproduce the no-memory packet exactly. + restored = copy.deepcopy(memory_packet) + restored.pop("memory") + for mine, base in zip(restored["results"], packet["results"]): + if mine.pop("reused", False): + mine["snippet"] = base["snippet"] + mine.pop("stale", None) + if restored != packet: + stat.page_mismatches += 1 + + returned_s: list[tuple[str, int]] = [] + saved = 0 + delivered_files: set[str] = set() + padded = list(candidates) + [None] * max(0, len(packet["results"]) - len(candidates)) + for base, mine, candidate in zip(packet["results"], memory_packet["results"], padded): + path = base["path"].replace("\\", "/") + tokens = int(base.get("token_est") or 0) + snippet = base.get("snippet") + if not snippet: + returned_s.append((path, tokens)) + continue + stat.deliveries += 1 + stat.b_tokens += tokens + delivered_files.add(path) + span_now = (files.lines(path) or [])[base["line_start"] - 1:base["line_end"]] + + # C: evidence memory + if mine.get("stale"): + stat.c_stale_flags += 1 + stat.example("stale_flag", path=path, lines=[base["line_start"], base["line_end"]], + chunk_kind=getattr(candidate, "kind", None), + source=getattr(candidate, "source", None), + content=(getattr(candidate, "content", None) or "")[:240], + span="\n".join(span_now)[:240]) + span_text = "\n".join(span_now) + if mine.get("reused"): + stat.c_reused += 1 + saved += tokens + if not session_holds(truth, path, snippet, span_text): + stat.c_stale_withheld += 1 + stat.example("c_stale_withheld", path=path, + lines=[base["line_start"], base["line_end"]], + source=getattr(candidate, "source", None), + skeletonized=base.get("skeletonized"), snippet=snippet[:400], + held=[h[:400] for h in sorted(truth.held.get(path, set()))[:3]]) + else: + stat.c_tokens += tokens + truth.held.setdefault(path, set()).add(snippet) + if snippet == redact_snippet(span_text): + truth.full_spans.setdefault(path, set()).add(span_text) + if not mine.get("stale"): + span = (files.lines(path) or [])[base["line_start"] - 1:base["line_end"]] + key = (path, _sha16(span)) + known = truth.atoms.get(key) + if known is None or known.invalid_at is not None: + truth.atoms[key] = Atom(path, int(base["line_start"]), span, position) + + # S: unsafe dedup by locator + locator = (path, int(base["line_start"]), int(base["line_end"])) + if locator in truth.seen_at: + stat.s_reused += 1 + if truth.seen_at[locator] != snippet: + stat.s_stale_withheld += 1 + if path in task.expected_files: + stat.s_stale_gold += 1 + continue # the session holds a stale version: not useful evidence + else: + truth.seen_at[locator] = snippet + stat.s_tokens += tokens + returned_s.append((path, tokens)) + + # A / A-mem: whole-file reads of every delivered result's file + for path in sorted(delivered_files): + raw = files.raw(path) or b"" + cost = estimate_tokens(raw.decode("utf-8", "ignore")) if raw else 0 + stat.a_tokens += cost + digest = hashlib.sha256(raw).hexdigest() + if truth.files_read.get(path) != digest: + stat.amem_tokens += cost + truth.files_read[path] = digest + + stat.b_packet_tokens += estimate_tokens(json.dumps(packet, separators=(",", ":"))) + stat.c_packet_tokens += estimate_tokens(json.dumps(memory_packet, separators=(",", ":"))) + stat.saved_per_task.append(float(saved)) + returned_b = [(r["path"].replace("\\", "/"), int(r.get("token_est") or 0)) + for r in packet["results"]] + stat.useful_b.append( + metrics.useful_context_at_budget(returned_b, task.expected_files, token_budget)) + stat.useful_s.append( + metrics.useful_context_at_budget(returned_s, task.expected_files, token_budget)) + + +# --- one corpus -------------------------------------------------------------------- + + +@dataclass +class CorpusResult: + name: str + tasks: int + totals: dict[Optional[int], Totals] + search_ms: list[float] + update_ms: list[float] + + +def replay_corpus(repo: Path, queries: Path, sessions: Sequence[Optional[int]], *, + limit: int = LIMIT, token_budget: int = BUDGET, + max_tasks: Optional[int] = None, + log: Callable[[str], None] = lambda _m: None) -> CorpusResult: + with tempfile.TemporaryDirectory() as tmp_name: + tmp = Path(tmp_name) + work = tmp / "work" + subprocess.run(["git", "clone", "-q", "--shared", "--no-checkout", str(repo), str(work)], + check=True, capture_output=True) + tasks, order = load_tasks(queries, work) + if max_tasks is not None: + tasks = tasks[:max_tasks] + cfg = corpus_config(work) + repo_id = ident.repo_id_for(work) + totals = {k: Totals() for k in sessions} + truths: dict[Optional[int], SessionTruth] = {k: SessionTruth() for k in sessions} + active: dict[Optional[int], int] = {} + stores = {k: MemoryStore.open(tmp / f"memory-{k or 'all'}.sqlite") for k in sessions} + search_ms: list[float] = [] + update_ms: list[float] = [] + db = Database(tmp / "index.sqlite").open() + try: + for index, task in enumerate(tasks): + git(work, "checkout", "-q", "-f", "--detach", task.parent) + started = time.perf_counter() + if index == 0: + build_index(cfg, db, root=work) + else: + update_index(cfg, db, root=work) + update_ms.append((time.perf_counter() - started) * 1000) + + captured: dict[str, Any] = {} + started = time.perf_counter() + packet = search(db.conn, task.query, mode="hybrid", limit=limit, + token_budget=token_budget, no_fallback=True, + evidence=lambda _p, c: captured.update(candidates=list(c))) + search_ms.append((time.perf_counter() - started) * 1000) + + files = FileCache(work) + now = _EPOCH + timedelta(minutes=index) + for k in sessions: + session_index = 0 if k is None else index // k + if active.get(k) != session_index: + if k in active: + close_session(truths[k], totals[k]) + truths[k] = SessionTruth() + totals[k].sessions += 1 + active[k] = session_index + tag = f"k{k or 'all'}-s{session_index}" + store = stores[k] + session_id = store.touch_session( + repo_id, ident.session_key(repo_id, tag), now=now) + processor = EvidenceProcessor( + root=work, config=cfg, now=now, + session=Session(tag=tag, repo_id=repo_id, store=store, + session_id=session_id), + index_sha=index_sha_lookup(db.conn)) + score_task(files=files, processor=processor, task=task, packet=packet, + candidates=captured.get("candidates", []), truth=truths[k], + stat=totals[k], position=order[task.parent], + token_budget=token_budget) + if (index + 1) % 25 == 0: + log(f" {index + 1}/{len(tasks)} tasks") + for k in sessions: + if k in active: + close_session(truths[k], totals[k]) + finally: + db.close() + for k, store in stores.items(): + store.conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") + totals[k].store_bytes = store.stats(repo_id)["bytes"] + store.close() + return CorpusResult(Path(repo).resolve().name, len(tasks), totals, search_ms, update_ms) + + +# --- reporting --------------------------------------------------------------------- + + +def _pct(part: float, whole: float) -> str: + return f"{100.0 * part / whole:.1f}%" if whole else "—" + + +def _ratio(num: int, den: int) -> str: + return f"{num / den:.3f}" if den else "—" + + +def summarise(label: str, t: Totals) -> dict: + mean_saved, lo, hi = metrics.paired_bootstrap_ci(t.saved_per_task, resamples=2000) + return { + "sessions_k": label, + "tasks": t.tasks, + "sessions": t.sessions, + "deliveries": t.deliveries, + "distinct_atoms": t.distinct_atoms, + "B_tokens": t.b_tokens, + "C_tokens": t.c_tokens, + "S_tokens": t.s_tokens, + "A_tokens": t.a_tokens, + "A_mem_tokens": t.amem_tokens, + "C_token_reuse_rate": (t.b_tokens - t.c_tokens) / t.b_tokens if t.b_tokens else 0.0, + "C_evidence_reuse_rate": t.c_reused / t.deliveries if t.deliveries else 0.0, + "C_saved_per_task": mean_saved, + "C_saved_per_task_ci95": [lo, hi], + "C_stale_withheld": t.c_stale_withheld, + "C_validated_reuse_rate": ( + (t.c_reused - t.c_stale_withheld) / t.c_reused if t.c_reused else None), + "S_token_reuse_rate": (t.b_tokens - t.s_tokens) / t.b_tokens if t.b_tokens else 0.0, + "S_stale_withheld": t.s_stale_withheld, + "S_stale_reuse_rate": t.s_stale_withheld / t.s_reused if t.s_reused else None, + "S_stale_gold": t.s_stale_gold, + "notices": t.notices, + "invalidation_precision": t.notice_tp / (t.notice_tp + t.notice_fp) + if (t.notice_tp + t.notice_fp) else None, + "invalidation_recall": t.notice_tp / (t.notice_tp + t.notice_fn) + if (t.notice_tp + t.notice_fn) else None, + "page_mismatches": t.page_mismatches, + "stale_flags": t.c_stale_flags, + "B_packet_tokens": t.b_packet_tokens, + "C_packet_tokens": t.c_packet_tokens, + "useful_B_equals_C": statistics.fmean(t.useful_b) if t.useful_b else 0.0, + "useful_S": statistics.fmean(t.useful_s) if t.useful_s else 0.0, + "survival": {h: (s / o if o else None, o) for h, (s, o) in t.survival.items()}, + "memory_ms_p50": metrics.percentile(t.memory_ms, 50), + "memory_ms_p95": metrics.percentile(t.memory_ms, 95), + "store_bytes": t.store_bytes, + "examples": t.examples, + } + + +def format_report(rows: list[dict]) -> str: + out = [ + "| K | tasks | sessions | B tokens | C tokens (reuse) | 95% CI saved/task | " + "C stale withheld | S tokens (reuse) | S stale withheld | notices P/R | " + "A tokens | A-mem tokens | packet tok B→C |", + "|" + "---|" * 13, + ] + for r in rows: + lo, hi = r["C_saved_per_task_ci95"] + precision = r["invalidation_precision"] + recall = r["invalidation_recall"] + out.append( + f"| {r['sessions_k']} | {r['tasks']} | {r['sessions']} | {r['B_tokens']} | " + f"{r['C_tokens']} ({100 * r['C_token_reuse_rate']:.1f}%) | " + f"{r['C_saved_per_task']:.1f} [{lo:.1f}, {hi:.1f}] | {r['C_stale_withheld']} | " + f"{r['S_tokens']} ({100 * r['S_token_reuse_rate']:.1f}%) | " + f"{r['S_stale_withheld']} | " + f"{'—' if precision is None else f'{precision:.3f}'}/" + f"{'—' if recall is None else f'{recall:.3f}'} | " + f"{r['A_tokens']} | {r['A_mem_tokens']} | " + f"{r['B_packet_tokens']}→{r['C_packet_tokens']} |" + ) + out.append("") + out.append("| K | useful@budget B=C | useful@budget S | S stale gold | page mismatches | " + "stale flags | distinct atoms | survival h=1/5/10/20 | memory ms p50/p95 | " + "store bytes |") + out.append("|" + "---|" * 10) + for r in rows: + surv = "/".join("—" if v[0] is None else f"{v[0]:.2f}" for v in r["survival"].values()) + out.append( + f"| {r['sessions_k']} | {r['useful_B_equals_C']:.3f} | {r['useful_S']:.3f} | " + f"{r['S_stale_gold']} | {r['page_mismatches']} | {r['stale_flags']} | " + f"{r['distinct_atoms']} | {surv} | {r['memory_ms_p50']:.2f}/{r['memory_ms_p95']:.2f} | " + f"{r['store_bytes']} |" + ) + return "\n".join(out) + + +def _parse_corpus(spec: str) -> tuple[Path, Path]: + head, sep, tail = spec.rpartition(":") + if not sep or (len(head) == 1 and head.isalpha()): + raise argparse.ArgumentTypeError(f"--corpus expects ':', got {spec!r}") + return Path(head).resolve(), Path(tail) + + +def _parse_sessions(spec: str) -> list[Optional[int]]: + out: list[Optional[int]] = [] + for part in spec.split(","): + part = part.strip().lower() + if part == "all": + out.append(None) + elif part: + out.append(max(1, int(part))) + return out + + +def main(argv: Optional[list[str]] = None) -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--corpus", action="append", default=[], metavar="REPO:QUERIES") + ap.add_argument("--sessions", default="1,5,10,25,all", + help="session lengths in tasks; 'all' = one session per corpus") + ap.add_argument("--limit", type=int, default=LIMIT) + ap.add_argument("--token-budget", type=int, default=BUDGET) + ap.add_argument("--max-tasks", type=int, default=None) + ap.add_argument("--json-out", default=None) + args = ap.parse_args(argv) + if sys.platform == "win32": + sys.stdout.reconfigure(encoding="utf-8") # type: ignore[union-attr] + + corpora = [_parse_corpus(s) for s in args.corpus] or [ + (REPO_ROOT, harness.QUERY_DIR / "self_repo_git.yml")] + sessions = _parse_sessions(args.sessions) + results: list[CorpusResult] = [] + for repo, queries in corpora: + print(f"replaying {repo.name} ({queries})", file=sys.stderr, flush=True) + results.append(replay_corpus( + repo, queries, sessions, limit=args.limit, token_budget=args.token_budget, + max_tasks=args.max_tasks, log=lambda m: print(m, file=sys.stderr, flush=True))) + + def label(k: Optional[int]) -> str: + return "all" if k is None else str(k) + + report: dict[str, Any] = {"corpora": {}, "pooled": []} + for res in results: + rows = [summarise(label(k), res.totals[k]) for k in sessions] + report["corpora"][res.name] = { + "tasks": res.tasks, + "rows": rows, + "search_ms_p50": metrics.percentile(res.search_ms, 50), + "search_ms_p95": metrics.percentile(res.search_ms, 95), + "update_ms_p50": metrics.percentile(res.update_ms, 50), + } + print(f"\n## {res.name} ({res.tasks} tasks; search p50 " + f"{report['corpora'][res.name]['search_ms_p50']:.1f} ms, update p50 " + f"{report['corpora'][res.name]['update_ms_p50']:.1f} ms)\n") + print(format_report(rows)) + + pooled_rows = [] + for k in sessions: + merged = Totals() + for res in results: + merged.merge(res.totals[k]) + pooled_rows.append(summarise(label(k), merged)) + report["pooled"] = pooled_rows + if len(results) > 1: + print(f"\n## POOLED ({sum(r.tasks for r in results)} tasks, {len(results)} corpora)\n") + print(format_report(pooled_rows)) + if args.json_out: + Path(args.json_out).write_text(json.dumps(report, indent=2, default=str), + encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/eval/metrics.py b/tests/eval/metrics.py index fac5df0..d88c70f 100644 --- a/tests/eval/metrics.py +++ b/tests/eval/metrics.py @@ -121,6 +121,38 @@ def useful_context_at_budget( return len(found) / len(rel) +# --- oracle / headroom ------------------------------------------------------ +# +# MRR alone cannot tell "the retrievers never found it" apart from "the ranker had +# it and buried it", yet those two failures have nothing in common: one is fixed by +# widening recall, the other by reranking. The pair below splits them, and the split +# is what justified spending 1.10.0 on the ranker rather than on new retrievers. + + +def oracle_reciprocal_rank(pool: Sequence[str], relevant: Iterable[str]) -> float: + """Reciprocal rank a *perfect* reranker would achieve over this candidate pool. + + A perfect reranker puts a relevant candidate first, so the answer is binary: + 1.0 when the pool contains any relevant item, 0.0 when retrieval never + surfaced one. Averaged over a query set this is the ceiling MRR can reach + without touching candidate generation, and `1 - oracle` is the share of the + query set that only better *recall* can ever fix. + """ + rel = set(relevant) + return 1.0 if any(item in rel for item in pool) else 0.0 + + +def rerank_efficiency(mrr: float, oracle_mrr: float) -> float: + """Fraction of the achievable ranking quality the ranker actually delivers. + + 1.0 means every answer the retrievers found is ranked first; 0.0 means none + are. This is the number 1.10.0 set out to move: at 1.9.0 it was 0.64, so a + third of the answers already in the pool were being ranked below something + else — an error mode roughly three times larger than the remaining recall gap. + """ + return mrr / oracle_mrr if oracle_mrr else 0.0 + + def percentile(values: Sequence[float], pct: float) -> float: """Nearest-rank percentile (no interpolation) — stable for small samples.""" if not values: diff --git a/tests/eval/results/1.10.0-perf.json b/tests/eval/results/1.10.0-perf.json new file mode 100644 index 0000000..fe26f24 --- /dev/null +++ b/tests/eval/results/1.10.0-perf.json @@ -0,0 +1,14 @@ +{ + "cold_index_s": 4.316, + "files": 311, + "chunks": 5636, + "symbols": 1242, + "db_bytes": 5820416, + "noop_update_s": 0.139, + "one_file_update_s": 0.251, + "one_file_update_indexed": 1, + "queries": 123, + "query_p50_ms": 38.14, + "query_p95_ms": 59.77, + "query_mean_ms": 39.28 +} \ No newline at end of file diff --git a/tests/eval/results/1.10.0-retrieval-8corpus.txt b/tests/eval/results/1.10.0-retrieval-8corpus.txt new file mode 100644 index 0000000..bf0b041 --- /dev/null +++ b/tests/eval/results/1.10.0-retrieval-8corpus.txt @@ -0,0 +1,58 @@ +corpus: codebase-index (../codebase-index) � 87 queries from research/data/codebase-index.yml +corpus: Civitas (../Civitas) � 64 queries from research/data/Civitas.yml +corpus: PoliternalSite (../PoliternalSite) � 120 queries from research/data/PoliternalSite.yml +corpus: PoliternalParkour (../PoliternalParkour) � 64 queries from research/data/PoliternalParkour.yml +corpus: TerraForge (../TerraForge) � 17 queries from research/data/TerraForge.yml +corpus: denfry.github.io (../denfry.github.io) � 39 queries from research/data/denfry.github.io.yml +corpus: DevGraph (../DevGraph) � 12 queries from research/data/DevGraph.yml +corpus: Windows-Cleaner-and-Optimizer-main (../Windows-Cleaner-and-Optimizer-main) � 17 queries from research/data/WinCleaner.yml +pooled: 420 queries across 8 corpora +building index for codebase-index (once, shared by all variants)... +building index for Civitas (once, shared by all variants)... +building index for PoliternalSite (once, shared by all variants)... +building index for PoliternalParkour (once, shared by all variants)... +building index for TerraForge (once, shared by all variants)... +building index for denfry.github.io (once, shared by all variants)... +building index for DevGraph (once, shared by all variants)... +building index for Windows-Cleaner-and-Optimizer-main (once, shared by all variants)... + +| label | recall@5 | recall@10 | MRR | oracle | eff | nDCG@10 | hit@3 | P@5 | MAP | useful@budget | tokens | dup% | p50_ms | p95_ms | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +| baseline (1.7.0) | 0.376 | 0.481 | 0.318 | 0.714 | 0.446 | 0.329 | 0.383 | 0.130 | 0.256 | 0.468 | 508.1 | 1.3 | 24.2 | 40.1 | +| previous release (1.9.0) | 0.589 (+0.214) | 0.623 (+0.142) | 0.560 (+0.242) | 0.900 (+0.186) | 0.622 (+0.177) | 0.522 (+0.193) | 0.655 (+0.271) | 0.190 (+0.059) | 0.452 (+0.196) | 0.583 (+0.115) | 1099.4 (+591.3) | 0.0 (-1.3) | 48.6 (+24.4) | 95.0 (+54.9) | +| default (all signals) | 0.610 (+0.235) | 0.687 (+0.205) | 0.580 (+0.262) | 0.900 (+0.186) | 0.645 (+0.199) | 0.554 (+0.225) | 0.667 (+0.283) | 0.197 (+0.067) | 0.475 (+0.219) | 0.613 (+0.144) | 1083.6 (+575.5) | 0.0 (-1.3) | 53.0 (+28.9) | 102.3 (+62.2) | + +paired comparison: default (all signals) vs previous release (1.9.0) (n=420) +| metric | delta | 95% CI | p | sig | +|---|---|---|---|---| +| recall@5 | +0.0210 | [+0.0111,+0.0329] | 0.000 | yes | +| recall@10 | +0.0631 | [+0.0429,+0.0861] | 0.000 | yes | +| MRR | +0.0200 | [+0.0076,+0.0326] | 0.002 | yes | +| nDCG@10 | +0.0316 | [+0.0213,+0.0425] | 0.000 | yes | +| hit@3 | +0.0119 | [+0.0000,+0.0262] | 0.175 | no | +| P@5 | +0.0075 | [+0.0043,+0.0113] | 0.000 | yes | +| MAP | +0.0230 | [+0.0127,+0.0348] | 0.000 | yes | +| useful@budget | +0.0296 | [+0.0056,+0.0538] | 0.011 | yes | +| oracle | +0.0000 | [+0.0000,+0.0000] | 1.000 | no | +| cand_recall | +0.0000 | [+0.0000,+0.0000] | 1.000 | no | + +Per-corpus (default config): +| label | recall@5 | recall@10 | MRR | oracle | eff | nDCG@10 | hit@3 | P@5 | MAP | useful@budget | tokens | dup% | p50_ms | p95_ms | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +| codebase-index | 0.548 | 0.634 | 0.649 | 0.931 | 0.697 | 0.552 | 0.736 | 0.226 | 0.459 | 0.617 | 1193.3 | 0.0 | 53.1 | 80.7 | +| Civitas | 0.546 | 0.633 | 0.572 | 0.938 | 0.610 | 0.529 | 0.656 | 0.175 | 0.462 | 0.548 | 921.5 | 0.0 | 55.0 | 137.8 | +| PoliternalSite | 0.560 | 0.631 | 0.548 | 0.875 | 0.626 | 0.516 | 0.625 | 0.184 | 0.441 | 0.570 | 1101.8 | 0.0 | 72.1 | 124.8 | +| PoliternalParkour | 0.695 | 0.809 | 0.488 | 0.875 | 0.557 | 0.561 | 0.609 | 0.147 | 0.479 | 0.680 | 931.9 | 0.0 | 47.8 | 112.0 | +| TerraForge | 0.706 | 0.740 | 0.669 | 1.000 | 0.669 | 0.646 | 0.824 | 0.282 | 0.571 | 0.549 | 933.8 | 0.0 | 43.3 | 114.9 | +| denfry.github.io | 0.656 | 0.720 | 0.577 | 0.795 | 0.725 | 0.574 | 0.667 | 0.191 | 0.505 | 0.720 | 1226.9 | 0.0 | 22.3 | 43.8 | +| DevGraph | 0.819 | 0.819 | 0.694 | 1.000 | 0.694 | 0.667 | 0.750 | 0.317 | 0.574 | 0.778 | 1260.8 | 0.0 | 33.6 | 52.7 | +| Windows-Cleaner-and-Optimizer-main | 0.863 | 0.863 | 0.675 | 0.941 | 0.718 | 0.677 | 0.647 | 0.261 | 0.583 | 0.578 | 1270.2 | 0.0 | 24.7 | 40.4 | + +Per-category MRR (default config): + bug 0.647 + change 0.529 + docs 0.576 + feature 0.643 + perf 1.000 + refactor 0.750 + test 0.494 diff --git a/tests/eval/results/2026-09-14-evidence-memory.json b/tests/eval/results/2026-09-14-evidence-memory.json new file mode 100644 index 0000000..7c6ebae --- /dev/null +++ b/tests/eval/results/2026-09-14-evidence-memory.json @@ -0,0 +1,960 @@ +{ + "corpora": { + "codebase-index": { + "tasks": 87, + "rows": [ + { + "sessions_k": "1", + "tasks": 87, + "sessions": 87, + "deliveries": 643, + "distinct_atoms": 643, + "B_tokens": 98222, + "C_tokens": 98222, + "S_tokens": 98222, + "A_tokens": 1928268, + "A_mem_tokens": 1928268, + "C_token_reuse_rate": 0.0, + "C_evidence_reuse_rate": 0.0, + "C_saved_per_task": 0.0, + "C_saved_per_task_ci95": [ + 0.0, + 0.0 + ], + "C_stale_withheld": 0, + "C_validated_reuse_rate": null, + "S_token_reuse_rate": 0.0, + "S_stale_withheld": 0, + "S_stale_reuse_rate": null, + "S_stale_gold": 0, + "notices": 0, + "invalidation_precision": null, + "invalidation_recall": null, + "page_mismatches": 0, + "stale_flags": 0, + "B_packet_tokens": 172851, + "C_packet_tokens": 174475, + "useful_B_equals_C": 0.27586206896551724, + "useful_S": 0.27586206896551724, + "survival": { + "1": [ + null, + 0 + ], + "5": [ + null, + 0 + ], + "10": [ + null, + 0 + ], + "20": [ + null, + 0 + ] + }, + "memory_ms_p50": 4.046000000016647, + "memory_ms_p95": 6.0016000006726244, + "store_bytes": 409600, + "examples": [] + }, + { + "sessions_k": "5", + "tasks": 87, + "sessions": 18, + "deliveries": 643, + "distinct_atoms": 591, + "B_tokens": 98222, + "C_tokens": 94358, + "S_tokens": 87633, + "A_tokens": 1928268, + "A_mem_tokens": 1291386, + "C_token_reuse_rate": 0.03933945551913014, + "C_evidence_reuse_rate": 0.05132192846034215, + "C_saved_per_task": 44.41379310344828, + "C_saved_per_task_ci95": [ + 12.89655172413793, + 88.70114942528735 + ], + "C_stale_withheld": 0, + "C_validated_reuse_rate": 1.0, + "S_token_reuse_rate": 0.10780680499277148, + "S_stale_withheld": 15, + "S_stale_reuse_rate": 0.28846153846153844, + "S_stale_gold": 2, + "notices": 36, + "invalidation_precision": 1.0, + "invalidation_recall": 0.972972972972973, + "page_mismatches": 0, + "stale_flags": 0, + "B_packet_tokens": 172851, + "C_packet_tokens": 171263, + "useful_B_equals_C": 0.27586206896551724, + "useful_S": 0.27011494252873564, + "survival": { + "1": [ + 0.9714912280701754, + 456 + ], + "5": [ + 0.8198757763975155, + 161 + ], + "10": [ + 0.65, + 100 + ], + "20": [ + 0.075, + 40 + ] + }, + "memory_ms_p50": 7.953800000905176, + "memory_ms_p95": 15.853599999900325, + "store_bytes": 372736, + "examples": [ + { + "kind": "notice_fn", + "path": "docs/FAQ.md", + "sha": "f481fe4537198f01", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 7, + "oracle_occurrences": 0 + } + ] + }, + { + "sessions_k": "10", + "tasks": 87, + "sessions": 9, + "deliveries": 643, + "distinct_atoms": 563, + "B_tokens": 98222, + "C_tokens": 92565, + "S_tokens": 83756, + "A_tokens": 1928268, + "A_mem_tokens": 1084176, + "C_token_reuse_rate": 0.05759402170593146, + "C_evidence_reuse_rate": 0.08242612752721618, + "C_saved_per_task": 65.02298850574712, + "C_saved_per_task_ci95": [ + 30.862068965517242, + 111.6896551724138 + ], + "C_stale_withheld": 0, + "C_validated_reuse_rate": 1.0, + "S_token_reuse_rate": 0.1472786137525198, + "S_stale_withheld": 20, + "S_stale_reuse_rate": 0.2597402597402597, + "S_stale_gold": 2, + "notices": 49, + "invalidation_precision": 1.0, + "invalidation_recall": 0.9245283018867925, + "page_mismatches": 0, + "stale_flags": 0, + "B_packet_tokens": 172851, + "C_packet_tokens": 169696, + "useful_B_equals_C": 0.27586206896551724, + "useful_S": 0.27586206896551724, + "survival": { + "1": [ + 0.9737373737373738, + 495 + ], + "5": [ + 0.9025069637883009, + 359 + ], + "10": [ + 0.7680412371134021, + 194 + ], + "20": [ + 0.3977272727272727, + 88 + ] + }, + "memory_ms_p50": 10.889099999985774, + "memory_ms_p95": 23.312499999519787, + "store_bytes": 368640, + "examples": [ + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "150e7a90d1bec372", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 49, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/FAQ.md", + "sha": "f481fe4537198f01", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 7, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/PRODUCT_UPGRADE_PLAN.md", + "sha": "2bbea03fa8d909c6", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 1, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/PRODUCT_UPGRADE_PLAN.md", + "sha": "802af198a186890e", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 141, + "oracle_occurrences": 0 + } + ] + }, + { + "sessions_k": "25", + "tasks": 87, + "sessions": 4, + "deliveries": 643, + "distinct_atoms": 542, + "B_tokens": 98222, + "C_tokens": 92268, + "S_tokens": 79119, + "A_tokens": 1928268, + "A_mem_tokens": 827557, + "C_token_reuse_rate": 0.06061778420313168, + "C_evidence_reuse_rate": 0.1104199066874028, + "C_saved_per_task": 68.4367816091954, + "C_saved_per_task_ci95": [ + 34.51724137931034, + 114.25287356321839 + ], + "C_stale_withheld": 0, + "C_validated_reuse_rate": 1.0, + "S_token_reuse_rate": 0.19448799657917779, + "S_stale_withheld": 30, + "S_stale_reuse_rate": 0.30303030303030304, + "S_stale_gold": 4, + "notices": 69, + "invalidation_precision": 1.0, + "invalidation_recall": 0.92, + "page_mismatches": 0, + "stale_flags": 0, + "B_packet_tokens": 172851, + "C_packet_tokens": 169861, + "useful_B_equals_C": 0.27586206896551724, + "useful_S": 0.27011494252873564, + "survival": { + "1": [ + 0.9778225806451613, + 496 + ], + "5": [ + 0.9225806451612903, + 465 + ], + "10": [ + 0.8777506112469438, + 409 + ], + "20": [ + 0.7800687285223368, + 291 + ] + }, + "memory_ms_p50": 19.56839999911608, + "memory_ms_p95": 38.00810000029742, + "store_bytes": 368640, + "examples": [ + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "039513f1b18c1364", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 38, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "e15784d0fcfe16d1", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 45, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "150e7a90d1bec372", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 49, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/FAQ.md", + "sha": "f481fe4537198f01", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 7, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/PRODUCT_UPGRADE_PLAN.md", + "sha": "2bbea03fa8d909c6", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 1, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/PRODUCT_UPGRADE_PLAN.md", + "sha": "802af198a186890e", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 141, + "oracle_occurrences": 0 + } + ] + }, + { + "sessions_k": "all", + "tasks": 87, + "sessions": 1, + "deliveries": 643, + "distinct_atoms": 473, + "B_tokens": 98222, + "C_tokens": 87634, + "S_tokens": 64318, + "A_tokens": 1928268, + "A_mem_tokens": 544122, + "C_token_reuse_rate": 0.10779662397426239, + "C_evidence_reuse_rate": 0.19440124416796267, + "C_saved_per_task": 121.70114942528735, + "C_saved_per_task_ci95": [ + 76.34482758620689, + 178.7816091954023 + ], + "C_stale_withheld": 0, + "C_validated_reuse_rate": 1.0, + "S_token_reuse_rate": 0.34517725153224327, + "S_stale_withheld": 54, + "S_stale_reuse_rate": 0.3157894736842105, + "S_stale_gold": 6, + "notices": 128, + "invalidation_precision": 1.0, + "invalidation_recall": 0.8707482993197279, + "page_mismatches": 0, + "stale_flags": 0, + "B_packet_tokens": 172851, + "C_packet_tokens": 166456, + "useful_B_equals_C": 0.27586206896551724, + "useful_S": 0.26436781609195403, + "survival": { + "1": [ + 0.9754464285714286, + 448 + ], + "5": [ + 0.9249448123620309, + 453 + ], + "10": [ + 0.8778280542986425, + 442 + ], + "20": [ + 0.8408071748878924, + 446 + ] + }, + "memory_ms_p50": 36.57369999928051, + "memory_ms_p95": 68.22990000000573, + "store_bytes": 352256, + "examples": [ + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "039513f1b18c1364", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 38, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "e15784d0fcfe16d1", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 45, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "150e7a90d1bec372", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 49, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "README.md", + "sha": "2353492da96d24f4", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 145, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "ROADMAP.md", + "sha": "1faec12ba8307640", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 49, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "ROADMAP.md", + "sha": "1fb676dca4d76452", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 37, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "ROADMAP.md", + "sha": "7b40838bf03af0d5", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 43, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "ROADMAP.md", + "sha": "ab9b49fee3918b21", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 31, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "ROADMAP.md", + "sha": "c64353992a16f581", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 25, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "afb56398eb3173a5", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 33, + "oracle_occurrences": 0 + } + ] + } + ], + "search_ms_p50": 31.602200000634184, + "search_ms_p95": 55.25899999884132, + "update_ms_p50": 154.52839999852586 + } + }, + "pooled": [ + { + "sessions_k": "1", + "tasks": 87, + "sessions": 87, + "deliveries": 643, + "distinct_atoms": 643, + "B_tokens": 98222, + "C_tokens": 98222, + "S_tokens": 98222, + "A_tokens": 1928268, + "A_mem_tokens": 1928268, + "C_token_reuse_rate": 0.0, + "C_evidence_reuse_rate": 0.0, + "C_saved_per_task": 0.0, + "C_saved_per_task_ci95": [ + 0.0, + 0.0 + ], + "C_stale_withheld": 0, + "C_validated_reuse_rate": null, + "S_token_reuse_rate": 0.0, + "S_stale_withheld": 0, + "S_stale_reuse_rate": null, + "S_stale_gold": 0, + "notices": 0, + "invalidation_precision": null, + "invalidation_recall": null, + "page_mismatches": 0, + "stale_flags": 0, + "B_packet_tokens": 172851, + "C_packet_tokens": 174475, + "useful_B_equals_C": 0.27586206896551724, + "useful_S": 0.27586206896551724, + "survival": { + "1": [ + null, + 0 + ], + "5": [ + null, + 0 + ], + "10": [ + null, + 0 + ], + "20": [ + null, + 0 + ] + }, + "memory_ms_p50": 4.046000000016647, + "memory_ms_p95": 6.0016000006726244, + "store_bytes": 409600, + "examples": [] + }, + { + "sessions_k": "5", + "tasks": 87, + "sessions": 18, + "deliveries": 643, + "distinct_atoms": 591, + "B_tokens": 98222, + "C_tokens": 94358, + "S_tokens": 87633, + "A_tokens": 1928268, + "A_mem_tokens": 1291386, + "C_token_reuse_rate": 0.03933945551913014, + "C_evidence_reuse_rate": 0.05132192846034215, + "C_saved_per_task": 44.41379310344828, + "C_saved_per_task_ci95": [ + 12.89655172413793, + 88.70114942528735 + ], + "C_stale_withheld": 0, + "C_validated_reuse_rate": 1.0, + "S_token_reuse_rate": 0.10780680499277148, + "S_stale_withheld": 15, + "S_stale_reuse_rate": 0.28846153846153844, + "S_stale_gold": 2, + "notices": 36, + "invalidation_precision": 1.0, + "invalidation_recall": 0.972972972972973, + "page_mismatches": 0, + "stale_flags": 0, + "B_packet_tokens": 172851, + "C_packet_tokens": 171263, + "useful_B_equals_C": 0.27586206896551724, + "useful_S": 0.27011494252873564, + "survival": { + "1": [ + 0.9714912280701754, + 456 + ], + "5": [ + 0.8198757763975155, + 161 + ], + "10": [ + 0.65, + 100 + ], + "20": [ + 0.075, + 40 + ] + }, + "memory_ms_p50": 7.953800000905176, + "memory_ms_p95": 15.853599999900325, + "store_bytes": 372736, + "examples": [ + { + "kind": "notice_fn", + "path": "docs/FAQ.md", + "sha": "f481fe4537198f01", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 7, + "oracle_occurrences": 0 + } + ] + }, + { + "sessions_k": "10", + "tasks": 87, + "sessions": 9, + "deliveries": 643, + "distinct_atoms": 563, + "B_tokens": 98222, + "C_tokens": 92565, + "S_tokens": 83756, + "A_tokens": 1928268, + "A_mem_tokens": 1084176, + "C_token_reuse_rate": 0.05759402170593146, + "C_evidence_reuse_rate": 0.08242612752721618, + "C_saved_per_task": 65.02298850574712, + "C_saved_per_task_ci95": [ + 30.862068965517242, + 111.6896551724138 + ], + "C_stale_withheld": 0, + "C_validated_reuse_rate": 1.0, + "S_token_reuse_rate": 0.1472786137525198, + "S_stale_withheld": 20, + "S_stale_reuse_rate": 0.2597402597402597, + "S_stale_gold": 2, + "notices": 49, + "invalidation_precision": 1.0, + "invalidation_recall": 0.9245283018867925, + "page_mismatches": 0, + "stale_flags": 0, + "B_packet_tokens": 172851, + "C_packet_tokens": 169696, + "useful_B_equals_C": 0.27586206896551724, + "useful_S": 0.27586206896551724, + "survival": { + "1": [ + 0.9737373737373738, + 495 + ], + "5": [ + 0.9025069637883009, + 359 + ], + "10": [ + 0.7680412371134021, + 194 + ], + "20": [ + 0.3977272727272727, + 88 + ] + }, + "memory_ms_p50": 10.889099999985774, + "memory_ms_p95": 23.312499999519787, + "store_bytes": 368640, + "examples": [ + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "150e7a90d1bec372", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 49, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/FAQ.md", + "sha": "f481fe4537198f01", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 7, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/PRODUCT_UPGRADE_PLAN.md", + "sha": "2bbea03fa8d909c6", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 1, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/PRODUCT_UPGRADE_PLAN.md", + "sha": "802af198a186890e", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 141, + "oracle_occurrences": 0 + } + ] + }, + { + "sessions_k": "25", + "tasks": 87, + "sessions": 4, + "deliveries": 643, + "distinct_atoms": 542, + "B_tokens": 98222, + "C_tokens": 92268, + "S_tokens": 79119, + "A_tokens": 1928268, + "A_mem_tokens": 827557, + "C_token_reuse_rate": 0.06061778420313168, + "C_evidence_reuse_rate": 0.1104199066874028, + "C_saved_per_task": 68.4367816091954, + "C_saved_per_task_ci95": [ + 34.51724137931034, + 114.25287356321839 + ], + "C_stale_withheld": 0, + "C_validated_reuse_rate": 1.0, + "S_token_reuse_rate": 0.19448799657917779, + "S_stale_withheld": 30, + "S_stale_reuse_rate": 0.30303030303030304, + "S_stale_gold": 4, + "notices": 69, + "invalidation_precision": 1.0, + "invalidation_recall": 0.92, + "page_mismatches": 0, + "stale_flags": 0, + "B_packet_tokens": 172851, + "C_packet_tokens": 169861, + "useful_B_equals_C": 0.27586206896551724, + "useful_S": 0.27011494252873564, + "survival": { + "1": [ + 0.9778225806451613, + 496 + ], + "5": [ + 0.9225806451612903, + 465 + ], + "10": [ + 0.8777506112469438, + 409 + ], + "20": [ + 0.7800687285223368, + 291 + ] + }, + "memory_ms_p50": 19.56839999911608, + "memory_ms_p95": 38.00810000029742, + "store_bytes": 368640, + "examples": [ + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "039513f1b18c1364", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 38, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "e15784d0fcfe16d1", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 45, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "150e7a90d1bec372", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 49, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/FAQ.md", + "sha": "f481fe4537198f01", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 7, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/PRODUCT_UPGRADE_PLAN.md", + "sha": "2bbea03fa8d909c6", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 1, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/PRODUCT_UPGRADE_PLAN.md", + "sha": "802af198a186890e", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 141, + "oracle_occurrences": 0 + } + ] + }, + { + "sessions_k": "all", + "tasks": 87, + "sessions": 1, + "deliveries": 643, + "distinct_atoms": 473, + "B_tokens": 98222, + "C_tokens": 87634, + "S_tokens": 64318, + "A_tokens": 1928268, + "A_mem_tokens": 544122, + "C_token_reuse_rate": 0.10779662397426239, + "C_evidence_reuse_rate": 0.19440124416796267, + "C_saved_per_task": 121.70114942528735, + "C_saved_per_task_ci95": [ + 76.34482758620689, + 178.7816091954023 + ], + "C_stale_withheld": 0, + "C_validated_reuse_rate": 1.0, + "S_token_reuse_rate": 0.34517725153224327, + "S_stale_withheld": 54, + "S_stale_reuse_rate": 0.3157894736842105, + "S_stale_gold": 6, + "notices": 128, + "invalidation_precision": 1.0, + "invalidation_recall": 0.8707482993197279, + "page_mismatches": 0, + "stale_flags": 0, + "B_packet_tokens": 172851, + "C_packet_tokens": 166456, + "useful_B_equals_C": 0.27586206896551724, + "useful_S": 0.26436781609195403, + "survival": { + "1": [ + 0.9754464285714286, + 448 + ], + "5": [ + 0.9249448123620309, + 453 + ], + "10": [ + 0.8778280542986425, + 442 + ], + "20": [ + 0.8408071748878924, + 446 + ] + }, + "memory_ms_p50": 36.57369999928051, + "memory_ms_p95": 68.22990000000573, + "store_bytes": 352256, + "examples": [ + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "039513f1b18c1364", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 38, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "e15784d0fcfe16d1", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 45, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "150e7a90d1bec372", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 49, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "README.md", + "sha": "2353492da96d24f4", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 145, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "ROADMAP.md", + "sha": "1faec12ba8307640", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 49, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "ROADMAP.md", + "sha": "1fb676dca4d76452", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 37, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "ROADMAP.md", + "sha": "7b40838bf03af0d5", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 43, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "ROADMAP.md", + "sha": "ab9b49fee3918b21", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 31, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "ROADMAP.md", + "sha": "c64353992a16f581", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 25, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "afb56398eb3173a5", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 33, + "oracle_occurrences": 0 + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/eval/results/2026-09-14-evidence-memory.md b/tests/eval/results/2026-09-14-evidence-memory.md new file mode 100644 index 0000000..bb7f4cd --- /dev/null +++ b/tests/eval/results/2026-09-14-evidence-memory.md @@ -0,0 +1,19 @@ +replaying codebase-index (C:\Projects\codebase-index\tests\eval\queries\self_repo_git.yml) + +## codebase-index (87 tasks; search p50 31.6 ms, update p50 154.5 ms) + +| K | tasks | sessions | B tokens | C tokens (reuse) | 95% CI saved/task | C stale withheld | S tokens (reuse) | S stale withheld | notices P/R | A tokens | A-mem tokens | packet tok B→C | +|---|---|---|---|---|---|---|---|---|---|---|---|---| +| 1 | 87 | 87 | 98222 | 98222 (0.0%) | 0.0 [0.0, 0.0] | 0 | 98222 (0.0%) | 0 | —/— | 1928268 | 1928268 | 172851→174475 | +| 5 | 87 | 18 | 98222 | 94358 (3.9%) | 44.4 [12.9, 88.7] | 0 | 87633 (10.8%) | 15 | 1.000/0.973 | 1928268 | 1291386 | 172851→171263 | +| 10 | 87 | 9 | 98222 | 92565 (5.8%) | 65.0 [30.9, 111.7] | 0 | 83756 (14.7%) | 20 | 1.000/0.925 | 1928268 | 1084176 | 172851→169696 | +| 25 | 87 | 4 | 98222 | 92268 (6.1%) | 68.4 [34.5, 114.3] | 0 | 79119 (19.4%) | 30 | 1.000/0.920 | 1928268 | 827557 | 172851→169861 | +| all | 87 | 1 | 98222 | 87634 (10.8%) | 121.7 [76.3, 178.8] | 0 | 64318 (34.5%) | 54 | 1.000/0.871 | 1928268 | 544122 | 172851→166456 | + +| K | useful@budget B=C | useful@budget S | S stale gold | page mismatches | stale flags | distinct atoms | survival h=1/5/10/20 | memory ms p50/p95 | store bytes | +|---|---|---|---|---|---|---|---|---|---| +| 1 | 0.276 | 0.276 | 0 | 0 | 0 | 643 | —/—/—/— | 4.05/6.00 | 409600 | +| 5 | 0.276 | 0.270 | 2 | 0 | 0 | 591 | 0.97/0.82/0.65/0.07 | 7.95/15.85 | 372736 | +| 10 | 0.276 | 0.276 | 2 | 0 | 0 | 563 | 0.97/0.90/0.77/0.40 | 10.89/23.31 | 368640 | +| 25 | 0.276 | 0.270 | 4 | 0 | 0 | 542 | 0.98/0.92/0.88/0.78 | 19.57/38.01 | 368640 | +| all | 0.276 | 0.264 | 6 | 0 | 0 | 473 | 0.98/0.92/0.88/0.84 | 36.57/68.23 | 352256 | diff --git a/tests/eval/run_eval.py b/tests/eval/run_eval.py index 3cad5fa..d022ef7 100644 --- a/tests/eval/run_eval.py +++ b/tests/eval/run_eval.py @@ -49,6 +49,7 @@ ABLATABLE = ( "soft_lexical", "query_expansion", + "name_cooccurrence", "fuzzy_symbols", "graph_source", "mmr", @@ -57,6 +58,11 @@ "file_agreement", ) +# Rows are looked up by label rather than position, so adding a variant cannot +# silently repoint the significance tests at the wrong column. +PREVIOUS_LABEL = "previous release (1.9.0)" +DEFAULT_LABEL = "default (all signals)" + def _parse_corpus(spec: str) -> tuple[Path, str]: """Split `:`, tolerating a Windows drive letter.""" @@ -112,7 +118,11 @@ def main(argv: list[str] | None = None) -> int: variants: list[tuple[str, RetrievalTuning]] = [ ("baseline (1.7.0)", RetrievalTuning.baseline()), - ("default (all signals)", RetrievalTuning()), + # The previous release, pinned. "Is this better than 1.7.0" and "is this + # better than what we shipped last" are different questions, and only the + # second one decides whether a change belongs in the next release. + (PREVIOUS_LABEL, RetrievalTuning.v190()), + (DEFAULT_LABEL, RetrievalTuning()), ] if args.ablate: default = RetrievalTuning() @@ -138,7 +148,7 @@ def main(argv: list[str] | None = None) -> int: repeats=args.repeats, ) collected[label].append(report) - if label == "default (all signals)": + if label == DEFAULT_LABEL: # Relabel so the per-corpus table identifies the repository # rather than repeating the variant name on every row. per_corpus_default[name] = replace(report, label=name) @@ -146,12 +156,16 @@ def main(argv: list[str] | None = None) -> int: db.close() reports = [harness.pool(collected[label], label=label) for label, _ in variants] + by_label = {rep.label: rep for rep in reports} + default = by_label[DEFAULT_LABEL] + previous = by_label[PREVIOUS_LABEL] + ablations = [rep for rep in reports if rep.label.startswith(" -")] if args.as_json: payload = { "pooled": [r.as_row() for r in reports], "per_corpus_default": {k: v.as_row() for k, v in per_corpus_default.items()}, - "per_category_default": reports[1].per_category, + "per_category_default": default.per_category, } print(json.dumps(payload, indent=2)) return 0 @@ -159,14 +173,15 @@ def main(argv: list[str] | None = None) -> int: print() print(harness.format_table(reports, baseline=reports[0])) - default = reports[1] + # The release decision is default vs the previous release, so that is the + # comparison printed first and in full. print() - print(harness.format_significance(reports[0], default, resamples=args.resamples)) + print(harness.format_significance(previous, default, resamples=args.resamples)) - if len(reports) > 2: + if ablations: print() print("Ablation significance (each row vs the shipped default):") - for rep in reports[2:]: + for rep in ablations: print() print(harness.format_significance(default, rep, resamples=args.resamples)) diff --git a/tests/fixtures/index-1.10.0/index.sqlite b/tests/fixtures/index-1.10.0/index.sqlite new file mode 100644 index 0000000..9b837b9 Binary files /dev/null and b/tests/fixtures/index-1.10.0/index.sqlite differ diff --git a/tests/golden/mcp_healthcheck.json b/tests/golden/mcp_healthcheck.json index 565a111..e20798a 100644 --- a/tests/golden/mcp_healthcheck.json +++ b/tests/golden/mcp_healthcheck.json @@ -7,6 +7,10 @@ "path": ".claude/cache/codebase-index/index.sqlite", "stale": false }, + "memory": { + "enabled": true, + "exists": false + }, "package_version": "", "root": "", "schema_version": 1, diff --git a/tests/golden/mcp_index_stats.json b/tests/golden/mcp_index_stats.json index d8dcf27..8e6c7c6 100644 --- a/tests/golden/mcp_index_stats.json +++ b/tests/golden/mcp_index_stats.json @@ -3,6 +3,10 @@ "exists": true, "files": 6, "head_commit": "", + "memory": { + "enabled": true, + "exists": false + }, "schema_version": 1, "symbols": 7, "tool": "index_stats", diff --git a/tests/golden/stats.json b/tests/golden/stats.json index bf445b5..c89a6c0 100644 --- a/tests/golden/stats.json +++ b/tests/golden/stats.json @@ -3,6 +3,10 @@ "exists": true, "files": 6, "head_commit": "", + "memory": { + "enabled": true, + "exists": false + }, "symbols": 7, "treesitter_coverage": [ { diff --git a/tests/test_discovery.py b/tests/test_discovery.py index b00817d..9108219 100644 --- a/tests/test_discovery.py +++ b/tests/test_discovery.py @@ -49,3 +49,27 @@ def read_bytes_should_not_be_used(self: Path) -> bytes: found = _walk_paths(root) assert "big.py" in found + + +def test_walk_skips_symlinks_pointing_outside_root(tmp_path): + import os + + import pytest + + outside = tmp_path / "outside" + (outside / "bazel-out").mkdir(parents=True) + (outside / "bazel-out" / "gen.py").write_text("x = 1\n", encoding="utf-8") + (outside / "leak.py").write_text("SECRET = 1\n", encoding="utf-8") + + root = tmp_path / "proj" + (root / "src").mkdir(parents=True) + (root / "src" / "keep.py").write_text("y = 2\n", encoding="utf-8") + try: + os.symlink(outside / "bazel-out", root / "bazel-out", target_is_directory=True) + os.symlink(outside / "leak.py", root / "src" / "leak.py") + except (OSError, NotImplementedError): + pytest.skip("symlinks unavailable") + + found = _walk_paths(root) + + assert set(found) == {"src/keep.py"} diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index e98412f..50b37d8 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -39,6 +39,7 @@ def test_mcp_server_has_expected_tools(): "path_between", "describe_symbol", "index_stats", + "verify_evidence", } diff --git a/tests/test_memory_eval_smoke.py b/tests/test_memory_eval_smoke.py new file mode 100644 index 0000000..8a7bbea --- /dev/null +++ b/tests/test_memory_eval_smoke.py @@ -0,0 +1,119 @@ +"""A miniature of the sequential memory benchmark on a synthetic git history. + +The invariants asserted here must hold on any corpus: memory never withholds text the +session does not hold, withheld packets restore to the no-memory packet exactly, every +invalidation notice matches an independent oracle, and a one-task session reuses nothing. +The scenario is built so the unsafe arm demonstrably goes stale where memory does not. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import yaml + +from tests.eval import memory_eval + +INVOICE = '''def compute_invoice_total(lines, tax_rate): + """Sum invoice line amounts and apply the tax rate.""" + subtotal = sum(line.amount for line in lines) + return round(subtotal * (1 + tax_rate), 2) +''' + +REFUND = '''def issue_refund(invoice, amount): + """Refund part of an invoice total.""" + if amount > invoice.total: + raise ValueError("refund exceeds invoice total") + return invoice.total - amount +''' + +TAX = '''def tax_rate_for(region): + """Look up the sales tax rate for a region.""" + return {"eu": 0.2, "us": 0.07}.get(region, 0.0) +''' + + +def _git(repo: Path, *args: str) -> str: + return subprocess.run( + ["git", "-c", "user.name=bench", "-c", "user.email=bench@example.com", + "-c", "core.autocrlf=false", "-C", str(repo), *args], + check=True, capture_output=True, text=True, + ).stdout.strip() + + +def _commit(repo: Path, message: str, files: dict[str, str]) -> str: + for rel, text in files.items(): + path = repo / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8", newline="\n") + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "-m", message) + return _git(repo, "rev-parse", "HEAD") + + +def _history(tmp_path: Path) -> tuple[Path, Path]: + repo = tmp_path / "origin" + repo.mkdir() + _git(repo, "init", "-q") + _commit(repo, "initial billing", {"billing/invoice.py": INVOICE, + "billing/refund.py": REFUND, "billing/tax.py": TAX}) + queries = [] + # Task 1 sees the original refund; its own commit then changes the signature and the + # check without moving any line, so the same (path, lines) now holds different text. + c1 = _commit(repo, "tighten refund check", {"billing/refund.py": REFUND.replace( + "def issue_refund(invoice, amount):", "def issue_refund(invoice, amount, reason=None):" + ).replace(">", ">=")}) + queries.append({"query": "refund exceeds invoice total", "commit": c1, + "expected_files": ["billing/refund.py"]}) + # Unrelated change: evidence from task 1 about invoices stays valid. + c2 = _commit(repo, "add region", {"billing/tax.py": TAX.replace('"us": 0.07', '"us": 0.07, "ca": 0.05')}) + queries.append({"query": "refund exceeds invoice total amount", "commit": c2, + "expected_files": ["billing/tax.py"]}) + c3 = _commit(repo, "round tax", {"billing/tax.py": TAX.replace("0.0)", "0.0) or 0.0")}) + queries.append({"query": "invoice total refund tax rate", "commit": c3, + "expected_files": ["billing/tax.py"]}) + path = tmp_path / "queries.yml" + path.write_text(yaml.safe_dump(queries), encoding="utf-8") + return repo, path + + +def test_replay_is_sound_and_reuses_only_unchanged_evidence(tmp_path): + repo, queries = _history(tmp_path) + head_before = _git(repo, "rev-parse", "HEAD") + result = memory_eval.replay_corpus(repo, queries, [None, 1]) + assert result.tasks == 3 + assert _git(repo, "rev-parse", "HEAD") == head_before # source repository untouched + assert _git(repo, "status", "--porcelain") == "" + + one_session = result.totals[None] + assert one_session.c_stale_withheld == 0 + assert one_session.page_mismatches == 0 + assert one_session.notice_fp == 0 and one_session.notice_fn == 0 + assert one_session.c_reused > 0 + assert one_session.c_tokens < one_session.b_tokens + assert one_session.s_stale_withheld > 0 # trusting old reads goes stale here + assert one_session.notices >= 1 # ...and memory said so instead + + per_task = result.totals[1] + assert per_task.c_reused == 0 and per_task.c_tokens == per_task.b_tokens + assert per_task.s_reused == 0 + + summary = memory_eval.summarise("all", one_session) + assert summary["C_validated_reuse_rate"] == 1.0 + assert memory_eval.format_report([summary]).count("|") > 20 + + +def test_oracle_helpers(): + assert memory_eval.span_present(["a", "b", "c"], ["b", "c"], 2) + assert memory_eval.span_present(["x", "b", "c"], ["b", "c"], 1) # moved, unique + assert not memory_eval.span_present(["b", "c", "b", "c"], ["b", "c"], 9) # ambiguous + assert not memory_eval.span_present(None, ["b"], 1) + truth = memory_eval.SessionTruth() + span = "export default function App() {\n return 1\n}" + truth.full_spans["a.tsx"] = {span} + truth.held["b.py"] = {"def g(x):"} + assert memory_eval.session_holds(truth, "a.tsx", "function App()", span) # derived excerpt + assert not memory_eval.session_holds(truth, "a.tsx", "function App()", span + " ") + assert memory_eval.session_holds(truth, "b.py", "def g(x):", "def g(x):\n return 2") + assert not memory_eval.session_holds(truth, "b.py", "def g(y):", "def g(y):") diff --git a/tests/test_memory_identity.py b/tests/test_memory_identity.py new file mode 100644 index 0000000..4425828 --- /dev/null +++ b/tests/test_memory_identity.py @@ -0,0 +1,156 @@ +"""Evidence identity: line model, hashing, references. Pure functions, so tested exhaustively.""" + +from __future__ import annotations + +import random +import sys + +import pytest + +from codebase_index.memory import identity as ident +from codebase_index.parsers.line_chunker import chunk_text + + +def test_crlf_and_lf_checkouts_are_the_same_evidence(): + lf = b"def f():\n return 1\n" + crlf = b"def f():\r\n return 1\r\n" + assert ident.split_lines(lf) == ident.split_lines(crlf) + assert ident.span_sha(ident.split_lines(lf), 1, 2) == ident.span_sha(ident.split_lines(crlf), 1, 2) + + +def test_line_model_matches_the_chunker_including_form_feeds(): + """Chunk content and the hashed span must be the same lines, or verification lies.""" + raw = b"a = 1\n\x0cb = 2\nc = 3\r\nd = 4\n" + text = raw.decode("utf-8").replace("\r\n", "\n") + chunk = chunk_text(text, window_lines=80, overlap_lines=0)[0] + lines = ident.split_lines(raw) + assert ident.span_text(lines, chunk.line_start, chunk.line_end) == chunk.content + + +def test_whitespace_and_comments_are_part_of_identity(): + base = ident.split_lines(b"x = 1\n") + for variant in (b"x = 1\n", b"x = 1 # note\n", b"X = 1\n"): + assert ident.span_sha(ident.split_lines(variant), 1, 1) != ident.span_sha(base, 1, 1) + + +def test_undecodable_bytes_still_change_the_hash(): + a = ident.split_lines(b"blob = '\xff'\n") + b = ident.split_lines(b"blob = '\xfe'\n") + assert ident.span_sha(a, 1, 1) != ident.span_sha(b, 1, 1) + # ...while the indexer-visible text (bytes dropped) is identical for both. + assert ident.visible_text(a[0]) == ident.visible_text(b[0]) + + +def test_span_identity_is_independent_of_position(): + body = [b"def g(x):", b" return x * 2"] + top = ident.split_lines(b"\n".join(body) + b"\n") + shifted = ident.split_lines(b"import os\n\n" + b"\n".join(body) + b"\n") + assert ident.span_sha(top, 1, 2) == ident.span_sha(shifted, 3, 4) + + +@pytest.mark.parametrize("start,end", [(0, 1), (2, 1), (1, 99)]) +def test_out_of_range_spans_have_no_identity(start, end): + assert ident.span_sha(["a", "b"], start, end) is None + + +def test_reference_round_trip_and_prefix_matching(): + lines = ident.split_lines(b"one\ntwo\nthree\n") + full = ident.span_sha(lines, 2, 3) + ref = ident.make_ref("src/mod.py", 2, 3, full) + printed = str(ref) + assert printed == f"src/mod.py:2-3@{full[:ident.REF_HASH_CHARS]}" + parsed = ident.parse_ref(printed) + assert parsed.path == "src/mod.py" and (parsed.line_start, parsed.line_end) == (2, 3) + assert parsed.matches(full) + assert not parsed.matches(ident.span_sha(lines, 1, 2)) + + +def test_reference_parsing_is_right_anchored_for_odd_paths(): + ref = ident.parse_ref("docs/a:b@c.md:10-12@0123456789abcdef") + assert ref.path == "docs/a:b@c.md" and ref.line_start == 10 and ref.line_end == 12 + + +def test_windows_separators_normalise(): + assert ident.parse_ref("src\\pkg\\m.py:1-2@0123456789ab").path == "src/pkg/m.py" + + +@pytest.mark.parametrize( + "text", + [ + "src/m.py:1-2", # no hash + "src/m.py:3-2@0123456789abcdef", # inverted range + "src/m.py:0-2@0123456789abcdef", # zero line + "src/m.py:1-2@0123456789", # hash too short + "src/m.py:1-2@zzzzzzzzzzzzzzzz", # not hex + "/etc/passwd:1-1@0123456789abcdef", # absolute + "C:/Windows/win.ini:1-1@0123456789abcdef", + "../outside.py:1-1@0123456789abcdef", + "src/../../outside.py:1-1@0123456789abcdef", + "src/m\x00.py:1-1@0123456789abcdef", + ], +) +def test_malformed_or_escaping_references_are_rejected(text): + with pytest.raises(ValueError): + ident.parse_ref(text) + + +def test_content_matches_equality_and_excerpts(): + span = "def f(a, b):\n return a + b" + assert ident.content_matches(span, span) + assert ident.is_full_span(span, span) + assert ident.content_matches("def f(a, b):", span) + assert not ident.is_full_span("def f(a, b):", span) + assert not ident.content_matches("def f(a):", span) + assert not ident.content_matches(None, span) + assert not ident.content_matches("", span) + assert ident.content_matches(span.replace("\n", "\r\n"), span) + + +def test_repo_ids_differ_per_root_and_fold_case_on_windows(tmp_path): + a, b = tmp_path / "a", tmp_path / "b" + a.mkdir() + b.mkdir() + assert ident.repo_id_for(a) != ident.repo_id_for(b) + assert ident.repo_id_for(a) == ident.repo_id_for(str(a)) + if sys.platform == "win32": + assert ident.repo_id_for(str(a).upper()) == ident.repo_id_for(str(a).lower()) + + +def test_session_keys_are_scoped_to_repository_and_hide_the_tag(): + key_a = ident.session_key("repoA", "auth-fix") + assert key_a != ident.session_key("repoB", "auth-fix") + assert key_a != ident.session_key("repoA", "auth-fix-2") + assert "auth" not in key_a + + +@pytest.mark.parametrize("tag", ["", " ", "-lead", "a b", "x" * 129, "emoji-😀", "a/b"]) +def test_invalid_session_tags(tag): + with pytest.raises(ValueError): + ident.validate_session_tag(tag) + + +def test_valid_session_tags(): + assert ident.validate_session_tag(" auth-fix.2026:09_10 ") == "auth-fix.2026:09_10" + + +def test_randomised_edits_change_identity_iff_span_bytes_change(): + """Property: for any edit, the span hash is unchanged exactly when the span text is.""" + rng = random.Random(20260910) + vocab = ["x = 1", "return x", "", " pass", "def f():", "}", "# c", "y = 'é'"] + for _ in range(400): + lines = [rng.choice(vocab) for _ in range(rng.randint(3, 25))] + start = rng.randint(1, len(lines)) + end = rng.randint(start, len(lines)) + before = ident.span_sha(lines, start, end) + edited = list(lines) + op = rng.choice(["replace", "insert", "delete", "noop"]) + pos = rng.randrange(len(edited)) + if op == "replace": + edited[pos] = rng.choice(vocab) + elif op == "insert": + edited.insert(pos, rng.choice(vocab)) + elif op == "delete" and len(edited) > 1: + del edited[pos] + after = ident.span_sha(edited, start, end) + same_text = ident.span_text(edited, start, end) == ident.span_text(lines, start, end) + assert (after == before) == same_text diff --git a/tests/test_memory_lifecycle.py b/tests/test_memory_lifecycle.py new file mode 100644 index 0000000..2af1f35 --- /dev/null +++ b/tests/test_memory_lifecycle.py @@ -0,0 +1,232 @@ +"""Evidence validity across real repository lifecycles, driven through real git. + +Validity is byte identity with the working tree, so every git operation is just another +way of changing (or restoring) bytes. These tests pin that down for the operations agents +actually perform, plus the security property that excluded content never reaches memory. +""" + +from __future__ import annotations + +import subprocess +from dataclasses import dataclass +from pathlib import Path + +import pytest + +from codebase_index.config import Config +from codebase_index.indexer.pipeline import build_index, update_index +from codebase_index.service import search_payload, verify_payload +from codebase_index.storage.db import Database + +REFUND = '''def issue_refund(invoice, amount): + """Refund part of an invoice total.""" + if amount > invoice.total: + raise ValueError("refund exceeds invoice total") + return invoice.total - amount +''' +INVOICE = '''def compute_invoice_total(lines, tax_rate): + """Sum invoice line amounts and apply the tax rate.""" + subtotal = sum(line.amount for line in lines) + return round(subtotal * (1 + tax_rate), 2) +''' +TAX = '''def tax_rate_for(region): + """Look up the sales tax rate for a region.""" + return {"eu": 0.2, "us": 0.07}.get(region, 0.0) +''' +QUERY = "refund exceeds invoice total" + + +def _git(root: Path, *args: str) -> str: + return subprocess.run( + ["git", "-c", "user.name=t", "-c", "user.email=t@example.com", + "-c", "core.autocrlf=false", "-C", str(root), *args], + check=True, capture_output=True, text=True, + ).stdout.strip() + + +@dataclass +class Repo: + root: Path + cfg: Config + db: Path + + def write(self, rel: str, text: str) -> None: + path = self.root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(text.encode("utf-8")) + + def commit(self, message: str) -> None: + _git(self.root, "add", "-A") + _git(self.root, "commit", "-q", "-m", message) + + def update(self) -> None: + with Database(self.db) as db: + update_index(self.cfg, db, root=self.root) + + def search(self, query: str = QUERY, session: str = "s1") -> dict: + return search_payload(self.db, self.cfg, query, mode="hybrid", limit=10, + token_budget=1500, no_fallback=False, session=session) + + def states(self, session: str = "s1") -> dict[str, str]: + payload = verify_payload(self.cfg, [], session) + return {v["ref"].rsplit("@", 1)[0].rsplit(":", 1)[0]: v["state"] + for v in payload["evidence"]} + + +def _make_repo(tmp_path: Path, monkeypatch, name: str = "repo") -> Repo: + for var in ("CBX_DB_PATH", "CBX_MEMORY", "CBX_ROOT"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("CBX_MEMORY_PATH", str(tmp_path / "memory.sqlite")) + root = tmp_path / name + root.mkdir() + _git(root, "init", "-q", "-b", "main") + repo = Repo(root, Config(root=str(root)), tmp_path / f"{name}.sqlite") + repo.cfg.root = str(root) + repo.write("billing/refund.py", REFUND) + repo.write("billing/invoice.py", INVOICE) + repo.write("billing/tax.py", TAX) + repo.commit("initial") + with Database(repo.db) as db: + build_index(repo.cfg, db, root=root) + return repo + + +@pytest.fixture +def repo(tmp_path, monkeypatch) -> Repo: + return _make_repo(tmp_path, monkeypatch) + + +def _delivered_paths(payload: dict) -> set[str]: + return {r["path"] for r in payload["results"] if r["snippet"]} + + +def test_unrelated_commit_keeps_evidence_valid_and_reused(repo): + delivered = _delivered_paths(repo.search()) + assert "billing/refund.py" in delivered + repo.write("billing/tax.py", TAX.replace("0.07", "0.08")) + repo.commit("adjust us tax") + repo.update() + assert set(repo.states().values()) == {"valid"} + again = repo.search() + assert again["memory"]["invalidated"] == [] + assert any(r.get("reused") for r in again["results"] if r["path"] == "billing/refund.py") + + +def test_branch_switch_invalidates_what_differs_and_restores_on_return(repo): + repo.search() + _git(repo.root, "checkout", "-q", "-b", "feature") + repo.write("billing/refund.py", REFUND.replace(">", ">=")) + repo.commit("tighten refund") + assert repo.states()["billing/refund.py"] == "changed" + _git(repo.root, "checkout", "-q", "main") + assert repo.states()["billing/refund.py"] == "valid" # identical bytes are identical evidence + + +def test_detached_head_is_validated_against_its_own_tree(repo): + first = _git(repo.root, "rev-parse", "HEAD") + repo.write("billing/refund.py", REFUND.replace("amount):", "amount, note=None):")) + repo.commit("add note") + repo.update() + repo.search() # evidence from the new commit + _git(repo.root, "checkout", "-q", "--detach", first) + assert repo.states()["billing/refund.py"] == "changed" + + +def test_worktree_is_a_separate_memory_scope(repo, tmp_path): + payload = repo.search() + ref = next(v["ref"] for v in verify_payload(repo.cfg, [], "s1")["evidence"]) + other = tmp_path / "wt" + _git(repo.root, "worktree", "add", "-q", "--detach", str(other), "HEAD") + cfg = Config() + cfg.root = str(other) + scoped = verify_payload(cfg, [ref], "s1") + assert scoped["session"]["found"] is False # sessions never leak across checkouts + assert scoped["evidence"][0]["state"] == "valid" # but the bytes there are the same + assert payload["memory"]["reused"] == 0 + + +def test_dirty_staged_and_unstaged_changes_are_what_is_validated(repo): + repo.search() + changed = REFUND.replace("refund exceeds", "refund is larger than") + repo.write("billing/refund.py", changed) + assert repo.states()["billing/refund.py"] == "changed" # unstaged edit + _git(repo.root, "add", "billing/refund.py") + assert repo.states()["billing/refund.py"] == "changed" # staged, same bytes + repo.write("billing/refund.py", REFUND) + assert repo.states()["billing/refund.py"] == "valid" # worktree wins over index + _git(repo.root, "reset", "-q", "billing/refund.py") + assert repo.states()["billing/refund.py"] == "valid" + + +def test_rename_and_delete_invalidate_the_old_paths(repo): + repo.search("refund invoice total tax rate region") + held = repo.states() + assert {"billing/refund.py", "billing/tax.py"} <= set(held) + _git(repo.root, "mv", "billing/refund.py", "billing/refunds.py") + (repo.root / "billing" / "tax.py").unlink() + states = repo.states() + assert states["billing/refund.py"] == "deleted" + assert states["billing/tax.py"] == "deleted" + + +def test_new_file_evidence_is_delivered_before_it_can_be_reused(repo): + repo.write("billing/credit.py", REFUND.replace("issue_refund", "issue_store_credit")) + repo.update() + first = repo.search("issue store credit refund exceeds invoice total") + hit = next(r for r in first["results"] if r["path"] == "billing/credit.py") + assert hit["snippet"] and not hit.get("reused") + + +def test_rebase_evidence_follows_bytes_not_history(repo): + repo.search("refund invoice total tax rate region") + _git(repo.root, "checkout", "-q", "-b", "topic") + repo.write("billing/invoice.py", INVOICE.replace("round(", "abs(round("). replace(", 2)", ", 2))")) + repo.commit("topic change to invoice") + _git(repo.root, "checkout", "-q", "main") + repo.write("billing/tax.py", TAX.replace("0.2", "0.21")) + repo.commit("main change to tax") + _git(repo.root, "checkout", "-q", "topic") + _git(repo.root, "rebase", "-q", "main") + states = repo.states() + assert states["billing/refund.py"] == "valid" + assert states["billing/invoice.py"] == "changed" + assert states.get("billing/tax.py", "changed") == "changed" + + +def test_crlf_checkout_is_not_a_change(repo): + repo.search() + (repo.root / "billing" / "refund.py").write_bytes(REFUND.replace("\n", "\r\n").encode()) + assert repo.states()["billing/refund.py"] == "valid" + + +CANARY = "CANARY_7f3a9e_do_not_persist" + + +def test_excluded_content_never_reaches_memory_or_verification(tmp_path, monkeypatch): + repo = _make_repo(tmp_path, monkeypatch) + repo.write(".env", f"API_KEY={CANARY}\n") + repo.write("config/secrets.json", f'{{"token": "{CANARY}"}}\n') + repo.write("node_modules/pkg/index.js", f"module.exports = '{CANARY}';\n") + repo.write("private/notes.py", f"NOTE = '{CANARY}'\n") + repo.write(".gitignore", "private/\n") + repo.write("billing/audit.py", f"def audit_refund():\n return '{CANARY} refund exceeds'\n") + repo.update() + + payload = repo.search(f"{CANARY} refund exceeds invoice total") + assert all(not r["path"].startswith((".env", "config/", "node_modules/", "private/")) + for r in payload["results"]) + # The file is indexed today; ignoring it afterwards must stop memory from using it. + assert any(r["path"] == "billing/audit.py" for r in payload["results"]) + repo.write(".gitignore", "private/\nbilling/audit.py\n") + later = repo.search(f"{CANARY} refund exceeds invoice total") + audit = [r for r in later["results"] if r["path"] == "billing/audit.py"] + assert audit and all(r.get("stale") for r in audit) + + refs = [f"{p}:1-1@0123456789abcdef" for p in + (".env", "config/secrets.json", "node_modules/pkg/index.js", "private/notes.py", + "billing/audit.py")] + verdicts = verify_payload(repo.cfg, refs)["evidence"] + assert [v["state"] for v in verdicts] == ["excluded"] * 5 + + blob = b"".join(p.read_bytes() for p in tmp_path.glob("memory.sqlite*")) + assert CANARY.encode() not in blob diff --git a/tests/test_memory_session.py b/tests/test_memory_session.py new file mode 100644 index 0000000..1f57de4 --- /dev/null +++ b/tests/test_memory_session.py @@ -0,0 +1,293 @@ +"""End-to-end evidence memory through the service layer: retrieve -> persist -> reuse -> +edit -> invalidate. The load-bearing assertions are equivalences with the no-memory +packet, because they are what make every measured saving attributable to memory alone. +""" + +from __future__ import annotations + +import copy +import sqlite3 + +import pytest + +from codebase_index.config import Config +from codebase_index.indexer.pipeline import build_index, update_index +from codebase_index.retrieval.pipeline import search +from codebase_index.service import search_payload +from codebase_index.storage.db import Database + +INVOICE = '''def compute_invoice_total(lines, tax_rate): + """Sum invoice line amounts and apply the tax rate.""" + subtotal = sum(line.amount for line in lines) + return round(subtotal * (1 + tax_rate), 2) + + +def format_invoice_number(prefix, sequence): + """Render an invoice number such as INV-000042.""" + return f"{prefix}-{sequence:06d}" +''' + +REFUND = '''def issue_refund(invoice, amount): + """Refund part of an invoice total.""" + if amount > invoice.total: + raise ValueError("refund exceeds invoice total") + return invoice.total - amount +''' + +QUERY = "compute invoice total" + + +@pytest.fixture +def repo(tmp_path, monkeypatch): + root = tmp_path / "repo" + (root / "src" / "billing").mkdir(parents=True) + (root / "src" / "billing" / "invoice.py").write_text(INVOICE, encoding="utf-8") + (root / "src" / "billing" / "refund.py").write_text(REFUND, encoding="utf-8") + monkeypatch.setenv("CBX_MEMORY_PATH", str(tmp_path / "memory.sqlite")) + monkeypatch.delenv("CBX_MEMORY", raising=False) + monkeypatch.delenv("CBX_DB_PATH", raising=False) + cfg = Config() + cfg.root = str(root) + db_path = tmp_path / "index.sqlite" + with Database(db_path) as db: + build_index(cfg, db, root=root) + return root, cfg, db_path + + +def _run(repo, query=QUERY, session=None): + _root, cfg, db_path = repo + return search_payload(db_path, cfg, query, mode="hybrid", limit=10, token_budget=1500, + no_fallback=False, session=session) + + +def _baseline(repo, monkeypatch, query=QUERY): + monkeypatch.setenv("CBX_MEMORY", "0") + try: + return _run(repo, query) + finally: + monkeypatch.delenv("CBX_MEMORY") + + +def _update(repo): + root, cfg, db_path = repo + with Database(db_path) as db: + update_index(cfg, db, root=root) + + +def _notice_states(payload): + return {n["ref"].rsplit("@", 1)[0].rsplit(":", 1)[0]: n["state"] + for n in payload["memory"]["invalidated"]} + + +def test_memory_disabled_is_identical_to_the_retrieval_pipeline(repo, monkeypatch): + root, cfg, db_path = repo + monkeypatch.setenv("CBX_MEMORY", "0") + via_service = _run(repo) + with Database(db_path) as db: + direct = search(db.conn, QUERY, mode="hybrid", limit=10, token_budget=1500, + no_fallback=False, root=root, config=cfg, compact=True, + compact_min_reduction=0.25) + assert via_service == direct + + +def test_enabled_without_session_on_a_fresh_index_is_identical(repo, monkeypatch): + assert _run(repo) == _baseline(repo, monkeypatch) + + +def test_first_session_call_is_the_baseline_plus_a_memory_block(repo, monkeypatch): + first = _run(repo, session="t1") + assert first.pop("memory") == {"session": "t1", "reused": 0, "tokens_saved": 0, + "invalidated": []} + assert first == _baseline(repo, monkeypatch) + + +def test_repeat_call_withholds_exactly_what_the_session_already_holds(repo, monkeypatch): + first = _run(repo, session="t1") + second = _run(repo, session="t1") + delivered = [r for r in first["results"] if r["snippet"]] + assert delivered + assert [r.get("reused", False) for r in second["results"]] == [ + bool(r["snippet"]) for r in first["results"] + ] + assert second["memory"]["reused"] == len(delivered) + assert second["memory"]["tokens_saved"] == sum(r["token_est"] for r in delivered) + + restored = copy.deepcopy(second) + restored.pop("memory") + for result, original in zip(restored["results"], first["results"]): + if result.pop("reused", False): + result["snippet"] = original["snippet"] + assert restored == _baseline(repo, monkeypatch) + + +def test_another_session_receives_everything(repo, monkeypatch): + _run(repo, session="t1") + other = _run(repo, session="t2") + assert other.pop("memory")["reused"] == 0 + assert other == _baseline(repo, monkeypatch) + + +def test_changed_evidence_is_reported_once_and_redelivered(repo): + root, _, _ = repo + first = _run(repo, session="t1") + body = next(r for r in first["results"] if r["path"] == "src/billing/refund.py") + assert "amount > invoice.total" in body["snippet"] + (root / "src/billing/refund.py").write_text(REFUND.replace(">", ">="), encoding="utf-8") + _update(repo) + second = _run(repo, session="t1") + assert _notice_states(second).get("src/billing/refund.py") == "changed" + redelivered = next(r for r in second["results"] if r["path"] == "src/billing/refund.py") + assert "amount >= invoice.total" in redelivered["snippet"] + assert not redelivered.get("reused") + # Unchanged evidence in the same packet is still withheld... + assert any(r.get("reused") for r in second["results"] + if r["path"] == "src/billing/invoice.py") + # ...and each invalidation is reported exactly once. + assert _run(repo, session="t1")["memory"]["invalidated"] == [] + + +def test_unindexed_edit_marks_results_stale_and_never_withholds_them(repo): + """The index still holds the old body; the snippet is flagged, never trusted or hidden.""" + root, _, _ = repo + _run(repo, session="t1") + (root / "src/billing/refund.py").write_text(REFUND.replace(">", ">="), encoding="utf-8") + second = _run(repo, session="t1") + stale = [r for r in second["results"] if r.get("stale")] + assert [r["path"] for r in stale] == ["src/billing/refund.py"] + assert "amount > invoice.total" in stale[0]["snippet"] + assert not stale[0].get("reused") + assert _notice_states(second).get("src/billing/refund.py") == "changed" + + +def test_excerpt_that_still_holds_is_not_stale(repo): + """A signature snippet stays accurate when only the body changes, so it is not flagged; + the span changed, so it is delivered again rather than withheld.""" + root, _, _ = repo + first = _run(repo, session="t1") + signature = next(r for r in first["results"] if r["path"] == "src/billing/invoice.py") + assert signature["snippet"] == "def compute_invoice_total(lines, tax_rate):" + (root / "src/billing/invoice.py").write_text(INVOICE.replace("round(", "floor("), + encoding="utf-8") + second = _run(repo, session="t1") + again = next(r for r in second["results"] if r["path"] == "src/billing/invoice.py") + assert not again.get("stale") and not again.get("reused") and again["snippet"] + + +def test_derived_index_text_is_flagged_only_when_the_file_really_changed(repo, tmp_path): + """Config-key and section summaries are derived, not copied from the file, so they are + not byte-verifiable: never recorded or withheld, and flagged stale only when the file's + bytes differ from what was indexed. (Ranking decides whether such a chunk surfaces, so + this drives the processor with the exact chunk the index stored.)""" + from datetime import datetime, timezone + from types import SimpleNamespace + + from codebase_index.memory import identity as ident + from codebase_index.memory.session import EvidenceProcessor, Session, index_sha_lookup + from codebase_index.memory.store import MemoryStore + + root, cfg, db_path = repo + (root / "config.json").write_text('{"refund": {"window_days": 30}}\n', encoding="utf-8") + _update(repo) + derived = "config key: refund.window_days = 30" + with Database(db_path) as db: + stored = [r[0] for r in db.conn.execute( + "SELECT c.content FROM chunks c JOIN files f ON f.id = c.file_id " + "WHERE f.path = 'config.json' AND c.kind = 'doc'")] + assert stored == [derived] + + def process(store: MemoryStore, session_id: int) -> dict: + result = {"rank": 1, "path": "config.json", "line_start": 1, "line_end": 1, + "snippet": derived, "token_est": 9, "skeletonized": False} + repo_id = ident.repo_id_for(root) + with Database(db_path) as db: + EvidenceProcessor( + root=root, config=cfg, now=datetime.now(timezone.utc), + session=Session(tag="t", repo_id=repo_id, store=store, session_id=session_id), + index_sha=index_sha_lookup(db.conn), + )({"results": [result]}, [SimpleNamespace(content=derived)]) + return result + + with MemoryStore.open(tmp_path / "derived.sqlite") as store: + session_id = store.touch_session("r", "t", now=datetime.now(timezone.utc)) + fresh = process(store, session_id) + assert "stale" not in fresh and fresh["snippet"] == derived and "reused" not in fresh + assert store.pending(session_id) == [] # never recorded + (root / "config.json").write_text('{"refund": {"window_days": 14}}\n', encoding="utf-8") + assert process(store, session_id).get("stale") is True + + +def test_code_that_only_moved_within_its_file_is_still_reused(repo): + root, _, _ = repo + first = _run(repo, session="t1") + target = next(r for r in first["results"] if r["path"] == "src/billing/invoice.py" + and r["snippet"] and "def compute_invoice_total" in r["snippet"]) + (root / "src/billing/invoice.py").write_text("import math\nimport decimal\n\n\n" + INVOICE, + encoding="utf-8") + _update(repo) + second = _run(repo, session="t1") + moved = [r for r in second["results"] if r["path"] == "src/billing/invoice.py" + and (r["line_start"], r["line_end"]) == (target["line_start"] + 4, + target["line_end"] + 4)] + assert moved and moved[0].get("reused") is True and moved[0]["snippet"] is None + assert second["memory"]["invalidated"] == [] + + +def test_deleted_file_evidence_is_reported(repo): + root, _, _ = repo + first = _run(repo, "issue refund exceeds invoice total", session="t1") + assert any(r["path"] == "src/billing/refund.py" and r["snippet"] for r in first["results"]) + (root / "src/billing/refund.py").unlink() + _update(repo) + assert _notice_states(_run(repo, session="t1")).get("src/billing/refund.py") == "deleted" + + +def test_locked_memory_store_degrades_to_the_full_packet(repo, monkeypatch, tmp_path): + import codebase_index.memory.store as store_mod + + monkeypatch.setattr(store_mod, "BUSY_TIMEOUT_MS", 50) + _run(repo, session="t1") + holder = sqlite3.connect(tmp_path / "memory.sqlite", isolation_level=None) + holder.execute("BEGIN IMMEDIATE") + try: + second = _run(repo, session="t1") + finally: + holder.execute("ROLLBACK") + holder.close() + block = second.pop("memory") + assert block["available"] is False and block["reused"] == 0 + assert second == _baseline(repo, monkeypatch) + + +def test_session_with_memory_disabled_says_so(repo, monkeypatch): + monkeypatch.setenv("CBX_MEMORY", "0") + assert _run(repo, session="t1")["memory"] == { + "session": "t1", "available": False, "reason": "memory is disabled"} + + +def test_malformed_session_tag_is_rejected(repo): + with pytest.raises(ValueError): + _run(repo, session="not a tag!") + + +def test_session_survives_an_index_rebuild_with_unchanged_sources(repo): + root, cfg, db_path = repo + first = _run(repo, session="t1") + for suffix in ("", "-wal", "-shm"): + sidecar = db_path.with_name(db_path.name + suffix) + if sidecar.exists(): + sidecar.unlink() + with Database(db_path) as db: + build_index(cfg, db, root=root) + second = _run(repo, session="t1") + assert second["memory"]["reused"] == sum(1 for r in first["results"] if r["snippet"]) + + +def test_memory_store_never_contains_delivered_text_or_the_tag(repo, tmp_path): + first = _run(repo, session="privacy-canary-tag") + _run(repo, session="privacy-canary-tag") + blob = b"".join(p.read_bytes() for p in tmp_path.glob("memory.sqlite*")) + assert b"privacy-canary-tag" not in blob + for result in first["results"]: + for line in (result["snippet"] or "").splitlines(): + if len(line.strip()) >= 16: + assert line.strip().encode() not in blob diff --git a/tests/test_memory_stale_context.py b/tests/test_memory_stale_context.py new file mode 100644 index 0000000..2e28b3d --- /dev/null +++ b/tests/test_memory_stale_context.py @@ -0,0 +1,124 @@ +"""Signature test for 2.0: a query-keyed cache serves a stale fact; evidence memory cannot. + + T0 an agent learns fact X (sessions last 3600 s) from evidence E + T1 an unrelated commit lands -> E is still true: both reuse it + T2 E changes (sessions now last 900 s) + T3 a semantically similar question is asked + query-keyed cache -> similar question, cache hit, serves X = 3600 (stale) + evidence memory -> E's bytes changed: reports E invalid, delivers 900 + +Deterministic: real git history, a token-overlap similarity for the cache, no model. +""" + +from __future__ import annotations + +import re +import subprocess +from pathlib import Path + +from codebase_index.config import Config +from codebase_index.indexer.pipeline import build_index, update_index +from codebase_index.service import search_payload, verify_payload +from codebase_index.storage.db import Database + +POLICY = '''"""Session lifetime policy.""" + +SESSION_TTL_SECONDS = 3600 # idle login session lifetime in seconds +''' +BILLING = "def invoice_total(lines):\n return sum(lines)\n" + + +class QueryKeyedCache: + """What a semantic cache does: reuse an answer when the question looks similar.""" + + def __init__(self, threshold: float = 0.5) -> None: + self.threshold = threshold + self.entries: list[tuple[set[str], dict]] = [] + + @staticmethod + def _words(query: str) -> set[str]: + return set(re.findall(r"[a-z0-9]+", query.lower())) + + def get(self, query: str): + words = self._words(query) + for key, packet in self.entries: + if len(words & key) / len(words | key) >= self.threshold: + return packet + return None + + def put(self, query: str, packet: dict) -> None: + self.entries.append((self._words(query), packet)) + + +def _git(root: Path, *args: str) -> None: + subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@example.com", + "-c", "core.autocrlf=false", "-C", str(root), *args], + check=True, capture_output=True) + + +def _policy_text(packet: dict) -> str: + return "\n".join(r["snippet"] or "" for r in packet["results"] + if r["path"] == "auth/policy.py") + + +def test_query_keyed_cache_goes_stale_where_evidence_memory_does_not(tmp_path, monkeypatch): + for var in ("CBX_DB_PATH", "CBX_MEMORY", "CBX_ROOT"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("CBX_MEMORY_PATH", str(tmp_path / "memory.sqlite")) + root = tmp_path / "repo" + (root / "auth").mkdir(parents=True) + (root / "billing").mkdir() + (root / "auth" / "policy.py").write_bytes(POLICY.encode()) + (root / "billing" / "invoice.py").write_bytes(BILLING.encode()) + _git(root, "init", "-q") + _git(root, "add", "-A") + _git(root, "commit", "-q", "-m", "initial") + cfg = Config() + cfg.root = str(root) + db_path = tmp_path / "index.sqlite" + with Database(db_path) as db: + build_index(cfg, db, root=root) + + def update() -> None: + with Database(db_path) as db: + update_index(cfg, db, root=root) + + def ask(query: str) -> dict: + return search_payload(db_path, cfg, query, mode="hybrid", limit=10, token_budget=1500, + no_fallback=False, session="agent") + + cache = QueryKeyedCache() + + # T0 — learn X from E + t0_query = "session ttl seconds lifetime" + t0 = ask(t0_query) + assert "3600" in _policy_text(t0) + cache.put(t0_query, t0) + + # T1 — unrelated commit: E still holds, so reuse is correct for both designs + (root / "billing" / "invoice.py").write_bytes(BILLING.replace("sum(lines)", "sum(lines, 0)").encode()) + _git(root, "commit", "-q", "-am", "unrelated billing change") + update() + t1 = ask(t0_query) + assert t1["memory"]["invalidated"] == [] + assert any(r.get("reused") for r in t1["results"] if r["path"] == "auth/policy.py") + assert cache.get(t0_query) is t0 + + # T2 — E changes: the fact is no longer true + (root / "auth" / "policy.py").write_bytes(POLICY.replace("3600", "900").encode()) + _git(root, "commit", "-q", "-am", "shorten sessions") + update() + + # T3 — a semantically similar question + t3_query = "session ttl lifetime in seconds" + cached = cache.get(t3_query) + assert cached is not None, "the cache must consider these questions similar" + assert "3600" in _policy_text(cached) # the cache serves the stale fact + assert "900" in (root / "auth" / "policy.py").read_text() + + t3 = ask(t3_query) + assert "900" in _policy_text(t3) and "3600" not in _policy_text(t3) + assert not any(r.get("reused") for r in t3["results"] if r["path"] == "auth/policy.py") + stale_refs = [n for n in t3["memory"]["invalidated"] if n["ref"].startswith("auth/policy.py:")] + assert [n["state"] for n in stale_refs] == ["changed"] + assert verify_payload(cfg, [stale_refs[0]["ref"]])["evidence"][0]["state"] == "changed" diff --git a/tests/test_memory_store.py b/tests/test_memory_store.py new file mode 100644 index 0000000..e77d289 --- /dev/null +++ b/tests/test_memory_store.py @@ -0,0 +1,208 @@ +"""memory.sqlite: schema/migrations, ledger semantics, scoping, GC, recovery, locking.""" + +from __future__ import annotations + +import sqlite3 +from datetime import datetime, timedelta, timezone + +import pytest + +from codebase_index.memory.store import ( + SCHEMA_VERSION, + MemoryStore, + MemoryUnavailable, + NewDelivery, +) + +NOW = datetime(2026, 9, 10, 12, 0, tzinfo=timezone.utc) + + +def _item(path="src/a.py", span="s" * 64, snippet="p" * 64, full=True, start=1, end=5, tokens=40): + return NewDelivery(path=path, span_sha=span, line_count=end - start + 1, + first_line_sha="f" * 16, snippet_sha=snippet, full=full, + line_start=start, line_end=end, token_est=tokens) + + +@pytest.fixture +def store(tmp_path): + with MemoryStore.open(tmp_path / "memory.sqlite") as s: + yield s + + +def test_open_creates_current_schema_and_is_idempotent(tmp_path): + path = tmp_path / "memory.sqlite" + with MemoryStore.open(path) as s: + assert s.schema_version == SCHEMA_VERSION + with MemoryStore.open(path) as s: + assert s.schema_version == SCHEMA_VERSION + assert s.recovered_from is None + + +def test_migrations_run_in_order_inside_transactions(tmp_path): + path = tmp_path / "memory.sqlite" + with MemoryStore.open(path): + pass + from codebase_index.memory.store import MIGRATIONS + + future = {**MIGRATIONS, 2: ("ALTER TABLE atoms ADD COLUMN note TEXT",)} + with MemoryStore.open(path, migrations=future, schema_version=2) as s: + assert s.schema_version == 2 + cols = [r[1] for r in s.conn.execute("PRAGMA table_info(atoms)")] + assert "note" in cols + + broken = {**MIGRATIONS, 2: ("ALTER TABLE nope ADD COLUMN x TEXT",)} + other = tmp_path / "other.sqlite" + with MemoryStore.open(other): + pass + with pytest.raises(MemoryUnavailable): + MemoryStore.open(other, migrations=broken, schema_version=2) + with MemoryStore.open(other) as s: + assert s.schema_version == 1 # failed migration rolled back, nothing half-applied + + +def test_newer_schema_is_refused_without_touching_the_file(tmp_path): + path = tmp_path / "memory.sqlite" + with MemoryStore.open(path) as s: + s.conn.execute("UPDATE meta SET value = '99' WHERE key = 'schema_version'") + before = path.read_bytes() + with pytest.raises(MemoryUnavailable, match="newer"): + MemoryStore.open(path) + assert path.read_bytes() == before + + +def test_corrupt_store_is_preserved_and_replaced(tmp_path): + path = tmp_path / "memory.sqlite" + path.write_bytes(b"this is not a sqlite database" * 100) + with MemoryStore.open(path) as s: + assert s.recovered_from is not None + assert (tmp_path / s.recovered_from).read_bytes().startswith(b"this is not") + assert s.schema_version == SCHEMA_VERSION + + +def test_known_requires_full_span_or_identical_snippet(store): + sid = store.touch_session("repo", "tag", now=NOW) + store.record("repo", sid, [_item(full=False, snippet="a" * 64)], now=NOW) + assert store.is_known(sid, "src/a.py", "s" * 64, "a" * 64) + assert not store.is_known(sid, "src/a.py", "s" * 64, "b" * 64) # different skeleton + store.record("repo", sid, [_item(full=True, snippet="c" * 64)], now=NOW) + assert store.is_known(sid, "src/a.py", "s" * 64, "b" * 64) # whole span delivered + assert not store.is_known(sid, "src/a.py", "t" * 64, "c" * 64) # other bytes + + +def test_invalidated_delivery_is_not_known_until_redelivered(store): + sid = store.touch_session("repo", "tag", now=NOW) + store.record("repo", sid, [_item()], now=NOW) + atom_id = store.pending(sid)[0].atom_id + store.mark_invalid(sid, [(atom_id, "changed")]) + assert not store.is_known(sid, "src/a.py", "s" * 64, "p" * 64) + assert store.pending(sid) == [] + store.record("repo", sid, [_item()], now=NOW) + assert store.is_known(sid, "src/a.py", "s" * 64, "p" * 64) + + +def test_sessions_and_repositories_are_isolated(store): + a = store.touch_session("repoA", "tag", now=NOW) + b = store.touch_session("repoB", "tag", now=NOW) + other = store.touch_session("repoA", "tag2", now=NOW) + store.record("repoA", a, [_item()], now=NOW) + assert not store.is_known(b, "src/a.py", "s" * 64, "p" * 64) + assert not store.is_known(other, "src/a.py", "s" * 64, "p" * 64) + assert store.find_session("repoB", "missing") is None + assert store.stats("repoA")["deliveries"] == 1 + assert store.stats("repoB")["deliveries"] == 0 + + +def test_counters_and_stats(store): + sid = store.touch_session("repo", "tag", now=NOW) + store.touch_session("repo", "tag", now=NOW) + store.record("repo", sid, [_item(), _item(path="src/b.py")], now=NOW) + store.add_counters(sid, tokens_delivered=80, tokens_saved=40, reused=1, invalidations=2) + stats = store.stats("repo") + assert stats["sessions"] == 1 and stats["atoms"] == 2 and stats["deliveries"] == 2 + assert (stats["tokens_delivered"], stats["tokens_saved"]) == (80, 40) + assert (stats["reused"], stats["invalidations"]) == (1, 2) + assert stats["bytes"] > 0 + + +def test_gc_expires_old_sessions_and_orphan_atoms_but_keeps_recent(store): + old = store.touch_session("repo", "old", now=NOW - timedelta(days=30)) + new = store.touch_session("repo", "new", now=NOW) + store.record("repo", old, [_item(path="src/old.py")], now=NOW - timedelta(days=30)) + store.record("repo", new, [_item(path="src/new.py")], now=NOW) + result = store.gc(now=NOW, retention_days=14, max_deliveries=1000) + assert result == {"expired_sessions": 1, "capped_sessions": 0, "orphan_atoms": 1} + assert store.is_known(new, "src/new.py", "s" * 64, "p" * 64) + assert store.stats("repo")["atoms"] == 1 + assert not store.gc_due(now=NOW) + + +def test_gc_caps_total_deliveries_by_least_recent_use(store): + for day in range(5): + sid = store.touch_session("repo", f"s{day}", now=NOW - timedelta(days=4 - day)) + store.record("repo", sid, [_item(path=f"src/{day}_{i}.py") for i in range(3)], + now=NOW) + result = store.gc(now=NOW, retention_days=365, max_deliveries=7) + assert result["capped_sessions"] == 3 + assert store.stats("repo")["deliveries"] == 6 + assert store.find_session("repo", "s4") is not None + + +def test_gc_only_shrinks_what_is_known(store): + """GC never adds knowledge: every key known after GC was known before it.""" + keys = [] + for day in range(4): + sid = store.touch_session("repo", f"s{day}", now=NOW - timedelta(days=day * 10)) + store.record("repo", sid, [_item(path=f"src/{day}.py")], now=NOW) + keys.append((sid, f"src/{day}.py")) + before = {k for k in keys if store.is_known(k[0], k[1], "s" * 64, "p" * 64)} + store.gc(now=NOW, retention_days=15, max_deliveries=2) + after = {k for k in keys if store.is_known(k[0], k[1], "s" * 64, "p" * 64)} + assert after <= before and len(after) < len(before) + + +def test_clear_one_session_or_whole_repository(store): + a = store.touch_session("repo", "a", now=NOW) + b = store.touch_session("repo", "b", now=NOW) + elsewhere = store.touch_session("other", "a", now=NOW) + for sid, repo in ((a, "repo"), (b, "repo"), (elsewhere, "other")): + store.record(repo, sid, [_item()], now=NOW) + assert store.clear("repo", "a") == 1 + assert store.find_session("repo", "b") is not None + assert store.clear("repo") == 1 + assert store.stats("repo")["atoms"] == 0 + assert store.stats("other")["deliveries"] == 1 + + +def test_first_line_hint_by_reference_prefix(store): + sid = store.touch_session("repo", "tag", now=NOW) + store.record("repo", sid, [_item(span="abcdef0123456789" + "0" * 48)], now=NOW) + assert store.first_line_hint("repo", "src/a.py", "abcdef0123456789") == "f" * 16 + assert store.first_line_hint("repo", "src/a.py", "ffff000011112222") is None + assert store.first_line_hint("other", "src/a.py", "abcdef0123456789") is None + + +def test_lock_contention_degrades_instead_of_blocking_forever(tmp_path): + path = tmp_path / "memory.sqlite" + with MemoryStore.open(path): + pass + holder = sqlite3.connect(path, isolation_level=None) + holder.execute("BEGIN IMMEDIATE") + try: + with MemoryStore.open(path, busy_timeout_ms=50) as s: + with pytest.raises(MemoryUnavailable, match="busy"): + s.touch_session("repo", "tag", now=NOW) + finally: + holder.execute("ROLLBACK") + holder.close() + + +def test_store_contains_no_source_text(tmp_path): + path = tmp_path / "memory.sqlite" + secret_line = "AWS_SECRET_ACCESS_KEY = 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'" + with MemoryStore.open(path) as s: + sid = s.touch_session("repo", "tag-sha", now=NOW) + s.record("repo", sid, [_item()], now=NOW) + s.conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") + blob = b"".join(p.read_bytes() for p in tmp_path.iterdir() if p.is_file()) + assert secret_line.encode() not in blob + assert b"wJalrXUtnFEMI" not in blob diff --git a/tests/test_memory_surfaces.py b/tests/test_memory_surfaces.py new file mode 100644 index 0000000..f79b3c2 --- /dev/null +++ b/tests/test_memory_surfaces.py @@ -0,0 +1,197 @@ +"""Evidence memory through every public surface: CLI, MCP, Markdown, stats and doctor.""" + +from __future__ import annotations + +import json +import os +import sqlite3 +from pathlib import Path +from unittest.mock import patch + +import pytest +from typer.testing import CliRunner + +from codebase_index.cli import app +from codebase_index.config import Config +from codebase_index.doctor import run_doctor +from codebase_index.output.markdown import render, render_verify + +runner = CliRunner() + +REFUND = '''def issue_refund(invoice, amount): + """Refund part of an invoice total.""" + if amount > invoice.total: + raise ValueError("refund exceeds invoice total") + return invoice.total - amount +''' +QUERY = "refund exceeds invoice total" + + +@pytest.fixture +def indexed(tmp_path, monkeypatch): + for name in ("CBX_DB_PATH", "CBX_MEMORY_PATH", "CBX_MEMORY", "CBX_ROOT"): + monkeypatch.delenv(name, raising=False) + root = tmp_path / "repo" + (root / "billing").mkdir(parents=True) + (root / "billing" / "refund.py").write_text(REFUND, encoding="utf-8") + assert runner.invoke(app, ["--root", str(root), "index"]).exit_code == 0 + return root + + +def _cli(root: Path, *args: str): + return runner.invoke(app, ["--root", str(root), *args]) + + +def _json(result) -> dict: + assert result.exit_code == 0, result.output + return json.loads(result.output) + + +def test_cli_session_search_reuses_and_verify_reports_changes(indexed): + first = _json(_cli(indexed, "search", QUERY, "--session", "cli-1", "--json")) + assert first["memory"]["reused"] == 0 + second = _json(_cli(indexed, "search", QUERY, "--session", "cli-1", "--json")) + assert second["memory"]["reused"] >= 1 + + verdict = _json(_cli(indexed, "verify", "--session", "cli-1", "--json")) + assert verdict["all_valid"] is True and verdict["session"]["found"] is True + + (indexed / "billing" / "refund.py").write_text(REFUND.replace(">", ">="), encoding="utf-8") + changed = _json(_cli(indexed, "verify", "--session", "cli-1", "--json")) + assert changed["all_valid"] is False + assert "changed" in changed["summary"] + ref = next(v["ref"] for v in changed["evidence"] if v["state"] == "changed") + + strict = _cli(indexed, "verify", ref, "--strict", "--json") + assert strict.exit_code == 1 + assert json.loads(strict.output)["evidence"][0]["state"] == "changed" + + +def test_cli_verify_arguments_and_session_tags_are_validated(indexed): + assert _cli(indexed, "verify").exit_code == 2 + assert _cli(indexed, "search", QUERY, "--session", "bad tag").exit_code == 2 + assert _cli(indexed, "explain", QUERY, "--session", "bad/tag").exit_code == 2 + bad = _json(_cli(indexed, "verify", "../../etc/passwd:1-2@0123456789abcdef", "--json")) + assert bad["all_valid"] is False and bad["errors"] + + +def test_cli_verify_of_unknown_session_is_not_all_valid(indexed): + payload = _json(_cli(indexed, "verify", "--session", "never-used", "--json")) + assert payload["all_valid"] is False and payload["session"]["found"] is False + + +def test_cli_stats_memory_block_and_human_line(indexed): + assert _json(_cli(indexed, "stats", "--json"))["memory"] == {"enabled": True, "exists": False} + _json(_cli(indexed, "search", QUERY, "--session", "s", "--json")) + _json(_cli(indexed, "search", QUERY, "--session", "s", "--json")) + memory = _json(_cli(indexed, "stats", "--json"))["memory"] + assert memory["sessions"] == 1 and memory["reused"] >= 1 and memory["tokens_saved"] > 0 + human = _cli(indexed, "stats") + assert "tokens not resent" in human.output + + +def test_cli_memory_gc_and_clear(indexed): + _json(_cli(indexed, "search", QUERY, "--session", "a", "--json")) + _json(_cli(indexed, "search", QUERY, "--session", "b", "--json")) + gc = _json(_cli(indexed, "memory", "gc", "--json")) + assert gc["exists"] and gc["sessions"] == 2 + assert _json(_cli(indexed, "memory", "clear", "--session", "a", "--yes", "--json"))[ + "removed_sessions"] == 1 + assert _json(_cli(indexed, "memory", "clear", "--yes", "--json"))["removed_sessions"] == 1 + assert _json(_cli(indexed, "stats", "--json"))["memory"]["sessions"] == 0 + + +def test_cli_memory_disabled_by_environment(indexed, monkeypatch): + monkeypatch.setenv("CBX_MEMORY", "0") + payload = _json(_cli(indexed, "search", QUERY, "--session", "s", "--json")) + assert payload["memory"]["available"] is False + assert not (indexed / ".claude" / "cache" / "codebase-index" / "memory.sqlite").exists() + + +def test_markdown_marks_reused_stale_and_invalidated(): + payload = { + "query": "q", "intent": "keyword", "confidence": "high", + "results": [ + {"rank": 1, "path": "a.py", "line_start": 1, "line_end": 3, "reason": "x", + "snippet": None, "reused": True}, + {"rank": 2, "path": "b.py", "line_start": 4, "line_end": 9, "reason": "y", + "snippet": "def b(): pass", "stale": True}, + ], + "recommended_reads": [], + "fallback_suggestions": {}, + "memory": {"session": "s", "reused": 1, "tokens_saved": 12, + "invalidated": [{"ref": "c.py:1-2@0123456789abcdef", "state": "changed"}]}, + } + text = render(payload) + assert "already delivered in this session" in text + assert "index is older than the file" in text + assert "c.py:1-2@0123456789abcdef` — changed" in text + + verdicts = render_verify({ + "all_valid": False, "summary": {"changed": 1}, + "session": {"session": "s", "found": True, "evidence": 1}, + "evidence": [{"ref": "c.py:1-2@0123456789abcdef", "state": "changed", "valid": False, + "line_start": None, "line_end": None, "reason": "content no longer occurs"}], + "errors": [{"ref": "nope", "error": "not an evidence reference"}], + }) + assert "CHANGED" in verdicts and "some evidence is not valid" in verdicts and "nope" in verdicts + + +try: + from codebase_index.mcp import server as mcp_server + MCP_AVAILABLE = True +except ImportError: # pragma: no cover + MCP_AVAILABLE = False + + +@pytest.mark.skipif(not MCP_AVAILABLE, reason="mcp extra not installed") +def test_mcp_session_reuse_and_verify_evidence(indexed): + db_path = indexed / ".claude" / "cache" / "codebase-index" / "index.sqlite" + env = {"CBX_ROOT": str(indexed), "CBX_DB_PATH": str(db_path)} + with patch.dict(os.environ, env, clear=False): + json.loads(mcp_server.search_code(query=QUERY, session="mcp-1")) + again = json.loads(mcp_server.search_code(query=QUERY, session="mcp-1")) + assert again["schema_version"] == 1 and again["memory"]["reused"] >= 1 + + checked = json.loads(mcp_server.verify_evidence(session="mcp-1")) + assert checked["tool"] == "verify_evidence" and checked["all_valid"] is True + ref = checked["evidence"][0]["ref"] + by_ref = json.loads(mcp_server.verify_evidence(refs=[ref, "garbage"])) + assert by_ref["evidence"][0]["valid"] is True and by_ref["errors"] + + assert "error" in json.loads(mcp_server.verify_evidence()) + assert "error" in json.loads(mcp_server.search_code(query=QUERY, session="a b")) + assert "error" in json.loads(mcp_server.explain_code(query=QUERY, session="a b")) + health = json.loads(mcp_server.healthcheck()) + assert health["memory"]["exists"] is True + + +def _memory_path(root: Path) -> Path: + return root / ".claude" / "cache" / "codebase-index" / "memory.sqlite" + + +def _doctor_memory(root: Path): + cfg = Config() + cfg.root = str(root) + return next(f for f in run_doctor(root, cfg) if f.id == "memory_store") + + +def test_doctor_reports_memory_health_without_repairing(indexed): + assert _doctor_memory(indexed).ok + _json(_cli(indexed, "search", QUERY, "--session", "s", "--json")) + healthy = _doctor_memory(indexed) + assert healthy.ok and "healthy" in healthy.detail + + path = _memory_path(indexed) + conn = sqlite3.connect(path) + conn.execute("UPDATE meta SET value = '99' WHERE key = 'schema_version'") + conn.commit() + conn.close() + newer = _doctor_memory(indexed) + assert not newer.ok and "newer" in newer.detail + + for sidecar in path.parent.glob("memory.sqlite*"): + sidecar.unlink() + path.write_bytes(b"not a database" * 64) + broken = _doctor_memory(indexed) + assert not broken.ok and path.read_bytes().startswith(b"not a database") # untouched diff --git a/tests/test_memory_validate.py b/tests/test_memory_validate.py new file mode 100644 index 0000000..0e99cdc --- /dev/null +++ b/tests/test_memory_validate.py @@ -0,0 +1,210 @@ +"""Working-tree validation: verdict states, relocation, and gate parity with the indexer.""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +import pytest + +from codebase_index.config import Config +from codebase_index.discovery.gates import PathGate +from codebase_index.discovery.walker import walk +from codebase_index.memory import identity as ident +from codebase_index.memory.validate import FileView, WorkingTree, locate, validate + +BODY = "def total(items):\n return sum(i.price for i in items)\n" + + +def _repo(tmp_path: Path, files: dict[str, str | bytes], **cfg) -> tuple[Path, WorkingTree]: + root = tmp_path / "repo" + for rel, content in files.items(): + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + if isinstance(content, bytes): + path.write_bytes(content) + else: + path.write_text(content, encoding="utf-8", newline="") + root.mkdir(exist_ok=True) + config = Config(**cfg) + config.root = str(root) + return root, WorkingTree(PathGate(root, config)) + + +def _ref(root: Path, rel: str, start: int, end: int) -> ident.EvidenceRef: + lines = ident.split_lines((root / rel).read_bytes()) + return ident.make_ref(rel, start, end, ident.span_sha(lines, start, end)) + + +def _fresh(root: Path, config_kwargs: dict | None = None) -> WorkingTree: + config = Config(**(config_kwargs or {})) + config.root = str(root) + return WorkingTree(PathGate(root, config)) + + +def test_unchanged_evidence_is_valid(tmp_path): + root, tree = _repo(tmp_path, {"src/cart.py": "import os\n\n" + BODY}) + ref = _ref(root, "src/cart.py", 3, 4) + verdict = validate(ref, tree) + assert verdict.state == "valid" and verdict.valid + assert (verdict.line_start, verdict.line_end) == (3, 4) + + +def test_edit_elsewhere_in_the_file_keeps_the_span_valid(tmp_path): + root, _ = _repo(tmp_path, {"src/cart.py": "import os\n\n" + BODY + "\nX = 1\n"}) + ref = _ref(root, "src/cart.py", 3, 4) + (root / "src/cart.py").write_text("import os\n\n" + BODY + "\nX = 2\n", encoding="utf-8") + assert validate(ref, _fresh(root)).state == "valid" + + +@pytest.mark.parametrize("with_hint", [True, False]) +def test_insertion_above_relocates(tmp_path, with_hint): + root, _ = _repo(tmp_path, {"src/cart.py": "import os\n\n" + BODY}) + ref = _ref(root, "src/cart.py", 3, 4) + hint = ident.line_sha("def total(items):") if with_hint else None + (root / "src/cart.py").write_text("import os\nimport sys\n\n\n" + BODY, encoding="utf-8") + verdict = validate(ref, _fresh(root), first_line_sha=hint) + assert verdict.state == "relocated" and verdict.valid + assert (verdict.line_start, verdict.line_end) == (5, 6) + + +def test_body_change_invalidates(tmp_path): + root, _ = _repo(tmp_path, {"src/cart.py": BODY}) + ref = _ref(root, "src/cart.py", 1, 2) + (root / "src/cart.py").write_text(BODY.replace("sum(", "max("), encoding="utf-8") + verdict = validate(ref, _fresh(root)) + assert verdict.state == "changed" and not verdict.valid + + +def test_whitespace_only_change_invalidates(tmp_path): + root, _ = _repo(tmp_path, {"src/cart.py": BODY}) + ref = _ref(root, "src/cart.py", 1, 2) + (root / "src/cart.py").write_text(BODY.replace(" return", " return"), encoding="utf-8") + assert validate(ref, _fresh(root)).state == "changed" + + +def test_crlf_conversion_is_not_a_change(tmp_path): + root, _ = _repo(tmp_path, {"src/cart.py": BODY}) + ref = _ref(root, "src/cart.py", 1, 2) + (root / "src/cart.py").write_bytes(BODY.replace("\n", "\r\n").encode()) + assert validate(ref, _fresh(root)).state == "valid" + + +def test_duplicated_content_is_ambiguous_not_valid(tmp_path): + root, _ = _repo(tmp_path, {"src/cart.py": "# a\n" + BODY}) + ref = _ref(root, "src/cart.py", 2, 3) + (root / "src/cart.py").write_text("# b\n\n" + BODY + BODY, encoding="utf-8") + assert validate(ref, _fresh(root)).state == "ambiguous" + + +def test_deleted_file(tmp_path): + root, _ = _repo(tmp_path, {"src/cart.py": BODY}) + ref = _ref(root, "src/cart.py", 1, 2) + (root / "src/cart.py").unlink() + assert validate(ref, _fresh(root)).state == "deleted" + + +def test_move_to_another_file_is_not_reused(tmp_path): + root, _ = _repo(tmp_path, {"src/cart.py": BODY}) + ref = _ref(root, "src/cart.py", 1, 2) + (root / "src/core").mkdir() + (root / "src/cart.py").replace(root / "src/core/cart.py") + assert validate(ref, _fresh(root)).state == "deleted" + + +def test_range_past_end_of_shortened_file_is_changed(tmp_path): + root, _ = _repo(tmp_path, {"src/cart.py": BODY + "x = 1\n"}) + ref = _ref(root, "src/cart.py", 2, 3) + (root / "src/cart.py").write_text("y = 2\n", encoding="utf-8") + assert validate(ref, _fresh(root)).state == "changed" + + +@pytest.mark.parametrize( + "rel,files,cfg", + [ + (".env", {".env": "API_KEY=abc\n"}, {}), + ("config/prod.pem", {"config/prod.pem": "-----BEGIN KEY-----\n"}, {}), + ("node_modules/pkg/index.js", {"node_modules/pkg/index.js": "x\n"}, {}), + ("build/out.py", {"build/out.py": "x\n"}, {}), + ("private/notes.py", {"private/notes.py": "x\n", ".gitignore": "private/\n"}, {}), + ("big.py", {"big.py": "x = 1\n" * 50}, {"max_file_bytes": 64}), + ("blob.py", {"blob.py": b"x = 1\n\x00\x01"}, {}), + ], +) +def test_gated_paths_are_excluded(tmp_path, rel, files, cfg): + root, _ = _repo(tmp_path, files) + ref = ident.EvidenceRef(rel, 1, 1, "0" * 16) + verdict = validate(ref, _fresh(root, cfg)) + assert verdict.state == "excluded" and not verdict.valid + + +def test_name_gated_files_are_never_opened(tmp_path, monkeypatch): + root, _ = _repo(tmp_path, {".env": "API_KEY=abc\n", "node_modules/a.js": "x\n"}) + opened: list[str] = [] + real = Path.read_bytes + + def spy(self): + opened.append(self.name) + return real(self) + + monkeypatch.setattr(Path, "read_bytes", spy) + tree = _fresh(root) + for rel in (".env", "node_modules/a.js"): + assert validate(ident.EvidenceRef(rel, 1, 1, "0" * 16), tree).state == "excluded" + assert opened == [] + + +@pytest.mark.skipif(sys.platform != "win32", reason="case-insensitive filesystem check") +def test_case_variant_cannot_bypass_directory_gate(tmp_path): + root, _ = _repo(tmp_path, {"node_modules/pkg/a.js": "module.exports = 1\n"}) + ref = ident.EvidenceRef("NODE_MODULES/pkg/a.js", 1, 1, "0" * 16) + assert validate(ref, _fresh(root)).state == "excluded" + + +def test_symlink_outside_repository_is_excluded(tmp_path): + outside = tmp_path / "outside.py" + outside.write_text("SECRET = 1\n", encoding="utf-8") + root, _ = _repo(tmp_path, {"src/keep.py": "x = 1\n"}) + try: + os.symlink(outside, root / "src" / "link.py") + except (OSError, NotImplementedError): + pytest.skip("symlinks unavailable") + ref = ident.EvidenceRef("src/link.py", 1, 1, "0" * 16) + verdict = validate(ref, _fresh(root)) + assert verdict.state == "excluded" and "outside" in verdict.reason + + +def test_gate_admits_exactly_what_the_walker_indexes(sample_repo): + """Security parity: validation can never read a file the indexer would refuse.""" + config = Config() + config.root = str(sample_repo) + walked = {c.rel_path for c in walk(sample_repo, config)} + gate = PathGate(sample_repo, config) + admitted = set() + for path in Path(sample_repo).rglob("*"): + if path.is_file(): + rel = path.relative_to(sample_repo).as_posix() + if gate.read(rel).state == "ok": + admitted.add(rel) + assert admitted == walked + assert "node_modules/leftpad/index.js" not in admitted + + +def test_locate_is_bounded_and_limited(): + view = FileView(["same"] * 50) + sha = ident.span_sha(view.lines, 1, 2) + assert locate(view, sha, 2, limit=2) == [1, 2] + assert locate(view, sha, 2, max_steps=10) is None + assert locate(view, sha, 99) == [] + + +def test_bounded_search_failure_is_reported_invalid(tmp_path, monkeypatch): + import codebase_index.memory.validate as mod + + root, _ = _repo(tmp_path, {"src/cart.py": "import os\n\n" + BODY}) + ref = _ref(root, "src/cart.py", 3, 4) + (root / "src/cart.py").write_text("import os\nimport sys\n\n\n" + BODY, encoding="utf-8") + monkeypatch.setattr(mod, "MAX_SCAN_STEPS", 1) + verdict = mod.validate(ref, _fresh(root)) + assert verdict.state == "changed" and "too large" in verdict.reason diff --git a/tests/test_migration_1x.py b/tests/test_migration_1x.py new file mode 100644 index 0000000..2713d5f --- /dev/null +++ b/tests/test_migration_1x.py @@ -0,0 +1,104 @@ +"""Upgrading from 1.x: a real 1.10.0 index keeps working, and memory attaches beside it. + +`tests/fixtures/index-1.10.0/index.sqlite` was built by the released 1.10.0 code (commit +abb67df) over `tests/fixtures/sample_repo`, not synthesised by 2.0, so these tests check the +upgrade path users actually take. 2.0 leaves the index schema at version 3; evidence memory +lives in its own `memory.sqlite`, so no reindex is needed. +""" + +from __future__ import annotations + +import json +import shutil +import sqlite3 +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from codebase_index.cli import app +from codebase_index.config import load +from codebase_index.storage.db import SCHEMA_VERSION, peek_schema_version + +FIXTURES = Path(__file__).parent / "fixtures" +runner = CliRunner() + + +@pytest.fixture +def upgraded(tmp_path, monkeypatch): + for name in ("CBX_DB_PATH", "CBX_MEMORY_PATH", "CBX_MEMORY", "CBX_ROOT"): + monkeypatch.delenv(name, raising=False) + root = tmp_path / "repo" + # Other tests index sample_repo in place; never carry their runtime cache along. + shutil.copytree(FIXTURES / "sample_repo", root, ignore=shutil.ignore_patterns(".claude")) + cache = root / ".claude" / "cache" / "codebase-index" + cache.mkdir(parents=True, exist_ok=True) + shutil.copy(FIXTURES / "index-1.10.0" / "index.sqlite", cache / "index.sqlite") + # The config 1.10.0 `init` writes: every section 2.0 knows, except `memory`. + (cache / "config.json").write_text(json.dumps({ + "root": ".", "languages": "auto", "max_file_bytes": 1048576, + "ignore_files": [".gitignore", ".cursorignore", ".claudeignore", ".codeindexignore"], + "extra_ignore": [], "chunk": {"window_lines": 80, "overlap_lines": 10}, + "retrieval": {"default_mode": "hybrid", "rrf_k": 60, "token_budget": 1500, "limit": 10, + "compact_snippets": True, "compact_min_reduction": 0.25}, + "embeddings": {"backend": "noop", "enabled": False, "model": "all-MiniLM-L6-v2", + "allow_external": False, "endpoint": None}, + "graph": {"max_depth": 2, "node_cap": 40}, "redaction": {"enabled": True}, + }), encoding="utf-8") + return root, cache / "index.sqlite" + + +def _meta(db: Path, key: str): + conn = sqlite3.connect(db) + try: + row = conn.execute("SELECT value FROM meta WHERE key = ?", (key,)).fetchone() + return row[0] if row else None + finally: + conn.close() + + +def _cli_json(root: Path, *args: str) -> dict: + result = runner.invoke(app, ["--root", str(root), *args]) + assert result.exit_code == 0, result.output + return json.loads(result.output) + + +def test_fixture_is_a_real_1_10_index_on_the_current_schema(upgraded): + _root, db = upgraded + assert peek_schema_version(db) == 3 == SCHEMA_VERSION + conn = sqlite3.connect(db) + try: + assert conn.execute("SELECT COUNT(*) FROM files").fetchone()[0] == 6 + finally: + conn.close() + + +def test_1x_index_is_used_in_place_without_a_rebuild(upgraded): + root, db = upgraded + built_at = _meta(db, "built_at") + search = _cli_json(root, "search", "token", "--json") + assert search["results"] and "memory" not in search + update = _cli_json(root, "--json", "update") + assert update["indexed"] == 0 and update["deleted"] == 0 # same bytes: nothing reparsed + assert _meta(db, "built_at") == built_at # the 1.x database survived + assert peek_schema_version(db) == 3 + + +def test_memory_attaches_to_an_upgraded_project(upgraded): + root, db = upgraded + assert load(root).memory.enabled is True # 1.x config, 2.0 defaults + first = _cli_json(root, "search", "token", "--session", "upgrade", "--json") + second = _cli_json(root, "search", "token", "--session", "upgrade", "--json") + assert first["memory"]["reused"] == 0 and second["memory"]["reused"] >= 1 + assert (db.parent / "memory.sqlite").is_file() + assert _cli_json(root, "verify", "--session", "upgrade", "--json")["all_valid"] is True + stats = _cli_json(root, "stats", "--json") + assert stats["memory"]["sessions"] == 1 and stats["files"] == 6 + + +def test_clean_removes_the_index_but_keeps_memory(upgraded): + root, db = upgraded + _cli_json(root, "search", "token", "--session", "keep", "--json") + result = runner.invoke(app, ["--root", str(root), "clean", "--yes", "--json"]) + assert result.exit_code == 0, result.output + assert not db.exists() and (db.parent / "memory.sqlite").is_file() diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 5297c2e..118c1b4 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -29,9 +29,9 @@ def test_packaged_cbx_whitelists_safe_subcommands_only(): cbx = (_template() / "scripts" / "cbx").read_text(encoding="utf-8") assert ( 'ALLOWED="search explain architecture symbol refs impact diff-impact path describe ' - 'graph stats doctor update index"' + 'verify graph stats doctor update index"' ) in cbx - for forbidden in ("clean", "init", "watch"): + for forbidden in ("clean", "init", "watch", "memory"): assert f" {forbidden} " not in f' {cbx.split("ALLOWED=")[1].splitlines()[0]} ' diff --git a/tests/test_ranking_features.py b/tests/test_ranking_features.py new file mode 100644 index 0000000..28c5011 --- /dev/null +++ b/tests/test_ranking_features.py @@ -0,0 +1,263 @@ +"""Contract tests for the 1.10.0 name-co-occurrence ranking signal. + +The feature is one short function, but it sits in the hot path of every query and +its whole justification is a measured interaction effect. These tests pin the +properties that made it worth shipping — superlinearity, name-zone scope, role +conditioning, determinism — plus the degenerate inputs that a query string can +actually contain. +""" + +from __future__ import annotations + +from codebase_index.retrieval.features import ( + name_cooccurrence, + name_zone, + query_profile, +) +from codebase_index.retrieval.rerank import rerank +from codebase_index.retrieval.tuning import DEFAULT_TUNING, RetrievalTuning +from codebase_index.retrieval.types import Candidate, Intent + + +def _cooc(query: str, path: str, symbol: str | None = None) -> float: + return name_cooccurrence(query_profile(query), name_zone(path, symbol))[1] + + +def _cand(path: str, **kw) -> Candidate: + base = dict( + path=path, line_start=1, line_end=10, source="symbol", score=1.0, kind="function" + ) + base.update(kw) + return Candidate(**base) # type: ignore[arg-type] + + +# --- the interaction itself ------------------------------------------------- + + +def test_query_terms_co_occurring_in_one_symbol_beat_a_single_term_match(): + """The measured failure this signal exists to fix. + + On "graph resolution + traversal accessors", 1.9.0 ranked `graph_candidates` + (one term) above `test_graph_accessors_resolve_and_walk` (three terms in one + name), because RRF sums per-term evidence and cannot see co-occurrence. + """ + query = "graph resolution traversal accessors" + one_term = _cooc(query, "src/graph/retrieval.py", "graph_candidates") + three_terms = _cooc(query, "src/storage/repo.py", "graph_accessors_resolve") + assert one_term == 0.0 + assert three_terms > one_term + + +def test_score_is_superlinear_not_additive(): + """Two terms in one name must beat one, and the first match earns nothing. + + A single matched term is already fully paid for by the retriever that + surfaced the candidate; crediting it again here would just re-weight lexical + matching, which is not what the benchmark showed was missing. + """ + query = "token budget applied results" + assert _cooc(query, "a/token.py") == 0.0 + assert 0.0 < _cooc(query, "a/token_budget.py") < _cooc(query, "a/token_budget_results.py") + assert _cooc(query, "a/token_budget_applied_results.py") == 1.0 + + +def test_single_term_queries_have_no_co_occurrence_to_observe(): + assert _cooc("budget", "src/retrieval/budget.py") == 0.0 + assert name_cooccurrence(query_profile("budget"), {"budget"}) == (0, 0.0) + + +def test_normalisation_makes_short_and_long_queries_comparable(): + """Both candidates match every salient term, so both must score 1.0.""" + assert _cooc("parse config", "a/parse_config.py") == 1.0 + assert _cooc("parse config values from disk", "a/parse_config_values_from_disk.py") == 1.0 + + +# --- name-zone scope -------------------------------------------------------- + + +def test_zone_is_basename_and_symbol_but_never_directories(): + """Directory components are shared by hundreds of files, so they carry no + evidence about which file answers the query — only co-occurrence noise.""" + zone = name_zone("src/main/java/net/denfry/newtowny/items/CustomItems.java", None) + assert zone == {"custom", "items"} + assert "denfry" not in zone and "java" not in zone and "src" not in zone + assert _cooc("newtowny items denfry", "src/net/denfry/newtowny/items/Foo.java") == 0.0 + + +def test_zone_splits_camel_snake_and_kebab_identifiers(): + assert name_zone("a/getUserById.ts", None) == {"get", "user", "by", "id"} + assert name_zone("a/refresh_access_token.py", None) == {"refresh", "access", "token"} + assert name_zone("a/my-http-server.js", None) == {"my", "http", "server"} + assert name_zone("a/x.py", "HTTPServerFactory") == {"x", "http", "server", "factory"} + + +def test_extension_is_not_part_of_the_zone(): + """Otherwise every query mentioning `py` or `ts` would fire on whole languages.""" + assert "py" not in name_zone("src/parse.py", None) + assert "java" not in name_zone("src/Parse.java", None) + assert _cooc("parse py files", "src/parse.py") == 0.0 + + +def test_dotted_and_multi_suffix_names_keep_their_stem(): + assert name_zone("web/app.module.ts", None) == {"app", "module"} + assert name_zone("a/schema.generated.ts", None) == {"schema", "generated"} + + +def test_windows_separators_are_normalised(): + assert name_zone(r"src\retrieval\token_budget.py", None) == {"token", "budget"} + + +# --- role conditioning ------------------------------------------------------ + + +def test_descriptive_test_names_get_a_discounted_bonus(): + """Test function names are sentences, so they harvest co-occurrences that real + identifiers never do. Discounted, not withheld: on git-derived ground truth the + changed file often *is* the test.""" + query = "secrets redacted before output" + impl = _cand("src/output/redact.py", symbol="redact_snippet") + test = _cand("tests/test_budget.py", symbol="test_compactor_output_is_redacted") + ranked = rerank([impl, test], query=query, intent=Intent.KEYWORD, tuning=DEFAULT_TUNING) + assert [c.path for c in ranked][0] == "src/output/redact.py" + + scale = DEFAULT_TUNING.name_cooccurrence_demoted_scale + assert 0.0 < scale < 1.0 + + +def test_an_explicit_test_query_is_not_discounted(): + query = "tests for redacted output" + graded = rerank( + [_cand("tests/test_redacted_output.py", symbol="test_redacted_output")], + query=query, + intent=Intent.KEYWORD, + tuning=DEFAULT_TUNING, + ) + assert "co-occur in name" in graded[0].reason + + +def test_generated_sources_are_discounted_too(): + query = "schema generated types" + plain = _cand("src/schema_generated_types.ts") + generated = _cand("src/schema_generated_types.ts", is_generated=True) + for c in (plain, generated): + rerank([c], query=query, intent=Intent.KEYWORD, tuning=DEFAULT_TUNING) + assert plain.score > generated.score + + +# --- ablatability and explainability --------------------------------------- + + +def test_signal_is_independently_ablatable(): + query = "token budget applied results" + on = _cand("src/token_budget.py", symbol="apply_token_budget") + off = _cand("src/token_budget.py", symbol="apply_token_budget") + rerank([on], query=query, intent=Intent.KEYWORD, tuning=DEFAULT_TUNING) + rerank([off], query=query, intent=Intent.KEYWORD, + tuning=DEFAULT_TUNING.without("name_cooccurrence")) + assert on.score > off.score + assert "co-occur in name" in on.reason + assert "co-occur in name" not in off.reason + + +def test_reason_reports_the_actual_counts(): + c = _cand("src/token_budget.py") + rerank([c], query="token budget applied results", intent=Intent.KEYWORD, + tuning=DEFAULT_TUNING) + assert "2/4 query terms co-occur in name" in c.reason + + +def test_weight_scales_the_bonus_monotonically(): + scores = [] + for weight in (0.0, 1.0, 2.0): + c = _cand("src/token_budget.py") + rerank( + [c], + query="token budget applied results", + intent=Intent.KEYWORD, + tuning=RetrievalTuning(name_cooccurrence_weight=weight), + ) + scores.append(c.score) + assert scores[0] < scores[1] < scores[2] + + +# --- determinism and degenerate input --------------------------------------- + + +def test_ranking_is_deterministic_and_stable_on_ties(): + query = "token budget applied results" + def run(): + cands = [ + _cand("src/token_budget.py", symbol="apply"), + _cand("src/budget_token.py", symbol="apply"), + _cand("src/other.py", symbol="apply"), + ] + return [c.path for c in rerank(cands, query=query, intent=Intent.KEYWORD, + tuning=DEFAULT_TUNING)] + first = run() + assert all(run() == first for _ in range(5)) + # The two equally-matching files tie on score, so input order decides and the + # sort must not reshuffle them. + assert first[:2] == ["src/token_budget.py", "src/budget_token.py"] + + +def test_duplicate_candidates_score_identically(): + query = "token budget applied" + a, b = _cand("src/token_budget.py"), _cand("src/token_budget.py") + rerank([a, b], query=query, intent=Intent.KEYWORD, tuning=DEFAULT_TUNING) + assert a.score == b.score + + +def test_empty_and_punctuation_only_queries_are_inert(): + for query in ("", " ", "???", "!!! ... ???", "- -- ---", "/////"): + assert query_profile(query).terms == () + assert _cooc(query, "src/token_budget.py") == 0.0 + + +def test_stopword_only_query_is_inert(): + assert _cooc("how does the", "src/how_does_the.py") == 0.0 + + +def test_punctuation_between_terms_does_not_break_matching(): + assert _cooc("token, budget; applied!", "src/token_budget.py") > 0.0 + + +def test_unicode_identifiers_and_queries_are_handled(): + assert name_zone("src/расчёт_налога.py", None) == {"расчёт", "налога"} + assert _cooc("расчёт налога ставка", "src/расчёт_налога.py") > 0.0 + # Mixed scripts must not raise or silently drop the ASCII half. + assert _cooc("расчёт tax rate", "src/расчёт_tax.py") > 0.0 + + +def test_pathological_long_query_stays_bounded_and_sane(): + query = " ".join(f"term{i}" for i in range(2000)) + profile = query_profile(query) + assert profile.n_terms == 2000 + # One matched term out of 2000 is not evidence of anything. + assert _cooc(query, "src/term7.py") == 0.0 + assert 0.0 < _cooc(query, "src/term7_term8_term9.py") < 0.01 + + +def test_pathological_long_identifier_stays_bounded(): + symbol = "_".join(f"part{i}" for i in range(2000)) + score = _cooc("part7 part8 part9", "src/x.py", symbol) + assert score == 1.0 + + +def test_missing_and_malformed_metadata_does_not_raise(): + query = "token budget applied" + for path in ("", ".", "/", "a/", "...", "no_extension"): + assert 0.0 <= _cooc(query, path, None) <= 1.0 + assert _cooc(query, "src/token_budget.py", None) > 0.0 + # A candidate with no symbol, no kind and no content must still rank. + c = Candidate(path="src/token_budget.py", line_start=0, line_end=0, + source="path", score=0.0) + ranked = rerank([c], query=query, intent=Intent.KEYWORD, tuning=DEFAULT_TUNING) + assert ranked[0].score > 0.0 + + +def test_score_is_always_in_the_unit_interval(): + queries = ("a b", "token budget applied results", "x" * 300, "расчёт налога") + paths = ("src/token_budget.py", "", "a/b/c/d.py", "src/" + "x" * 300 + ".py") + for query in queries: + for path in paths: + assert 0.0 <= _cooc(query, path, "some_symbol_name") <= 1.0 diff --git a/tests/test_tuning_contract.py b/tests/test_tuning_contract.py index b335997..ba891ec 100644 --- a/tests/test_tuning_contract.py +++ b/tests/test_tuning_contract.py @@ -45,10 +45,13 @@ def test_baseline_disables_every_post_170_signal(): "dedup", "source_priors", "file_agreement", + "name_cooccurrence", ): assert getattr(baseline, flag) is False, flag - # 1.7.0 took the page size verbatim; over-fetching is a later addition. + # 1.7.0 took the page size verbatim and allowed three hits per file; + # over-fetching and single-hit packing are later additions. assert baseline.candidate_pool_multiplier == 1 + assert baseline.max_per_file == 3 def test_shipped_defaults_are_the_measured_configuration(): @@ -62,6 +65,27 @@ def test_shipped_defaults_are_the_measured_configuration(): assert DEFAULT_TUNING.mmr is False +def test_v190_reproduces_the_previous_release(): + """The 1.10.0 comparison point must stay pinned, not track the default.""" + v190 = RetrievalTuning.v190() + assert v190.name_cooccurrence is False + assert v190.max_per_file == 3 + # Everything 1.9.0 shipped on stays on, so the "before" column is 1.9.0 and + # not an accidental partial ablation of it. + for flag in ("soft_lexical", "dedup", "source_priors", "file_agreement", "fuzzy_symbols"): + assert getattr(v190, flag) is True, flag + + +def test_name_cooccurrence_defaults_are_the_measured_configuration(): + assert DEFAULT_TUNING.name_cooccurrence is True + assert DEFAULT_TUNING.name_cooccurrence_weight == pytest.approx(1.8) + # Withholding the bonus entirely from tests over-corrects; granting it in full + # lets descriptive test function names outrank implementations. + assert 0.0 < DEFAULT_TUNING.name_cooccurrence_demoted_scale < 1.0 + # The page is packed with distinct files, one hit each. + assert DEFAULT_TUNING.max_per_file == 1 + + def test_tuning_is_immutable_and_hashable(): tuning = RetrievalTuning() with pytest.raises(Exception):