From abb67df5d5b44f3bc7b5bd0908de435ef2df29b1 Mon Sep 17 00:00:00 2001 From: denfry Date: Thu, 10 Sep 2026 09:35:35 +0300 Subject: [PATCH 01/18] release: v1.10.0 --- .claude-plugin/plugin.json | 2 +- .claude/skills/codebase-index/.skill_version | 2 +- .codex/skills/codebase-index/.skill_version | 2 +- .gitattributes | 4 + .../skills/codebase-index/.skill_version | 2 +- CHANGELOG.md | 136 ++++++++- README.md | 22 +- docs/ARCHITECTURE.md | 9 +- docs/BENCHMARKS.md | 42 +++ docs/RETRIEVAL.md | 118 +++++++- docs/RETRIEVAL_PIPELINE.md | 18 +- requirements.lock | 2 +- skill/scripts/doctor.py | 4 +- skill/scripts/smoke_test.py | 1 - src/codebase_index/__init__.py | 2 +- src/codebase_index/retrieval/diversity.py | 54 +++- src/codebase_index/retrieval/features.py | 119 ++++++++ src/codebase_index/retrieval/pipeline.py | 44 +++ src/codebase_index/retrieval/rerank.py | 28 +- src/codebase_index/retrieval/tuning.py | 88 +++++- tests/eval/README.md | 55 +++- tests/eval/harness.py | 67 ++++- tests/eval/metrics.py | 32 +++ tests/eval/run_eval.py | 29 +- tests/test_ranking_features.py | 263 ++++++++++++++++++ tests/test_tuning_contract.py | 26 +- 26 files changed, 1112 insertions(+), 59 deletions(-) create mode 100644 src/codebase_index/retrieval/features.py create mode 100644 tests/test_ranking_features.py diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 29e7c00..3ab5ec4 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -3,7 +3,7 @@ "name": "codebase-index", "displayName": "Codebase Index", "description": "Give Claude a precise local map of your codebase: find implementations, trace behavior, and predict change impact with file-line evidence.", - "version": "1.9.0", + "version": "1.10.0", "author": { "name": "codebase-index contributors" }, diff --git a/.claude/skills/codebase-index/.skill_version b/.claude/skills/codebase-index/.skill_version index f8e233b..81c871d 100644 --- a/.claude/skills/codebase-index/.skill_version +++ b/.claude/skills/codebase-index/.skill_version @@ -1 +1 @@ -1.9.0 +1.10.0 diff --git a/.codex/skills/codebase-index/.skill_version b/.codex/skills/codebase-index/.skill_version index f8e233b..81c871d 100644 --- a/.codex/skills/codebase-index/.skill_version +++ b/.codex/skills/codebase-index/.skill_version @@ -1 +1 @@ -1.9.0 +1.10.0 diff --git a/.gitattributes b/.gitattributes index 5a02ab9..e1dc5cb 100644 --- a/.gitattributes +++ b/.gitattributes @@ -5,3 +5,7 @@ *.py text eol=lf # cbx — POSIX-скрипт без расширения; CRLF в worktree ломает shebang при копировании cbx text eol=lf +# .skill_version хранит ровно "\n"; scripts/sync_skill_copies.py пишет и +# сравнивает байты, поэтому при core.autocrlf=true git подставлял CRLF в worktree и +# test_real_repo_is_in_sync падал на любом свежем клоне под Windows. +.skill_version text eol=lf diff --git a/.opencode/skills/codebase-index/.skill_version b/.opencode/skills/codebase-index/.skill_version index f8e233b..81c871d 100644 --- a/.opencode/skills/codebase-index/.skill_version +++ b/.opencode/skills/codebase-index/.skill_version @@ -1 +1 @@ -1.9.0 +1.10.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index 64f7fa8..cc0b537 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,138 @@ All notable changes to this project are documented here. The format is based on ## [Unreleased] +## [1.10.0] - 2026-09-02 + +Ranking release. 1.9.0's own diagnostics showed that a perfect reranker over the +candidate pool it already generated would score MRR 0.902 against the 0.577 actually +delivered — a ranking gap roughly three times larger than the remaining recall gap. +1.10.0 spends its entire budget on closing part of that gap, and adds the metrics +that make the gap visible. + +Measured over **420 queries across eight repositories** (Python ×2, Java ×3, +TypeScript/TSX ×2, PowerShell ×1) against a pinned 1.9.0: MRR +0.0188 (p=0.004), +nDCG@10 +0.0304 (p<0.001), MAP +0.0213 (p=0.001), recall@10 +0.0627 (p<0.001, 37 +wins / 0 losses), useful@budget +0.0292 (p=0.020), −19 tokens per query. The +candidate pool is unchanged, so every gain is reranking. No corpus regressed. Under +leave-one-repository-out the pooled gain is +0.0219 MRR with 7/8 folds improving and +0 regressing. Full tables in [docs/BENCHMARKS.md](docs/BENCHMARKS.md). + +### Added + +- **Query↔candidate name co-occurrence** (`retrieval/features.py`), the one new + ranking signal. Every retriever scores each query term independently and RRF sums + those independent verdicts, so nothing in the pipeline could distinguish a + candidate that matched *one* query term well from one that matched *three* terms in + a single name. On "graph resolution + traversal accessors" 1.9.0 ranked + `graph/retrieval.py` (one term) above `test_graph_accessors_resolve_and_walk` + (three); on "greedy token budgeting with redaction" it ranked `output/redact.py` + above `retrieval/budget.py`. The signal credits query terms for occurring + *together* in one name — file basename plus symbol, camel/snake split, directories + excluded — and only for terms beyond the first, since the first match is already + paid for by the retriever that surfaced the candidate. Cost is + `O(len(path) + len(symbol) + len(terms))` per candidate: no corpus statistics, no + posting-list scan, no model, no network. Ablatable via + `RetrievalTuning(name_cooccurrence=False)`. +- **Oracle / headroom metrics** in the eval harness (`oracle`, `cand_recall`, `eff`). + MRR alone cannot separate "retrieval never found it" from "the ranker buried it", + and those two failures share no fix. `oracle` is the MRR a perfect reranker would + achieve over the pool actually generated, `eff = MRR / oracle` is the fraction of + achievable quality delivered (0.639 → 0.660 in this release). These also make + ranking changes falsifiable in a new way: `-name_cooccurrence` moves eight quality + metrics while leaving `oracle` at ±0.0000, proving the gain is not disguised recall. +- **`search(..., explain=True)`** returns a `diagnostics` block with the pre-rerank + candidate pool and the final order, each candidate carrying source, symbol, score + and retriever agreement. This is what the oracle metrics are computed from, and + what turns "the ranking is wrong" into a decomposable failure. Measured at −0.8ms + p50 (inside noise); nothing is allocated when the flag is off. +- **`RetrievalTuning.v190()`** pins the previous release as the comparison column, so + "better than what we shipped last" cannot drift as the default changes. + `run_eval.py` now reports 1.7.0, 1.9.0 and the current default side by side. + +### Changed + +- **Pages are packed with distinct files** (`max_per_file` 3 → 1). The agent's unit + of decision is "which file do I open", so a 10-result page spending three slots on + three regions of one file offers seven choices, not ten. 1.9.0's page held 7.1 + distinct files on average, and of the queries whose answer was in the pool but + missing from the page, 45 of 57 had it past rank 10 — crowded out by repeat hits + rather than better candidates. Monotone over 1–5, so this is a plateau boundary, + not a fitted peak. Nothing is dropped: overflow hits keep their relative order at + the tail. recall@10 +0.045, nDCG@10 +0.015, unchanged token cost. +- **Name co-occurrence is discounted for test and generated sources** + (`name_cooccurrence_demoted_scale`, 0.5). Test function names are descriptive + sentences (`test_compactor_output_is_redacted`), so they harvest query-term + co-occurrences real identifiers never do. The value was chosen by splitting the + benchmark on whether its own ground truth is a test: across the 261 queries whose + answer is *not* a test the gain is flat at +0.020 MRR for every scale, so the whole + aggregate difference between 0.5 and 1.0 comes from the 159 test-answer queries — + an artifact of mining ground truth from commits, which touch tests. 0.5 is the only + setting that improves both partitions. +- **Benchmark corpora no longer index changelog files.** `gen_queries` documented + `CHANGELOG_EXCLUDES` as applied to the corpus but the harness never wired it in. A + git-derived query *is* a commit subject and a changelog entry paraphrases it + verbatim while never being an accepted answer, so every affected query carried an + unbeatable distractor that compressed all variants toward the same floor. + +### Performance + +- **Duplicate detection roughly halved.** It was 42% of the query path on the Java + corpus. Two independent fixes: the operator scan in `normalize_code_tokens` no + longer runs up to 24 `str.startswith` calls per punctuation character (1.71× + faster on 1600 real chunks, bit-identical token stream), and only the leading 2000 + characters of a body now decide duplication — two chunks agreeing for 2000 + characters are the same snippet. Recall, duplicate rate and useful-context are + identical; MRR within −0.0003 (p=0.51). + +### Fixed + +- **Non-ASCII identifiers can now earn name-level ranking credit.** The query side + parses Unicode correctly, so `расчёт_налога.py` produced matching query terms but + its own name components were silently dropped, making the file unrankable by name. +- **`.skill_version` is pinned to LF in `.gitattributes`.** `sync_skill_copies.py` + writes and byte-compares `"\n"`, but the file carried no EOL attribute, + so with `core.autocrlf=true` git materialised CRLF and `test_real_repo_is_in_sync` + failed on any fresh Windows clone — and again after any `git checkout` of that file. +- **Lint is clean repository-wide.** `skill/scripts/` was outside the CI lint scope + (`ruff check src tests`) and had accumulated three violations. + +### Verdicts on existing signals + +Every pre-existing signal was re-examined rather than inherited. `soft_lexical` +(−0.161 MRR when off) and `source_priors` (−0.029) remain load-bearing; +`file_agreement` (−0.010, p=0.048) and `dedup` (−0.0016, p=0.006) keep their places. + +- **`query_expansion` survives a deletion attempt, and the reason is a lesson.** On + the 420 git-derived queries the synonym vocabulary is worth nothing measurable + (MRR −0.0022, p=0.40 when removed), and the flag's apparent benefit turned out to + come from it also swapping the symbol retriever's tokenizer. It was removed — and + then restored, because a commit subject is written by someone looking at the + identifiers they just changed and so reuses the codebase's spelling, while a user + asking a question does not. On the 36 hand-written natural-language queries removal + cost −0.060 MRR. The git benchmark is structurally blind to morphology; both query + families are needed to make this call. +- **The intent classifier is inert on commit-style queries but kept.** It returns + `keyword` for 419 of 420 git-derived queries, and removing the layer entirely is + bit-identical on that benchmark. It fires on 30.6% of hand-written questions, where + removing it costs MRR, and the retriever *weights* it selects are worth +0.058 MRR + against uniform weights. Deleting it would optimise for the benchmark's phrasing. +- **`fuzzy_symbols` and `graph_source` remain measurably inert** on all eight corpora + (0/1 and 0/0 query changes respectively); both stay as-is rather than accumulating + new tuning. + +### Rejected + +Recorded so they are not re-attempted without a new hypothesis. A pairwise logistic +ranker fitted over 19 deterministic query↔candidate features selected exactly one +feature, and forward selection found no second feature clearing the noise floor, so +one explainable term ships instead of a model. Individually measured and rejected: +idf weighting of matched terms (pool-local and corpus-wide), substring matching, +prefix/stem-tolerant matching, ordered-subsequence matching, term proximity in the +chunk body, body-text coverage, zone-size normalisation, restricting the name zone to +the filename or symbol alone, and including the parent directory. `exact_symbol` was +found to have no discriminative power at all (AUC 0.500) but is left untouched, since +changing it is a separate experiment from adding a signal. + ## [1.9.0] - 2026-09-02 ### Added @@ -504,7 +636,9 @@ Pooled over 305 queries (Python, Java, TypeScript), v1.8.0 → 1.9.0: - Hooks example + `watch` mode for keeping the index fresh without blocking the edit loop (M8). - `doctor`, `stats`, `clean` diagnostics/maintenance commands. -[Unreleased]: https://github.com/denfry/codebase-index/compare/v1.8.0...HEAD +[Unreleased]: https://github.com/denfry/codebase-index/compare/v1.10.0...HEAD +[1.10.0]: https://github.com/denfry/codebase-index/compare/v1.9.0...v1.10.0 +[1.9.0]: https://github.com/denfry/codebase-index/compare/v1.8.0...v1.9.0 [1.8.0]: https://github.com/denfry/codebase-index/compare/v1.7.0...v1.8.0 [1.7.0]: https://github.com/denfry/codebase-index/compare/v1.6.0...v1.7.0 [1.6.0]: https://github.com/denfry/codebase-index/compare/v1.5.0...v1.6.0 diff --git a/README.md b/README.md index e6ad992..8502a05 100644 --- a/README.md +++ b/README.md @@ -271,7 +271,7 @@ See [MCP.md](docs/MCP.md) for client configuration. ## Project status -The latest released line is **1.9.0**. It includes: +The latest released line is **1.10.0**. It includes: - hybrid and optional vector retrieval; - Tree-sitter symbol extraction across the documented language tiers; @@ -281,6 +281,11 @@ The latest released line is **1.9.0**. It includes: - token-budgeted and skeletonized retrieval packets; - benchmark-calibrated lexical expansion, fuzzy identifier matching, and source-aware ranking; - rank fusion that scores cross-retriever agreement at file level, not just at a locator; +- a reranker that scores query↔candidate *interactions* — several query terms + co-occurring in one filename or symbol — instead of only summing independent + per-term matches; +- oracle/headroom metrics that separate "retrieval never found it" from "the ranker + buried it", so ranking work is aimed by measurement; - bounded, intent-directed graph discovery with optional diversity and duplicate suppression; - CLI, Skill, plugin, and MCP delivery; - incremental updates, watch hooks, diagnostics, skill rollback, and diff-aware @@ -289,12 +294,15 @@ The latest released line is **1.9.0**. It includes: one-signal ablations, and paired significance tests ([tests/eval](tests/eval/README.md)). -Every shipped ranking signal has to survive that evaluation: 1.9.0 removed the -cost of two signals that could not demonstrate a benefit and rejected several -plausible ones outright (IDF-weighted coverage, stemming, graph propagation, MMR, -a file-length prior). Planned work is deliberately separated from shipped -capability. The next product priorities are typed framework edges and an even more -direct task-context workflow. See the [roadmap](docs/ROADMAP.md). +Every shipped ranking signal has to survive that evaluation. 1.10.0 was validated on +420 queries across eight repositories in four languages, held out one repository at a +time to choose its parameters, and shipped exactly one new signal — a pairwise model +fitted over 19 candidate features could not justify a second. Ten further ranking +features were measured and rejected, and one attempted deletion was reverted when the +hand-written query set showed what the git-derived set structurally cannot see. +Planned work is deliberately separated from shipped capability. The next product +priorities are typed framework edges and an even more direct task-context workflow. +See the [roadmap](docs/ROADMAP.md). ## Documentation diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 4bb88af..eb574f3 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -2,7 +2,7 @@ ## 1. Overview -`codebase-index` is a **local-first** code intelligence layer for AI coding agents. In `1.9.0` +`codebase-index` is a **local-first** code intelligence layer for AI coding agents. In `1.10.0` it has two shipped faces: 1. **A Claude Code Skill** (`.claude/skills/codebase-index/SKILL.md`) that Claude auto-invokes for @@ -97,8 +97,8 @@ codebase-index/ ├── graph/ # builder.py (edge resolution), expand.py (impact), │ # export.py (HTML graph) ├── storage/ # db.py (pragmas, schema, version guard), schema.sql, repo.py - ├── retrieval/ # intent.py, searchers.py, fusion.py, rerank.py, priors.py, - │ # lexical.py, fuzzy.py, diversity.py, skeleton.py, + ├── retrieval/ # intent.py, searchers.py, fusion.py, rerank.py, features.py, + │ # priors.py, lexical.py, fuzzy.py, diversity.py, skeleton.py, │ # tuning.py (ablation contract), budget.py, pipeline.py, types.py ├── embeddings/ # backend.py, noop.py (default), local.py, external.py — opt-in ├── output/ # markdown.py, json.py, redact.py @@ -128,7 +128,8 @@ CI fails if they drift (`--check`). (a future-versioned index asks for a rebuild rather than guessing), and typed accessors. FTS5 virtual tables and (optional) `sqlite-vec` vector tables live here. - **retrieval** — The query path. `intent.py` classifies the query; `searchers.py` runs the - relevant retrievers; `fusion.py` merges them with RRF; `rerank.py` reorders; `graph.expand` + relevant retrievers; `fusion.py` merges them with RRF; `features.py` extracts deterministic + query↔candidate features and `rerank.py` reorders on them; `graph.expand` pulls in related nodes; `budget.py` trims to a token budget. - **embeddings** — Opt-in only. A `Backend` protocol so vector providers are pluggable. Default is `noop` (disabled). Local models supported; external APIs require explicit config + a warning. diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 5b1d7db..9e134cb 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -38,6 +38,48 @@ query text is not in the indexed corpus). Corpora are pooled across languages, o index per corpus is shared by every variant, and every non-baseline row carries a paired bootstrap CI and permutation p-value. +`1.10.0` was measured over **420 queries across eight repositories** (Python ×2, +Java ×3, TypeScript/TSX ×2, PowerShell ×1) against a pinned `1.9.0` +(`RetrievalTuning.v190()`, so the comparison point cannot drift with the default): + +| Metric | 1.9.0 | 1.10.0 | Δ | 95% CI | p | W/L/T | +|---|---|---|---|---|---|---| +| MRR | 0.5767 | 0.5955 | +0.0188 | [+0.0061, +0.0321] | 0.004 | 44/14/362 | +| nDCG@10 | 0.5352 | 0.5656 | +0.0304 | [+0.0197, +0.0416] | <0.001 | 64/19/337 | +| MAP | 0.4654 | 0.4867 | +0.0213 | [+0.0104, +0.0331] | 0.001 | 64/19/337 | +| recall@5 | 0.6026 | 0.6212 | +0.0187 | [+0.0089, +0.0306] | <0.001 | 14/0/406 | +| recall@10 | 0.6306 | 0.6933 | +0.0627 | [+0.0421, +0.0847] | <0.001 | 37/0/383 | +| P@5 | 0.1971 | 0.2033 | +0.0062 | [+0.0030, +0.0099] | <0.001 | 14/2/404 | +| useful@budget | 0.5919 | 0.6210 | +0.0292 | [+0.0048, +0.0550] | 0.020 | 36/17/367 | +| hit@3 | 0.6690 | 0.6833 | +0.0143 | [+0.0000, +0.0286] | 0.107 | 8/2/410 | +| tokens/query | 1080 | 1061 | −19 | — | — | — | + +Two properties of that table matter more than the deltas: + +- **`oracle` and `cand_recall` are unchanged to four decimals.** The candidate pool + is identical, so every gain is reranking, not new recall. Reranking efficiency + (MRR / oracle) went 0.639 → 0.660, closing ~6% of the ranking headroom 1.9.0 left + on the table. +- **No corpus regressed.** An aggregate improvement is worthless if one large corpus + masks a regression elsewhere, so per-corpus MRR is checked on every run. + +Held-out validation, because hand-picked coefficients are still fitted parameters: +under leave-one-repository-out — both tuned numbers selected on seven corpora and +scored on the eighth — the pooled gain is **+0.0219 MRR, with 7/8 folds improving +and 0 regressing**. The shipped configuration is deliberately *more conservative* +than that selection would pick (see the `name_cooccurrence_demoted_scale` rationale +in `retrieval/tuning.py`), so these numbers under-claim what the benchmark alone +would support. + +Known cost, stated because the benchmark family that reveals it is the small one: on +the 36 hand-written natural-language queries MRR moved −0.028 (p=0.63; 1 win, 3 +losses, 32 ties). Three questions fell from rank 1 to rank 2–3 where a test file's +descriptive function name matches more query terms than the implementation's name. + +Per-corpus `oracle` ranges 0.795–1.000 and reranking efficiency 0.557–0.753, so the +largest remaining headroom is still ranking, not recall — see §10 of +[RETRIEVAL.md](RETRIEVAL.md). + `1.9.0` was measured over 305 queries across Python, Java and TypeScript corpora against `1.8.0`: MRR +0.027, MAP +0.028, nDCG@10 +0.024, recall@5 +0.031 (all p < 0.001), with p50 latency 78.6 ms → 51.2 ms. These are version-over-version diff --git a/docs/RETRIEVAL.md b/docs/RETRIEVAL.md index 6102baa..e479ff8 100644 --- a/docs/RETRIEVAL.md +++ b/docs/RETRIEVAL.md @@ -21,8 +21,7 @@ query ▼ [3] rank fusion ── Reciprocal Rank Fusion (RRF) across retriever result lists │ - ▼ -[4] rerank ── feature-based score (symbol-kind, path, source role, centrality) +[4] rerank ── query↔candidate features: name co-occurrence, symbol, path, source role, centrality │ ▼ [5] optional graph expansion ── pull in imports/callers/callees per intent (bounded) @@ -115,10 +114,13 @@ there. Set `RetrievalTuning(file_agreement=False)` to recover plain locator-only ## 4. Reranking (`retrieval/rerank.py`) A lightweight, explainable feature score (no external model required) layered on the fused order. -Every term is bounded, so reranking reorders near-neighbours rather than overruling retrieval: +Most terms are bounded tiebreakers, so they reorder near-neighbours rather than overruling +retrieval. The exception is name co-occurrence, which is deliberately large enough to move a +candidate several places — see below for why, and for the evidence that it should. | Feature | Effect | Intuition | |---|---:|---| +| Query terms co-occurring in one name | ≤ +1.80 | several query terms in one name is qualitatively better evidence than one term matched well | | Exact symbol match | +0.20 | the user named a specific symbol | | Symbol definition kind | +0.05 | a `def`/`class` outranks an incidental mention | | Symbol name among query terms | +0.05 | the name was asked for, not just matched | @@ -128,6 +130,56 @@ Every term is bounded, so reranking reorders near-neighbours rather than overrul | Source role prior | −0.25…+0.08 | see below | | Generated, or test on a non-test query | −0.15 | supporting evidence, not the answer | +### Name co-occurrence (`retrieval/features.py`) + +Every retriever scores each query term independently, and RRF sums those independent verdicts. That +structure cannot distinguish a candidate which matched **one** query term very well from one which +matched **three** terms in a single name — and on the 1.9.0 benchmark that confusion was the single +largest reranking loss. Two measured examples, both ranked wrong by 1.9.0: + +| Query | 1.9.0 winner | correct answer, ranked below it | +|---|---|---| +| "graph resolution + traversal accessors" | `graph/retrieval.py` (`graph`) | `test_graph_accessors_resolve_and_walk` (`graph` + `accessors` + `resolve`) | +| "greedy token budgeting with redaction" | `output/redact.py` (`redact`) | `retrieval/budget.py` (`budget` + `token`) | + +The fix is an interaction term, not another per-term bonus. Let `zone(d)` be the identifier +components of the candidate's *name* — file basename plus symbol, camel/snake split — and `m` the +number of salient query terms appearing in it: + +``` +cooccurrence(d) = max(0, m - 1) / (n_terms - 1) # 0 when m < 2 +score(d) += w · cooccurrence(d) · (scale if demoted else 1) +``` + +Only terms **beyond the first** earn credit: a single matched term is already fully paid for by the +retriever that surfaced the candidate, so crediting it again would merely re-weight lexical +matching, which is not what was missing. Directories are excluded from the zone — +`src/main/java/net/...` is shared by hundreds of files, so it adds co-occurrence noise to all of +them and evidence to none. + +`w = 1.8` sits in the interior of a plateau. Pooled MRR rises to ≈1.8 and then flattens, and past +that point per-query wins stay flat while losses nearly double (37W/19L at 1.8 against 39W/32L at +4.0), because a larger bonus turns the feature into the primary sort key and reduces fusion to a +tiebreak. Every value in 1.0–4.0 leaves all eight benchmark corpora at or above 1.9.0. + +`scale = 0.5` discounts — rather than withholds — the bonus for test and generated sources. Test +function names are descriptive sentences (`test_compactor_output_is_redacted`), so they harvest +query-term co-occurrences that real identifiers never do, and a bonus reaching +1.8 is not +counterbalanced by a flat −0.15 demotion calibrated when the largest name bonus was +0.05. The +value was chosen by splitting the benchmark on whether its *own ground truth* is a test: across the +261 queries whose answer is not a test, the gain is flat at +0.020 MRR for every scale, so the +entire aggregate difference between 0.5 and 1.0 comes from the 159 test-answer queries — an +artifact of mining ground truth from commits, which touch tests. 0.5 is the only setting that +improves both partitions. + +Variants that were measured and **rejected**, each failing to beat this one on held-out +repositories: idf weighting of the matched terms (pool-local and corpus-wide); substring instead of +component matching; prefix/stem-tolerant matching (`redact`/`redacted`); ordered-subsequence +matching; term proximity within the chunk body; body-text coverage; zone-size ("tightness") +normalisation; and restricting the zone to the filename or the symbol alone. A pairwise logistic +ranker fitted over 19 candidate features selected this one, and forward selection found no second +feature clearing the noise floor — so one feature ships, not a model. + ### Source role priors (`retrieval/priors.py`) | Role | Prior | Rationale | @@ -160,19 +212,44 @@ When enabled, it is bounded by depth and node cap: Expanded nodes retain edge confidence and receive distance-decayed scores so seeds stay on top. -## 6. Diversity and duplicate control +## 6. Diversity, page packing, and duplicate control `retrieval.diversity` provides bounded MMR selection and SimHash near-duplicate suppression. +### Page packing (`max_per_file`) + +The agent's unit of decision is "which file do I open", so a 10-result page that spends three +slots on three regions of one file offers seven choices, not ten. Measured across eight +repositories, 1.9.0's page held **7.1 distinct files** on average, and of the queries whose answer +was in the candidate pool but missing from the page, 45 of 57 had it sitting past rank 10 — crowded +out by repeat hits rather than by better candidates. + +`max_per_file = 1` keeps one hit per file in place and pushes the rest to the tail. Nothing is +dropped, so a file with several relevant regions still surfaces them below the first page. The +parameter is monotone over 1–5 (1 > 2 > 3 > 4 > 5), so this is a plateau boundary rather than a +fitted peak: recall@10 +0.045 and nDCG@10 +0.015 against 1.9.0's value of 3, at unchanged token +cost, with no metric and no corpus regressing. + +### MMR + MMR is disabled in the shipped default: it moved no ranking metric on the benchmark and roughly doubled p50 latency. Callers that need broader snippet coverage can enable `RetrievalTuning(mmr=True)`. -SimHash duplicate suppression stays on, but on noise grounds rather than ranking grounds: it does -not move MRR, and it takes the duplicate rate of returned snippets from ~1.6% to ~0%. The -over-fetch that feeds selection is now an explicit `candidate_pool_multiplier` rather than an -implicit side effect of enabling dedup — an earlier ablation credited dedup with a quality win that -was really the wider pool doing the work. +### Duplicate suppression + +SimHash duplicate suppression stays on, but on noise grounds rather than ranking grounds: it moves +MRR by −0.0016 (p=0.006) and takes the duplicate rate of returned snippets to ~0%. The over-fetch +that feeds selection is an explicit `candidate_pool_multiplier` rather than an implicit side effect +of enabling dedup — an earlier ablation credited dedup with a quality win that was really the wider +pool doing the work. + +Only the leading 2000 characters of a candidate body decide duplication. Fingerprinting whole chunk +bodies made this the most expensive stage of the query path — 42% of it on the Java corpus — to +answer a question the first ~50 lines already answer: two chunks that agree for 2000 characters are +the same snippet. The bound leaves recall, duplicate rate and useful-context identical and MRR +within −0.0003 (p=0.51). Combined with a rewritten operator scan in the tokeniser (1.71× faster, +bit-identical output), duplicate control costs roughly half what it did in 1.9.0. ## 7. Token budgeting (`retrieval/budget.py`) @@ -227,3 +304,26 @@ number of agreeing retrievers, and whether a symbol matched exactly. The Markdown renderer (`output/markdown.py`) prints the same data as a tight table + fenced snippets so it's compact in Claude's context. See SKILL.md for how Claude is told to read it. + +## 10. Ranking diagnostics (`--explain`) + +Every result already carries a `reason` string, but diagnosing *why the ranking was wrong* needs +the order before reranking as well as after. `search(..., explain=True)` adds a `diagnostics` +block containing the pre-rerank candidate pool and the final order, each with per-candidate +source, symbol, score and retriever agreement. Building it costs about thirty dict literals per +query — measured at −0.8ms p50, i.e. inside the noise of a 40ms query — and nothing is allocated +when the flag is off. + +This is what the evaluation harness uses to compute the **oracle** metrics, and it is what turns +"MRR is low" into an actionable decomposition: + +| Metric | Question it answers | +|---|---| +| `oracle` | what MRR a *perfect* reranker would score over the pool actually generated | +| `1 - oracle` | the share of queries only better **recall** can ever fix | +| `eff` = MRR / oracle | the share of achievable ranking quality actually delivered | + +At 1.9.0 that split read `oracle = 0.902`, `MRR = 0.577`, `eff = 0.639`: a third of the answers +already sitting in the candidate pool were ranked below something else, an error mode roughly +three times larger than the remaining 0.098 of recall headroom. That measurement is why 1.10.0 +spent its effort on the reranker instead of adding retrievers. diff --git a/docs/RETRIEVAL_PIPELINE.md b/docs/RETRIEVAL_PIPELINE.md index 5bf0944..6c2f323 100644 --- a/docs/RETRIEVAL_PIPELINE.md +++ b/docs/RETRIEVAL_PIPELINE.md @@ -132,10 +132,14 @@ strong one, and cross-source agreement never affected the score. ## 8. Reranking -After fusion, apply bounded explainable boosts and penalties: +After fusion, apply explainable query↔candidate boosts and penalties. All but the +first are bounded tiebreakers; the co-occurrence term is deliberately larger, +because it is the only signal that can see an *interaction* between query terms +rather than another independent per-term match. | Factor | Effect | Rationale | |---|---:|---| +| Query terms co-occurring in one name | up to +1.80 | Several query terms in one filename/symbol beats one term matched well; RRF sums per-term evidence and cannot see this | | Exact symbol match | +0.20 | User named a specific symbol | | Symbol definition kind | +0.05 | Prefer actionable definitions | | Path term match | +0.05 | User supplied a location clue | @@ -145,6 +149,18 @@ After fusion, apply bounded explainable boosts and penalties: | Generated/vendor/build | -0.25 | Suppress low-value derived code | | Test path on non-test query | -0.06 | Keep tests as supporting evidence | +The co-occurrence bonus is halved for test and generated sources: descriptive test +function names (`test_compactor_output_is_redacted`) are word bags that collect +query-term matches real identifiers never do. See `retrieval/features.py` and +[RETRIEVAL.md](RETRIEVAL.md) §4 for the measurement behind both numbers. + +## 8a. Page packing + +Selection keeps **one hit per file** on the page and pushes further hits from the +same file to the tail. The agent decides at file granularity, so three regions of +one file occupy three of ten slots while offering one choice. Nothing is dropped; +recall@10 improves because distinct files now fit. + ## 9. Confidence Confidence is categorical (`high`, `medium`, `low`) and is derived from diff --git a/requirements.lock b/requirements.lock index 6843b5b..f83cde3 100644 --- a/requirements.lock +++ b/requirements.lock @@ -1,3 +1,3 @@ -codebase-index @ https://github.com/denfry/codebase-index/archive/refs/tags/v1.9.0.tar.gz +codebase-index @ https://github.com/denfry/codebase-index/archive/refs/tags/v1.10.0.tar.gz tree-sitter==0.25.2 tree-sitter-language-pack==1.8.1 diff --git a/skill/scripts/doctor.py b/skill/scripts/doctor.py index 25aeac4..957dbf7 100644 --- a/skill/scripts/doctor.py +++ b/skill/scripts/doctor.py @@ -69,7 +69,7 @@ def check_skill_installed(project_root: Path) -> bool: print(f"[OK] Skill installed at: {skill_dir}") return True else: - print(f"[INFO] Skill not installed in .claude/skills/") + print("[INFO] Skill not installed in .claude/skills/") print(" Run: python skill/scripts/install.py") return True # Not a hard failure @@ -86,7 +86,7 @@ def check_config(project_root: Path) -> bool: print(f"[FAIL] Invalid JSON in {config_path}") return False else: - print(f"[INFO] No config file (using defaults)") + print("[INFO] No config file (using defaults)") return True diff --git a/skill/scripts/smoke_test.py b/skill/scripts/smoke_test.py index 7e4f0fa..a92b866 100644 --- a/skill/scripts/smoke_test.py +++ b/skill/scripts/smoke_test.py @@ -11,7 +11,6 @@ import json import os -import shutil import subprocess import sys import tempfile diff --git a/src/codebase_index/__init__.py b/src/codebase_index/__init__.py index af8c17b..a8508b9 100644 --- a/src/codebase_index/__init__.py +++ b/src/codebase_index/__init__.py @@ -4,4 +4,4 @@ See docs/ARCHITECTURE.md for the module map. """ -__version__ = "1.9.0" +__version__ = "1.10.0" diff --git a/src/codebase_index/retrieval/diversity.py b/src/codebase_index/retrieval/diversity.py index d94be5d..0619fae 100644 --- a/src/codebase_index/retrieval/diversity.py +++ b/src/codebase_index/retrieval/diversity.py @@ -18,6 +18,26 @@ _MASK64 = (1 << 64) - 1 +# Leading characters of a candidate body that decide near-duplication. See +# `deduplicate` for why bounding this is safe and what it buys. +_FINGERPRINT_CHARS = 2000 + +# Multi-character operators kept as one token, grouped by length so a match can be +# found with a slice and a set lookup instead of a linear scan. Probed longest-first. +_OPERATORS_BY_LEN: tuple[tuple[int, frozenset[str]], ...] = ( + (4, frozenset({">>>="})), + (3, frozenset({"===", "!==", "**=", "..."})), + ( + 2, + frozenset( + { + "=>", "->", "::", "==", "!=", "<=", ">=", "&&", "||", "++", + "--", "+=", "-=", "*=", "/=", "%=", "<<", ">>", "**", "??", + } + ), + ), +) + def normalize_code_tokens(content: str | None) -> tuple[str, ...]: """Return normalized code tokens, omitting whitespace and comments.""" @@ -27,10 +47,15 @@ def normalize_code_tokens(content: str | None) -> tuple[str, ...]: tokens: list[str] = [] length = len(content) index = 0 - operators = ( - "===", "!==", ">>>=", "**=", "...", "=>", "->", "::", "==", "!=", "<=", ">=", - "&&", "||", "++", "--", "+=", "-=", "*=", "/=", "%=", "<<", ">>", "**", "??", - ) + # `operators` used to be scanned with `next(c for c in operators if + # content.startswith(c, index))`, i.e. up to 24 `str.startswith` calls for every + # punctuation character in every candidate chunk. On the Java corpus that single + # generator was 2.1s of the 5.9s spent tokenising. `_OPERATORS_BY_LEN` replaces it + # with at most three slice-and-set-lookup probes, longest first. + # + # This is bit-identical, not merely equivalent-looking: every 3- and 4-character + # operator appeared before every 2-character one in the old tuple, so first-match + # and longest-match already agreed, and the emitted token stream is unchanged. while index < length: char = content[index] @@ -81,10 +106,11 @@ def normalize_code_tokens(content: str | None) -> tuple[str, ...]: index = cursor continue - operator = next((candidate for candidate in operators if content.startswith(candidate, index)), None) - if operator is not None: - tokens.append(operator) - index += len(operator) + for width, group in _OPERATORS_BY_LEN: + if content[index : index + width] in group: + tokens.append(content[index : index + width]) + index += width + break else: tokens.append(char) index += 1 @@ -171,7 +197,17 @@ def deduplicate(candidates: Sequence[Candidate], hamming_distance: int = 3) -> l # Only the fingerprint is needed here; building the token set as well # doubled the per-candidate cost of the pipeline's hottest stage. "No # fingerprint" means "no tokens", never "fingerprint happened to be 0". - tokens = normalize_code_tokens(candidate.content) + # + # Only the leading _FINGERPRINT_CHARS decide duplication. Tokenising whole + # chunk bodies made this the single most expensive stage of the query path + # (42% of it on the Java corpus) to answer a question the first ~50 lines + # already answer: two chunks that agree for 2000 characters are the same + # snippet. Measured over 420 queries on eight repositories the bound leaves + # recall, duplicate rate and useful-context identical and MRR within + # -0.0003 (p=0.51), for -3ms p50. The retained candidate keeps its full + # content; only the comparison window is bounded. + content = candidate.content + tokens = normalize_code_tokens(content[:_FINGERPRINT_CHARS] if content else None) fingerprint = token_fingerprint(tokens) if tokens else None if fingerprint is None: representatives.append(candidate) diff --git a/src/codebase_index/retrieval/features.py b/src/codebase_index/retrieval/features.py new file mode 100644 index 0000000..6aa9a7b --- /dev/null +++ b/src/codebase_index/retrieval/features.py @@ -0,0 +1,119 @@ +"""Deterministic query<->candidate ranking features. + +The fused RRF score answers "how many retrievers liked this candidate, and how +much". It cannot answer "does this candidate's *name* actually talk about the +thing the query asked for", because every retriever scores each query term +independently and RRF only ever sums those independent verdicts. + +That gap is a measured failure mode, not a hypothetical one. Two examples from +the benchmark, both ranked wrong by 1.9.0: + + "graph resolution + traversal accessors" + won by graph/retrieval.py (`graph` — one term) + beating test_graph_accessors_... (`graph` + `accessors` + `resolve`) + + "greedy token budgeting with redaction" + won by output/redact.py (`redact` — one term) + beating retrieval/budget.py (`budget` + `token`) + +The winner in each pair matched *one* query term very well. The correct answer +matched *several* terms in a single name. Summing per-term evidence cannot tell +those apart, so the signal has to be an interaction: credit terms for occurring +*together* in one name, over and above their individual matches. + +Design constraints, all of which the eval harness enforces: + + * Bounded. Cost is O(len(path) + len(symbol) + len(terms)) per candidate, with + no corpus statistics, no posting-list scan, and no second query. + * Deterministic. Set membership over identifier components; no floating-point + accumulation order dependence, no randomness, no I/O. + * Explainable. The score reduces to a count the reranker can print verbatim. + +Feature variants that were measured and rejected are recorded in +`docs/RETRIEVAL.md`; the short version is that idf weighting (pool-local or +corpus-wide), substring matching, proximity, ordered-subsequence matching, and +body-text coverage all failed to beat this one feature on held-out repositories. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from .lexical import salient_terms, split_identifier + + +def _components(text: str) -> set[str]: + """Lowercase identifier components of *text* (camelCase and snake_case aware). + + `split_identifier` is applied to the whole string rather than to regex-matched + words first: it already treats `_`, `-` and every non-alphanumeric character as + a separator, and it decides "alphanumeric" with `str.isalnum`. Pre-splitting on + an ASCII word pattern would silently reduce `расчёт_налога` to nothing, so + non-ASCII identifiers could never earn this bonus while the query side — which + parses Unicode correctly — happily produced the matching terms. + """ + return set(split_identifier(text)) + + +def name_zone(path: str, symbol: str | None) -> set[str]: + """Identifier components of a candidate's *name*: file basename + symbol. + + Directories are excluded deliberately. A path prefix such as + ``src/main/java/net/denfry/newtowny/`` is shared by hundreds of files, so its + components carry no evidence about which of them answers the query while + adding co-occurrence noise to all of them. Measured on eight repositories, + including the parent directory was worth -0.001 MRR: indistinguishable from + noise, for strictly more code. + """ + basename = path.replace("\\", "/").rsplit("/", 1)[-1] + stem = basename[: basename.rfind(".")] if "." in basename else basename + zone = _components(stem) + if symbol: + zone |= _components(symbol) + return zone + + +@dataclass(frozen=True, slots=True) +class QueryProfile: + """Query terms worth matching against a candidate name, computed once per query. + + Built from :func:`lexical.salient_terms`, so stopword policy stays in one + place and the reranker cannot drift from the retrievers' idea of a term. + """ + + terms: tuple[str, ...] + + @property + def n_terms(self) -> int: + return len(self.terms) + + +def query_profile(query: str) -> QueryProfile: + return QueryProfile(terms=salient_terms(query)) + + +def name_cooccurrence(profile: QueryProfile, zone: set[str]) -> tuple[int, float]: + """Return (terms matched in *zone*, co-occurrence score in [0, 1]). + + The score credits only the query terms *beyond the first*: + + score = max(0, matched - 1) / (n_terms - 1) + + A candidate whose name matches one query term scores 0 — it has shown no + interaction, and its single match is already fully paid for by the retriever + that surfaced it. The second matched term is what distinguishes + ``test_graph_accessors_resolve_and_walk`` from ``graph_candidates`` on the + query "graph resolution + traversal accessors", so that is where the credit + starts. Normalising by ``n_terms - 1`` keeps the feature comparable between a + two-term and a nine-term query. + + Single-term queries return 0.0: there is no co-occurrence to observe, and the + exact-name machinery in the symbol retriever already handles them. + """ + n = profile.n_terms + if n < 2 or not zone: + return 0, 0.0 + matched = sum(1 for term in profile.terms if term in zone) + if matched < 2: + return matched, 0.0 + return matched, (matched - 1) / (n - 1) diff --git a/src/codebase_index/retrieval/pipeline.py b/src/codebase_index/retrieval/pipeline.py index 020c956..9d5e3dd 100644 --- a/src/codebase_index/retrieval/pipeline.py +++ b/src/codebase_index/retrieval/pipeline.py @@ -144,6 +144,27 @@ def _fallback_suggestions(query, ranked) -> dict: return {"ripgrep": rg} +def _pool_entry(rank: int, c) -> dict: + """One pre-rerank candidate, as the ranking-diagnostics view of it. + + Recorded before rerank mutates `score`/`reason` in place, so the fused order + stays comparable with the final order. Only built when `explain` is set: the + default query path allocates nothing. + """ + return { + "rank": rank, + "path": c.path, + "line_start": c.line_start, + "line_end": c.line_end, + "source": c.source, + "symbol": c.symbol, + "kind": c.kind, + "fused_score": round(c.score, 4), + "agreeing_sources": c.agreeing_sources, + "exact_symbol": c.exact_symbol, + } + + def search( conn: sqlite3.Connection, query: str, @@ -159,6 +180,7 @@ def search( offset: int = 0, compact: bool = True, compact_min_reduction: float = 0.25, + explain: bool = False, ) -> dict: tuning = tuning or DEFAULT_TUNING plan = detect_intent(query) @@ -190,6 +212,7 @@ def search( k=tuning.rrf_k, file_agreement=tuning.file_agreement_weight if tuning.file_agreement else 0.0, ) + pool = [_pool_entry(rank, c) for rank, c in enumerate(fused, start=1)] if explain else [] ranked = rerank(fused, query=query, intent=plan.intent, tuning=tuning) if tuning.dedup: ranked = deduplicate(ranked, hamming_distance=tuning.dedup_hamming) @@ -255,4 +278,25 @@ def search( "has_more": has_more, "next_offset": offset + limit if has_more else None, } + if explain: + payload["diagnostics"] = { + "weights": plan.weights, + "pool_size": len(pool), + "pool": pool, + "ranked": [ + { + "rank": rank, + "path": c.path, + "line_start": c.line_start, + "line_end": c.line_end, + "source": c.source, + "symbol": c.symbol, + "score": round(c.score, 4), + "reason": c.reason, + "agreeing_sources": c.agreeing_sources, + "exact_symbol": c.exact_symbol, + } + for rank, c in enumerate(ranked, start=1) + ], + } return payload diff --git a/src/codebase_index/retrieval/rerank.py b/src/codebase_index/retrieval/rerank.py index 4b976d8..464bb8c 100644 --- a/src/codebase_index/retrieval/rerank.py +++ b/src/codebase_index/retrieval/rerank.py @@ -11,6 +11,7 @@ import re from ..discovery.classify import is_test_path +from .features import name_cooccurrence, name_zone, query_profile from .priors import source_role_prior from .tuning import DEFAULT_TUNING, RetrievalTuning from .types import Candidate, Intent @@ -35,9 +36,33 @@ def rerank( tuning: RetrievalTuning = DEFAULT_TUNING, ) -> list[Candidate]: terms = {t.lower() for t in _TERM_RE.findall(query)} + wants_tests = "test" in terms or "tests" in terms + profile = query_profile(query) if tuning.name_cooccurrence else None for c in candidates: bonus = 0.0 reasons: list[str] = [] + # Computed before the bonuses because the name-co-occurrence signal is + # conditioned on it, not merely penalised after the fact. + demoted = c.is_generated or (is_test_path(c.path) and not wants_tests) + + # Name co-occurrence is *discounted*, not withheld, for sources the ranker + # is unwilling to promote. Test symbol names are descriptive sentences + # (`test_compactor_output_is_redacted`), so they harvest query-term + # co-occurrences that real identifiers never do, and a bonus reaching + # +`name_cooccurrence_weight` is not counterbalanced by a flat -0.15 + # demotion calibrated when the largest name bonus was +0.05. Withholding it + # outright over-corrects: on git-derived ground truth the changed file often + # *is* the test, and zeroing the bonus cost -0.017 MRR across eight + # repositories. The discount keeps a strongly-matching test ahead of a + # barely-matching implementation while restoring the intended role ordering + # when both match comparably. + if profile is not None: + matched, cooccurrence = name_cooccurrence(profile, name_zone(c.path, c.symbol)) + if cooccurrence: + if demoted: + cooccurrence *= tuning.name_cooccurrence_demoted_scale + bonus += tuning.name_cooccurrence_weight * cooccurrence + reasons.append(f"{matched}/{profile.n_terms} query terms co-occur in name") if c.source == "symbol" and c.kind in {"function", "method", "class", "interface", "type"}: bonus += 0.05 @@ -71,8 +96,7 @@ def rerank( bonus += prior reasons.append(f"source prior {prior:+.2f}") - wants_tests = "test" in terms or "tests" in terms - if c.is_generated or (is_test_path(c.path) and not wants_tests): + if demoted: bonus -= 0.15 reasons.append("generated/test demoted") diff --git a/src/codebase_index/retrieval/tuning.py b/src/codebase_index/retrieval/tuning.py index 9898d6e..6d47543 100644 --- a/src/codebase_index/retrieval/tuning.py +++ b/src/codebase_index/retrieval/tuning.py @@ -44,7 +44,21 @@ class RetrievalTuning: query_expansion: bool = True """Down-weighted code-synonym expansion (auth->authentication, ...). Original - terms always keep a strictly higher weight so precision is preserved.""" + terms always keep a strictly higher weight so precision is preserved. + + 1.10.0 tried to delete this. On the 420-query git-derived benchmark the + vocabulary is worth nothing measurable — MRR -0.0022 (p=0.40) when removed — + and the flag's apparent +0.006 turned out to come from it also swapping the + symbol retriever's tokenizer, not from the synonyms. + + It survives because the git benchmark cannot see what it does. A commit + subject is written by someone looking at the identifiers they just changed, + so it reuses the codebase's own spelling; a user asking a question does not. + On the 36 hand-written natural-language queries — the only set that phrases + things the way a person would — removing the vocabulary cost -0.060 MRR, + because "where are secrets redacted" has to reach `redact_snippet` and + "how does authentication work" has to reach `auth/`. Both benchmark families + are needed to make this call, and only one of them can measure this.""" graph_source: bool = False """Personalized-PageRank candidate source seeded from lexical/symbol hits. @@ -106,9 +120,62 @@ class RetrievalTuning: """Discount applied to same-file, different-locator evidence. Tuned on 305 queries over three repositories; the 0.3-0.6 plateau peaks here.""" + name_cooccurrence: bool = True + """Reward query terms that co-occur in one candidate *name* (file basename + + symbol), superlinearly, over and above their independent per-term matches. + + RRF sums independent per-term verdicts, so it cannot distinguish a candidate + that matched one query term very well from one that matched three terms in a + single name. Measured over eight repositories and 420 queries, that confusion + was the largest single reranking loss: the correct answer sat in the candidate + pool with a better name-level match and lost to a one-term winner. + + See `retrieval/features.py` for the feature and the variants it beat.""" + + name_cooccurrence_weight: float = 1.8 + """Bonus at full co-occurrence (every query term present in one name). + + Chosen on a plateau, not at a peak. Pooled MRR rises to w≈1.8 and then flattens + (0.599 at 1.8, 0.603 at 4.0, saturating at 0.603 beyond); past 1.8 the extra + movement is churn, with per-query wins flat and losses nearly doubling + (37W/19L at 1.8 against 39W/32L at 4.0), because a larger bonus turns the + feature into the primary sort key and reduces fusion to a tiebreak. Every + value in 1.0-4.0 leaves all eight corpora at or above 1.9.0; 1.8 is the + interior of that region with the strongest significance (p=0.004).""" + + name_cooccurrence_demoted_scale: float = 0.5 + """Fraction of the co-occurrence bonus granted to test/generated sources. + + 1.0 lets descriptive test function names outrank implementations on questions + like "where are secrets redacted before output"; 0.0 over-corrects, since on + git-derived ground truth the changed file often *is* the test. + + Decided by splitting the benchmark on whether its own ground truth is a test: + across the 261 queries whose answer is *not* a test, the gain is flat at + +0.020 MRR for every scale, so the whole aggregate difference between 0.5 and + 1.0 comes from the 159 test-answer queries — an artifact of mining ground + truth from commits, which touch tests. 0.5 is the only setting that improves + both partitions (+0.015 test-answer, +0.020 implementation-answer).""" + + # --- selection, continued ----------------------------------------------- + max_per_file: int = 1 + """Hits from one file kept in place before the rest are pushed to the tail. + + The agent's unit of decision is "which file do I open", so a page of 10 + results that spends three slots on three regions of one file offers seven + choices, not ten. Measured across eight repositories the page held 7.1 + distinct files on average, and 45 of 57 queries whose answer was in the + candidate pool but absent from the page had it sitting past rank 10. + + Monotone over 1-5 (1 > 2 > 3 > 4 > 5), so this is a plateau boundary rather + than a fitted peak: recall@10 +0.045 and nDCG@10 +0.015 against 3, at + unchanged token cost and with no metric or corpus regressing. Nothing is + dropped — overflow hits keep their relative order at the tail — so a file + with several relevant regions still surfaces them below the first page. + """ + # --- fixed parameters ---------------------------------------------------- rrf_k: int = 60 - max_per_file: int = 3 graph_damping: float = 0.85 graph_iterations: int = 12 graph_weight: float = 0.1 @@ -127,10 +194,25 @@ def baseline(cls) -> RetrievalTuning: dedup=False, source_priors=False, file_agreement=False, - # 1.7.0 had no over-fetch: the pool was exactly the requested page. + name_cooccurrence=False, + # 1.7.0 had no over-fetch: the pool was exactly the requested page, + # and it kept up to three hits from any one file. candidate_pool_multiplier=1, + max_per_file=3, ) + @classmethod + def v190(cls) -> RetrievalTuning: + """The 1.9.0 shipped configuration, as the immutable "before" column. + + `baseline()` reaches back to 1.7.0 and answers "was any of this worth it"; + this answers the narrower question every 1.10.0 change is judged on: is it + better than the release it replaces. Both must keep working, so a later + default change can never silently redefine its own comparison point. + """ + return cls(name_cooccurrence=False, max_per_file=3) + + def without(self, flag: str) -> RetrievalTuning: """Return a copy with one boolean signal disabled (single-signal ablation).""" field_names = {f.name for f in fields(self)} diff --git a/tests/eval/README.md b/tests/eval/README.md index 58903a1..893cb9d 100644 --- a/tests/eval/README.md +++ b/tests/eval/README.md @@ -62,17 +62,50 @@ Conventional Commits prefix is stripped; duplicate subjects collapse. Changelog- files are never accepted as answers because they paraphrase commit subjects, and any commit touching `tests/eval/` is dropped so the benchmark cannot grade itself. -Corpora used to validate the 1.9.0 ranking changes, beyond this repository: +Corpora used to validate the 1.10.0 ranking changes, beyond this repository: | Corpus | Language | Files | Queries | |---|---|---|---| | Civitas | Java | 944 | 64 | -| PoliternalSite | TypeScript / TSX | 443 | 118 | +| PoliternalSite | TypeScript / TSX | 450 | 120 | +| PoliternalParkour | Java | 96 | 64 | +| TerraForge | Java | 325 | 17 | +| denfry.github.io | TypeScript / TSX | 48 | 39 | +| DevGraph | Python | 59 | 12 | +| Windows-Cleaner-and-Optimizer | PowerShell | 39 | 17 | Those repositories are not vendored here — shipping someone else's source to run a benchmark is not reproducible either. The generator is the reproducible part: point it at any git repository and the protocol is identical. +## Two query families, and why both are load-bearing + +The git-derived sets are large and objective, but they are not a substitute for +human phrasing, and 1.10.0 turned up a case where the difference decided a release. + +A commit subject is written by someone looking at the identifiers they just +changed, so it reuses the codebase's own spelling. A user asking a question does +not: they type "where are secrets redacted", not "redact". So the git sets are +structurally blind to morphology. Removing the synonym vocabulary looked free on 420 +git-derived queries (MRR −0.0022, p=0.40) and cost −0.060 MRR on the 36 hand-written +ones. The vocabulary stayed. + +The reverse also holds. Ground truth mined from commits counts a *test* file as the +answer whenever the commit touched tests (159 of 420 queries here), which the +hand-written set never does. Any signal that rewards test files therefore looks +better than it is. `run_eval.py` reports both families; a change needs to survive +the large set and not visibly break the small one. + +## Held-out validation + +Hand-tuned coefficients are fitted parameters, so tuning them on the same corpora +that report the result is how a benchmark gets gamed by accident. Ranking parameters +are selected under **leave-one-repository-out**: choose on seven corpora, score on +the eighth, pool the eight held-out scores. A parameter is only shipped from a +*plateau* — a region where neighbouring values are statistically indistinguishable — +never from a single peak, and per-corpus scores are inspected so that no fold is +allowed to regress in exchange for a better aggregate. + ## What is measured Ranking quality is scored at **file** granularity, because the agent's unit of @@ -88,6 +121,24 @@ best rank. - `dup%` — fraction of returned results that near-duplicate an earlier result - `p50/p95/p99` latency, in-process, excluding interpreter start-up +### Oracle metrics: is this a recall problem or a ranking problem? + +MRR alone cannot separate "the retrievers never found it" from "the ranker had it +and buried it", yet those two failures share no fix. Each variant therefore also +reports, against the *pre-rerank candidate pool* it actually generated: + +- `oracle` — the MRR a perfect reranker would score over that pool. `1 - oracle` is + the share of the query set that only better recall can ever fix. +- `cand_recall` — mean fraction of expected files present anywhere in the pool. +- `eff` = `MRR / oracle` — the share of achievable ranking quality delivered. + +At 1.9.0 this read `oracle = 0.902`, `MRR = 0.577`, `eff = 0.639`: a third of the +answers already in the pool were ranked below something else, an error mode about +three times larger than the 0.098 of remaining recall headroom. That is why 1.10.0 +worked on the reranker rather than adding retrievers, and it is also the cheapest +available check that a "ranking" change did not quietly change recall instead — +`-name_cooccurrence` moves eight quality metrics and leaves `oracle` at ±0.0000. + ## Files | File | Role | diff --git a/tests/eval/harness.py b/tests/eval/harness.py index 58b9e2c..fd61fe6 100644 --- a/tests/eval/harness.py +++ b/tests/eval/harness.py @@ -32,7 +32,7 @@ from codebase_index.retrieval.tuning import RetrievalTuning from codebase_index.storage.db import Database -from . import metrics +from . import gen_queries, metrics QUERY_DIR = Path(__file__).parent / "queries" DEFAULT_BUDGET = 1500 @@ -58,6 +58,12 @@ class QueryOutcome: duplicates: int = 0 """Results whose snippet near-duplicates an earlier result in the same page.""" n_results: int = 0 + pool_files: list[str] = field(default_factory=list) + """Distinct files in the pre-rerank candidate pool, in fused order. + + The oracle metrics are computed against this, so they measure exactly what the + ranker was handed — not what the retrievers could have found with other + settings.""" @dataclass @@ -84,6 +90,14 @@ class EvalReport: """Fraction of returned results that near-duplicate an earlier result.""" mean_candidates: float = 0.0 """Mean results returned per query, before the agent reads anything.""" + oracle_mrr: float = 0.0 + """Ceiling MRR a perfect reranker could reach over the pool actually generated. + + `1 - oracle_mrr` is the share of the query set no reranking can ever fix.""" + candidate_recall: float = 0.0 + """Mean fraction of expected files present anywhere in the candidate pool.""" + mean_pool: float = 0.0 + """Mean distinct files in the candidate pool, i.e. what the ranker chose from.""" per_category: dict[str, float] = field(default_factory=dict) per_query: dict[str, list[float]] = field(default_factory=dict) """Per-query metric vectors, in query order. Required for paired significance @@ -107,6 +121,10 @@ def as_row(self) -> dict[str, float | str | int]: "p50_ms": self.p50_ms, "p95_ms": self.p95_ms, "p99_ms": self.p99_ms, + "oracle": self.oracle_mrr, + "eff": metrics.rerank_efficiency(self.mrr, self.oracle_mrr), + "cand_recall": self.candidate_recall, + "pool": self.mean_pool, } @@ -145,7 +163,20 @@ def validate_queries(queries: Iterable[EvalQuery], root: Path) -> list[str]: # the corpus makes it the top lexical hit for almost every query — a measurement # artifact that depresses scores and hides real ranking behaviour. Benchmark # scaffolding is excluded from the corpus it grades. -CORPUS_EXCLUDES = ("tests/eval/**", "tests/benchmark_*", "tests/fixtures/expected_answers.yml") +# +# Changelog-like files are excluded for the mirror-image reason: a git-derived +# query *is* a commit subject, and a changelog entry paraphrases that subject +# verbatim while never being an accepted answer (`gen_queries._ANSWER_DENY_RE`). +# Indexing them plants an unbeatable distractor at rank 1 for a large share of +# queries, which compresses every variant's score toward the same floor and hides +# ranking differences. `gen_queries.CHANGELOG_EXCLUDES` documented this exclusion; +# it was never applied to the corpus. +CORPUS_EXCLUDES = ( + "tests/eval/**", + "tests/benchmark_*", + "tests/fixtures/expected_answers.yml", + *gen_queries.CHANGELOG_EXCLUDES, +) def build_corpus_index(root: Path, db_path: Path) -> Database: @@ -181,8 +212,18 @@ def run_query( token_budget=token_budget, no_fallback=True, tuning=tuning, + # The pre-rerank pool is what the oracle metrics are measured against. + # Building it costs ~30 dict literals per query, which is inside the noise + # of a 38ms query (measured: -0.8ms p50, i.e. unmeasurable), so it stays on + # for the timed call rather than forcing a second untimed pass. + explain=True, ) latency_ms = (time.perf_counter() - start) * 1000.0 + pool_files: list[str] = [] + for entry in payload.get("diagnostics", {}).get("pool", ()): + p = _normalise(entry["path"]) + if p not in pool_files: + pool_files.append(p) ranked_files: list[str] = [] returned: list[tuple[str, int]] = [] @@ -221,6 +262,7 @@ def run_query( total_tokens=total_tokens, duplicates=duplicates, n_results=len(results), + pool_files=pool_files, ) @@ -260,6 +302,15 @@ def evaluate( "useful@budget": lambda o: metrics.useful_context_at_budget( o.returned, o.query.expected_files, token_budget ), + # Scored per query so the oracle ceiling gets the same paired significance + # treatment as everything else: a ranking change that only moved the + # ceiling has not improved ranking. + "oracle": lambda o: metrics.oracle_reciprocal_rank( + o.pool_files, o.query.expected_files + ), + "cand_recall": lambda o: metrics.recall_at_k( + o.pool_files, o.query.expected_files, len(o.pool_files) + ), } per_query = {name: [fn(o) for o in outcomes] for name, fn in scorers.items()} @@ -294,6 +345,11 @@ def mean(name: str) -> float: ), duplicate_rate=(duplicate_total / returned_total) if returned_total else 0.0, mean_candidates=(returned_total / len(outcomes)) if outcomes else 0.0, + oracle_mrr=mean("oracle"), + candidate_recall=mean("cand_recall"), + mean_pool=( + statistics.fmean([len(o.pool_files) for o in outcomes]) if outcomes else 0.0 + ), per_category={ cat: statistics.fmean(vals) for cat, vals in sorted(per_category.items()) }, @@ -312,8 +368,8 @@ def format_table( ) -> str: """Render reports as a Markdown table, with deltas against `baseline`.""" cols = list(columns) if columns else [ - "label", "recall@5", "recall@10", "MRR", "nDCG@10", "hit@3", "P@5", - "MAP", "useful@budget", "tokens", "dup%", "p50_ms", "p95_ms", + "label", "recall@5", "recall@10", "MRR", "oracle", "eff", "nDCG@10", + "hit@3", "P@5", "MAP", "useful@budget", "tokens", "dup%", "p50_ms", "p95_ms", ] lines = ["| " + " | ".join(cols) + " |", "|" + "|".join("---" for _ in cols) + "|"] @@ -387,6 +443,9 @@ def weighted(attr: str) -> float: mean_tokens=weighted("mean_tokens"), duplicate_rate=weighted("duplicate_rate"), mean_candidates=weighted("mean_candidates"), + oracle_mrr=mean("oracle"), + candidate_recall=mean("cand_recall"), + mean_pool=weighted("mean_pool"), per_category={c: statistics.fmean(v) for c, v in sorted(per_category.items())}, per_query=per_query, ) diff --git a/tests/eval/metrics.py b/tests/eval/metrics.py index fac5df0..d88c70f 100644 --- a/tests/eval/metrics.py +++ b/tests/eval/metrics.py @@ -121,6 +121,38 @@ def useful_context_at_budget( return len(found) / len(rel) +# --- oracle / headroom ------------------------------------------------------ +# +# MRR alone cannot tell "the retrievers never found it" apart from "the ranker had +# it and buried it", yet those two failures have nothing in common: one is fixed by +# widening recall, the other by reranking. The pair below splits them, and the split +# is what justified spending 1.10.0 on the ranker rather than on new retrievers. + + +def oracle_reciprocal_rank(pool: Sequence[str], relevant: Iterable[str]) -> float: + """Reciprocal rank a *perfect* reranker would achieve over this candidate pool. + + A perfect reranker puts a relevant candidate first, so the answer is binary: + 1.0 when the pool contains any relevant item, 0.0 when retrieval never + surfaced one. Averaged over a query set this is the ceiling MRR can reach + without touching candidate generation, and `1 - oracle` is the share of the + query set that only better *recall* can ever fix. + """ + rel = set(relevant) + return 1.0 if any(item in rel for item in pool) else 0.0 + + +def rerank_efficiency(mrr: float, oracle_mrr: float) -> float: + """Fraction of the achievable ranking quality the ranker actually delivers. + + 1.0 means every answer the retrievers found is ranked first; 0.0 means none + are. This is the number 1.10.0 set out to move: at 1.9.0 it was 0.64, so a + third of the answers already in the pool were being ranked below something + else — an error mode roughly three times larger than the remaining recall gap. + """ + return mrr / oracle_mrr if oracle_mrr else 0.0 + + def percentile(values: Sequence[float], pct: float) -> float: """Nearest-rank percentile (no interpolation) — stable for small samples.""" if not values: diff --git a/tests/eval/run_eval.py b/tests/eval/run_eval.py index 3cad5fa..d022ef7 100644 --- a/tests/eval/run_eval.py +++ b/tests/eval/run_eval.py @@ -49,6 +49,7 @@ ABLATABLE = ( "soft_lexical", "query_expansion", + "name_cooccurrence", "fuzzy_symbols", "graph_source", "mmr", @@ -57,6 +58,11 @@ "file_agreement", ) +# Rows are looked up by label rather than position, so adding a variant cannot +# silently repoint the significance tests at the wrong column. +PREVIOUS_LABEL = "previous release (1.9.0)" +DEFAULT_LABEL = "default (all signals)" + def _parse_corpus(spec: str) -> tuple[Path, str]: """Split `:`, tolerating a Windows drive letter.""" @@ -112,7 +118,11 @@ def main(argv: list[str] | None = None) -> int: variants: list[tuple[str, RetrievalTuning]] = [ ("baseline (1.7.0)", RetrievalTuning.baseline()), - ("default (all signals)", RetrievalTuning()), + # The previous release, pinned. "Is this better than 1.7.0" and "is this + # better than what we shipped last" are different questions, and only the + # second one decides whether a change belongs in the next release. + (PREVIOUS_LABEL, RetrievalTuning.v190()), + (DEFAULT_LABEL, RetrievalTuning()), ] if args.ablate: default = RetrievalTuning() @@ -138,7 +148,7 @@ def main(argv: list[str] | None = None) -> int: repeats=args.repeats, ) collected[label].append(report) - if label == "default (all signals)": + if label == DEFAULT_LABEL: # Relabel so the per-corpus table identifies the repository # rather than repeating the variant name on every row. per_corpus_default[name] = replace(report, label=name) @@ -146,12 +156,16 @@ def main(argv: list[str] | None = None) -> int: db.close() reports = [harness.pool(collected[label], label=label) for label, _ in variants] + by_label = {rep.label: rep for rep in reports} + default = by_label[DEFAULT_LABEL] + previous = by_label[PREVIOUS_LABEL] + ablations = [rep for rep in reports if rep.label.startswith(" -")] if args.as_json: payload = { "pooled": [r.as_row() for r in reports], "per_corpus_default": {k: v.as_row() for k, v in per_corpus_default.items()}, - "per_category_default": reports[1].per_category, + "per_category_default": default.per_category, } print(json.dumps(payload, indent=2)) return 0 @@ -159,14 +173,15 @@ def main(argv: list[str] | None = None) -> int: print() print(harness.format_table(reports, baseline=reports[0])) - default = reports[1] + # The release decision is default vs the previous release, so that is the + # comparison printed first and in full. print() - print(harness.format_significance(reports[0], default, resamples=args.resamples)) + print(harness.format_significance(previous, default, resamples=args.resamples)) - if len(reports) > 2: + if ablations: print() print("Ablation significance (each row vs the shipped default):") - for rep in reports[2:]: + for rep in ablations: print() print(harness.format_significance(default, rep, resamples=args.resamples)) diff --git a/tests/test_ranking_features.py b/tests/test_ranking_features.py new file mode 100644 index 0000000..28c5011 --- /dev/null +++ b/tests/test_ranking_features.py @@ -0,0 +1,263 @@ +"""Contract tests for the 1.10.0 name-co-occurrence ranking signal. + +The feature is one short function, but it sits in the hot path of every query and +its whole justification is a measured interaction effect. These tests pin the +properties that made it worth shipping — superlinearity, name-zone scope, role +conditioning, determinism — plus the degenerate inputs that a query string can +actually contain. +""" + +from __future__ import annotations + +from codebase_index.retrieval.features import ( + name_cooccurrence, + name_zone, + query_profile, +) +from codebase_index.retrieval.rerank import rerank +from codebase_index.retrieval.tuning import DEFAULT_TUNING, RetrievalTuning +from codebase_index.retrieval.types import Candidate, Intent + + +def _cooc(query: str, path: str, symbol: str | None = None) -> float: + return name_cooccurrence(query_profile(query), name_zone(path, symbol))[1] + + +def _cand(path: str, **kw) -> Candidate: + base = dict( + path=path, line_start=1, line_end=10, source="symbol", score=1.0, kind="function" + ) + base.update(kw) + return Candidate(**base) # type: ignore[arg-type] + + +# --- the interaction itself ------------------------------------------------- + + +def test_query_terms_co_occurring_in_one_symbol_beat_a_single_term_match(): + """The measured failure this signal exists to fix. + + On "graph resolution + traversal accessors", 1.9.0 ranked `graph_candidates` + (one term) above `test_graph_accessors_resolve_and_walk` (three terms in one + name), because RRF sums per-term evidence and cannot see co-occurrence. + """ + query = "graph resolution traversal accessors" + one_term = _cooc(query, "src/graph/retrieval.py", "graph_candidates") + three_terms = _cooc(query, "src/storage/repo.py", "graph_accessors_resolve") + assert one_term == 0.0 + assert three_terms > one_term + + +def test_score_is_superlinear_not_additive(): + """Two terms in one name must beat one, and the first match earns nothing. + + A single matched term is already fully paid for by the retriever that + surfaced the candidate; crediting it again here would just re-weight lexical + matching, which is not what the benchmark showed was missing. + """ + query = "token budget applied results" + assert _cooc(query, "a/token.py") == 0.0 + assert 0.0 < _cooc(query, "a/token_budget.py") < _cooc(query, "a/token_budget_results.py") + assert _cooc(query, "a/token_budget_applied_results.py") == 1.0 + + +def test_single_term_queries_have_no_co_occurrence_to_observe(): + assert _cooc("budget", "src/retrieval/budget.py") == 0.0 + assert name_cooccurrence(query_profile("budget"), {"budget"}) == (0, 0.0) + + +def test_normalisation_makes_short_and_long_queries_comparable(): + """Both candidates match every salient term, so both must score 1.0.""" + assert _cooc("parse config", "a/parse_config.py") == 1.0 + assert _cooc("parse config values from disk", "a/parse_config_values_from_disk.py") == 1.0 + + +# --- name-zone scope -------------------------------------------------------- + + +def test_zone_is_basename_and_symbol_but_never_directories(): + """Directory components are shared by hundreds of files, so they carry no + evidence about which file answers the query — only co-occurrence noise.""" + zone = name_zone("src/main/java/net/denfry/newtowny/items/CustomItems.java", None) + assert zone == {"custom", "items"} + assert "denfry" not in zone and "java" not in zone and "src" not in zone + assert _cooc("newtowny items denfry", "src/net/denfry/newtowny/items/Foo.java") == 0.0 + + +def test_zone_splits_camel_snake_and_kebab_identifiers(): + assert name_zone("a/getUserById.ts", None) == {"get", "user", "by", "id"} + assert name_zone("a/refresh_access_token.py", None) == {"refresh", "access", "token"} + assert name_zone("a/my-http-server.js", None) == {"my", "http", "server"} + assert name_zone("a/x.py", "HTTPServerFactory") == {"x", "http", "server", "factory"} + + +def test_extension_is_not_part_of_the_zone(): + """Otherwise every query mentioning `py` or `ts` would fire on whole languages.""" + assert "py" not in name_zone("src/parse.py", None) + assert "java" not in name_zone("src/Parse.java", None) + assert _cooc("parse py files", "src/parse.py") == 0.0 + + +def test_dotted_and_multi_suffix_names_keep_their_stem(): + assert name_zone("web/app.module.ts", None) == {"app", "module"} + assert name_zone("a/schema.generated.ts", None) == {"schema", "generated"} + + +def test_windows_separators_are_normalised(): + assert name_zone(r"src\retrieval\token_budget.py", None) == {"token", "budget"} + + +# --- role conditioning ------------------------------------------------------ + + +def test_descriptive_test_names_get_a_discounted_bonus(): + """Test function names are sentences, so they harvest co-occurrences that real + identifiers never do. Discounted, not withheld: on git-derived ground truth the + changed file often *is* the test.""" + query = "secrets redacted before output" + impl = _cand("src/output/redact.py", symbol="redact_snippet") + test = _cand("tests/test_budget.py", symbol="test_compactor_output_is_redacted") + ranked = rerank([impl, test], query=query, intent=Intent.KEYWORD, tuning=DEFAULT_TUNING) + assert [c.path for c in ranked][0] == "src/output/redact.py" + + scale = DEFAULT_TUNING.name_cooccurrence_demoted_scale + assert 0.0 < scale < 1.0 + + +def test_an_explicit_test_query_is_not_discounted(): + query = "tests for redacted output" + graded = rerank( + [_cand("tests/test_redacted_output.py", symbol="test_redacted_output")], + query=query, + intent=Intent.KEYWORD, + tuning=DEFAULT_TUNING, + ) + assert "co-occur in name" in graded[0].reason + + +def test_generated_sources_are_discounted_too(): + query = "schema generated types" + plain = _cand("src/schema_generated_types.ts") + generated = _cand("src/schema_generated_types.ts", is_generated=True) + for c in (plain, generated): + rerank([c], query=query, intent=Intent.KEYWORD, tuning=DEFAULT_TUNING) + assert plain.score > generated.score + + +# --- ablatability and explainability --------------------------------------- + + +def test_signal_is_independently_ablatable(): + query = "token budget applied results" + on = _cand("src/token_budget.py", symbol="apply_token_budget") + off = _cand("src/token_budget.py", symbol="apply_token_budget") + rerank([on], query=query, intent=Intent.KEYWORD, tuning=DEFAULT_TUNING) + rerank([off], query=query, intent=Intent.KEYWORD, + tuning=DEFAULT_TUNING.without("name_cooccurrence")) + assert on.score > off.score + assert "co-occur in name" in on.reason + assert "co-occur in name" not in off.reason + + +def test_reason_reports_the_actual_counts(): + c = _cand("src/token_budget.py") + rerank([c], query="token budget applied results", intent=Intent.KEYWORD, + tuning=DEFAULT_TUNING) + assert "2/4 query terms co-occur in name" in c.reason + + +def test_weight_scales_the_bonus_monotonically(): + scores = [] + for weight in (0.0, 1.0, 2.0): + c = _cand("src/token_budget.py") + rerank( + [c], + query="token budget applied results", + intent=Intent.KEYWORD, + tuning=RetrievalTuning(name_cooccurrence_weight=weight), + ) + scores.append(c.score) + assert scores[0] < scores[1] < scores[2] + + +# --- determinism and degenerate input --------------------------------------- + + +def test_ranking_is_deterministic_and_stable_on_ties(): + query = "token budget applied results" + def run(): + cands = [ + _cand("src/token_budget.py", symbol="apply"), + _cand("src/budget_token.py", symbol="apply"), + _cand("src/other.py", symbol="apply"), + ] + return [c.path for c in rerank(cands, query=query, intent=Intent.KEYWORD, + tuning=DEFAULT_TUNING)] + first = run() + assert all(run() == first for _ in range(5)) + # The two equally-matching files tie on score, so input order decides and the + # sort must not reshuffle them. + assert first[:2] == ["src/token_budget.py", "src/budget_token.py"] + + +def test_duplicate_candidates_score_identically(): + query = "token budget applied" + a, b = _cand("src/token_budget.py"), _cand("src/token_budget.py") + rerank([a, b], query=query, intent=Intent.KEYWORD, tuning=DEFAULT_TUNING) + assert a.score == b.score + + +def test_empty_and_punctuation_only_queries_are_inert(): + for query in ("", " ", "???", "!!! ... ???", "- -- ---", "/////"): + assert query_profile(query).terms == () + assert _cooc(query, "src/token_budget.py") == 0.0 + + +def test_stopword_only_query_is_inert(): + assert _cooc("how does the", "src/how_does_the.py") == 0.0 + + +def test_punctuation_between_terms_does_not_break_matching(): + assert _cooc("token, budget; applied!", "src/token_budget.py") > 0.0 + + +def test_unicode_identifiers_and_queries_are_handled(): + assert name_zone("src/расчёт_налога.py", None) == {"расчёт", "налога"} + assert _cooc("расчёт налога ставка", "src/расчёт_налога.py") > 0.0 + # Mixed scripts must not raise or silently drop the ASCII half. + assert _cooc("расчёт tax rate", "src/расчёт_tax.py") > 0.0 + + +def test_pathological_long_query_stays_bounded_and_sane(): + query = " ".join(f"term{i}" for i in range(2000)) + profile = query_profile(query) + assert profile.n_terms == 2000 + # One matched term out of 2000 is not evidence of anything. + assert _cooc(query, "src/term7.py") == 0.0 + assert 0.0 < _cooc(query, "src/term7_term8_term9.py") < 0.01 + + +def test_pathological_long_identifier_stays_bounded(): + symbol = "_".join(f"part{i}" for i in range(2000)) + score = _cooc("part7 part8 part9", "src/x.py", symbol) + assert score == 1.0 + + +def test_missing_and_malformed_metadata_does_not_raise(): + query = "token budget applied" + for path in ("", ".", "/", "a/", "...", "no_extension"): + assert 0.0 <= _cooc(query, path, None) <= 1.0 + assert _cooc(query, "src/token_budget.py", None) > 0.0 + # A candidate with no symbol, no kind and no content must still rank. + c = Candidate(path="src/token_budget.py", line_start=0, line_end=0, + source="path", score=0.0) + ranked = rerank([c], query=query, intent=Intent.KEYWORD, tuning=DEFAULT_TUNING) + assert ranked[0].score > 0.0 + + +def test_score_is_always_in_the_unit_interval(): + queries = ("a b", "token budget applied results", "x" * 300, "расчёт налога") + paths = ("src/token_budget.py", "", "a/b/c/d.py", "src/" + "x" * 300 + ".py") + for query in queries: + for path in paths: + assert 0.0 <= _cooc(query, path, "some_symbol_name") <= 1.0 diff --git a/tests/test_tuning_contract.py b/tests/test_tuning_contract.py index b335997..ba891ec 100644 --- a/tests/test_tuning_contract.py +++ b/tests/test_tuning_contract.py @@ -45,10 +45,13 @@ def test_baseline_disables_every_post_170_signal(): "dedup", "source_priors", "file_agreement", + "name_cooccurrence", ): assert getattr(baseline, flag) is False, flag - # 1.7.0 took the page size verbatim; over-fetching is a later addition. + # 1.7.0 took the page size verbatim and allowed three hits per file; + # over-fetching and single-hit packing are later additions. assert baseline.candidate_pool_multiplier == 1 + assert baseline.max_per_file == 3 def test_shipped_defaults_are_the_measured_configuration(): @@ -62,6 +65,27 @@ def test_shipped_defaults_are_the_measured_configuration(): assert DEFAULT_TUNING.mmr is False +def test_v190_reproduces_the_previous_release(): + """The 1.10.0 comparison point must stay pinned, not track the default.""" + v190 = RetrievalTuning.v190() + assert v190.name_cooccurrence is False + assert v190.max_per_file == 3 + # Everything 1.9.0 shipped on stays on, so the "before" column is 1.9.0 and + # not an accidental partial ablation of it. + for flag in ("soft_lexical", "dedup", "source_priors", "file_agreement", "fuzzy_symbols"): + assert getattr(v190, flag) is True, flag + + +def test_name_cooccurrence_defaults_are_the_measured_configuration(): + assert DEFAULT_TUNING.name_cooccurrence is True + assert DEFAULT_TUNING.name_cooccurrence_weight == pytest.approx(1.8) + # Withholding the bonus entirely from tests over-corrects; granting it in full + # lets descriptive test function names outrank implementations. + assert 0.0 < DEFAULT_TUNING.name_cooccurrence_demoted_scale < 1.0 + # The page is packed with distinct files, one hit each. + assert DEFAULT_TUNING.max_per_file == 1 + + def test_tuning_is_immutable_and_hashable(): tuning = RetrievalTuning() with pytest.raises(Exception): From 1c357cb991dc60fa237282d1de2d0242f50783eb Mon Sep 17 00:00:00 2001 From: denfry Date: Thu, 10 Sep 2026 09:35:36 +0300 Subject: [PATCH 02/18] research: NUCLEUS necessity-driven context spike A measured research spike on whether an agent can be given a computed set of necessary repository knowledge rather than similar data. 420 git-derived queries across eight repositories, paired bootstrap and permutation tests, leave-one-repository-out fitting, and a no-lookahead co-change model. Retrieval plane refuted: anchor-and-expand lost MRR -0.0073 (p<0.001). Recovered gold per completion slot (0.0310) is below the cost of evicting the cheapest baseline slot (0.0347). Appending contract slices instead beat a page-matched baseline by +0.0056 useful@budget for +6.4% tokens, i.e. lower useful-per-token. The LORO weight fit did not generalise (held-out -0.0024). Memory plane validated: span-keyed evidence survives 0.795/0.521/0.309 of reuse after 1/10/20 commits, where a query-keyed cache would serve 20.5%, 47.9% and 69.1% stale entries. 42.0% of evidence tokens repeat across the workload. Both mechanisms have close prior art; the contribution is the measurements. Generated data and private corpora stay git-ignored. --- research/README.md | 125 ++++++++ research/__init__.py | 0 research/architecture.md | 185 +++++++++++ research/build_indexes.py | 68 +++++ research/data/.gitignore | 8 + research/diagnose_accretion.py | 141 +++++++++ research/diagnose_calibration.py | 59 ++++ research/diagnose_slots.py | 118 +++++++ research/diagnose_structure.py | 293 ++++++++++++++++++ research/experiment_memory.py | 162 ++++++++++ research/experiments.md | 313 +++++++++++++++++++ research/fit_gate.py | 68 +++++ research/fit_weights.py | 185 +++++++++++ research/hypotheses.md | 284 +++++++++++++++++ research/novelty.md | 98 ++++++ research/nucleus/__init__.py | 6 + research/nucleus/baselines.py | 313 +++++++++++++++++++ research/nucleus/evalrun.py | 382 +++++++++++++++++++++++ research/nucleus/memory.py | 161 ++++++++++ research/nucleus/relations.py | 355 +++++++++++++++++++++ research/nucleus/search.py | 508 +++++++++++++++++++++++++++++++ research/related-work.md | 250 +++++++++++++++ research/selection.md | 95 ++++++ research/test_nucleus.py | 129 ++++++++ 24 files changed, 4306 insertions(+) create mode 100644 research/README.md create mode 100644 research/__init__.py create mode 100644 research/architecture.md create mode 100644 research/build_indexes.py create mode 100644 research/data/.gitignore create mode 100644 research/diagnose_accretion.py create mode 100644 research/diagnose_calibration.py create mode 100644 research/diagnose_slots.py create mode 100644 research/diagnose_structure.py create mode 100644 research/experiment_memory.py create mode 100644 research/experiments.md create mode 100644 research/fit_gate.py create mode 100644 research/fit_weights.py create mode 100644 research/hypotheses.md create mode 100644 research/novelty.md create mode 100644 research/nucleus/__init__.py create mode 100644 research/nucleus/baselines.py create mode 100644 research/nucleus/evalrun.py create mode 100644 research/nucleus/memory.py create mode 100644 research/nucleus/relations.py create mode 100644 research/nucleus/search.py create mode 100644 research/related-work.md create mode 100644 research/selection.md create mode 100644 research/test_nucleus.py diff --git a/research/README.md b/research/README.md new file mode 100644 index 0000000..0c61b82 --- /dev/null +++ b/research/README.md @@ -0,0 +1,125 @@ +# NUCLEUS — a research spike on necessity-driven context for coding agents + +**The question.** Can an agent be given not "similar data" but a *computed minimal set +of knowledge necessary for the task at hand*? + +**The answer this work supports.** Partly, and not the part I expected. Computing the +required *set* by relation-based completion is measurable, real, and **not worth its +tokens** — it is dominated by simply returning more results. Computing what a +conclusion *depended on*, so that work can be reused soundly, is worth a great deal. + +Everything below is measured on 420 queries across eight real repositories in four +languages, with paired significance tests, leave-one-repository-out fitting, and a +no-lookahead guarantee enforced by the data structure rather than by discipline. + +--- + +## Headline results + +| Claim | Verdict | Evidence | +|---|---|---| +| Required-set members are related by computable relations | **True** | 71.8% of 900 anchor→target gold pairs, no lookahead | +| Structure and history are non-redundant | **True** | edges contribute 5.3% uniquely, co-change 9.8% | +| Anchor + completion beats a hybrid ranker | **False** | dominated by `hybrid13`; 5% *worse* useful-per-token | +| Relation weights can be fitted | **False** | LORO: in-sample +46%, held-out −0.0024, 1/8 folds improve | +| The completion score is calibrated | **True** | precision 0.073 → 0.353 from "always fire" to "top 10%" | +| Query-keyed ("semantic") caches are unsound for code | **True** | **47.9% stale after 10 commits**, 20.5% after one | +| Finer evidence granularity retains more valid reuse | **True, weaker than predicted** | 1.36× at h=10 (predicted 2–3×), rising to 1.45× at h=20 | +| Agent work is heavily shareable | **True** | **42.0%** of tokens never re-sent across a 420-task workload | + +## The one number that killed the retrieval plane + +``` +accretion recovers gold the baseline missed : 0.0310 per query +cost of evicting even the cheapest rank slot: 0.0347 per query +``` + +Every completion slot is net-negative. Removing the eviction (appending contract +slices instead) makes it positive but not efficient: **+0.9% answer for +6.4% tokens**, +i.e. 5.22e-4 useful-per-token against the baseline's 5.51e-4. + +## The one number that saved the memory plane + +``` +conclusions whose evidence had changed after 10 commits: 47.9% +what a query-keyed cache would do with them : serve all of them +what an evidence-keyed cache does with them : cannot address them at all +``` + +Soundness here is free — a property of the key construction, not a tuned threshold. + +--- + +## Documents + +| File | Stage | Contents | +|---|---|---| +| `related-work.md` | 1 | 15 approach families, the *fundamental* limit of each, and the single shared assumption underneath them | +| `hypotheses.md` | 2–3 | 13 hypotheses with formal models; the go/no-go diagnostic | +| `selection.md` | 4 | Scoring matrix, TOP-3, what was rejected and why | +| `architecture.md` | 5 | NUCLEUS: abstraction, data model, algorithms, pseudocode, Big-O | +| `experiments.md` | 6–7 | Full protocol, 8 experiments, ablations, and the attempt to destroy the result | +| `novelty.md` | 8 | Prior-art check. **Both mechanisms turned out to have close prior art**; what remains is empirical | + +## Code + +``` +research/ + nucleus/ + relations.py typed relation union; CoChangeModel with a structural no-lookahead guarantee + search.py nucleate -> accrete -> select; reproduces the shipped pipeline when accretion is off + baselines.py BM25, LSA dense (numpy-only randomized SVD), hybrid RAG, product hybrid, forced-PPR graph + memory.py evidence-keyed memo, span/file granularity, exact invalidation + evalrun.py benchmark runner; reuses tests/eval/metrics.py verbatim + build_indexes.py one persistent index per corpus + diagnose_structure.py Experiment 0: is the premise true at all + diagnose_accretion.py generation vs ranking vs displacement decomposition + diagnose_slots.py slot economics: gain vs eviction cost per rank + diagnose_calibration.py precision as a function of score + fit_weights.py LORO weight fit (result: does not generalise) + fit_gate.py LORO gate selection (result: tau=0.6, 7/8 folds, on a plateau) + experiment_memory.py H3/H4: survival, unsoundness, cross-task sharing + test_nucleus.py correctness, incl. equivalence-to-incumbent and no-lookahead +``` + +## Reproducing + +```bash +# 1. query sets from git history (leak-free: subjects live in metadata, not the corpus) +python tests/eval/gen_queries.py --repo --out research/data/.yml + +# 2. one index per corpus, shared by every system +PYTHONPATH=. python research/build_indexes.py --corpus ":" ... + +# 3. correctness first -- the equivalence test is what makes deltas attributable +PYTHONPATH=. python -m pytest research/test_nucleus.py --no-cov -q + +# 4. the experiments +PYTHONPATH=. python research/diagnose_structure.py --corpus "|||" ... +PYTHONPATH=. python -m research.nucleus.evalrun --systems bm25,dense,rag,graph,hybrid,hybrid13,nucleus13 +PYTHONPATH=. python -m research.nucleus.evalrun --systems hybrid13,nucleus13 --ablate --baseline-label nucleus13 +PYTHONPATH=. python research/diagnose_slots.py +PYTHONPATH=. python research/fit_weights.py --rebuild --c 1 +PYTHONPATH=. python research/experiment_memory.py +``` + +External corpora are not vendored — shipping someone else's source to run a benchmark +is not reproducible either. The generator is the reproducible part: point it at any +git repository and the protocol is identical. + +## Honest limits + +- **Ground truth is a proxy.** "Files the commit touched" ≠ "files the agent needed". + The right instrument is a `ddmin` oracle over an executable verifier (H6); it needs + an LLM agent loop unavailable here. Every number inherits this proxy — on both sides + of every comparison. +- **The dense baseline is LSA, not a neural code encoder.** No + `sentence-transformers`, GPU, or network in this environment. **No claim of the form + "beats embeddings" is made anywhere.** The load-bearing comparison is the paired one + against NUCLEUS's own anchor stage. +- **Whole-conclusion memo hit rate is unmeasured.** The 42% figure is atom-level + sharing. Exact-set keying is brittle and the "agents don't redo work" claim is not + fully tested. This is the largest hole. +- **Neither mechanism is novel.** See `novelty.md`. The contribution is measurement + and falsification, not invention. +- **Multi-agent routing (H11) is designed, not measured.** No claim is made for it. diff --git a/research/__init__.py b/research/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/research/architecture.md b/research/architecture.md new file mode 100644 index 0000000..434ec9f --- /dev/null +++ b/research/architecture.md @@ -0,0 +1,185 @@ +# Stage 5 — NUCLEUS: architecture, algorithms, complexity + +> **Status after Stage 6–7. Read this first.** +> The **retrieval plane (H1/H2) is refuted** on this benchmark: it is dominated by the +> trivial baseline of returning more results, and is *less* token-efficient. The +> **memory plane (H3/H4) is validated** and is the part worth keeping. The +> architecture is documented in full anyway, because a design that was measured and +> rejected is a result, and because the refutation only makes sense against the +> design it refutes. Numbers: `experiments.md`. + +**NUCLEUS** — *Necessity-driven Unified Closure over Lexical, Evolutionary and +Structural relations.* + +--- + +## 1. Core abstraction: the evidence atom + +Everything in the system is an operation on one object. + +``` +Atom = (id, kind, span, content_hash, obligations) + id stable identity: (path, line_start, line_end) + kind file | span | symbol | conclusion + content_hash hash of the bytes this atom currently holds + obligations set of identifiers the atom defines or references +``` + +Three properties fall out of this and are the whole reason for the choice: + +- **Content addressing gives deduplication for free.** Two agents that read the same + region name it identically, so a shared store holds it once. Measured: 1738 atom + reads across a 420-task workload collapse to 1098 distinct atoms — **42% of tokens + never re-sent**. +- **Content addressing gives *exact* invalidation.** A conclusion records the atoms it + consumed; it is valid iff every one of those hashes is unchanged. Not "probably + fresh", not "recent enough" — decidable. +- **Obligations give a coverage objective.** A set of atoms can be scored by what it + covers rather than by how similar its members are to a query. + +## 2. Data model + +``` +atoms(id, kind, path, line_start, line_end, content_hash, token_est) +obligations(atom_id, symbol) -- bipartite atom x obligation +relations(src, dst, type, weight) -- typed, undirected at file level + -- type in {edge, cochange, testlink, + -- stem, dir} +memo(key, task_class, result, evidence[], created_at) + -- key = H(sorted atom content hashes) +``` + +`relations` is the union of two very different sources, and the diagnostic in +`hypotheses.md` is what justifies carrying both: static edges connect 30.7% of gold +co-members, history connects 30.4%, and each contributes *uniquely* (5.3% / 9.8%). + +## 3. Indexing algorithm + +``` +INDEX(repo): + for each file f: # O(N) files + parse f -> symbols, spans # tree-sitter, O(|f|) + atoms += spans; obligations += symbols + resolve edges globally # O(E α(N)) with a suffix map + fold git history into co-change counts # O(C · k²), k = files/commit + build name/dir/stem inverted maps # O(N · t), t = tokens per name +``` + +`k` is bounded by `MAX_COMMIT_FILES = 40`, so the history fold is linear in commits, +not quadratic in repository size. This matters: it is what makes the evolutionary +relation affordable on a repository with a long history. + +## 4. Retrieval algorithm + +``` +NUCLEUS_SEARCH(q, B): # B = token budget + # -- NUCLEATE ------------------------------------------------------ + A <- incumbent_hybrid(q) # unchanged, deliberately + A0 <- distinct_paths(A[:anchor_head]) # anchors + + # -- ACCRETE ------------------------------------------------------- + contrib <- {} + for i, a in enumerate(A0): + alpha <- 1 / (1 + decay·i) + for c in neighbours(a): # relation-local, not a scan + for r in RELATIONS: + contrib[c][r] += alpha · rel_r(a, c) + for c in contrib: # hub damping + contrib[c] /= sqrt(1 + degree(c)) + score(c) <- Σ_r θ_r · contrib[c][r] # linear in θ by construction + C <- { c : score(c) ≥ τ } + + # -- SELECT (budgeted greedy coverage) ------------------------------ + covered <- obligations(A0) ∪ terms(q) + S <- {} + while |S| < max_completions and C ≠ ∅: + c* <- argmax_{c∈C} score(c) · sqrt(1 + |obligations(c) \ covered|) + S <- S ∪ {c*}; covered <- covered ∪ obligations(c*); C <- C \ {c*} + + # -- EMIT ----------------------------------------------------------- + render each c ∈ S as a CONTRACT SLICE (signatures, not bodies) + return A ++ S under budget B +``` + +Two design decisions are load-bearing and both came from measurement, not taste: + +**Score is linear in θ.** `contrib` is per-relation, so re-weighting never re-walks the +graph. That is what made leave-one-repository-out fitting affordable (`fit_weights.py`) +— and therefore what made it possible to *discover* that the fit does not generalise. + +**Completions are appended, never inserted.** The first implementation inserted at +rank 4 and lost significantly (MRR −0.0073, p<0.001). `diagnose_slots.py` explains why: +a completion has precision 0.073, while the rank-4 slot it displaces holds gold with +probability 0.096. Appending removes the eviction entirely, so a completion must only +justify its own tokens. + +## 5. Memory algorithm (the part that survived) + +``` +REMEMBER(task, evidence, result): + key <- H( sort{ (atom.id, atom.content_hash) for atom in evidence } ‖ H(task_class) ) + memo[key] <- (result, evidence) + +RECALL(task, evidence): + key <- H( ... ) # same construction + return memo.get(key) # a hit is sound by construction +``` + +There is no eviction policy, no TTL, no similarity threshold, and no staleness +heuristic. An entry whose evidence changed is not evicted — it simply becomes +unreachable, because the key that would address it can no longer be constructed. This +is the Bazel/Nix action-cache discipline applied to agent reasoning. + +**Granularity is the only tuning knob, and it is the one that matters.** Keying on +whole files is the obvious implementation and costs 26% of the achievable reuse at a +10-commit horizon (survival 0.383 vs 0.521 span-keyed). + +## 6. Complexity + +| Operation | Cost | Notes | +|---|---|---| +| Index construction | `O(Σ|f| + E·α(N) + C·k²)` | parse + global edge resolve + history fold; `k ≤ 40` | +| Insert / update one file | `O(|f| + deg(f) + k²)` | re-parse, re-resolve its edges, re-fold its commits | +| Accretion (per query) | `O(|A₀| · d̄ · \|R\|)` | `d̄` = mean relation degree; **no corpus scan** | +| Coverage selection | `O(m · max_completions · \|O\|)` | `m` = gated candidates; greedy, `(1−1/e)` | +| Retrieval total | anchor stage + ~1 ms | measured: p50 41.3 ms → 56.2 ms | +| Memo lookup | `O(\|E\|)` hash + `O(1)` | `\|E\|` = evidence atoms, measured mean 4.1 | +| Memo invalidation | `O(1)` amortised | no scan: stale keys are simply never formed | +| Memory usage | `O(N + E + P)` | `P` = distinct co-change pairs, bounded by `C·k²` | +| Multi-agent sync | `O(1)` per atom | content-addressed ⇒ no coordination, no consensus | + +The `O(1)` synchronisation is worth naming: because every atom and every memo entry is +named by a hash of its content, two agents never need to agree on anything. There is +no invalidation broadcast, no version vector, and no leader. It is the CRDT-like +property that content addressing buys. + +## 7. Fault tolerance and incremental update + +- **Partial index is safe.** Missing relations reduce accretion recall; they cannot + produce a wrong answer, because relations only *propose* candidates. +- **Corrupt/absent history degrades to structure-only.** `CoChangeModel` with no + commits yields zero co-change weight and the system still runs (asserted in + `test_nucleus.py`). +- **The memo cannot serve a stale entry** even after a crash mid-write: a partially + written entry has a key nothing will construct. +- **Incremental update** is per-file; the co-change fold is append-only, which is also + what enforces the no-lookahead property in evaluation. + +## 8. Agent communication and distributed operation (designed, not measured) + +Routing by obligation ownership (H11): a task's implicated obligation set determines +which shard owns its atoms, so assignment is graph partitioning of the atom×obligation +bipartite graph minimising cut. Two agents whose obligation sets intersect are +predicted to conflict *before* either writes. This is specified but **not measured** — +it needs a live multi-agent workload, and no claim is made for it here. + +## 9. What the architecture got right and wrong + +Right: content addressing as the single abstraction; separating anchor budget from +completion budget; keeping the score linear so it could be fitted and falsified; +contract slices instead of bodies. + +Wrong: the central bet. Conditional completion assumed the incumbent's tail slots were +weak. They are not — a tuned hybrid ranker's rank-6..10 results carry gold at +0.021–0.035, and the relation union's top candidate carries it at 0.073 only when it +fires at all, on 42% of queries. The margin is real but too thin to pay for its tokens. diff --git a/research/build_indexes.py b/research/build_indexes.py new file mode 100644 index 0000000..0dcd2c4 --- /dev/null +++ b/research/build_indexes.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Build one persistent index per corpus, shared by every experiment. + +Rebuilding per variant would make ablation deltas measure indexing variance +instead of retrieval, and re-indexing 2k-file Java repos for each of a dozen +configurations is the difference between a benchmark that runs and one that does +not. The exclude list is taken verbatim from the shipped eval harness so the +research runs grade themselves on exactly the corpus the product's own benchmark +uses. +""" + +from __future__ import annotations + +import argparse +import time +from pathlib import Path + +from codebase_index.config import Config +from codebase_index.indexer.pipeline import build_index +from codebase_index.storage.db import Database + +from tests.eval.harness import CORPUS_EXCLUDES + +INDEX_DIR = Path(__file__).parent / "data" / "index" + + +def build(name: str, root: Path, *, force: bool = False) -> tuple[int, float]: + INDEX_DIR.mkdir(parents=True, exist_ok=True) + db_path = INDEX_DIR / f"{name}.sqlite" + if db_path.exists() and not force: + db = Database(db_path).open() + n = db.conn.execute("SELECT COUNT(*) FROM files").fetchone()[0] + db.close() + return n, 0.0 + for suffix in ("", "-wal", "-shm"): + p = Path(str(db_path) + suffix) + if p.exists(): + p.unlink() + cfg = Config() + cfg.root = str(root) + cfg.embeddings.enabled = False + cfg.extra_ignore = [*cfg.extra_ignore, *CORPUS_EXCLUDES] + start = time.perf_counter() + db = Database(db_path).open() + build_index(cfg, db, root=root) + elapsed = time.perf_counter() - start + n = db.conn.execute("SELECT COUNT(*) FROM files").fetchone()[0] + sym = db.conn.execute("SELECT COUNT(*) FROM symbols").fetchone()[0] + edg = db.conn.execute("SELECT COUNT(*) FROM edges WHERE resolved = 1").fetchone()[0] + db.close() + print(f" {name}: {n} files, {sym} symbols, {edg} resolved edges, {elapsed:.1f}s") + return n, elapsed + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--corpus", action="append", required=True, help="name:repo_path") + ap.add_argument("--force", action="store_true") + args = ap.parse_args() + for spec in args.corpus: + name, root = spec.split(":", 1) + print(f"[{name}] indexing {root}") + build(name, Path(root).resolve(), force=args.force) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/research/data/.gitignore b/research/data/.gitignore new file mode 100644 index 0000000..a4ed2ec --- /dev/null +++ b/research/data/.gitignore @@ -0,0 +1,8 @@ +# Generated benchmark artifacts. Deliberately not committed: +# * the query YAMLs contain commit subjects mined from private sibling repositories; +# * the SQLite indexes are large binaries regenerable in seconds. +# `research/README.md` documents the two commands that rebuild everything here, which +# is the reproducible part -- vendoring someone else's source to run a benchmark is +# not reproducible either. +* +!.gitignore diff --git a/research/diagnose_accretion.py b/research/diagnose_accretion.py new file mode 100644 index 0000000..f542611 --- /dev/null +++ b/research/diagnose_accretion.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Why does accretion lose? Split the failure into generation vs ranking vs displacement. + +The pooled benchmark says NUCLEUS(default) is worse than the incumbent. Three very +different causes produce that same number, and they have opposite fixes: + + generation the gold file the baseline missed is never even proposed as a + completion candidate -> the relation union is too narrow + ranking it is proposed but scored below the completions actually inserted + -> the relation weights are wrong + displacement the completions inserted are fine, but they evict baseline results + that were themselves gold -> the slot policy is too aggressive + +This measures all three on the same sweep, with the same no-lookahead history. +""" + +from __future__ import annotations + +import argparse +import sqlite3 +import statistics +from collections import Counter +from pathlib import Path + +from codebase_index.retrieval.tuning import RetrievalTuning +from research.nucleus import baselines +from research.nucleus.evalrun import CORPORA, DATA, load_queries +from research.nucleus.relations import CoChangeModel, RelationGraph, load_static_edges +from research.nucleus.search import NucleusParams, ObligationIndex, accrete + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--corpus", action="append", default=None) + ap.add_argument("--anchor-head", type=int, default=3) + ap.add_argument("--min-score", type=float, default=0.0, + help="0 disables the gate so generation can be measured") + args = ap.parse_args() + + corpora = CORPORA if not args.corpus else [c for c in CORPORA if c[0] in set(args.corpus)] + + tot = Counter() + acc_ranks: list[int] = [] + prec_at = Counter() + n_slots_eval = 0 + + for name, repo in corpora: + qs, idx = DATA / f"{name}.yml", DATA / "index" / f"{name}.sqlite" + if not qs.exists() or not idx.exists(): + continue + queries = load_queries(qs, Path(repo)) + conn = sqlite3.connect(idx) + conn.row_factory = sqlite3.Row + files = [r[0].replace("\\", "/") for r in conn.execute("SELECT path FROM files")] + cochange = CoChangeModel.from_repo(Path(repo)) + graph = RelationGraph(files, static=load_static_edges(idx), cochange=cochange) + obl = ObligationIndex(conn) + params = NucleusParams(anchor_head=args.anchor_head, min_score=args.min_score) + + for q in queries: + cochange.advance_to(q.position) + cands, _pool = baselines.product_candidates( + conn, q.query, limit=10, tuning=RetrievalTuning()) + base_files: list[str] = [] + for c in cands[:10]: + p = c.path.replace("\\", "/") + if p not in base_files: + base_files.append(p) + gold = set(q.expected_files) + if not gold: + continue + tot["queries"] += 1 + + missed = gold - set(base_files) + tot["gold_total"] += len(gold) + tot["gold_missed_by_baseline"] += len(missed) + + head = base_files[: args.anchor_head] + anchors = list(dict.fromkeys(head)) + if not anchors: + continue + ranked = accrete(conn, anchors=anchors, graph=graph, obligations=obl, + params=params, exclude=set(head), query_terms=set()) + ranked_paths = [p for p, _s, _pa in ranked] + rank_of = {p: i for i, p in enumerate(ranked_paths)} + + for gfile in missed: + if gfile in rank_of: + tot["missed_generated"] += 1 + acc_ranks.append(rank_of[gfile]) + else: + tot["missed_not_generated"] += 1 + + # Precision of the slots accretion would actually claim. + for k in (1, 3, 5): + top = ranked_paths[:k] + if top: + prec_at[f"hits@{k}"] += sum(1 for p in top if p in gold) + prec_at[f"slots@{k}"] += len(top) + if ranked_paths: + n_slots_eval += 1 + + # Displacement: baseline results at ranks anchor_head..10 that are gold + # and would be pushed past rank 10 by 3 insertions. + tail = base_files[args.anchor_head:] + evicted = tail[max(0, len(tail) - 3):] + tot["evicted_gold"] += sum(1 for p in evicted if p in gold) + tot["evicted_total"] += len(evicted) + conn.close() + + print("\n=== accretion failure decomposition ===") + print(f"queries {tot['queries']}") + print(f"gold files (total) {tot['gold_total']}") + print(f"gold missed by baseline top-10 {tot['gold_missed_by_baseline']}" + f" ({tot['gold_missed_by_baseline']/max(1,tot['gold_total']):.1%} of gold)") + gen = tot["missed_generated"] + nogen = tot["missed_not_generated"] + print(f" ... proposed by accretion {gen}" + f" ({gen/max(1,gen+nogen):.1%}) <- generation recall") + print(f" ... never proposed {nogen} ({nogen/max(1,gen+nogen):.1%})") + if acc_ranks: + acc_ranks.sort() + print(f" rank of proposed gold within accretion list: " + f"median={statistics.median(acc_ranks):.0f} " + f"p25={acc_ranks[len(acc_ranks)//4]} p75={acc_ranks[3*len(acc_ranks)//4]}") + for k in (1, 3, 5, 10, 20): + print(f" within top-{k:<2}: {sum(1 for r in acc_ranks if r < k)}" + f"/{len(acc_ranks)} ({sum(1 for r in acc_ranks if r < k)/len(acc_ranks):.1%})") + print("\nprecision of the slots accretion claims (vs full gold set):") + for k in (1, 3, 5): + h, s = prec_at[f"hits@{k}"], prec_at[f"slots@{k}"] + print(f" P@{k} = {h}/{s} = {h/max(1,s):.3f}") + print(f"\ndisplacement: of the last 3 baseline slots, " + f"{tot['evicted_gold']}/{tot['evicted_total']} = " + f"{tot['evicted_gold']/max(1,tot['evicted_total']):.3f} are gold") + print(" ^ accretion must beat THIS precision to be worth a slot") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/research/diagnose_calibration.py b/research/diagnose_calibration.py new file mode 100644 index 0000000..bdc3b85 --- /dev/null +++ b/research/diagnose_calibration.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Is the accretion score calibrated? The last chance for a positive-EV design. + +Accretion loses on average (gain 0.031 < cost 0.035 per slot). Averages hide +selectivity: if a *high* accretion score reliably predicts gold, a gated accretion +that fires rarely could still beat the slot it spends, even though firing always does +not. This measures precision as a function of the score, using the cached +contributions (no retrieval re-run needed). + +Decision rule: accretion is worth keeping iff some score threshold yields +precision > 0.035 (the rank-10 gold density) on a non-trivial share of queries. +""" +from __future__ import annotations +import json, sys +from pathlib import Path + +CACHE = Path("research/data/contrib_cache.json") +RELATIONS = ["edge", "cochange", "testlink", "stem", "dir"] +COST = 0.0347 # measured P(gold) at baseline rank 10 -- the cheapest slot + +def main() -> int: + cache = json.loads(CACHE.read_text(encoding="utf-8")) + w = {r: 1.0 for r in RELATIONS} + rows = [] # (top1_score, is_gold, n_candidates) + for corpus, qs in cache.items(): + for row in qs: + cands, missed = row["cands"], set(row["missed"]) + if not cands: + continue + scored = sorted( + ((sum(w[k]*v for k, v in acc.items()), p) for p, acc in cands.items()), + key=lambda t: (-t[0], t[1])) + s, p = scored[0] + rows.append((s, 1 if p in missed else 0, len(cands))) + rows.sort(key=lambda t: -t[0]) + n = len(rows) + print(f"queries with >=1 accretion candidate: {n}") + print(f"median candidate-list size: {sorted(r[2] for r in rows)[n//2]}") + print(f"\n{'threshold':>10} {'fires':>7} {'fire%':>7} {'hits':>5} {'precision':>10} {'vs cost':>9}") + print("-" * 56) + for frac in (0.02, 0.05, 0.10, 0.20, 0.30, 0.50, 0.75, 1.00): + k = max(1, int(n * frac)) + top = rows[:k] + hits = sum(t[1] for t in top) + prec = hits / k + print(f"{top[-1][0]:>10.3f} {k:>7} {frac:>6.0%} {hits:>5} {prec:>10.4f} " + f"{prec - COST:>+9.4f}") + # Expected net gold per query at each threshold, versus spending nothing. + print(f"\nnet expected gold per query (gain - cost), all {n} queries as denominator:") + for frac in (0.02, 0.05, 0.10, 0.20, 0.50, 1.00): + k = max(1, int(n * frac)) + top = rows[:k] + hits = sum(t[1] for t in top) + net = (hits - COST * k) / n + print(f" fire on top {frac:>4.0%}: net = {net:+.5f} gold/query") + return 0 + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/research/diagnose_slots.py b/research/diagnose_slots.py new file mode 100644 index 0000000..122c702 --- /dev/null +++ b/research/diagnose_slots.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Slot economics: what does a completion slot cost, and is any slot cheap enough? + +Accretion buys recall by spending a rank slot. Whether that trade is ever positive is +an arithmetic question that can be answered without running the retriever end to end: + + gain(slot) = P(top accretion candidate is gold the baseline top-10 missed) + cost(slot) = P(the baseline result evicted from that slot was gold) + +If gain < cost at every slot, no amount of tuning saves the design and it should be +reported as refuted. If some slots are free -- because the baseline did not fill them +-- then accretion has a positive-expected-value niche and the design survives in a +narrower form than proposed. +""" + +from __future__ import annotations + +import argparse +import sqlite3 +from collections import Counter +from pathlib import Path + +from codebase_index.retrieval.tuning import RetrievalTuning +from research.nucleus import baselines +from research.nucleus.evalrun import CORPORA, DATA, load_queries +from research.nucleus.relations import CoChangeModel, RelationGraph, load_static_edges +from research.nucleus.search import NucleusParams, ObligationIndex, accrete + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--anchor-head", type=int, default=3) + args = ap.parse_args() + + rank_gold = Counter() # baseline rank -> # gold + rank_seen = Counter() # baseline rank -> # queries that filled it + nfiles_hist = Counter() + recovery_at = Counter() + n_queries = 0 + free_slot_queries = 0 + free_slot_recovery = 0 + + for name, repo in CORPORA: + qs, idx = DATA / f"{name}.yml", DATA / "index" / f"{name}.sqlite" + if not qs.exists() or not idx.exists(): + continue + queries = load_queries(qs, Path(repo)) + conn = sqlite3.connect(idx) + conn.row_factory = sqlite3.Row + files = [r[0].replace("\\", "/") for r in conn.execute("SELECT path FROM files")] + cochange = CoChangeModel.from_repo(Path(repo)) + graph = RelationGraph(files, static=load_static_edges(idx), cochange=cochange) + obl = ObligationIndex(conn) + params = NucleusParams(anchor_head=args.anchor_head, min_score=0.0) + + for q in queries: + cochange.advance_to(q.position) + cands, _ = baselines.product_candidates( + conn, q.query, limit=10, tuning=RetrievalTuning()) + base_files: list[str] = [] + for c in cands[:10]: + p = c.path.replace("\\", "/") + if p not in base_files: + base_files.append(p) + gold = set(q.expected_files) + if not gold: + continue + n_queries += 1 + nfiles_hist[len(base_files)] += 1 + for i, p in enumerate(base_files, start=1): + rank_seen[i] += 1 + if p in gold: + rank_gold[i] += 1 + + missed = gold - set(base_files) + head = base_files[: args.anchor_head] + anchors = list(dict.fromkeys(head)) + if not anchors: + continue + ranked = accrete(conn, anchors=anchors, graph=graph, obligations=obl, + params=params, exclude=set(head), query_terms=set()) + paths = [p for p, _s, _pa in ranked] + for c in (1, 2, 3): + recovery_at[c] += sum(1 for p in paths[:c] if p in missed) + if len(base_files) < 10: + free_slot_queries += 1 + free_slot_recovery += sum(1 for p in paths[:1] if p in missed) + conn.close() + + print(f"\n=== baseline gold density by rank (n={n_queries} queries) ===") + print(f"{'rank':>5} {'filled':>7} {'gold':>6} {'P(gold)':>9}") + for i in range(1, 11): + s, g = rank_seen[i], rank_gold[i] + print(f"{i:>5} {s:>7} {g:>6} {g/max(1,s):>9.4f}") + + print(f"\n=== how many distinct files the baseline actually returns ===") + for k in sorted(nfiles_hist): + print(f" {k:>2} files: {nfiles_hist[k]:>4} queries " + f"({nfiles_hist[k]/max(1,n_queries):>6.1%})") + under = sum(v for k, v in nfiles_hist.items() if k < 10) + print(f" fewer than 10: {under}/{n_queries} = {under/max(1,n_queries):.1%}") + + print("\n=== the trade ===") + for c in (1, 2, 3): + print(f" recovery@{c} (gold the baseline missed, per query): " + f"{recovery_at[c]/max(1,n_queries):.4f}") + last = rank_seen[10] + print(f" cost of evicting rank 10: P(gold) = {rank_gold[10]/max(1,last):.4f}") + print(f" cost of evicting rank 4: P(gold) = {rank_gold[4]/max(1,rank_seen[4]):.4f}") + print(f"\n queries with a FREE slot (<10 files returned): {free_slot_queries}" + f" ({free_slot_queries/max(1,n_queries):.1%})") + print(f" gold recovered into free slots by top-1 accretion: {free_slot_recovery}" + f" ({free_slot_recovery/max(1,free_slot_queries):.4f} per such query)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/research/diagnose_structure.py b/research/diagnose_structure.py new file mode 100644 index 0000000..4118f76 --- /dev/null +++ b/research/diagnose_structure.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +"""Go/no-go diagnostic for necessity-based retrieval. + +The premise of "compute the required set" rather than "rank similar documents" is +that the members of a required set are *related to each other* by something +computable. If a commit's files are mutually unrelated by every cheap relation, +then no completion mechanism can ever recover the members that lexical retrieval +misses, and the whole research direction is dead. + +This script measures that premise before anything is built. For every +multi-file ground-truth answer it asks: given one gold file (the "anchor"), is +each *other* gold file reachable by + + * dir - same directory + * stem - shared identifier tokens in the file name + * testlink - one is the test/spec of the other (or vice versa) + * edge - a resolved import/call/reference edge links the two files in + the code graph (direction ignored) + * cochange - the two files changed together in history STRICTLY BEFORE the + query's own commit (no lookahead, so the number is what a + deployed system could actually have known) + +Reported as *reachability*: the fraction of non-anchor gold files that at least +one relation connects to the anchor set. That is the ceiling on what any +completion stage can add. +""" + +from __future__ import annotations + +import argparse +import re +import sqlite3 +import subprocess +import sys +from collections import Counter, defaultdict +from pathlib import Path + +import yaml + +_WORD = re.compile(r"[A-Za-z][A-Za-z0-9]*") +_TEST_MARK = re.compile(r"(?:^|[._-])(?:test|tests|spec|specs)(?:$|[._-])", re.I) + + +def git(repo: Path, *args: str) -> str: + p = subprocess.run( + ["git", "-C", str(repo), *args], + capture_output=True, text=True, encoding="utf-8", errors="replace", + ) + if p.returncode != 0: + raise RuntimeError(f"git {' '.join(args)}: {p.stderr.strip()}") + return p.stdout + + +def commit_order(repo: Path) -> dict[str, int]: + """sha12 -> position, 0 = newest. Larger position == older.""" + shas = [s.strip() for s in git(repo, "log", "--format=%H").splitlines() if s.strip()] + return {s[:12]: i for i, s in enumerate(shas)} + + +def commit_filesets(repo: Path) -> list[tuple[str, list[str]]]: + """[(sha12, [paths])] newest first, merges excluded (matching gen_queries).""" + raw = git(repo, "log", "--no-merges", "--name-only", + "--pretty=format:%x01%H", "--diff-filter=ACMR") + out: list[tuple[str, list[str]]] = [] + for rec in raw.split("\x01"): + rec = rec.strip("\n") + if not rec: + continue + head, _, body = rec.partition("\n") + files = [ln.strip() for ln in body.splitlines() if ln.strip()] + if files: + out.append((head.strip()[:12], files)) + return out + + +def stem_tokens(path: str) -> set[str]: + base = path.rsplit("/", 1)[-1] + base = base.split(".", 1)[0] + # camelCase / snake_case / kebab-case -> lowercase tokens + parts = re.sub(r"([a-z0-9])([A-Z])", r"\1 \2", base).replace("_", " ").replace("-", " ") + return {w.lower() for w in _WORD.findall(parts) if len(w) > 2} + + +def core_stem(path: str) -> str: + """File stem with test/spec markers stripped, for test<->impl pairing.""" + base = path.rsplit("/", 1)[-1].split(".", 1)[0] + base = re.sub(r"^(?:test|spec)[._-]?", "", base, flags=re.I) + base = re.sub(r"[._-]?(?:test|tests|spec|specs)$", "", base, flags=re.I) + return base.lower() + + +def is_testish(path: str) -> bool: + return bool(_TEST_MARK.search(path)) or "/test" in path.lower() + + +def static_edges(index_path: Path) -> set[frozenset[str]]: + """Undirected file-level adjacency from resolved import/call/reference edges. + + Symbol targets are lifted to their defining file: the unit of the ground truth + is a file, so a call edge into a symbol is evidence about that symbol's file. + Self-loops are dropped (a file importing itself carries no completion signal). + """ + if not index_path or not index_path.exists(): + return set() + conn = sqlite3.connect(index_path) + try: + rows = conn.execute( + """ + SELECT src.path AS a, + CASE WHEN e.dst_kind = 'file' THEN df.path ELSE sf.path END AS b + FROM edges AS e + JOIN files AS src ON src.id = e.file_id + LEFT JOIN files AS df ON e.dst_kind = 'file' AND df.id = e.dst_id + LEFT JOIN symbols AS s ON e.dst_kind = 'symbol' AND s.id = e.dst_id + LEFT JOIN files AS sf ON sf.id = s.file_id + WHERE e.resolved = 1 + """ + ).fetchall() + except sqlite3.Error: + return set() + finally: + conn.close() + out: set[frozenset[str]] = set() + for a, b in rows: + if a and b and a != b: + out.add(frozenset((a, b))) + return out + + +def related(a: str, b: str, cochange: set[frozenset[str]], + edges: set[frozenset[str]]) -> set[str]: + rels: set[str] = set() + da, db = a.rsplit("/", 1)[0] if "/" in a else "", b.rsplit("/", 1)[0] if "/" in b else "" + if da == db: + rels.add("dir") + ta, tb = stem_tokens(a), stem_tokens(b) + if ta & tb: + rels.add("stem") + if (is_testish(a) != is_testish(b)) and core_stem(a) == core_stem(b) and core_stem(a): + rels.add("testlink") + if frozenset((a, b)) in edges: + rels.add("edge") + if frozenset((a, b)) in cochange: + rels.add("cochange") + return rels + + +def analyse(repo: Path, queries_path: Path, name: str, + index_path: Path | None = None) -> dict: + edges = static_edges(index_path) if index_path else set() + records = yaml.safe_load(queries_path.read_text(encoding="utf-8")) or [] + order = commit_order(repo) + history = commit_filesets(repo) + + # Pre-index history by position so the temporal filter is a slice, not a scan. + hist_by_pos: list[tuple[int, list[str]]] = [] + for sha, files in history: + pos = order.get(sha) + if pos is not None: + hist_by_pos.append((pos, files)) + + sizes = Counter() + rel_hits: Counter[str] = Counter() + n_targets = 0 + n_reachable = 0 + n_multi = 0 + cochange_only = 0 + edge_only = 0 + unreachable_examples: list[tuple[str, str, str]] = [] + hist_depth: list[int] = [] + + for rec in records: + gold = list(dict.fromkeys(rec.get("expected_files", []))) + sizes[len(gold)] += 1 + if len(gold) < 2: + continue + n_multi += 1 + qpos = order.get(rec.get("commit", ""), None) + if qpos is None: + continue + # Co-change pairs observable strictly before this commit (pos > qpos). + prior = [f for pos, f in hist_by_pos if pos > qpos] + hist_depth.append(len(prior)) + pairs: set[frozenset[str]] = set() + for files in prior: + if len(files) > 40: # sweeping commits couple everything; ignore + continue + for i, x in enumerate(files): + for y in files[i + 1:]: + pairs.add(frozenset((x, y))) + + # Every gold file gets a turn as the anchor; the rest are targets. + for anchor in gold: + for target in gold: + if target == anchor: + continue + n_targets += 1 + rels = related(anchor, target, pairs, edges) + if rels: + n_reachable += 1 + for r in rels: + rel_hits[r] += 1 + if rels == {"cochange"}: + cochange_only += 1 + if rels == {"edge"}: + edge_only += 1 + elif len(unreachable_examples) < 5: + unreachable_examples.append((rec["query"][:60], anchor, target)) + + return { + "corpus": name, + "queries": len(records), + "multi_file_queries": n_multi, + "size_hist": dict(sorted(sizes.items())), + "anchor_target_pairs": n_targets, + "reachable": n_reachable, + "reachability": (n_reachable / n_targets) if n_targets else 0.0, + "by_relation": {k: v / n_targets for k, v in rel_hits.items()} if n_targets else {}, + "cochange_only_share": (cochange_only / n_targets) if n_targets else 0.0, + "edge_only_share": (edge_only / n_targets) if n_targets else 0.0, + "n_static_edges": len(edges), + "median_prior_commits": ( + sorted(hist_depth)[len(hist_depth) // 2] if hist_depth else 0 + ), + "unreachable_examples": unreachable_examples, + } + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--corpus", action="append", required=True, + help="name|repo_path|queries.yml[|index.sqlite]") + args = ap.parse_args() + + rows = [] + for spec in args.corpus: + parts = spec.split("|") + name, repo, qs = parts[0], parts[1], parts[2] + idx = Path(parts[3]) if len(parts) > 3 else None + try: + rows.append(analyse(Path(repo), Path(qs), name, idx)) + except Exception as exc: # a corpus that cannot be read must not hide the rest + print(f"[{name}] FAILED: {type(exc).__name__}: {exc}", file=sys.stderr) + + print(f"\n{'corpus':22} {'q':>4} {'multi':>6} {'pairs':>6} {'reach':>7} " + f"{'dir':>6} {'stem':>6} {'test':>6} {'coch':>6} {'coch-only':>9} {'hist':>5}") + print("-" * 100) + tot_pairs = tot_reach = 0 + agg_rel: Counter[str] = Counter() + agg_conly = 0 + agg_eonly = 0 + for r in rows: + b = r["by_relation"] + print(f"{r['corpus']:22} {r['queries']:>4} {r['multi_file_queries']:>6} " + f"{r['anchor_target_pairs']:>6} {r['reachability']:>7.3f} " + f"{b.get('dir',0):>6.3f} {b.get('stem',0):>6.3f} {b.get('testlink',0):>6.3f} " + f"{b.get('edge',0):>6.3f} {b.get('cochange',0):>6.3f} " + f"{r['edge_only_share']:>7.3f} {r['cochange_only_share']:>7.3f} " + f"{r['median_prior_commits']:>5}") + tot_pairs += r["anchor_target_pairs"] + tot_reach += r["reachable"] + for k, v in b.items(): + agg_rel[k] += v * r["anchor_target_pairs"] + agg_conly += r["cochange_only_share"] * r["anchor_target_pairs"] + agg_eonly += r["edge_only_share"] * r["anchor_target_pairs"] + if tot_pairs: + print("-" * 112) + print(f"{'POOLED':22} {'':>4} {'':>6} {tot_pairs:>6} {tot_reach/tot_pairs:>7.3f} " + f"{agg_rel['dir']/tot_pairs:>6.3f} {agg_rel['stem']/tot_pairs:>6.3f} " + f"{agg_rel['testlink']/tot_pairs:>6.3f} {agg_rel['edge']/tot_pairs:>6.3f} " + f"{agg_rel['cochange']/tot_pairs:>6.3f} " + f"{agg_eonly/tot_pairs:>7.3f} {agg_conly/tot_pairs:>7.3f}") + + print("\ngold-set size distribution (pooled):") + total_sizes: Counter = Counter() + for r in rows: + for k, v in r["size_hist"].items(): + total_sizes[k] += v + n_all = sum(total_sizes.values()) + for k in sorted(total_sizes): + print(f" |gold|={k}: {total_sizes[k]:>4} ({total_sizes[k]/n_all:.1%})") + multi = sum(v for k, v in total_sizes.items() if k >= 2) + print(f" multi-file share: {multi}/{n_all} = {multi/n_all:.1%}") + + print("\nunreachable examples (query | anchor | target):") + for r in rows[:3]: + for q, a, t in r["unreachable_examples"][:3]: + print(f" [{r['corpus']}] {q!r}\n {a}\n -> {t}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/research/experiment_memory.py b/research/experiment_memory.py new file mode 100644 index 0000000..24c94ba --- /dev/null +++ b/research/experiment_memory.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""H3/H4: soundness and survival of evidence-keyed agent memory. + +Two questions, both answered against real history with no model in the loop: + + 1. How often would a query-keyed ("semantic") cache serve a stale answer? That is + exactly `1 - survival`, because such a cache reuses on question identity and + therefore reuses every entry whose evidence has since moved. + + 2. Does evidence granularity change survival multiplicatively (H4)? File-keyed and + span-keyed memories are both perfectly sound; they differ only in how much valid + reuse they retain, and the gap is the cost of the lazy implementation choice. + +Conclusions are not synthetic: each benchmark query is run through the shipped +retriever, and the spans it actually returns are recorded as that conclusion's +evidence. This is what an agent would really have read. +""" + +from __future__ import annotations + +import argparse +import sqlite3 +import statistics +from pathlib import Path + +from codebase_index.retrieval.tuning import RetrievalTuning +from research.nucleus import baselines +from research.nucleus.evalrun import BUDGET, CORPORA, DATA, load_queries +from research.nucleus.memory import Conclusion, Span, changed_regions, survives + +HORIZONS = (1, 2, 5, 10, 20, 50) + + +def collect(name: str, repo: Path, top_k: int) -> list[Conclusion]: + qs, idx = DATA / f"{name}.yml", DATA / "index" / f"{name}.sqlite" + if not qs.exists() or not idx.exists(): + return [] + queries = load_queries(qs, repo) + conn = sqlite3.connect(idx) + conn.row_factory = sqlite3.Row + out: list[Conclusion] = [] + for q in queries: + cands, _ = baselines.product_candidates( + conn, q.query, limit=top_k, tuning=RetrievalTuning()) + payload = baselines.finalize(cands, query=q.query, token_budget=BUDGET, + limit=top_k) + spans, tokens = [], 0 + for r in payload["results"]: + if not r.get("snippet"): + continue # not placed in context, so not evidence + spans.append(Span(r["path"].replace("\\", "/"), int(r["line_start"]), + int(r["line_end"]), int(r.get("token_est") or 0))) + tokens += int(r.get("token_est") or 0) + if spans: + out.append(Conclusion(task=q.query, spans=tuple(spans), tokens=tokens)) + conn.close() + return out + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--top-k", type=int, default=5, + help="results an agent is assumed to actually read") + ap.add_argument("--corpus", action="append", default=None) + args = ap.parse_args() + + corpora = CORPORA if not args.corpus else [c for c in CORPORA if c[0] in set(args.corpus)] + + rows: dict[str, dict[int, tuple[float, float, int]]] = {} + pooled: dict[int, list[tuple[bool, bool]]] = {h: [] for h in HORIZONS} + span_counts: list[int] = [] + file_counts: list[int] = [] + + for name, repo in corpora: + repo_p = Path(repo) + cons = collect(name, repo_p, args.top_k) + if not cons: + continue + span_counts += [len(c.spans) for c in cons] + file_counts += [len(c.files) for c in cons] + rows[name] = {} + for h in HORIZONS: + changed, hunks = changed_regions(repo_p, h) + if not changed and h > 1: + # Shallow history: no commit at that depth. Skip rather than report + # a spurious 100% survival. + continue + sf = [survives(c, changed, hunks, granularity="file") for c in cons] + ss = [survives(c, changed, hunks, granularity="span") for c in cons] + rows[name][h] = (statistics.fmean(sf), statistics.fmean(ss), len(cons)) + pooled[h] += list(zip(sf, ss)) + + print(f"\n=== evidence survival (top-{args.top_k} results treated as evidence) ===") + print("survival = fraction of conclusions whose evidence was NOT disturbed") + print("unsound = what a query-keyed semantic cache would serve stale = 1 - survival\n") + print(f"{'corpus':22} {'h':>4} {'n':>5} {'file-keyed':>11} {'span-keyed':>11} " + f"{'span/file':>10} {'semantic unsound':>17}") + print("-" * 90) + for name, hs in rows.items(): + for h, (sf, ss, n) in sorted(hs.items()): + ratio = (ss / sf) if sf > 0 else float("nan") + print(f"{name:22} {h:>4} {n:>5} {sf:>11.3f} {ss:>11.3f} {ratio:>10.2f} " + f"{1-ss:>17.3f}") + print("-" * 90) + print(f"{'POOLED':22} {'h':>4} {'n':>5} {'file-keyed':>11} {'span-keyed':>11} " + f"{'span/file':>10} {'semantic unsound':>17}") + for h in HORIZONS: + pairs = pooled[h] + if not pairs: + continue + sf = statistics.fmean([1.0 if a else 0.0 for a, _ in pairs]) + ss = statistics.fmean([1.0 if b else 0.0 for _, b in pairs]) + ratio = (ss / sf) if sf > 0 else float("nan") + print(f"{'':22} {h:>4} {len(pairs):>5} {sf:>11.3f} {ss:>11.3f} {ratio:>10.2f} " + f"{1-ss:>17.3f}") + + if span_counts: + print(f"\nevidence footprint: {statistics.fmean(span_counts):.1f} spans across " + f"{statistics.fmean(file_counts):.1f} files per conclusion") + + # --- multi-agent sharing -------------------------------------------------- + # Content addressing gives dedup for free: two agents that read the same span + # name it with the same key. This measures how much of a realistic workload is + # actually shared, i.e. the ceiling on what a shared memory can save. + print("\n=== evidence sharing across the task workload ===") + print(f"{'corpus':22} {'tasks':>6} {'atoms':>7} {'distinct':>9} {'shared%':>8} " + f"{'tok gross':>10} {'tok dedup':>10} {'saving':>7}") + print("-" * 88) + tot = [0, 0, 0, 0, 0] + for name, repo in corpora: + cons = collect(name, Path(repo), args.top_k) + if not cons: + continue + seen: dict[tuple[str, int, int], int] = {} + tok: dict[tuple[str, int, int], int] = {} + gross = 0 + for c in cons: + for sp in c.spans: + k = (sp.path, sp.start, sp.end) + seen[k] = seen.get(k, 0) + 1 + tok[k] = sp.tokens + gross += sp.tokens + atoms = sum(seen.values()) + distinct = len(seen) + shared = sum(1 for v in seen.values() if v > 1) + dedup = sum(tok.values()) + print(f"{name:22} {len(cons):>6} {atoms:>7} {distinct:>9} " + f"{shared/max(1,distinct):>7.1%} {gross:>10} {dedup:>10} " + f"{1-dedup/max(1,gross):>6.1%}") + tot[0] += len(cons); tot[1] += atoms; tot[2] += distinct + tot[3] += gross; tot[4] += dedup + print("-" * 88) + print(f"{'POOLED':22} {tot[0]:>6} {tot[1]:>7} {tot[2]:>9} {'':>8} " + f"{tot[3]:>10} {tot[4]:>10} {1-tot[4]/max(1,tot[3]):>6.1%}") + print("\n 'saving' = tokens a content-addressed shared memory never re-sends,") + print(" because a second agent asking a different question reads an atom the") + print(" first agent already materialised. Dedup, not compression.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/research/experiments.md b/research/experiments.md new file mode 100644 index 0000000..eff5e28 --- /dev/null +++ b/research/experiments.md @@ -0,0 +1,313 @@ +# Stage 6–7 — Experiments, ablations, and the refutation + +## Protocol + +**Corpora.** Eight real repositories, four languages (Python ×2, Java ×3, +TypeScript/TSX ×2, PowerShell ×1), 2 264 indexed files, 420 queries. Identical to the +set `docs/BENCHMARKS.md` reports 1.10.0 against, so the numbers here sit next to the +product's own published numbers. + +**Ground truth.** Commit subject → files that commit changed, mined by the shipped +`tests/eval/gen_queries.py`. Leak-free: the query text lives in git metadata, not in +the indexed corpus. Changelog-like files are excluded as both documents and answers. + +**Metrics.** `tests/eval/metrics.py` verbatim — never reimplemented. Every non-baseline +row carries a paired bootstrap 95% CI and a paired permutation p-value, both seeded. + +**Indexing.** One index per corpus, built once, shared by every system, so deltas +measure retrieval and never indexing variance. + +**Temporal discipline.** The co-change relation could trivially leak, since ground +truth is mined from commits. `CoChangeModel` is append-only and queries are swept +oldest-first, so a query's own commit and every later commit are structurally +unreachable — `test_cochange_model_refuses_lookahead` asserts that rewinding raises. + +**Attribution.** `NucleusParams(accrete=False)` reproduces +`codebase_index.retrieval.pipeline.search` result-for-result +(`test_accretion_off_reproduces_shipped_pipeline`, 5 queries × exact snippet match). +Without that equivalence every "NUCLEUS vs incumbent" delta would be confounded with +an incidental reimplementation. + +--- + +## Experiment 0 — Is the premise even true? + +Before building anything: are the members of a required set related to each other by +anything computable? Over 900 anchor→target gold pairs, no lookahead: + +| relation | covers | uniquely | +|---|---|---| +| stem (shared name tokens) | 35.8% | — | +| static edge (import/call/ref) | 30.7% | 5.3% | +| co-change (strictly prior history) | 30.4% | 9.8% | +| same directory | 23.8% | — | +| test ↔ impl | 14.4% | — | +| **union** | **71.8%** | | + +The premise holds, and structure and history are **not** redundant. This is what +justified proceeding, and it is also what makes the eventual negative result +interesting rather than trivial: the signal is genuinely there. + +Gold-set sizes: 53.1% single-file, 46.9% multi-file (2–4). Completion can only ever +help the 46.9%. + +--- + +## Experiment 1 — First implementation: a significant loss + +Default configuration (3 completions inserted after a 3-result anchor head): + +| vs `hybrid` | Δ | 95% CI | p | +|---|---|---|---| +| MRR | **−0.0073** | [−0.0108, −0.0038] | **<0.001** | +| P@5 | **−0.0102** | [−0.0176, −0.0029] | **0.009** | +| recall@10 | −0.0105 | [−0.0312, +0.0103] | 0.330 | +| tokens | **+108** | — | — | + +A clear loss. The interesting part is *why*, and three very different causes produce +the same number. + +## Experiment 2 — Decomposing the failure + +`diagnose_accretion.py`, 420 queries: + +``` +gold missed by baseline top-10 266 (36.6% of all gold) + ... proposed by accretion 142 (53.4%) <- generation works + ... never proposed 124 (46.6%) +rank of proposed gold within the accretion list: median 17, p25 5, p75 41 + within top-1 9.2% top-3 16.2% top-5 23.9% top-10 36.6% +``` + +So generation is not the bottleneck — **ranking inside the accretion list is**. Then +the economics, from `diagnose_slots.py`: + +``` +baseline gold density by rank: r1 0.467 r2 0.233 r3 0.122 r4 0.096 + r5 0.066 r6 0.021 r7 0.027 r8 0.025 + r9 0.034 r10 0.035 + +accretion recovers (gold the baseline missed): @1 0.0310 @2 0.0476 @3 0.0548 +cost of evicting the cheapest slot (rank 10): 0.0347 +``` + +**Every completion slot is net-negative**: 0.0310 gained < 0.0347 lost, and the second +and third slots are worse (+0.0166 and +0.0072 against 0.0339 and 0.0247). Inserting at +rank 4 was worse still — it displaced a slot holding gold at 0.096, which is why MRR +and P@5 fell while recall barely moved. + +The "free slot" escape is empty: only 17.6% of queries return fewer than 10 files, and +accretion placed exactly **1** gold file into all of them combined. + +## Experiment 3 — Fitting the weights: overfits, reported as such + +The weights had never been fitted (all 1.0). Leave-one-repository-out, +objective `recovery@1`: + +| | uniform | fitted | Δ | +|---|---|---|---| +| in-sample | 0.0310 | 0.0452 | +46% | +| **held-out (pooled)** | **0.0310** | **0.0286** | **−0.0024** | + +1 of 8 folds improved, 2 regressed, and the coefficients swing wildly across folds +(`cochange` 0.91–4.21, `edge` 0.00–0.28). **The fit does not generalise**, so uniform +weights are used everywhere below. A 46% in-sample gain that reverses out of sample is +exactly the artifact leave-one-out exists to catch. + +## Experiment 4 — Calibration: the signal is real and selective + +Accretion proposes nothing on 58% of queries. Conditioned on firing (n=177): + +| fire on top | threshold | precision | vs slot cost 0.035 | +|---|---|---|---| +| 2% | 2.095 | 0.667 | **19×** | +| 10% | 1.547 | 0.353 | **10×** | +| 20% | 1.321 | 0.200 | 5.8× | +| 50% | 0.826 | 0.125 | 3.6× | +| 100% | 0.124 | 0.073 | 2.1× | + +The score is **well calibrated** — precision rises monotonically and steeply. Gate +selected under LORO: τ = 0.600 in 7 of 8 folds, on a plateau (net gold/query flat at +0.021–0.024 for τ ∈ [0, 1.1]). This is a plateau interior, not a peak. + +## Experiment 5 — Fixing the slot policy: a null result + +Tail placement, one gated completion: + +| vs `hybrid` | Δ | 95% CI | p | +|---|---|---|---| +| recall@10 | +0.0062 | [−0.0085, +0.0212] | 0.431 | +| MRR | −0.0005 | [−0.0019, +0.0007] | 0.732 | +| useful@budget | +0.0038 | [−0.0062, +0.0145] | 0.501 | + +Nothing significant either way. The regression is repaired; no gain replaces it. + +## Experiment 5b — Comparison against the required baseline families + +All seven systems, same 420 queries, same indexes, same 1500-token budget, same +`apply_budget` and compactor, so token accounting is symmetric: + +| system | recall@5 | recall@10 | MRR | nDCG@10 | useful@budget | tokens | useful/token | p50 ms | +|---|---|---|---|---|---|---|---|---| +| `bm25` (FTS5 Okapi) | 0.452 | 0.497 | 0.368 | 0.370 | 0.423 | 1184.6 | 3.57e-4 | 13.5 | +| `dense` (LSA-160) | 0.351 | 0.386 | 0.243 | 0.262 | 0.333 | 741.3 | 4.49e-4 | 6.3 | +| `rag` (BM25 ⊕ dense, RRF) | 0.448 | 0.512 | 0.324 | 0.351 | 0.426 | 1131.7 | 3.76e-4 | 15.1 | +| `graph` (PPR, forced on) | 0.616 | 0.698 | 0.587 | 0.563 | 0.617 | 1083.2 | **5.70e-4** | 63.5 | +| `hybrid` (shipped 1.10.0) | 0.618 | 0.693 | 0.583 | 0.559 | 0.617 | 1093.0 | 5.64e-4 | 45.9 | +| `hybrid13` | 0.629 | 0.695 | 0.589 | 0.564 | 0.644 | 1168.4 | 5.51e-4 | 55.9 | +| `nucleus13` | 0.629 | **0.696** | **0.590** | **0.565** | **0.649** | 1243.0 | 5.22e-4 | 61.4 | + +Three things worth naming: + +- **The dense baseline is LSA, not a neural encoder** (no `sentence-transformers`, GPU + or network here). It is weak — MRR 0.243 — and I therefore make **no claim of + beating "embeddings"**. Its role is to keep the hybrid-RAG row honest, not to stand + in for a modern code encoder. +- **`graph` with PPR forced on is essentially tied with `hybrid`** (MRR +0.004, ns) but + raises `cand_recall` +0.049 (p<0.001). Structural expansion genuinely widens the + pool and genuinely fails to convert that into ranking gains — independent + corroboration of this repository's decision to ship `graph_source=False`, and the + first sign that Experiment 6 was going to come out the way it did. +- **Token efficiency falls monotonically with page size** (5.70 → 5.64 → 5.51 → 5.22 + e-4). "Useful per token" alone always favours returning less, so it cannot be used + as a lone objective; the honest object is the recall-versus-tokens curve, on which + NUCLEUS sits *below* the baseline. + +## Experiment 6 — The decisive comparison + +If the constraint is tokens rather than rank slots, completions can be *appended* as +contract slices (signatures, not bodies) instead of evicting. But then the baseline +must be allowed the same page growth, or "bigger page wins" is indistinguishable from +"better page wins". Hence `hybrid13`. + +| system | recall@10 | recall@15 | MRR | useful@budget | tokens | useful/token | +|---|---|---|---|---|---|---| +| `hybrid` (limit 10) | 0.693 | — | 0.583 | 0.617 | 1093 | 5.65e-4 | +| `hybrid13` (limit 13) | 0.695 | 0.723 | 0.589 | **0.644** | 1168 | **5.51e-4** | +| `nucleus` (10 + 3) | 0.694 | 0.717 | 0.584 | 0.629 | 1189 | 5.29e-4 | +| `nucleus13` (13 + 3) | 0.696 | **0.741** | 0.590 | **0.649** | 1243 | 5.22e-4 | + +Paired, against the page-matched baseline: + +| `nucleus` vs `hybrid13` | Δ | p | | `nucleus13` vs `hybrid13` | Δ | p | +|---|---|---|---|---|---|---| +| useful@1000 | **−0.0171** | **0.038** | | useful@budget | +0.0056 | **0.030** | +| cand_recall | **−0.0190** | **<0.001** | | useful@500 | +0.0062 | **0.029** | +| useful@budget | −0.0149 | 0.075 | | recall@15 | +0.0181 | **<0.001** | +| MRR | −0.0051 | 0.343 | | MAP | +0.0025 | **<0.001** | + +**This is the refutation.** + +1. `nucleus` (spending slots) is **worse** than simply returning three more results. +2. `nucleus13` (spending tokens) does beat the page-matched baseline significantly — + but by +0.0056 useful@budget for **+75 tokens (+6.4%)**. Its token efficiency is + **5.22e-4 vs 5.51e-4 useful-per-token: NUCLEUS is 5% *less* efficient than the + baseline it beats.** + +On the axis the whole research programme was aimed at — more answer per token — the +mechanism is negative. A +0.9% relative gain bought with +6.4% more tokens is not +minimal sufficient context; it is a slightly bigger context. + +## Experiment 7 — Ablations + +Every component ablated independently against `nucleus13` (420 queries): + +| variant | useful@budget Δ | tokens Δ | +|---|---|---| +| `-accretion` (= `hybrid13`) | −0.006 | −74.7 | +| `-cochange` | 0.000 | −0.5 | +| `-edges` | +0.001 | −16.7 | +| `-testlink` | −0.001 | −2.6 | +| `-stem` | −0.002 | −65.9 | +| `-dir` | 0.000 | −4.4 | +| `-coverage_select` (H2 off) | −0.001 | −4.0 | +| `-hub_penalty` | +0.009 | +67.6 | +| `-compact_completions` (H8 off) | −0.003 | +14.7 | +| `combine=max` | +0.003 | −17.6 | +| `gate=0.0` | +0.001 | +77.1 | +| `slots=1` | −0.001 | −36.5 | + +**No single relation carries the effect.** Every individual ablation moves +useful@budget by ≤0.003 — within noise on this query set. The only ablation that moves +anything is removing accretion wholesale. H2's coverage selection contributes −0.001, +i.e. nothing measurable: the elegant submodular objective is doing no work here, +because with `max_completions = 3` from a gated pool there is rarely a redundancy to +resolve. + +`-compact_completions` costs −0.003 useful@budget while *adding* 15 tokens, which is +the one place H8 (contracts instead of bodies) shows its expected sign. + +## Experiment 8 — Memory plane (H3/H4): validated + +Evidence = the spans the shipped retriever actually returns for each of the 420 +queries. Survival measured retrospectively against real history via +`git diff --unified=0 HEAD~h HEAD` new-side hunks, which are already in HEAD +coordinates — exact overlap, no line mapping, no re-indexing. + +| horizon (commits) | file-keyed survival | span-keyed survival | span/file | **semantic-cache unsound rate** | +|---|---|---|---|---| +| 1 | 0.729 | 0.795 | 1.09 | **20.5%** | +| 2 | 0.624 | 0.740 | 1.19 | 26.0% | +| 5 | 0.474 | 0.629 | 1.33 | 37.1% | +| 10 | 0.383 | 0.521 | 1.36 | **47.9%** | +| 20 | 0.213 | 0.309 | 1.45 | 69.1% | +| 50 | 0.099 | 0.130 | 1.32 | 87.0% | + +Two results: + +**Query-keyed caches are unsound at a rate that makes them unusable for code.** A +semantic cache reuses on question identity, so it would serve every one of these +entries; after 10 commits **47.9% of them are stale**, and after a *single* commit, +20.5%. There is no similarity threshold that fixes this, because the failure is not in +the similarity estimate — the cache key omits the dependency entirely. Evidence keying +drives this to zero by construction, not by tuning. + +**H4 holds, but weaker than predicted.** I predicted a large multiplicative retention +gain from finer granularity. Measured: **1.36× at h=10**, rising to 1.45× at h=20 — +real, and growing with horizon as predicted, but not the 2–3× I claimed. Per-repo +variance is large (1.00 for `denfry.github.io` and `DevGraph`, 2.75 for +`codebase-index` at h=10) and tracks whether commits touch files at a granularity +finer than the whole file. The mean evidence footprint is 4.1 spans across 4.1 files — +one span per file — which caps how much span-keying can possibly recover. + +**Cross-agent sharing.** Over the 420-task workload, 1738 atom reads collapse to 1098 +distinct atoms: **42.0% of tokens are never re-sent** by a content-addressed shared +memory (range 13.1% `TerraForge` – 65.4% `denfry.github.io`). This is deduplication, +not compression, and it is exact. + +--- + +## Stage 7 — Trying to destroy the surviving result + +**Adversarial corpora.** `denfry.github.io` (a website: 48 files, 30 symbols) and +`WinCleaner` (PowerShell: 39 files, **0** symbols, **0** edges) were kept in +deliberately. They break the retrieval plane (reachability 0.333 and structure-free) +and they do *not* break the memory plane, whose mechanism needs no symbols at all. +`PoliternalParkour` is the pathological case for memory: survival 0.203 at h=1, +because its commits are enormous — exactly the "highly dynamic codebase" failure mode. +Reported, not hidden. + +**Is the survival result just "code changes"?** Partly, and that is the point: the +contribution is the *magnitude* on real repositories and the demonstration that +granularity is a lever. But it means the numbers are corpus-specific and would differ +on a slow-moving repository. They are not a universal constant. + +**Does the memo help if evidence sets never repeat exactly?** This is the strongest +objection. Exact-set keying is brittle: two agents must read *identically* to share an +entry. The 42% dedup figure is measured at *atom* granularity (sub-conclusion sharing), +which is the level where reuse actually occurs; whole-conclusion reuse will be far +rarer. **I did not measure whole-conclusion hit rate under a realistic multi-agent +workload, and the architecture's `MemoStore.get_or_compute` is therefore untested at +the level that matters most for the "don't redo work" claim.** That is the largest +remaining hole in this work. + +**Is the retrieval refutation an artifact of file-level ground truth?** Plausibly, and +this is the fairest defence of H1. The benchmark scores at file granularity, so a +completion that supplies exactly the right *function* in a file the baseline already +returned earns nothing. Symbol-level ground truth might change the verdict. It would +not change the token-efficiency arithmetic, which is granularity-independent. + +**Threat I cannot close.** Ground truth is "files the commit touched", a proxy for +"files the agent needed". H6 (a `ddmin` oracle over an executable verifier) is the +right instrument and needs an LLM agent loop I do not have here. Every number above +inherits that proxy, on both sides of every comparison. diff --git a/research/fit_gate.py b/research/fit_gate.py new file mode 100644 index 0000000..5fa75d7 --- /dev/null +++ b/research/fit_gate.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Select the accretion gate under leave-one-repository-out. + +The calibration curve is broad and monotone, so a threshold can be chosen from a +*plateau* rather than a peak -- the same discipline `retrieval/tuning.py` applies to +the product's own coefficients. Objective is net expected gold per query: + + net(tau) = ( hits@1(score >= tau) - COST * n_fire(tau) ) / n_queries + +with COST = 0.0347, the measured gold density of the rank-10 slot a completion evicts. +""" +from __future__ import annotations +import json +from pathlib import Path + +CACHE = Path("research/data/contrib_cache.json") +RELATIONS = ["edge", "cochange", "testlink", "stem", "dir"] +COST = 0.0347 + +def rows_for(qs, w): + out = [] + for row in qs: + cands, missed = row["cands"], set(row["missed"]) + if not cands: + continue + best = max(((sum(w[k]*v for k, v in acc.items()), p) for p, acc in cands.items()), + key=lambda t: (t[0], t[1])) + out.append((best[0], 1 if best[1] in missed else 0)) + return out + +def net(rows, n_queries, tau): + fire = [r for r in rows if r[0] >= tau] + return (sum(r[1] for r in fire) - COST * len(fire)) / max(1, n_queries) + +def main() -> int: + cache = json.loads(CACHE.read_text(encoding="utf-8")) + w = {r: 1.0 for r in RELATIONS} + names = list(cache) + grid = [0.0, 0.2, 0.4, 0.6, 0.7, 0.826, 0.9, 1.0, 1.1, 1.3, 1.5, 2.0] + + print(f"{'held-out':22} {'tau*':>6} {'net(tau*)':>10} {'net(0)':>10} {'delta':>9}") + print("-" * 62) + tot_g = tot_0 = 0.0; N = 0 + taus = [] + for held in names: + tr = [r for k in names if k != held for r in cache[k]] + te = cache[held] + tr_rows, te_rows = rows_for(tr, w), rows_for(te, w) + tau = max(grid, key=lambda t: net(tr_rows, len(tr), t)) + g, z = net(te_rows, len(te), tau), net(te_rows, len(te), 0.0) + taus.append(tau) + tot_g += g*len(te); tot_0 += z*len(te); N += len(te) + print(f"{held:22} {tau:>6.3f} {g:>10.5f} {z:>10.5f} {g-z:>+9.5f}") + print("-" * 62) + print(f"{'POOLED (held-out)':22} {'':>6} {tot_g/N:>10.5f} {tot_0/N:>10.5f} {(tot_g-tot_0)/N:>+9.5f}") + print(f"chosen tau across folds: min={min(taus):.3f} median={sorted(taus)[len(taus)//2]:.3f} max={max(taus):.3f}") + + all_rows = rows_for([r for k in names for r in cache[k]], w) + nq = sum(len(cache[k]) for k in names) + print(f"\nplateau on all corpora (net gold/query vs tau):") + for t in grid: + fire = [r for r in all_rows if r[0] >= t] + print(f" tau={t:>5.3f} fires={len(fire):>4} hits={sum(r[1] for r in fire):>3} " + f"net={net(all_rows, nq, t):+.5f}") + return 0 + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/research/fit_weights.py b/research/fit_weights.py new file mode 100644 index 0000000..8112652 --- /dev/null +++ b/research/fit_weights.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +"""Fit the relation weights under leave-one-repository-out. + +The first benchmark run used weights of 1.0 for everything, which is not a +configuration so much as an absence of one, and the accretion ranking was +correspondingly poor (median rank 17 for gold that *was* generated). Hand-picking +coefficients on the same corpora that report the result is how a benchmark gets gamed +by accident, so weights are selected exactly the way this repository selects its +ranking parameters: choose on N-1 corpora, score on the held-out one, pool the folds. + +Objective: `recovery@C` -- the mean number of gold files, per query, that the top-C +accretion candidates recover *from among the gold the baseline's top-10 already +missed*. That is precisely the quantity a completion slot exists to buy, and it maps +monotonically onto recall@10. + +Because `contributions()` is linear in the weights, the whole fit runs over a cached +table of per-relation contributions: the graph is walked once, then tens of thousands +of weight vectors are scored as dot products. +""" + +from __future__ import annotations + +import argparse +import json +import sqlite3 +from dataclasses import replace +from pathlib import Path + +from codebase_index.retrieval.tuning import RetrievalTuning +from research.nucleus import baselines +from research.nucleus.evalrun import CORPORA, DATA, load_queries +from research.nucleus.relations import ( + CoChangeModel, RelationGraph, RelationWeights, load_static_edges, +) +from research.nucleus.search import NucleusParams, ObligationIndex, contributions + +RELATIONS = ["edge", "cochange", "testlink", "stem", "dir"] +CACHE = DATA / "contrib_cache.json" + + +def build_cache(anchor_head: int = 3) -> dict: + """corpus -> [ {gold_missed: [...], cands: {path: {rel: value}}} ] per query.""" + cache: dict[str, list] = {} + for name, repo in CORPORA: + qs, idx = DATA / f"{name}.yml", DATA / "index" / f"{name}.sqlite" + if not qs.exists() or not idx.exists(): + continue + queries = load_queries(qs, Path(repo)) + conn = sqlite3.connect(idx) + conn.row_factory = sqlite3.Row + files = [r[0].replace("\\", "/") for r in conn.execute("SELECT path FROM files")] + cochange = CoChangeModel.from_repo(Path(repo)) + graph = RelationGraph(files, static=load_static_edges(idx), cochange=cochange) + params = NucleusParams(anchor_head=anchor_head, min_score=0.0) + rows = [] + for q in queries: + cochange.advance_to(q.position) + cands, _ = baselines.product_candidates( + conn, q.query, limit=10, tuning=RetrievalTuning()) + base_files: list[str] = [] + for c in cands[:10]: + p = c.path.replace("\\", "/") + if p not in base_files: + base_files.append(p) + gold = set(q.expected_files) + missed = sorted(gold - set(base_files)) + head = base_files[:anchor_head] + anchors = list(dict.fromkeys(head)) + if not anchors or not missed: + # Queries whose gold the baseline already has contribute nothing to + # the objective: there is nothing left for a completion to recover. + # They still matter for *cost*, which the gate (fitted separately) + # and the end-to-end benchmark account for. + rows.append({"missed": missed, "cands": {}}) + continue + contrib = contributions(anchors=anchors, graph=graph, params=params, + exclude=set(head)) + rows.append({"missed": missed, + "cands": {p: {k: round(v, 6) for k, v in acc.items()} + for p, acc in contrib.items()}}) + cache[name] = rows + conn.close() + print(f"[{name}] cached {len(rows)} queries") + return cache + + +def recovery(rows: list, w: dict[str, float], *, c: int) -> float: + """Mean gold files recovered per query by the top-`c` accretion candidates.""" + total = 0.0 + for row in rows: + cands = row["cands"] + if not cands: + continue + missed = set(row["missed"]) + if not missed: + continue + scored = sorted( + ((sum(w.get(k, 0.0) * v for k, v in acc.items()), p) for p, acc in cands.items()), + key=lambda t: (-t[0], t[1]), + ) + total += sum(1 for _s, p in scored[:c] if p in missed) + return total / max(1, len(rows)) + + +def coordinate_ascent(train: list, *, c: int, passes: int = 4) -> dict[str, float]: + grid = [0.0, 0.15, 0.3, 0.5, 0.75, 1.0, 1.5, 2.0, 3.0, 4.0] + w = {r: 1.0 for r in RELATIONS} + best = recovery(train, w, c=c) + for _ in range(passes): + improved = False + for r in RELATIONS: + cur = w[r] + for v in grid: + if v == cur: + continue + trial = dict(w) + trial[r] = v + # L1-normalise so the fitted vector's *scale* stays comparable and the + # gate threshold keeps meaning the same thing across folds. + s = sum(trial.values()) or 1.0 + trial = {k: val * len(RELATIONS) / s for k, val in trial.items()} + score = recovery(train, trial, c=c) + if score > best + 1e-9: + best, w, improved = score, trial, True + if not improved: + break + return w + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--rebuild", action="store_true") + ap.add_argument("--c", type=int, default=1, help="completion slots to fit for") + args = ap.parse_args() + + if args.rebuild or not CACHE.exists(): + cache = build_cache() + CACHE.write_text(json.dumps(cache), encoding="utf-8") + else: + cache = json.loads(CACHE.read_text(encoding="utf-8")) + + names = list(cache) + uniform = {r: 1.0 for r in RELATIONS} + + print(f"\n=== leave-one-repository-out, objective = recovery@{args.c} ===") + print(f"{'held-out':22} {'uniform':>9} {'fitted':>9} {'delta':>9} weights chosen on the rest") + print("-" * 108) + tot_u = tot_f = 0.0 + n = 0 + folds = [] + for held in names: + train = [r for k in names if k != held for r in cache[k]] + test = cache[held] + w = coordinate_ascent(train, c=args.c) + u, f = recovery(test, uniform, c=args.c), recovery(test, w, c=args.c) + folds.append((held, u, f, w)) + wq = len(test) + tot_u += u * wq + tot_f += f * wq + n += wq + print(f"{held:22} {u:>9.4f} {f:>9.4f} {f-u:>+9.4f} " + + " ".join(f"{k}={v:.2f}" for k, v in w.items())) + print("-" * 108) + print(f"{'POOLED (held-out)':22} {tot_u/n:>9.4f} {tot_f/n:>9.4f} {(tot_f-tot_u)/n:>+9.4f}") + print(f"folds improved: {sum(1 for _h,u,f,_w in folds if f > u)}/{len(folds)}; " + f"regressed: {sum(1 for _h,u,f,_w in folds if f < u)}") + + full = coordinate_ascent([r for k in names for r in cache[k]], c=args.c) + print("\nweights fitted on ALL corpora (what would ship, reported as fitted):") + print(" " + " ".join(f"{k}={v:.3f}" for k, v in full.items())) + print(f" in-sample recovery@{args.c}: " + f"{recovery([r for k in names for r in cache[k]], full, c=args.c):.4f} " + f"(uniform {recovery([r for k in names for r in cache[k]], uniform, c=args.c):.4f})") + + # Stability across folds is the honest read on whether these weights mean + # anything: a coefficient that swings wildly between folds is noise. + print("\nper-relation weight across folds (min / median / max):") + for r in RELATIONS: + vals = sorted(w[r] for _h, _u, _f, w in folds) + print(f" {r:10} {vals[0]:.2f} / {vals[len(vals)//2]:.2f} / {vals[-1]:.2f}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/research/hypotheses.md b/research/hypotheses.md new file mode 100644 index 0000000..ff65081 --- /dev/null +++ b/research/hypotheses.md @@ -0,0 +1,284 @@ +# Stage 2–3 — Thirteen hypotheses + +Each entry states what is proposed, why the field has not already done it, and a +formal model. They are deliberately drawn from different parent disciplines; the +selection matrix in `selection.md` then cuts them down by feasibility *on evidence +available here*, not by appeal. + +Notation: corpus of atoms `A`; task `t`; retrieved set `S ⊆ A`; token cost +`c: A → ℕ`; budget `B`; gold/necessary set `N(t) ⊆ A`. + +--- + +## H1 — Retrieval as conditional set completion ("nucleate and accrete") + +**Hypothesis.** Replace `top-k argmax_d P(d|q)` with a two-stage set construction: +find a small high-precision *anchor* set `A₀` from the query, then **complete** it +using relations conditioned on the anchor rather than on the query: + +``` +A₀ = argtop_a P(a | q) (lexical/symbol; high precision, low recall) +S = A₀ ∪ argtop_c P(c ∈ N(t) | A₀, c ∉ A₀) (completion; query-independent) +``` + +**Why it may work / why it is not already done.** Every mainstream system scores +documents *independently given the query*, so the factor `P(c | A₀)` — by far the +strongest available signal for "needed together" — is structurally inexpressible. +Where systems do inject structure (PPR, graph retrievers) they inject it as *another +pointwise opinion into the same fusion*, which forces architectural neighbours to +compete with direct hits for the same ranked slots. This repository ran that +experiment and shipped `graph_source = False` because it lost MRR. The hypothesis is +that the signal was fine and the **composition** was wrong. + +**Mathematical intuition.** The true objective is a set posterior which does not +factorise: + +``` +P(N(t) = S | q) ≠ Π_{d∈S} P(d ∈ N(t) | q) +``` + +Model the dependence with a first-order chain (Chow–Liu style tree over the +change-set): each non-anchor member attaches to some already-selected member, + +``` +P(S | q) ≈ max_{a∈A₀} P(a | q) · Π_{c ∈ S\A₀} max_{s ∈ S, s≺c} P(c | s) +``` + +so `log P(S|q)` decomposes into an anchor term plus **pairwise completion terms** +`log P(c|s)` estimated over a typed relation union. Empirically (see +`diagnose_structure.py`) `P(∃ relation | c,s both gold) = 0.718` pooled across eight +repositories, with no temporal lookahead, versus a base rate of `O(1/|A|)` for a +random pair — a likelihood ratio of roughly `10²–10³`. + +--- + +## H2 — Selection as budgeted maximum coverage over latent obligations + +**Hypothesis.** Replace "take top-k, then MMR for diversity" with an explicit +**budgeted maximum coverage** program. Posit latent *obligations* `O` (things the task +must account for: a symbol, a contract, a config key). Each atom `a` covers +`cov(a) ⊆ O`; the task implicates `O(t) ⊆ O`. Then + +``` +maximise f(S) = | O(t) ∩ ⋃_{a∈S} cov(a) | subject to Σ_{a∈S} c(a) ≤ B +``` + +**Why it may work.** `f` is monotone submodular, so greedy by *coverage gain per +token* is a `(1 − 1/e)`-approximation (Nemhauser; Khuller–Moss–Naor for the budgeted +variant). More importantly it is the *correct* objective: MMR maximises pairwise +dissimilarity, which is a proxy that actively misfires here — two files implementing +the same contract are highly similar and both necessary, exactly the pair MMR +suppresses. No mainstream retriever optimises coverage of task obligations because +obligations are never materialised. + +**Mathematical intuition.** Diminishing returns: `f(S∪{a}) − f(S)` is non-increasing +in `S`. Greedy picks `argmax_a (f(S∪{a}) − f(S)) / c(a)`. Contrast with top-k, which +picks `argmax_a score(a)` — identical only when `cov` are disjoint and equal-cost, +i.e. never. + +--- + +## H3 — Evidence-keyed memoisation: agent reasoning as a content-addressed build + +**Hypothesis.** Store every agent conclusion keyed not by its *question* but by a hash +of the **evidence it consumed**: + +``` +key(r) = H( sort{ (atom_id, content_hash(atom)) : atom ∈ evidence(r) } ‖ H(prompt_class) ) +``` + +A cached conclusion is reusable **iff every atom in its key still hashes to the same +value**. Otherwise it is not "probably stale" — it is *definitely* invalid, and is +evicted deterministically. + +**Why it may work / why it is not already done.** Semantic caches key on query +similarity, which carries zero information about whether the world changed. Build +systems (Bazel, Nix, `ccache`) solved exactly this problem for compilation two decades +ago with content-addressed action caches, and self-adjusting computation (Acar et al.) +solved it for general programs. Agent memory has not adopted it because agent +conclusions are usually stored as prose with no recorded provenance — the dependency +set is thrown away at write time. It is an *engineering omission that has a +correctness consequence*, which is the most promising kind. + +**Mathematical intuition.** Let `V(r,τ)` be the event "r is still valid at time τ". +Under evidence keying, `P(unsound reuse) = 0` exactly (a hash collision aside), while a +similarity cache has `P(unsound reuse) = P(evidence changed | q ≈ q_cached) > 0` and +*uncorrelated with the threshold*. The interesting quantity is therefore not +soundness — which is free — but **survival**: `P(V(r,τ))` as a function of evidence +granularity. That is H4. + +--- + +## H4 — The granularity law of memory survival + +**Hypothesis.** Reuse survival is governed by the *measure* of the evidence footprint, +not its semantic breadth. If a conclusion depends on `m` atoms each independently +invalidated at per-commit rate `λ`, survival after `n` commits is + +``` +P(valid after n commits) ≈ (1 − λ)^{m·n} +``` + +so shrinking atom granularity (file → symbol) shrinks `λ` roughly in proportion to the +fraction of the file a typical commit touches. **Prediction: symbol-level evidence keys +retain multiplicatively more valid reuse than file-level keys**, and the multiplier is +measurable directly from git history with no model in the loop. + +**Why it matters.** It converts "choose your memory granularity" from taste into an +estimable quantity, and it predicts that the naive choice (file-level provenance, +which is what any straightforward implementation would do) is the expensive one. + +--- + +## H5 — Retrieval as entropy minimisation over the answer set + +**Hypothesis.** Choose the next atom that maximally reduces uncertainty about the +answer: `a* = argmax_a I(Y ; a | S)` where `Y` is the answer random variable. + +**Mathematical intuition.** `H(Y|S)` decreasing is the ideal stopping criterion: +retrieve until `H(Y|S) < ε`, which yields *adaptive* `k` — few atoms for a sharp +question, many for a diffuse one. Beautiful, and it subsumes H2 (coverage is a +tractable surrogate for mutual information under a set-cover likelihood). + +**Why it is hard here.** Estimating `I(Y;a|S)` needs either a scoring LLM in the loop +(cost, non-determinism, no API in this environment) or a strong proxy. Kept as theory: +it is the *justification* for H2's objective rather than an independently testable +mechanism. + +--- + +## H6 — A delta-debugging oracle for minimal sufficient context + +**Hypothesis.** Ground truth for "minimal necessary context" can be *computed*, not +guessed: given a task with an executable verifier (test suite), run `ddmin` over +candidate context sets to find a 1-minimal `S` such that the agent still succeeds. +Then distil a cheap predictor from those labels. + +**Why it may work.** Every code-retrieval benchmark today uses a weak proxy label +("the files the commit touched"). ddmin would produce true minimal sufficient sets and +expose how wrong the proxy is. + +**Why it is not selected.** Requires `O(|S| log |S|)` *agent executions per query* with +a working LLM and per-repo test harness. Out of reach in this environment; recorded as +the highest-value follow-up in `further-work`. + +--- + +## H7 — Context prefetching from agent access traces + +**Hypothesis.** An agent's file-access sequence is a memory reference stream. Apply +correlation/Markov prefetching: `P(next atom | last k atoms)`, prefetch on anchor +resolution, and bound the achievable gain with a Belady-style offline optimum. + +**Why it may work.** Hardware prefetching theory is mature and directly transferable; +"working set" and reuse-distance (Mattson stack distance) give principled context-window +sizing. **Why it is not selected now:** it needs a corpus of real agent traces, which I +do not have. Note the co-change relation in H1 is a *degenerate offline form* of this +(commits as access traces), which is the part that is testable today. + +--- + +## H8 — Obligation-level granularity: index contracts, return slices + +**Hypothesis.** The retrievable unit should be an *obligation* (signature + +pre/post-conditions + error modes), with full bodies returned only for atoms the task +must modify. Expected 3–10× token reduction on "how do I call X" tasks. + +**Mathematical intuition.** `tokens(signature) / tokens(body) ≈ 0.05–0.2`. If a +fraction `ρ` of retrieved atoms are needed only as *callees* (understand the contract) +rather than *editees*, the bill falls to `ρ·0.1 + (1−ρ)` of baseline. + +**Status.** Compatible with, and partially already implemented by, this repository's +`retrieval/skeleton.py` compactor. Folded into the architecture as a cost model rather +than claimed as novel. + +--- + +## H9 — Index the derivative, not the state + +**Hypothesis.** Make the indexed document a *change* (commit: subject + diff + +co-changed set), not a file. Retrieval then answers "what changed like this before", +returning both the precedent and the files it touched. + +**Why it may work.** Causal/intentional information ("why") lives in change records +and is absent from the tree. **Risk:** benchmark circularity — the ground truth here is +itself derived from commits, so indexing commits would let the system read the answer. +Only admissible under a strict temporal split; that discipline is adopted for the +co-change relation and this hypothesis is otherwise deferred. + +--- + +## H10 — Self-organising index (Hebbian re-weighting) + +**Hypothesis.** Relation weights adapt online: atoms co-retrieved in successful tasks +strengthen their link, `w ← w + η(1−w)`; unsuccessful, decay. The index reorganises +toward the workload. + +**Mathematical intuition.** Stochastic approximation on the completion model of H1 — +`P(c|s)` becomes an online-estimated parameter. Convergence under Robbins–Monro +conditions; the risk is a rich-get-richer collapse without exploration. + +**Status.** A natural extension of H1 once feedback exists. Not testable without a +live agent workload; the architecture leaves the parameter slot open for it. + +--- + +## H11 — Routing by obligation ownership + +**Hypothesis.** Route multi-agent work by *which obligations a task implicates*, not by +task-description similarity. Two agents whose implicated obligation sets intersect are +on a collision course and must share memory or serialise. + +**Mathematical intuition.** Predicted conflict = `|O(t₁) ∩ O(t₂)| > 0`. Assignment +becomes graph partitioning of the obligation hypergraph minimising cut (shared +obligations) — the classic distributed-systems objective, with obligations as the +shared state. + +**Status.** Designed into the architecture; measurable only in simulation here. + +--- + +## H12 — Negative results as first-class atoms + +**Hypothesis.** Store *refuted* hypotheses and failed attempts with their evidence +keys. A second agent asking a question whose evidence key matches a recorded failure +gets the refutation instead of repeating the search. + +**Why it may work.** Failed work is the largest silent cost in multi-agent systems and +is never recorded because it produces no artifact. Under H3's keying it is free: a +failure is just a conclusion with a negative polarity. + +--- + +## H13 — Speculative closure precomputation + +**Hypothesis.** Precompute and cache completion closures for the `O(k)` most likely +near-future tasks (derived from open diffs, recent commits, TODOs), so the closure is +warm when the task arrives. + +**Mathematical intuition.** Amortises closure cost; value = `hit_rate × +closure_latency`. Only worthwhile if closure latency is a bottleneck — measured below, +it is ~1 ms, so this is premature. **Rejected on measurement**, which is the correct +reason to reject something. + +--- + +## What the diagnostic already tells us + +Before any selection, `diagnose_structure.py` (run on 900 anchor→target gold pairs, +eight repositories, strict temporal split) reports: + +| relation | covers | uniquely | +|---|---|---| +| stem (shared name tokens) | 35.8% | — | +| static edge (import/call/ref) | 30.7% | 5.3% | +| co-change (history before the query commit) | 30.4% | 9.8% | +| same directory | 23.8% | — | +| test↔impl link | 14.4% | — | +| **union** | **71.8%** | | + +Two facts decide Stage 4. First, the union is far above any single relation, so the +completer must be **multi-relation**. Second, structure and history each contribute +*uniquely* (5.3% and 9.8%), so neither subsumes the other and a system with only one of +them leaves measurable recall on the table. H1 is therefore not merely plausible — its +central premise is already measured, on the same corpora the final benchmark uses. diff --git a/research/novelty.md b/research/novelty.md new file mode 100644 index 0000000..a377c9e --- /dev/null +++ b/research/novelty.md @@ -0,0 +1,98 @@ +# Stage 8 — Novelty assessment + +The brief forbids claiming "nobody has done this" without a real check. I ran one, and +the honest answer is uncomfortable: **neither of the two mechanisms I designed is +novel.** Both have close, recent prior art that I was unaware of when designing them. +What survives as a contribution is narrower and mostly empirical. + +Searches run 2026-09-09 across arXiv/Scholar-indexed literature and the code-retrieval +ecosystem. Phrasing throughout is "novel relative to the reviewed literature" or, where +prior art was found, no novelty claim at all. + +--- + +## The retrieval plane (H1 + H2) — **not novel** + +| Prior work | Overlap | +|---|---| +| **RepoHyper** (Search-Expand-Refine on a Repo-level Semantic Graph, arXiv 2403.06095) | This is essentially my architecture. "Search-then-Expand" = nucleate-then-accrete; a link predictor refines the expanded set = my scored relation union. Published 2024. | +| **CoCoMIC** | Method-level dependency graph for cross-file completion; the anchor-plus-dependency-context pattern. | +| **GRACE** (2509.05980), **RANGER** (2509.25257) | Graph-guided / graph-enhanced repository-level retrieval, same family. | +| **PACMS** (2606.20047) | "Recasts agent context assembly as budget-constrained submodular selection", facility-location coverage, and reports it beats both MMR-style diversification and pure relevance. This is H2, done first and done better. | +| **Context-Picker** (2512.14465) | "Shifts the paradigm from similarity-based ranking to minimal sufficient subset selection", mining minimal sufficient sets via leave-one-out. This is both my framing *and* my rejected H6 oracle. | +| **Zimmermann et al. ROSE** (ICSE'04), **Ying et al.** (TSE'04) | Co-change association-rule mining — the evolutionary relation, twenty years prior. | +| **Aider repo-map** | PageRank over the symbol graph for LLM context. | + +I independently rederived a design that the field converged on, and the literature is +ahead of it on the selection objective in particular. That is worth stating plainly +rather than hedging. + +## The memory plane (H3 + H4) — **not novel as a mechanism** + +| Prior work | Overlap | +|---|---| +| **Invalidation Contracts for Cross-Episode Agent Memory** (2609.00243) | Version stamps + cacheability hints attached to cached agent conclusions so stale entries can be evicted without trial and error. Same problem, same diagnosis. | +| **Fresh Memory, Stale Plans: Dependency-Scoped Validation for Distributed LLM-Agent Memory** (2609.03340) | PlanFence: plans cite the exact records they used; the executor validates only records that can affect the pending action. This is dependency-keyed validity for agent memory. | +| **From Agent Traces to Trust: Evidence Tracing and Execution Provenance** (2606.04990) | Provenance over retrieved passages, tool outputs, memory items. | +| **Bazel / Nix / ccache; Adapton, self-adjusting computation** | Content-addressed action caches with dependency-keyed invalidation — the actual ancestor, decades old. | + +The transplant of build-system content addressing to agent memory, which I framed as +the novel step, was published at least twice in 2026 before this work. + +--- + +## What is left, stated conservatively + +Novel relative to the reviewed literature, and all of it **empirical rather than +mechanistic**: + +1. **A quantified refutation of anchor-and-expand retrieval under a page-matched + baseline.** The graph-expansion literature (RepoHyper, GRACE, RANGER) reports gains + against a fixed-size retrieval baseline. I find that on commit-derived ground truth + across eight repositories, the gain is dominated by the trivial control of + *returning more results*, and that the mechanism is 5% **less** token-efficient than + the baseline it beats. I did not find this control reported in the reviewed papers. + It is cheap, and it should be standard. + +2. **Slot economics as a decision procedure.** Expressing an expansion mechanism's + value as `gain(slot) vs P(gold at the displaced rank)` — measured here as 0.031 + vs 0.035 — turns "should we add graph expansion" into arithmetic. This framing, and + the per-rank gold-density table it needs, I did not find in the reviewed literature. + +3. **Magnitude of query-keyed cache unsoundness on real code history**: 20.5% stale + after 1 commit, 47.9% after 10, 69.1% after 20. The invalidation-contract papers + argue the problem exists; I have not seen it *measured* against version history on + real repositories. + +4. **The granularity/retention measurement**: span-keyed evidence retains 1.36× more + valid reuse than file-keyed at a 10-commit horizon, growing to 1.45× at 20, with + the multiplier tracking file size versus commit size. Apparently novel as a + measurement; the underlying idea (finer dependencies invalidate less) is folklore. + +5. **42.0% cross-task evidence deduplication** on a 420-task workload over eight + repositories — a concrete ceiling for shared agent memory. + +6. **A reproducible negative-result protocol**: no-lookahead history enforced by the + data structure rather than by discipline, equivalence-to-incumbent asserted by test, + and leave-one-repository-out applied to a fit that then *failed* to generalise and + was reported anyway. + +## What I would have done differently knowing the prior art + +Read PACMS and Context-Picker before Stage 5. PACMS in particular reports the +submodular selection result I set out to obtain, which would have redirected the +effort toward the memory plane — where the measurements turned out to be worth having — +several hours earlier. + +## Sources + +- [RepoHyper: Search-Expand-Refine on Semantic Graphs](https://arxiv.org/abs/2403.06095) +- [PACMS: Submodular Context Selection as a Pluggable Engine for LLM Agents](https://arxiv.org/html/2606.20047) +- [Context-Picker: Dynamic Context Selection Using Multi-stage RL](https://arxiv.org/html/2512.14465) +- [GRACE: Graph-Guided Repository-Aware Code Completion](https://arxiv.org/pdf/2509.05980) +- [RANGER: Repository-level Agent for Graph-Enhanced Retrieval](https://arxiv.org/html/2509.25257) +- [Invalidation Contracts for Cross-Episode Agent Memory](https://arxiv.org/html/2609.00243) +- [Fresh Memory, Stale Plans: Dependency-Scoped Validation](https://arxiv.org/html/2609.03340) +- [From Agent Traces to Trust: Evidence Tracing and Execution Provenance](https://arxiv.org/html/2606.04990v1) +- [Enhancing Software Maintenance: Learning to Rank for Co-changed Method Identification](https://arxiv.org/pdf/2411.19099) +- [Retrieval-Augmented Code Generation: A Survey with Focus on Repository-Level Approaches](https://arxiv.org/html/2510.04905v1) diff --git a/research/nucleus/__init__.py b/research/nucleus/__init__.py new file mode 100644 index 0000000..1d4e9b7 --- /dev/null +++ b/research/nucleus/__init__.py @@ -0,0 +1,6 @@ +"""NUCLEUS research prototype: necessity-driven closure retrieval + evidence memory. + +Kept out of `src/codebase_index/` on purpose. This is a research spike measured +against the shipped pipeline; promoting any part of it into the product is a +separate decision that the numbers in `research/experiments.md` have to earn. +""" diff --git a/research/nucleus/baselines.py b/research/nucleus/baselines.py new file mode 100644 index 0000000..d2c0a8e --- /dev/null +++ b/research/nucleus/baselines.py @@ -0,0 +1,313 @@ +"""Comparison systems, all emitting the shipped `Candidate` shape. + +Every baseline goes through the same `apply_budget` with the same budget and the same +compactor, so `tokens` and `useful@budget` are measured symmetrically. A comparison +where one side pays for its context and the other does not is not a comparison. + + bm25 FTS5 Okapi BM25 over the same chunks. Pure lexical, no reranking. + dense LSA: tf-idf -> randomized truncated SVD -> cosine. Classical dense + retrieval. NOT a neural code encoder -- see the honesty note below. + rag bm25 + dense fused by RRF. The standard hybrid-RAG configuration. + hybrid the shipped 1.10.0 pipeline. The strongest incumbent available here. + graph the shipped pipeline with `graph_source=True` AND PPR expansion forced + on for every query. Forcing matters: commit-subject queries almost all + classify as KEYWORD intent, whose plan sets `graph_strategy="none"`, so + an unforced graph baseline would silently never activate and I would be + beating a system that was switched off. + +Honesty note on `dense` +----------------------- +No `sentence-transformers`, GPU, or network access exists in this environment, so the +dense baseline is Latent Semantic Analysis, not a modern neural encoder. LSA is a +legitimate dense retriever and the original one, but a code-tuned neural encoder would +very likely score higher. Therefore **no claim of the form "NUCLEUS beats embeddings" +is made anywhere in this work**. The load-bearing comparison is the paired one against +NUCLEUS's own anchor stage, which isolates accretion from retriever quality and would +hold on top of a better encoder too. +""" + +from __future__ import annotations + +import math +import re +import sqlite3 +from dataclasses import dataclass +from typing import Callable, Optional + +import numpy as np + +from codebase_index.retrieval.budget import apply_budget +from codebase_index.retrieval.diversity import deduplicate +from codebase_index.retrieval.fusion import fuse +from codebase_index.retrieval.intent import detect_intent +from codebase_index.retrieval.pipeline import _diversify, _run_retrievers +from codebase_index.retrieval.rerank import rerank +from codebase_index.retrieval.skeleton import make_compactor +from codebase_index.retrieval.tuning import DEFAULT_TUNING, RetrievalTuning +from codebase_index.retrieval.types import Candidate + +_TOK = re.compile(r"[A-Za-z][A-Za-z0-9_]*") + + +def _subtokens(word: str) -> list[str]: + parts = re.sub(r"([a-z0-9])([A-Z])", r"\1 \2", word).replace("_", " ").split() + return [p.lower() for p in parts if len(p) >= 3] + + +def tokenize(text: str, *, cap: int = 4000) -> list[str]: + out: list[str] = [] + for w in _TOK.findall(text or "")[:cap]: + lw = w.lower() + if len(lw) >= 3: + out.append(lw) + subs = _subtokens(w) + if len(subs) > 1: + out.extend(subs) + return out + + +def finalize( + candidates: list[Candidate], *, query: str, token_budget: int, limit: int, + pool_paths: Optional[list[str]] = None, +) -> dict: + """Shared tail: budget + payload, identical for every system under test. + + `pool_paths` is the *pre-truncation candidate pool*, which the oracle metrics are + measured against. It must mean the same thing for every system or `oracle` and + `cand_recall` become incomparable across the table; when a system does not + distinguish a pool from its ranking, its full untruncated candidate list is the + honest answer. + """ + plan = detect_intent(query) + ranked = candidates[:limit] + compactor = make_compactor(intent=plan.intent, query=query, enabled=True, + min_reduction=0.25) + results, recommended = apply_budget(ranked, token_budget=token_budget, + compactor=compactor) + if pool_paths is None: + seen: list[str] = [] + for c in candidates: + if c.path not in seen: + seen.append(c.path) + pool_paths = seen + return { + "query": query, "intent": plan.intent.value, "results": results, + "recommended_reads": recommended, "confidence": "medium", + "diagnostics": {"pool": [{"path": p} for p in pool_paths]}, + } + + +# --- BM25 -------------------------------------------------------------------- + + +def _match_query(query: str) -> str: + terms = {t.lower() for t in _TOK.findall(query) if len(t) >= 3} + if not terms: + return "" + return " OR ".join(f'"{t}"' for t in sorted(terms)) + + +def bm25_candidates(conn: sqlite3.Connection, query: str, *, limit: int) -> list[Candidate]: + match = _match_query(query) + if not match: + return [] + try: + rows = conn.execute( + """ + SELECT f.path, c.line_start, c.line_end, c.content, c.token_est, + bm25(fts_chunks) AS s + FROM fts_chunks + JOIN chunks c ON c.id = fts_chunks.rowid + JOIN files f ON f.id = c.file_id + WHERE fts_chunks MATCH ? + ORDER BY s + LIMIT ? + """, + (match, limit * 8), + ).fetchall() + except sqlite3.Error: + return [] + out = [] + for path, ls, le, content, tok, s in rows: + out.append(Candidate( + path=path.replace("\\", "/"), line_start=ls, line_end=le, source="fts", + score=-float(s), content=content, token_est=int(tok or 0), reason="bm25", + )) + return out + + +# --- dense (LSA) ------------------------------------------------------------- + + +@dataclass +class LSAIndex: + """tf-idf -> randomized truncated SVD. numpy only, no scipy/sklearn available.""" + + vocab: dict[str, int] + idf: np.ndarray + doc_emb: np.ndarray # (n_docs, k), L2-normalised + V: np.ndarray # (n_terms, k) + meta: list[tuple[str, int, int, str, int]] + + @classmethod + def build(cls, conn: sqlite3.Connection, *, k: int = 160, seed: int = 20260909) -> "LSAIndex": + rows = conn.execute( + """ + SELECT f.path, c.line_start, c.line_end, c.content, c.token_est + FROM chunks c JOIN files f ON f.id = c.file_id + ORDER BY c.id + """ + ).fetchall() + meta = [(r[0].replace("\\", "/"), int(r[1]), int(r[2]), r[3] or "", int(r[4] or 0)) + for r in rows] + n = len(meta) + if n == 0: + return cls({}, np.zeros(0), np.zeros((0, k)), np.zeros((0, k)), meta) + + df: dict[str, int] = {} + per_doc: list[dict[str, int]] = [] + for _, _, _, content, _ in meta: + tf: dict[str, int] = {} + for t in tokenize(content): + tf[t] = tf.get(t, 0) + 1 + per_doc.append(tf) + for t in tf: + df[t] = df.get(t, 0) + 1 + + # min_df 2 removes hapax noise; the upper bound removes terms so common they + # carry no discrimination (language keywords, license headers). + hi = max(3, int(0.30 * n)) + vocab = {t: i for i, t in enumerate( + sorted(t for t, d in df.items() if 2 <= d <= hi) + )} + if not vocab: + return cls({}, np.zeros(0), np.zeros((n, k)), np.zeros((0, k)), meta) + idf = np.zeros(len(vocab), dtype=np.float32) + for t, i in vocab.items(): + idf[i] = math.log(1.0 + n / df[t]) + + rows_i: list[int] = [] + cols_i: list[int] = [] + vals_f: list[float] = [] + for d, tf in enumerate(per_doc): + acc = [] + for t, c in tf.items(): + j = vocab.get(t) + if j is not None: + acc.append((j, (1.0 + math.log(c)) * idf[j])) + if not acc: + continue + norm = math.sqrt(sum(v * v for _, v in acc)) or 1.0 + for j, v in acc: + rows_i.append(d) + cols_i.append(j) + vals_f.append(v / norm) + R = np.asarray(rows_i, dtype=np.int32) + C = np.asarray(cols_i, dtype=np.int32) + Vl = np.asarray(vals_f, dtype=np.float32) + m = len(vocab) + k = min(k, max(2, min(n, m) - 1)) + + rng = np.random.default_rng(seed) + + def A_dot(X: np.ndarray) -> np.ndarray: # (m,k) -> (n,k) + out = np.zeros((n, X.shape[1]), dtype=np.float32) + for s in range(0, len(R), 200_000): + e = s + 200_000 + np.add.at(out, R[s:e], Vl[s:e, None] * X[C[s:e]]) + return out + + def AT_dot(X: np.ndarray) -> np.ndarray: # (n,k) -> (m,k) + out = np.zeros((m, X.shape[1]), dtype=np.float32) + for s in range(0, len(R), 200_000): + e = s + 200_000 + np.add.at(out, C[s:e], Vl[s:e, None] * X[R[s:e]]) + return out + + Omega = rng.standard_normal((m, k)).astype(np.float32) + Y = A_dot(Omega) + Y = A_dot(AT_dot(Y)) # one power iteration + Q, _ = np.linalg.qr(Y) + BT = AT_dot(Q) # (m,k) == B^T + Ub, Sb, Vbt = np.linalg.svd(BT, full_matrices=False) + V = Ub # (m,k) right singular vecs of A + doc = A_dot(V) + norms = np.linalg.norm(doc, axis=1, keepdims=True) + doc = doc / np.maximum(norms, 1e-8) + return cls(vocab, idf, doc.astype(np.float32), V.astype(np.float32), meta) + + def query(self, text: str, *, limit: int) -> list[Candidate]: + if not self.vocab or self.doc_emb.size == 0: + return [] + tf: dict[str, int] = {} + for t in tokenize(text): + tf[t] = tf.get(t, 0) + 1 + vec = np.zeros(len(self.vocab), dtype=np.float32) + for t, c in tf.items(): + j = self.vocab.get(t) + if j is not None: + vec[j] = (1.0 + math.log(c)) * self.idf[j] + nrm = float(np.linalg.norm(vec)) + if nrm <= 0: + return [] + q = (vec / nrm) @ self.V + qn = float(np.linalg.norm(q)) + if qn <= 0: + return [] + sims = self.doc_emb @ (q / qn) + take = min(limit * 8, sims.shape[0]) + idx = np.argpartition(-sims, take - 1)[:take] + idx = idx[np.argsort(-sims[idx])] + out = [] + for i in idx: + path, ls, le, content, tok = self.meta[int(i)] + out.append(Candidate( + path=path, line_start=ls, line_end=le, source="vector", + score=float(sims[int(i)]), content=content, token_est=tok, + reason="lsa cosine", + )) + return out + + +# --- shipped pipeline variants ---------------------------------------------- + + +def product_candidates( + conn: sqlite3.Connection, query: str, *, limit: int, tuning: RetrievalTuning, + force_graph: bool = False, +) -> tuple[list[Candidate], list[str]]: + """Returns (ranked candidates, pre-rerank pool paths).""" + plan = detect_intent(query) + pool_mult = max(1, tuning.candidate_pool_multiplier) + pool_limit = limit if pool_mult == 1 else max(limit * pool_mult, 20) + lists, weights = _run_retrievers( + conn, query, mode="hybrid", limit=pool_limit, weights=plan.weights, + backend=None, tuning=tuning, graph_depth=tuning.graph_depth, + graph_node_cap=tuning.graph_node_cap, + graph_strategy=( + ("both" if force_graph else plan.graph_strategy) + if tuning.graph_source else "none" + ), + ) + fused = fuse(lists, weights=weights, k=tuning.rrf_k, + file_agreement=tuning.file_agreement_weight if tuning.file_agreement else 0.0) + pool_paths: list[str] = [] + for c in fused: + p = c.path.replace("\\", "/") + if p not in pool_paths: + pool_paths.append(p) + ranked = rerank(fused, query=query, intent=plan.intent, tuning=tuning) + if tuning.dedup: + ranked = deduplicate(ranked, hamming_distance=tuning.dedup_hamming) + ranked = _diversify(ranked, per_file=tuning.max_per_file) + return ranked, pool_paths + + +def rrf_hybrid(a: list[Candidate], b: list[Candidate], *, k: int = 60) -> list[Candidate]: + return fuse({"fts": a, "vector": b}, weights={"fts": 1.0, "vector": 1.0}, k=k, + file_agreement=0.0) + + +# --- graph PPR baseline uses the shipped pipeline with graph_source on ------- + +GRAPH_TUNING = RetrievalTuning(graph_source=True) diff --git a/research/nucleus/evalrun.py b/research/nucleus/evalrun.py new file mode 100644 index 0000000..1075f2e --- /dev/null +++ b/research/nucleus/evalrun.py @@ -0,0 +1,382 @@ +"""Benchmark runner: many systems, one query sweep, paired significance. + +Design constraints inherited from `tests/eval/harness.py`, deliberately, so that the +numbers here are comparable with the ones the product publishes: + + * one index per corpus, shared by every variant -- deltas measure retrieval, not + indexing variance; + * metrics come from `tests.eval.metrics` verbatim, never reimplemented; + * every variant sees the identical query in the identical order, so per-query + vectors stay aligned and the paired bootstrap / permutation tests are valid. + +One constraint is new here. The co-change relation must never see the query's own +commit, so queries are swept **oldest-first** and the history model is advanced +monotonically. Running all variants inside one sweep is therefore not just faster -- +it is what guarantees every variant sees exactly the same revealed history. +""" + +from __future__ import annotations + +import argparse +import json +import statistics +import sqlite3 +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable, Optional + +import yaml + +from codebase_index.retrieval.diversity import ( + normalize_code_tokens, simhash_distance, token_fingerprint, +) +from codebase_index.retrieval.tuning import RetrievalTuning +from tests.eval import metrics +from tests.eval.harness import EvalReport, format_significance, format_table, pool + +from . import baselines +from .relations import CoChangeModel, RelationGraph, RelationWeights, load_static_edges +from .search import NucleusParams, ObligationIndex, nucleus_search + + +@dataclass(frozen=True) +class Query: + query: str + category: str + expected_files: tuple[str, ...] + commit: str + position: int # git log position; larger == older + + +@dataclass +class Ctx: + """Everything a variant may need for one corpus.""" + conn: sqlite3.Connection + graph: RelationGraph + obligations: ObligationIndex + lsa: Optional[baselines.LSAIndex] = None + + +Variant = Callable[[Ctx, str], dict] + + +@dataclass +class Outcome: + ranked_files: list[str] + returned: list[tuple[str, int]] + latency_ms: float + total_tokens: int + duplicates: int + n_results: int + pool_files: list[str] + + +def _norm(p: str) -> str: + return p.replace("\\", "/") + + +def load_queries(path: Path, repo: Path) -> list[Query]: + import subprocess + + raw = subprocess.run(["git", "-C", str(repo), "log", "--format=%H"], + capture_output=True, text=True, encoding="utf-8", + errors="replace") + order = {s.strip()[:12]: i for i, s in enumerate(raw.stdout.splitlines()) if s.strip()} + out: list[Query] = [] + for e in yaml.safe_load(path.read_text(encoding="utf-8")) or []: + commit = e.get("commit", "") + pos = order.get(commit) + if pos is None: + continue + out.append(Query( + query=e["query"], category=e.get("category", "uncategorized"), + expected_files=tuple(_norm(f) for f in e.get("expected_files", ())), + commit=commit, position=pos, + )) + # Oldest first: the co-change model is append-only and must never see the future. + out.sort(key=lambda q: -q.position) + return out + + +def outcome_from_payload(payload: dict, latency_ms: float) -> Outcome: + """Mirrors tests/eval/harness.run_query's accounting exactly. + + Only snippets are billed (results past the budget carry a token_est for a chunk + the agent never receives); file-level ranking collapses several hits in one file + to its best rank; duplicates are measured with the pipeline's own fingerprint. + """ + pool_files: list[str] = [] + for entry in payload.get("diagnostics", {}).get("pool", ()): + p = _norm(entry["path"]) + if p not in pool_files: + pool_files.append(p) + + ranked_files: list[str] = [] + returned: list[tuple[str, int]] = [] + total_tokens = 0 + duplicates = 0 + fingerprints: list[int] = [] + results = payload.get("results", []) + for r in results: + p = _norm(r["path"]) + if p not in ranked_files: + ranked_files.append(p) + tokens = int(r.get("token_est") or 0) + returned.append((p, tokens)) + snippet = r.get("snippet") + if snippet: + total_tokens += tokens + chunk_tokens = normalize_code_tokens(snippet or "") + if chunk_tokens: + fp = token_fingerprint(chunk_tokens) + if any(simhash_distance(fp, seen) <= 3 for seen in fingerprints): + duplicates += 1 + fingerprints.append(fp) + return Outcome(ranked_files, returned, latency_ms, total_tokens, duplicates, + len(results), pool_files) + + +def aggregate(label: str, queries: list[Query], outcomes: list[Outcome], + *, token_budget: int) -> EvalReport: + scorers = { + "recall@5": lambda q, o: metrics.recall_at_k(o.ranked_files, q.expected_files, 5), + "recall@10": lambda q, o: metrics.recall_at_k(o.ranked_files, q.expected_files, 10), + "recall@15": lambda q, o: metrics.recall_at_k(o.ranked_files, q.expected_files, 15), + "MRR": lambda q, o: metrics.reciprocal_rank(o.ranked_files, q.expected_files), + "nDCG@10": lambda q, o: metrics.ndcg_at_k(o.ranked_files, q.expected_files, 10), + "hit@3": lambda q, o: metrics.hit_rate_at_k(o.ranked_files, q.expected_files, 3), + "P@5": lambda q, o: metrics.precision_at_k(o.ranked_files, q.expected_files, 5), + "MAP": lambda q, o: metrics.average_precision(o.ranked_files, q.expected_files), + "useful@budget": lambda q, o: metrics.useful_context_at_budget( + o.returned, q.expected_files, token_budget), + # Token-normalised recall at three budgets. This is the agent-centric view: + # not "is the answer ranked somewhere" but "is it inside the context the + # agent can afford", which is the only axis on which a system returning a + # bigger page can be compared fairly with one returning a smaller page. + "useful@500": lambda q, o: metrics.useful_context_at_budget( + o.returned, q.expected_files, 500), + "useful@1000": lambda q, o: metrics.useful_context_at_budget( + o.returned, q.expected_files, 1000), + "oracle": lambda q, o: metrics.oracle_reciprocal_rank(o.pool_files, q.expected_files), + "cand_recall": lambda q, o: metrics.recall_at_k( + o.pool_files, q.expected_files, len(o.pool_files)), + } + per_query = {n: [fn(q, o) for q, o in zip(queries, outcomes)] for n, fn in scorers.items()} + + def mean(n: str) -> float: + v = per_query[n] + return statistics.fmean(v) if v else 0.0 + + per_cat: dict[str, list[float]] = {} + for q, rr in zip(queries, per_query["MRR"]): + per_cat.setdefault(q.category, []).append(rr) + lat = [o.latency_ms for o in outcomes] + returned_total = sum(o.n_results for o in outcomes) + dup_total = sum(o.duplicates for o in outcomes) + return EvalReport( + label=label, n_queries=len(outcomes), + recall_at_5=mean("recall@5"), recall_at_10=mean("recall@10"), mrr=mean("MRR"), + ndcg_at_10=mean("nDCG@10"), hit_rate_at_3=mean("hit@3"), + precision_at_5=mean("P@5"), map_score=mean("MAP"), + useful_context=mean("useful@budget"), + p50_ms=metrics.percentile(lat, 50), p95_ms=metrics.percentile(lat, 95), + p99_ms=metrics.percentile(lat, 99), + mean_ms=statistics.fmean(lat) if lat else 0.0, + mean_tokens=statistics.fmean([o.total_tokens for o in outcomes]) if outcomes else 0.0, + duplicate_rate=(dup_total / returned_total) if returned_total else 0.0, + mean_candidates=(returned_total / len(outcomes)) if outcomes else 0.0, + oracle_mrr=mean("oracle"), candidate_recall=mean("cand_recall"), + mean_pool=statistics.fmean([len(o.pool_files) for o in outcomes]) if outcomes else 0.0, + per_category={c: statistics.fmean(v) for c, v in sorted(per_cat.items())}, + per_query=per_query, + ) + + +# --- variants ---------------------------------------------------------------- + + +LIMIT = 10 +BUDGET = 1500 + + +def make_variants(names: set[str], nucleus_params: dict[str, NucleusParams]) -> dict[str, Variant]: + v: dict[str, Variant] = {} + if "bm25" in names: + v["bm25"] = lambda ctx, q: baselines.finalize( + baselines.bm25_candidates(ctx.conn, q, limit=LIMIT), + query=q, token_budget=BUDGET, limit=LIMIT) + if "dense" in names: + v["dense"] = lambda ctx, q: baselines.finalize( + ctx.lsa.query(q, limit=LIMIT) if ctx.lsa else [], + query=q, token_budget=BUDGET, limit=LIMIT) + if "rag" in names: + v["rag"] = lambda ctx, q: baselines.finalize( + baselines.rrf_hybrid( + baselines.bm25_candidates(ctx.conn, q, limit=LIMIT), + ctx.lsa.query(q, limit=LIMIT) if ctx.lsa else []), + query=q, token_budget=BUDGET, limit=LIMIT) + if "hybrid13" in names: + def _hybrid13(ctx: Ctx, q: str) -> dict: + # The incumbent, allowed exactly the page growth NUCLEUS takes. Without + # this row a "bigger page wins" result would be indistinguishable from a + # "better page wins" result. + cands, pool_paths = baselines.product_candidates( + ctx.conn, q, limit=13, tuning=RetrievalTuning()) + return baselines.finalize(cands, query=q, token_budget=BUDGET, + limit=13, pool_paths=pool_paths) + v["hybrid13"] = _hybrid13 + if "hybrid" in names: + def _hybrid(ctx: Ctx, q: str) -> dict: + cands, pool_paths = baselines.product_candidates( + ctx.conn, q, limit=LIMIT, tuning=RetrievalTuning()) + return baselines.finalize(cands, query=q, token_budget=BUDGET, + limit=LIMIT, pool_paths=pool_paths) + v["hybrid"] = _hybrid + if "graph" in names: + def _graph(ctx: Ctx, q: str) -> dict: + cands, pool_paths = baselines.product_candidates( + ctx.conn, q, limit=LIMIT, tuning=baselines.GRAPH_TUNING, + force_graph=True) + return baselines.finalize(cands, query=q, token_budget=BUDGET, + limit=LIMIT, pool_paths=pool_paths) + v["graph"] = _graph + for label, params in nucleus_params.items(): + def mk(p: NucleusParams, lim: int) -> Variant: + return lambda ctx, q: nucleus_search( + ctx.conn, q, limit=lim, token_budget=BUDGET, graph=ctx.graph, + obligations=ctx.obligations, params=p, explain=True) + # `nucleus13` asks the sharpest question available: does accretion add + # anything ON TOP OF simply returning more results, which is the cheap + # alternative that `hybrid13` shows is already effective? + v[label] = mk(params, 13 if label.endswith("13") else LIMIT) + return v + + +def run_corpus( + name: str, repo: Path, queries_path: Path, index_path: Path, + variants_wanted: set[str], nucleus_params: dict[str, NucleusParams], + *, need_dense: bool, +) -> tuple[list[Query], dict[str, list[Outcome]]]: + queries = load_queries(queries_path, repo) + conn = sqlite3.connect(index_path) + conn.row_factory = sqlite3.Row + files = [r[0].replace("\\", "/") for r in conn.execute("SELECT path FROM files")] + cochange = CoChangeModel.from_repo(repo) + graph = RelationGraph(files, static=load_static_edges(index_path), cochange=cochange) + obligations = ObligationIndex(conn) + lsa = baselines.LSAIndex.build(conn) if need_dense else None + ctx = Ctx(conn=conn, graph=graph, obligations=obligations, lsa=lsa) + + variants = make_variants(variants_wanted, nucleus_params) + out: dict[str, list[Outcome]] = {k: [] for k in variants} + for q in queries: + cochange.advance_to(q.position) # reveal only strictly-older commits + for label, fn in variants.items(): + t0 = time.perf_counter() + payload = fn(ctx, q.query) + dt = (time.perf_counter() - t0) * 1000.0 + out[label].append(outcome_from_payload(payload, dt)) + conn.close() + return queries, out + + +CORPORA = [ + ("codebase-index", "."), + ("Civitas", "../Civitas"), + ("PoliternalSite", "../PoliternalSite"), + ("PoliternalParkour", "../PoliternalParkour"), + ("TerraForge", "../TerraForge"), + ("denfry.github.io", "../denfry.github.io"), + ("DevGraph", "../DevGraph"), + ("WinCleaner", "../Windows-Cleaner-and-Optimizer-main"), +] + +DATA = Path(__file__).parent.parent / "data" + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--systems", default="hybrid,nucleus", + help="comma list of bm25,dense,rag,hybrid,graph,nucleus") + ap.add_argument("--ablate", action="store_true") + ap.add_argument("--baseline-label", default="hybrid") + ap.add_argument("--corpus", action="append", default=None) + ap.add_argument("--json-out", default=None) + ap.add_argument("--per-corpus", action="store_true") + args = ap.parse_args() + + wanted = {s.strip() for s in args.systems.split(",") if s.strip()} + nucleus_params: dict[str, NucleusParams] = {} + if "nucleus" in wanted: + nucleus_params["nucleus"] = NucleusParams() + if "nucleus13" in wanted: + nucleus_params["nucleus13"] = NucleusParams() + if args.ablate: + base = NucleusParams() + nucleus_params.update({ + "-accretion13": NucleusParams(accrete=False), + "-cochange13": NucleusParams(weights=RelationWeights(cochange=0.0)), + "-edges13": NucleusParams(weights=RelationWeights(edge=0.0)), + "-testlink13": NucleusParams(weights=RelationWeights(testlink=0.0)), + "-stem13": NucleusParams(weights=RelationWeights(stem=0.0)), + "-dir13": NucleusParams(weights=RelationWeights(dirw=0.0)), + "-coverage_sel13": NucleusParams(coverage_select=False), + "-hub_penalty13": NucleusParams(hub_penalty=False), + "-compact13": NucleusParams(compact_completions=False), + "combine=max13": NucleusParams(combine="max"), + "gate=0.0__13": NucleusParams(min_score=0.0), + "slots=1__13": NucleusParams(max_completions=1), + }) + need_dense = bool({"dense", "rag"} & wanted) + + corpora = CORPORA + if args.corpus: + keep = set(args.corpus) + corpora = [c for c in CORPORA if c[0] in keep] + + all_reports: dict[str, list[EvalReport]] = {} + for name, repo in corpora: + qs = DATA / f"{name}.yml" + idx = DATA / "index" / f"{name}.sqlite" + if not qs.exists() or not idx.exists(): + print(f"[{name}] skipped (missing data)") + continue + t0 = time.perf_counter() + queries, outcomes = run_corpus( + name, Path(repo), qs, idx, wanted, nucleus_params, need_dense=need_dense) + for label, oc in outcomes.items(): + all_reports.setdefault(label, []).append( + aggregate(f"{label}", queries, oc, token_budget=BUDGET)) + print(f"[{name}] {len(queries)} queries, {len(outcomes)} systems, " + f"{time.perf_counter()-t0:.1f}s") + if args.per_corpus: + reps = [aggregate(label, queries, oc, token_budget=BUDGET) + for label, oc in outcomes.items()] + base = next((r for r in reps if r.label == args.baseline_label), None) + print(format_table(reps, baseline=base)) + print() + + pooled = {label: pool(reps, label=label) for label, reps in all_reports.items()} + order = [l for l in ("bm25", "dense", "rag", "graph", "hybrid", "hybrid13") + if l in pooled] + order += [l for l in pooled if l not in order] + reports = [pooled[l] for l in order] + base = pooled.get(args.baseline_label) + print("\n=== POOLED ===") + print(format_table(reports, baseline=base)) + if base is not None: + for label in order: + if label == args.baseline_label: + continue + print() + print(format_significance(base, pooled[label])) + + if args.json_out: + Path(args.json_out).write_text(json.dumps( + {l: pooled[l].as_row() for l in order}, indent=2), encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/research/nucleus/memory.py b/research/nucleus/memory.py new file mode 100644 index 0000000..abb3cd4 --- /dev/null +++ b/research/nucleus/memory.py @@ -0,0 +1,161 @@ +"""Evidence-keyed memory: sound reuse of agent work, and the granularity law. + +Hypotheses H3/H4. Unlike the retrieval plane, the claim here is about a *correctness* +property, so the experiment is not "does quality go up" but "how often does the +incumbent design return an answer that is silently wrong, and what does soundness +cost". + +The three cache designs compared: + + query-keyed the semantic cache. Key = the question. Reuse when the question + repeats (or is similar). Carries no information about whether the + code the answer was derived from still exists in that form. + + file-keyed evidence-keyed at file granularity. Key = content hashes of every + file the conclusion read. Sound: any edit to any of those files + invalidates. This is what a straightforward implementation does. + + span-keyed evidence-keyed at the granularity actually read -- the retrieved + line spans. Equally sound, but invalidated only by edits that + overlap the specific regions the conclusion depended on. + +Survival is measured retrospectively against real history: for a conclusion whose +evidence is a set of spans at HEAD, was that evidence disturbed over the last `h` +commits? Because `git diff HEAD~h HEAD` reports hunks on the new side in HEAD's +coordinates, the overlap test is exact -- no re-indexing of historical trees and no +line-number mapping heuristics. + +The semantic cache's unsound-hit rate is then exactly `1 - survival`: it would have +served every one of those conclusions, and each one whose evidence moved was stale. +""" + +from __future__ import annotations + +import hashlib +import re +import subprocess +from dataclasses import dataclass, field +from pathlib import Path + +_HUNK = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@") + + +@dataclass(frozen=True) +class Span: + path: str + start: int + end: int + tokens: int = 0 + + +@dataclass +class Conclusion: + """One unit of agent work, with the evidence it consumed recorded at write time.""" + + task: str + spans: tuple[Span, ...] + tokens: int = 0 + + @property + def files(self) -> frozenset[str]: + return frozenset(s.path for s in self.spans) + + def key(self, content_hash: dict[str, str], *, granularity: str) -> str: + """Content-addressed key. This is the whole mechanism. + + A conclusion is identified by what it *depended on*, not by what it was + asked. Two agents that read the same evidence share a key and therefore + share the work; an edit to that evidence changes the key, so the stale entry + is not evicted by a heuristic -- it is simply never looked up again. + """ + if granularity == "file": + parts = sorted(f"{p}:{content_hash.get(p, '')}" for p in self.files) + else: + parts = sorted( + f"{s.path}:{s.start}-{s.end}:{content_hash.get(s.path, '')}" + for s in self.spans + ) + return hashlib.sha256("\x00".join(parts).encode()).hexdigest()[:16] + + +def _git(repo: Path, *args: str) -> str: + p = subprocess.run(["git", "-C", str(repo), *args], capture_output=True, + text=True, encoding="utf-8", errors="replace") + if p.returncode != 0: + raise RuntimeError(f"git {' '.join(args)}: {p.stderr.strip()}") + return p.stdout + + +def changed_regions(repo: Path, horizon: int) -> tuple[set[str], dict[str, list[tuple[int, int]]]]: + """Files and new-side line ranges touched by the last `horizon` commits. + + Returns (changed_files, {path: [(start, end), ...]}) with ranges in HEAD + coordinates, which is what makes the overlap test against HEAD spans exact. + """ + try: + diff = _git(repo, "diff", "--unified=0", f"HEAD~{horizon}", "HEAD") + except RuntimeError: + return set(), {} + files: set[str] = set() + hunks: dict[str, list[tuple[int, int]]] = {} + current: str | None = None + for line in diff.splitlines(): + if line.startswith("+++ b/"): + current = line[6:].strip() + if current == "/dev/null": + current = None + else: + files.add(current) + hunks.setdefault(current, []) + elif line.startswith("@@") and current: + m = _HUNK.match(line) + if m: + start = int(m.group(1)) + length = int(m.group(2) or 1) + if length == 0: # pure deletion: mark the seam + hunks[current].append((start, start + 1)) + else: + hunks[current].append((start, start + length - 1)) + return files, hunks + + +def survives( + c: Conclusion, changed_files: set[str], hunks: dict[str, list[tuple[int, int]]], + *, granularity: str, +) -> bool: + if granularity == "file": + return not (c.files & changed_files) + for s in c.spans: + if s.path not in changed_files: + continue + for lo, hi in hunks.get(s.path, ()): + if s.start <= hi and lo <= s.end: + return False + # A file listed as changed with no parsable hunks (rename, mode change, + # binary) is treated as disturbing every span in it. Conservative in the + # direction that costs the mechanism hit-rate rather than soundness. + if not hunks.get(s.path): + return False + return True + + +@dataclass +class MemoStore: + """Content-addressed store of completed work, with exact invalidation.""" + + granularity: str = "span" + entries: dict[str, Conclusion] = field(default_factory=dict) + hits: int = 0 + misses: int = 0 + tokens_saved: int = 0 + + def get_or_compute(self, c: Conclusion, content_hash: dict[str, str]) -> bool: + """True on reuse. The caller does the work only on False.""" + k = c.key(content_hash, granularity=self.granularity) + if k in self.entries: + self.hits += 1 + self.tokens_saved += c.tokens + return True + self.entries[k] = c + self.misses += 1 + return False diff --git a/research/nucleus/relations.py b/research/nucleus/relations.py new file mode 100644 index 0000000..3aeda19 --- /dev/null +++ b/research/nucleus/relations.py @@ -0,0 +1,355 @@ +"""Typed relation union: the completion model P(c | anchor). + +The architecture's central bet is that the members of a required set are related to +*each other* by something computable, so that a high-precision anchor found by lexical +retrieval can be completed into a set. This module estimates those relations. + +Five relations, chosen because `research/diagnose_structure.py` measured each of them +carrying signal on 900 real anchor->target gold pairs, and because static structure and +history were shown to contribute *uniquely* (5.3% and 9.8% of pairs respectively) — +neither subsumes the other, so a single-relation completer leaves recall on the table. + +Temporal honesty +---------------- +`CoChange` is the one relation that can leak. Ground truth here is mined from commits, +so a co-change model that has seen the query's own commit would be reading the answer. +`CoChangeModel.advance_to()` therefore exposes counts from commits **strictly older** +than a given commit, and the benchmark sweeps queries oldest-first so the observable +history only ever grows. There is no code path that can see a commit at or after the +query's own; the filter is a property of the data structure, not a discipline the +caller has to remember. +""" + +from __future__ import annotations + +import math +import re +import sqlite3 +import subprocess +from collections import defaultdict +from dataclasses import dataclass, field +from pathlib import Path + +_WORD = re.compile(r"[A-Za-z][A-Za-z0-9]*") +_TEST_MARK = re.compile(r"(?:^|[._-])(?:test|tests|spec|specs)(?:$|[._-])", re.I) + +# A commit touching more than this many files couples everything to everything and is +# evidence about a release process, not about a design relation. +MAX_COMMIT_FILES = 40 + +# Additive smoothing on co-change confidence. Without it a file that appears in one +# prior commit alongside one other file scores confidence 1.0 on a single observation. +COCHANGE_PRIOR = 2.0 + + +def _norm(p: str) -> str: + return p.replace("\\", "/") + + +def stem_tokens(path: str) -> set[str]: + base = _norm(path).rsplit("/", 1)[-1].split(".", 1)[0] + parts = re.sub(r"([a-z0-9])([A-Z])", r"\1 \2", base).replace("_", " ").replace("-", " ") + return {w.lower() for w in _WORD.findall(parts) if len(w) > 2} + + +def core_stem(path: str) -> str: + base = _norm(path).rsplit("/", 1)[-1].split(".", 1)[0] + base = re.sub(r"^(?:test|spec)[._-]?", "", base, flags=re.I) + base = re.sub(r"[._-]?(?:test|tests|spec|specs)$", "", base, flags=re.I) + return base.lower() + + +def is_testish(path: str) -> bool: + p = _norm(path) + return bool(_TEST_MARK.search(p)) or "/test" in p.lower() + + +def directory(path: str) -> str: + p = _norm(path) + return p.rsplit("/", 1)[0] if "/" in p else "" + + +# --- static structure ------------------------------------------------------- + + +def load_static_edges(index_path: Path) -> dict[str, dict[str, float]]: + """file -> {file: weight} from resolved import/call/reference edges. + + Symbol targets are lifted to their defining file because the unit of the ground + truth, and of the agent's decision, is a file. Edge confidence is carried through + as the weight: an `inferred` import-path match is weaker evidence than an + `extracted` unique-symbol match, and the completer should know the difference. + """ + conf_w = {"extracted": 1.0, "inferred": 0.75, "ambiguous": 0.35} + adj: dict[str, dict[str, float]] = defaultdict(dict) + if not index_path.exists(): + return adj + conn = sqlite3.connect(index_path) + try: + rows = conn.execute( + """ + SELECT src.path AS a, + CASE WHEN e.dst_kind = 'file' THEN df.path ELSE sf.path END AS b, + e.confidence + FROM edges AS e + JOIN files AS src ON src.id = e.file_id + LEFT JOIN files AS df ON e.dst_kind = 'file' AND df.id = e.dst_id + LEFT JOIN symbols AS s ON e.dst_kind = 'symbol' AND s.id = e.dst_id + LEFT JOIN files AS sf ON sf.id = s.file_id + WHERE e.resolved = 1 + """ + ).fetchall() + except sqlite3.Error: + return adj + finally: + conn.close() + for a, b, conf in rows: + if not a or not b or a == b: + continue + a, b = _norm(a), _norm(b) + w = conf_w.get((conf or "extracted").lower(), 0.5) + # Undirected: "must I also look at this" is symmetric even though the edge + # is not. Direction is kept out of the weight because the diagnostic found + # no consistent asymmetry at file granularity. + adj[a][b] = max(adj[a].get(b, 0.0), w) + adj[b][a] = max(adj[b].get(a, 0.0), w) + return adj + + +# --- history ---------------------------------------------------------------- + + +def _git(repo: Path, *args: str) -> str: + p = subprocess.run( + ["git", "-C", str(repo), *args], + capture_output=True, text=True, encoding="utf-8", errors="replace", + ) + if p.returncode != 0: + raise RuntimeError(f"git {' '.join(args)}: {p.stderr.strip()}") + return p.stdout + + +@dataclass +class CoChangeModel: + """Incrementally-revealed co-change counts with a hard no-lookahead guarantee. + + Commits are held oldest-last (position 0 = newest, matching `git log` order). + `advance_to(pos)` folds in every commit strictly older than `pos`. Because the + benchmark visits queries oldest-first, `pos` decreases monotonically and the fold + is a single linear sweep over history per corpus rather than a rescan per query. + """ + + commits: list[tuple[int, list[str]]] = field(default_factory=list) # (pos, files) + _pair: dict[tuple[str, str], int] = field(default_factory=lambda: defaultdict(int)) + _single: dict[str, int] = field(default_factory=lambda: defaultdict(int)) + _folded: int = 0 # index into self.commits (which is sorted oldest-first) + _current_pos: int | None = None + + @classmethod + def from_repo(cls, repo: Path) -> "CoChangeModel": + order: dict[str, int] = {} + for i, sha in enumerate( + s.strip() for s in _git(repo, "log", "--format=%H").splitlines() if s.strip() + ): + order[sha[:12]] = i + raw = _git(repo, "log", "--no-merges", "--name-only", + "--pretty=format:%x01%H", "--diff-filter=ACMR") + commits: list[tuple[int, list[str]]] = [] + for rec in raw.split("\x01"): + rec = rec.strip("\n") + if not rec: + continue + head, _, body = rec.partition("\n") + pos = order.get(head.strip()[:12]) + if pos is None: + continue + files = [_norm(ln.strip()) for ln in body.splitlines() if ln.strip()] + if 0 < len(files) <= MAX_COMMIT_FILES: + commits.append((pos, files)) + # Oldest first, so advancing toward newer queries is an append-only fold. + commits.sort(key=lambda x: -x[0]) + return cls(commits=commits) + + def advance_to(self, pos: int) -> None: + """Reveal every commit strictly older than `pos` (i.e. position > pos).""" + if self._current_pos is not None and pos > self._current_pos: + raise ValueError( + "CoChangeModel is append-only; queries must be visited oldest-first " + f"(asked for pos={pos} after pos={self._current_pos})" + ) + self._current_pos = pos + while self._folded < len(self.commits) and self.commits[self._folded][0] > pos: + _, files = self.commits[self._folded] + uniq = sorted(set(files)) + for f in uniq: + self._single[f] += 1 + for i, a in enumerate(uniq): + for b in uniq[i + 1:]: + self._pair[(a, b)] += 1 + self._folded += 1 + + def confidence(self, a: str, b: str) -> float: + """P(b changes | a changes), smoothed. Asymmetric by construction.""" + if a == b: + return 0.0 + key = (a, b) if a < b else (b, a) + joint = self._pair.get(key, 0) + if not joint: + return 0.0 + return joint / (self._single.get(a, 0) + COCHANGE_PRIOR) + + def neighbours(self, a: str) -> list[str]: + """Files ever seen co-changing with `a` in the revealed history.""" + # Kept simple: the pair table is small enough (<2e5) that a per-anchor scan + # is cheaper than maintaining a second adjacency index. + out = [] + for (x, y), n in self._pair.items(): + if n and x == a: + out.append(y) + elif n and y == a: + out.append(x) + return out + + +class CoChangeIndex: + """Adjacency view over CoChangeModel, rebuilt lazily when history advances.""" + + def __init__(self, model: CoChangeModel) -> None: + self.model = model + self._adj: dict[str, list[str]] = {} + self._built_at = -1 + + def neighbours(self, a: str) -> list[str]: + if self._built_at != self.model._folded: + adj: dict[str, list[str]] = defaultdict(list) + for (x, y), n in self.model._pair.items(): + if n: + adj[x].append(y) + adj[y].append(x) + self._adj = adj + self._built_at = self.model._folded + return self._adj.get(a, []) + + +# --- the relation union ----------------------------------------------------- + + +@dataclass(frozen=True) +class RelationWeights: + """Linear weights over the relation union. + + Defaults are the leave-one-repository-out selection from `fit_weights.py`; they + are fitted parameters and are reported as such. + """ + + edge: float = 1.0 + cochange: float = 1.0 + testlink: float = 1.0 + stem: float = 1.0 + dirw: float = 0.3 + + def as_dict(self) -> dict[str, float]: + return {"edge": self.edge, "cochange": self.cochange, + "testlink": self.testlink, "stem": self.stem, "dir": self.dirw} + + +class RelationGraph: + """All five relations for one corpus, with candidate generation. + + Generation is *anchor-local*: candidates come from the anchors' relation + neighbourhoods, never from a scan of the corpus. That is what keeps completion at + O(|A0| * deg) instead of O(|corpus|) and is why the closure stage costs ~1ms. + """ + + def __init__( + self, + files: list[str], + *, + static: dict[str, dict[str, float]], + cochange: CoChangeModel, + ) -> None: + self.files = [_norm(f) for f in files] + self.static = static + self.cochange = cochange + self.cochange_index = CoChangeIndex(cochange) + + self._dir: dict[str, list[str]] = defaultdict(list) + self._by_stem_token: dict[str, list[str]] = defaultdict(list) + self._core_stem: dict[str, list[str]] = defaultdict(list) + self._tokens: dict[str, set[str]] = {} + for f in self.files: + self._dir[directory(f)].append(f) + toks = stem_tokens(f) + self._tokens[f] = toks + for t in toks: + self._by_stem_token[t].append(f) + self._core_stem[core_stem(f)].append(f) + + n = max(1, len(self.files)) + # IDF over file-name tokens: `index` in a repo full of `*_index.py` is not + # evidence of anything, while a token appearing in two names is strong. + self._idf = { + t: math.log(n / len(fs)) for t, fs in self._by_stem_token.items() + } + self._max_idf = max(self._idf.values(), default=1.0) or 1.0 + + # -- individual relation strengths, all normalised to [0, 1] -------------- + + def rel_edge(self, a: str, c: str) -> float: + return self.static.get(a, {}).get(c, 0.0) + + def rel_cochange(self, a: str, c: str) -> float: + return min(1.0, self.cochange.confidence(a, c)) + + def rel_testlink(self, a: str, c: str) -> float: + if is_testish(a) == is_testish(c): + return 0.0 + cs = core_stem(a) + return 1.0 if cs and cs == core_stem(c) else 0.0 + + def rel_stem(self, a: str, c: str) -> float: + shared = self._tokens.get(a, set()) & self._tokens.get(c, set()) + if not shared: + return 0.0 + # Best shared token by IDF, normalised. Sum would let three generic tokens + # outweigh one highly specific one, which is the wrong ordering. + return max(self._idf.get(t, 0.0) for t in shared) / self._max_idf + + def rel_dir(self, a: str, c: str) -> float: + d = directory(a) + if d != directory(c): + return 0.0 + # A 3-file package is a strong grouping; a 200-file dumping ground is not. + return 1.0 / math.log2(2 + len(self._dir.get(d, ()))) + + # -- candidate generation ------------------------------------------------- + + def candidates(self, anchor: str, *, dir_cap: int = 12) -> set[str]: + """Files related to `anchor` by at least one relation.""" + out: set[str] = set() + out.update(self.static.get(anchor, {}).keys()) + out.update(self.cochange_index.neighbours(anchor)) + out.update(self._core_stem.get(core_stem(anchor), ())) + for t in self._tokens.get(anchor, ()): + # Generic tokens fan out to hundreds of files and contribute ~0 IDF + # anyway; capping keeps generation bounded without changing the ranking. + fs = self._by_stem_token.get(t, ()) + if len(fs) <= 30: + out.update(fs) + siblings = self._dir.get(directory(anchor), ()) + if len(siblings) <= dir_cap: + out.update(siblings) + out.discard(anchor) + return out + + def score(self, anchor: str, c: str, w: RelationWeights) -> tuple[float, dict[str, float]]: + parts = { + "edge": self.rel_edge(anchor, c), + "cochange": self.rel_cochange(anchor, c), + "testlink": self.rel_testlink(anchor, c), + "stem": self.rel_stem(anchor, c), + "dir": self.rel_dir(anchor, c), + } + wd = w.as_dict() + total = sum(wd[k] * v for k, v in parts.items()) + return total, parts diff --git a/research/nucleus/search.py b/research/nucleus/search.py new file mode 100644 index 0000000..47b1849 --- /dev/null +++ b/research/nucleus/search.py @@ -0,0 +1,508 @@ +"""NUCLEUS retrieval: nucleate -> accrete -> select. + +The pipeline deliberately reuses the shipped retrieval stack for the *anchor* stage. +That is not laziness: the claim under test is about composition, not about lexical +matching, so the anchor stage must be the incumbent's best configuration or the +comparison is rigged. With `accrete=False` this module reproduces +`codebase_index.retrieval.pipeline.search` result-for-result — `test_nucleus.py` +asserts it — so every measured delta is attributable to accretion and selection alone. + +Three stages: + + NUCLEATE the incumbent hybrid retriever produces a ranked candidate list; the + head of it is treated as anchors (high precision, low recall). + + ACCRETE candidates related to the anchors are scored by the relation union + (`relations.RelationGraph`) *conditioned on the anchors, not the query*. + This is the stage that can retrieve a file sharing no vocabulary with + the question. + + SELECT accreted candidates compete for their own slot budget by coverage gain + per token, so completions do not duplicate each other's obligations. + +The budget separation is the point. Completions are appended behind a frozen anchor +head, so a completion can only ever take a slot a *lower-ranked baseline result* +would have held. The documented failure mode of graph retrieval in this codebase +(`tuning.graph_source = False`: "architectural neighbors displaced direct lexical +hits") is excluded structurally rather than by tuning. +""" + +from __future__ import annotations + +import re +import sqlite3 +from dataclasses import dataclass +from typing import Optional + +from codebase_index.retrieval.budget import apply_budget +from codebase_index.retrieval.diversity import deduplicate, mmr_select +from codebase_index.retrieval.fusion import fuse +from codebase_index.retrieval.intent import detect_intent +from codebase_index.retrieval.pipeline import _confidence, _diversify, _run_retrievers +from codebase_index.retrieval.rerank import rerank +from codebase_index.retrieval.skeleton import make_compactor +from codebase_index.retrieval.tuning import DEFAULT_TUNING, RetrievalTuning +from codebase_index.retrieval.types import Candidate + +from .relations import RelationGraph, RelationWeights, stem_tokens + +_TERM_RE = re.compile(r"[A-Za-z0-9_]+") + + +@dataclass(frozen=True) +class NucleusParams: + """Accretion configuration. Every field is an ablation switch.""" + + accrete: bool = True + anchor_head: int = 3 + """Baseline results whose rank is frozen. Completions are inserted after these.""" + + max_completions: int = 3 + """Completions to append. + + Under the eviction policy this had to be 1: `diagnose_slots.py` showed the + marginal precision of the second and third completion (0.0166 and 0.0072 + gold/query) fell below the cost of the rank-9 and rank-8 slots they evicted + (0.0339, 0.0247). Appending removes the eviction cost, so the binding constraint + becomes tokens and more completions become affordable. Ablated at 1/2/3.""" + + append_only: bool = True + """Add completions beyond the baseline page instead of evicting for them. + + The slot framing was wrong. `diagnose_slots.py` shows every eviction is a losing + trade, but that is an artifact of a fixed 10-slot page: the agent's real budget is + tokens, and the incumbent spends only ~1093 of 1500. Rendered as contract slices + rather than code chunks (`_completion_slice`), a completion costs a fraction of a + chunk and evicts nothing, so its expected value no longer has to beat a rank slot + -- only its own token cost. Compared honestly against a baseline allowed the same + page size and the same budget.""" + + compact_completions: bool = True + """Render completions as signatures (H8) rather than bodies.""" + + insert_at: str = "tail" + """Where completions go. + + 'after_head' inserts at `anchor_head + 1`, which shifts every baseline result + below it down by one. That is what made the first benchmark run lose: it left + recall unchanged while damaging MRR/nDCG/P@5, because a gold file at rank 4 + (density 0.096) was being pushed to rank 5+ to make room for a completion with + precision 0.073. 'tail' spends only the last slot, whose gold density is the + lowest on the page (0.035), and cannot reorder anything above it.""" + + min_score: float = 0.6 + """Gate, selected under leave-one-repository-out by `fit_gate.py`. + + Accretion proposes nothing at all on 58% of queries, and where it does propose, + its score is well calibrated: precision rises from 0.073 (fire always) to 0.353 + (fire on the top-scoring 10%). The gate converts a signal that is only 2.1x the + slot cost on average into one that is several times the slot cost when it spends. + + 0.6 was chosen in 7 of 8 leave-one-repository-out folds (the eighth chose 0.7) + and sits on a plateau: net gold/query is flat at 0.021-0.024 for every tau in + [0, 1.1]. It is a plateau interior, not a peak, which is the only kind of fitted + value worth shipping.""" + + weights: RelationWeights = RelationWeights() + + anchor_decay: float = 1.0 + """Anchor i gets weight 1/(1 + decay*i): rank-1 evidence outweighs rank-3.""" + + combine: str = "sum" + """'sum' rewards a candidate related to several anchors (evidence of belonging to + one change-set); 'max' treats relations as independent. Ablated.""" + + hub_penalty: bool = True + """Divide by sqrt(relation degree): a file related to everything is evidence for + nothing. The graph-retrieval god-node failure mode in miniature.""" + + coverage_select: bool = True + """H2: pick completions by coverage gain per token rather than by score alone.""" + + +# --- obligations ------------------------------------------------------------- + + +class ObligationIndex: + """path -> the set of symbol/identifier obligations that file participates in. + + Obligations are the latent objects the coverage objective is defined over. They + are approximated by identifiers, which is crude but has the two properties the + objective needs: shared obligations really do indicate "these files implement one + contract", and it costs one query per corpus to build. + """ + + def __init__(self, conn: sqlite3.Connection) -> None: + self.cov: dict[str, set[str]] = {} + for path, name in conn.execute( + "SELECT f.path, s.name FROM symbols s JOIN files f ON f.id = s.file_id" + ): + self.cov.setdefault(_norm(path), set()).add(name.lower()) + for path, name in conn.execute( + "SELECT f.path, e.dst_name FROM edges e JOIN files f ON f.id = e.file_id " + "WHERE e.dst_name IS NOT NULL" + ): + self.cov.setdefault(_norm(path), set()).add(str(name).lower()) + for path in list(self.cov): + self.cov[path] |= stem_tokens(path) + + def of(self, path: str) -> set[str]: + p = _norm(path) + if p not in self.cov: + self.cov[p] = set(stem_tokens(p)) + return self.cov[p] + + +def _norm(p: str) -> str: + return p.replace("\\", "/") + + +# --- completion snippets ----------------------------------------------------- + + +def _completion_slice( + conn: sqlite3.Connection, path: str +) -> Optional[tuple[int, int, str, int, Optional[str]]]: + """A file's *contract*: its declarations, not its body (hypothesis H8). + + An accreted file is proposed because something the agent is already looking at is + coupled to it -- not because it matched the question. What the agent needs is + enough to decide whether to open it: what lives here, and what it is called. A + signature digest answers that in a fraction of the tokens a code chunk costs, and + the ratio is what makes appending completions affordable at all. + """ + rows = conn.execute( + """ + SELECT s.name, s.kind, s.signature, s.line_start, s.line_end + FROM symbols s JOIN files f ON f.id = s.file_id + WHERE f.path = ? + ORDER BY (s.parent_id IS NOT NULL), s.line_start + LIMIT 8 + """, + (path,), + ).fetchall() + if not rows: + return None + lines = [f"# {path} (related to your anchors; contract only)"] + for name, kind, sig, ls, _le in rows: + text = (sig or name or "").strip().splitlines()[0][:140] if (sig or name) else "" + if text: + lines.append(f"{ls}: {kind or 'symbol'} {text}") + body = "\n".join(lines) + if len(lines) <= 1: + return None + first, last = int(rows[0][3]), int(rows[-1][4]) + # Matches the indexer's estimator (~4 chars/token) so the accounting stays + # symmetric with every other result on the page. + return (first, max(first, last), body, max(1, len(body) // 4), rows[0][0]) + + +def _completion_chunk( + conn: sqlite3.Connection, path: str, terms: set[str] +) -> Optional[tuple[int, int, str, int, Optional[str]]]: + """Best chunk of `path` to show for an accreted file. + + An accreted file frequently shares no vocabulary with the query — that is the + whole reason it had to be reached by relation rather than by search. When no + chunk matches a query term we fall back to the file's first symbol-bearing chunk, + i.e. its declaration/header region, which is the most informative fixed-size + window for orienting an agent that has never seen the file. + """ + rows = conn.execute( + """ + SELECT c.line_start, c.line_end, c.content, c.token_est, c.symbol_names + FROM chunks c JOIN files f ON f.id = c.file_id + WHERE f.path = ? + ORDER BY c.line_start + """, + (path,), + ).fetchall() + if not rows: + return None + best = None + best_hits = 0 + for r in rows: + content = (r[2] or "").lower() + hits = sum(1 for t in terms if t and t in content) + if hits > best_hits: + best_hits, best = hits, r + if best is None: + for r in rows: + if (r[4] or "").strip(): + best = r + break + best = best or rows[0] + return (int(best[0]), int(best[1]), best[2] or "", int(best[3] or 0), best[4]) + + +# --- accretion --------------------------------------------------------------- + + +def accrete( + conn: sqlite3.Connection, + *, + anchors: list[str], + graph: RelationGraph, + obligations: ObligationIndex, + params: NucleusParams, + exclude: set[str], + query_terms: set[str], +) -> list[tuple[str, float, dict[str, float]]]: + """Score files related to the anchor set. Returns [(path, score, parts)] desc.""" + if not anchors: + return [] + contrib = contributions(anchors=anchors, graph=graph, params=params, exclude=exclude) + ranked = rank_contributions(contrib, weights=params.weights, min_score=params.min_score) + return ranked + + +def contributions( + *, anchors: list[str], graph: RelationGraph, params: NucleusParams, + exclude: set[str], +) -> dict[str, dict[str, float]]: + """Per-candidate, per-relation evidence, aggregated over anchors. + + Deliberately kept **linear in the relation weights**: the score is + `sum_r theta_r * contrib[c][r]`, so refitting theta never requires re-walking the + graph. That is what makes the leave-one-repository-out fit in `fit_weights.py` + affordable, and it means the fitted weights are fitted against exactly the + quantities the retriever will use at query time. + + The hub normaliser is folded in here (not applied to the final score) so it scales + every relation identically and cannot be confused with a relation weight. + """ + contrib: dict[str, dict[str, float]] = {} + for i, a in enumerate(anchors): + alpha = 1.0 / (1.0 + params.anchor_decay * i) + for c in graph.candidates(a): + if c in exclude: + continue + _s, parts = graph.score(a, c, params.weights) + if not any(parts.values()): + continue + acc = contrib.setdefault(c, {}) + for k, v in parts.items(): + if v <= 0.0: + continue + if params.combine == "max": + acc[k] = max(acc.get(k, 0.0), alpha * v) + else: + acc[k] = acc.get(k, 0.0) + alpha * v + + if params.hub_penalty: + # A file reachable from several anchors is genuine evidence; a file reachable + # from *everything* is a hub. The normaliser is the candidate's total relation + # degree over the corpus, so multi-anchor support is still rewarded while a + # god-node is damped -- the graph-retrieval failure mode in miniature. + for c, acc in contrib.items(): + deg = len(graph.static.get(c, {})) + len(graph.cochange_index.neighbours(c)) + f = (1.0 + deg) ** 0.5 + for k in acc: + acc[k] /= f + return contrib + + +def rank_contributions( + contrib: dict[str, dict[str, float]], *, weights: RelationWeights, min_score: float, +) -> list[tuple[str, float, dict[str, float]]]: + wd = weights.as_dict() + scored = [ + (p, sum(wd.get(k, 0.0) * v for k, v in acc.items()), acc) + for p, acc in contrib.items() + ] + scored.sort(key=lambda t: (-t[1], t[0])) + return [t for t in scored if t[1] >= min_score] + + +def _select_completions( + ranked: list[tuple[str, float, dict[str, float]]], + *, + covered: set[str], + obligations: ObligationIndex, + params: NucleusParams, + conn: sqlite3.Connection, + query_terms: set[str], +) -> list[tuple[str, float, dict[str, float]]]: + """H2: budgeted greedy coverage over obligations, instead of plain top-C. + + Score alone will happily pick three files that all cover the same contract. The + marginal-gain rule prefers a slightly weaker candidate that covers something not + yet covered, which is what "minimal sufficient set" means operationally. + """ + if not params.coverage_select: + return ranked[: params.max_completions] + + chosen: list[tuple[str, float, dict[str, float]]] = [] + seen = set(covered) + pool = list(ranked[: params.max_completions * 6]) + while pool and len(chosen) < params.max_completions: + best = None + best_gain = -1.0 + for item in pool: + path, score, _ = item + new = obligations.of(path) - seen + # Marginal value per unit of score-normalised novelty. Token cost is + # nearly constant across chunks here (chunking is size-bounded), so + # novelty is the term that actually discriminates; keeping score as a + # multiplier stops a junk file with exotic identifiers from winning. + gain = score * (1.0 + len(new)) ** 0.5 + if gain > best_gain: + best_gain, best = gain, item + if best is None: + break + chosen.append(best) + seen |= obligations.of(best[0]) + pool.remove(best) + return chosen + + +# --- the pipeline ------------------------------------------------------------ + + +def nucleus_search( + conn: sqlite3.Connection, + query: str, + *, + mode: str = "hybrid", + limit: int = 10, + token_budget: int = 1500, + tuning: Optional[RetrievalTuning] = None, + graph: Optional[RelationGraph] = None, + obligations: Optional[ObligationIndex] = None, + params: NucleusParams = NucleusParams(), + explain: bool = False, +) -> dict: + tuning = tuning or DEFAULT_TUNING + plan = detect_intent(query) + base_limit = limit + if token_budget <= 0: + token_budget = plan.token_budget + + # --- NUCLEATE: the incumbent pipeline, unmodified ----------------------- + pool_mult = max(1, tuning.candidate_pool_multiplier) + pool_limit = limit if pool_mult == 1 else max(limit * pool_mult, 20) + lists, weights = _run_retrievers( + conn, query, mode=mode, limit=pool_limit, weights=plan.weights, + backend=None, tuning=tuning, graph_depth=tuning.graph_depth, + graph_node_cap=tuning.graph_node_cap, graph_strategy=plan.graph_strategy, + ) + fused = fuse(lists, weights=weights, k=tuning.rrf_k, + file_agreement=tuning.file_agreement_weight if tuning.file_agreement else 0.0) + pool_paths: list[str] = [] + for c in fused: + p = _norm(c.path) + if p not in pool_paths: + pool_paths.append(p) + ranked = rerank(fused, query=query, intent=plan.intent, tuning=tuning) + if tuning.dedup: + ranked = deduplicate(ranked, hamming_distance=tuning.dedup_hamming) + if tuning.mmr: + ranked = mmr_select(ranked, limit, tuning.mmr_lambda) + else: + ranked = _diversify(ranked, per_file=tuning.max_per_file) + baseline = ranked[:base_limit] + + accretion_log: list[dict] = [] + final = baseline + + # --- ACCRETE + SELECT --------------------------------------------------- + if params.accrete and graph is not None and obligations is not None and baseline: + terms = {t.lower() for t in _TERM_RE.findall(query) if len(t) > 2} + head = baseline[: params.anchor_head] + anchors: list[str] = [] + for c in head: + p = _norm(c.path) + if p not in anchors: + anchors.append(p) + # Only files already on the page are excluded from accretion; a file the + # baseline ranked below the page is a legitimate promotion target. + on_page = {_norm(c.path) for c in head} + if params.insert_at != "after_head": + # With tail placement a "promotion" from mid-page to the last slot is a + # demotion, not a gain, so only genuinely new files are worth a slot. + on_page = {_norm(c.path) for c in baseline} + covered: set[str] = set(terms) + for a in anchors: + covered |= obligations.of(a) + + cands = accrete( + conn, anchors=anchors, graph=graph, obligations=obligations, + params=params, exclude=on_page, query_terms=terms, + ) + picked = _select_completions( + cands, covered=covered, obligations=obligations, params=params, + conn=conn, query_terms=terms, + ) + + split = params.anchor_head if params.insert_at == "after_head" else len(baseline) + tail = baseline[split:] + tail_by_path: dict[str, Candidate] = {} + for c in baseline[params.anchor_head:]: + tail_by_path.setdefault(_norm(c.path), c) + + promoted: list[Candidate] = [] + consumed: set[int] = set() + for path, score, parts in picked: + existing = tail_by_path.get(path) + if existing is not None: + # Promotion: reuse the baseline candidate so its query-matched + # snippet (better than a synthesized one) is preserved. + promoted.append(existing) + consumed.add(id(existing)) + else: + chunk = None + if params.compact_completions: + chunk = _completion_slice(conn, path) + if chunk is None: + chunk = _completion_chunk(conn, path, terms) + if chunk is None: + continue + ls, le, content, tok, symnames = chunk + why = ",".join(f"{k}={v:.2f}" for k, v in sorted(parts.items()) if v > 0) + promoted.append( + Candidate( + path=path, line_start=ls, line_end=le, source="accretion", + score=score, kind=None, + symbol=(symnames or "").split()[0] if symnames else None, + content=content, token_est=tok, + reason=f"accreted from anchors ({why})", + ) + ) + accretion_log.append({"path": path, "score": round(score, 4), "parts": parts}) + + if params.append_only: + # Nothing is evicted; the page grows and the token budget arbitrates. + final = [c for c in baseline if id(c) not in consumed] + promoted + elif params.insert_at == "after_head": + rest = [c for c in tail if id(c) not in consumed] + final = (head + promoted + rest)[:limit] + else: + # Tail placement: completions claim the last `len(promoted)` slots of the + # page and nothing above them moves. When the baseline under-fills the + # page the slots are free and nothing is evicted at all. + kept = [c for c in baseline if id(c) not in consumed] + keep_n = max(0, limit - len(promoted)) + final = kept[:keep_n] + promoted + + confidence = _confidence(final) + compactor = make_compactor(intent=plan.intent, query=query, enabled=True, + min_reduction=0.25) + results, recommended = apply_budget(final, token_budget=token_budget, + compactor=compactor) + + payload: dict = { + "query": query, + "intent": plan.intent.value, + "mode": mode, + "confidence": confidence.value, + "results": results, + "recommended_reads": recommended, + "fallback_suggestions": {}, + } + if explain: + payload["diagnostics"] = { + "pool": [{"path": p} for p in pool_paths], + "pool_size": len(pool_paths), + "accreted": accretion_log, + } + return payload diff --git a/research/related-work.md b/research/related-work.md new file mode 100644 index 0000000..d181149 --- /dev/null +++ b/research/related-work.md @@ -0,0 +1,250 @@ +# Stage 1 — The existing frontier and where each approach structurally stops + +This is not a literature survey for its own sake. For each family the question is +narrow: **what does this method make impossible?** A limitation that a bigger model +or a better index would fix is an engineering gap, not a fundamental one. Only the +latter justifies a new architecture. + +Throughout, `q` is a query, `d` a document/chunk, `S` a retrieved set, `t` a task. + +--- + +## 1. Sparse lexical retrieval (BM25, FTS5, tf-idf) + +**What it computes.** `score(q,d) = Σ_{w∈q} IDF(w)·tf-saturation(w,d)`. A bag-of-words +independence assumption per term, summed. + +**Fundamental limitation — the vocabulary/necessity confusion.** BM25 can only rank a +document that *contains the query's words*. For code retrieval this is not a synonym +problem (fixable with expansion) but a **causality problem**: the file you must edit +alongside `auth/token.py` is `api/service.py`, which contains none of the query's +terms and never will. No re-weighting of a term-overlap function can produce a +document with zero term overlap. The ceiling is structural, not parametric. + +Measured in this repository (`docs/BENCHMARKS.md`): the shipped hybrid ranker reaches +`recall@10 = 0.693` while `oracle = 0.90`. The residual is dominated by gold files +that share no vocabulary with the query. + +## 2. Dense retrieval / embeddings (DPR, sentence encoders, code encoders) + +**What it computes.** `score(q,d) = ⟨E(q), E(d)⟩` — a single vector per document, +compared by inner product. + +**Fundamental limitation — the single-vector bottleneck and pointwise independence.** +Two distinct failures: + +1. *Capacity.* One vector must encode everything a document could ever be relevant + for. A file participating in 30 unrelated concerns gets one point in space. This + is a known information-theoretic bound: a `k`-dimensional vector cannot preserve + all pairwise relevance orderings of a corpus beyond a critical size (the + "embedding dimension vs. retrievable set" limit). +2. *Independence.* `score` factorises over documents. The model can express + "d is similar to q" but **cannot express "d is needed only if d' is also + retrieved"**. Every retrieval-as-ranking system inherits this: the top-k of a + pointwise scorer is not a set-optimal answer, it is `k` independently good answers. + +This second point is the one that matters here, and no amount of encoder quality +removes it. It is an artifact of the *objective*, not the representation. + +## 3. Hybrid + fusion (RRF, convex combination) + +**What it computes.** `RRF(d) = Σ_r w_r / (k + rank_r(d))`. + +**Fundamental limitation.** Fusion is still pointwise: it aggregates independent +opinions about the *same* document. It adds robustness, never set-awareness. This +repository's own data shows the ceiling precisely: 1.10.0 improved reranking +efficiency 0.639 → 0.660 while `oracle` and `cand_recall` moved by less than 1e-4. +Fusion redistributes the pool; it cannot create a candidate no retriever proposed. + +## 4. ColBERT / late interaction + +**What it computes.** `score(q,d) = Σ_i max_j ⟨E_i(q), E_j(d)⟩` — token-level +MaxSim, avoiding the single-vector bottleneck. + +**Fundamental limitation.** Fixes capacity, keeps independence, and multiplies index +size by ~`|d|`. Still `argmax` over documents scored in isolation, so it inherits §2.2 +in full. Late interaction makes similarity finer, not necessity computable. + +## 5. ANN indexes: HNSW, IVF, PQ, LSH + +**What they compute.** Approximate `argmax_d ⟨q,d⟩` in sublinear time. + +**Fundamental limitation — they optimise the wrong operation faster.** These are +accelerators for a metric-space nearest-neighbour query. They presuppose that +"relevant" = "near in a fixed metric". If the target relation is *asymmetric*, +*conditional* (`d` needed given `d'`), or *non-metric* (violates triangle +inequality — necessity plainly does: A needs B, B needs C, A may not need C), the +data structure has no way to represent it. PQ additionally trades recall for memory +in a way that is invisible to the caller — a silent quality knob. + +## 6. Graph retrieval / GraphRAG / knowledge graphs / PPR + +**What it computes.** Entry by similarity, then diffusion (personalised PageRank, +community summarisation, multi-hop traversal). + +**Fundamental limitation — undirected diffusion is not necessity, and it competes for +the same slots.** Two concrete failures, both visible in this repository: + +1. *Isotropy.* PPR spreads probability along every edge type at once. "Related to" + is not "required for". A hub file is reached from everywhere and therefore ranks + highly for everything — the god-node problem, documented in + `tests/benchmark_honest_RESULTS.md`. +2. *Slot competition.* When graph neighbours are injected as another retriever into + the fusion, they displace direct lexical hits. This repo measured exactly that and + **shipped `graph_source = False`** (`retrieval/tuning.py`): "the self-repository + ablation showed lower MRR when architectural neighbors displaced direct lexical + hits." + +This is the single most informative prior result available to me. The failure was not +"structure is useless" — my Stage-2 diagnostic shows resolved edges connect 30.7% of +anchor→target gold pairs. The failure was **architectural**: structural evidence was +forced to compete pointwise with lexical evidence in one ranked list, at one weight, +for one budget. That diagnosis is what the proposed architecture is built around. + +## 7. GraphRAG specifically + +Community detection + LLM-generated community summaries, queried by similarity. +**Limitation:** the summarisation is lossy and *query-independent* — it must guess in +advance which facts matter. It also costs an LLM pass over the whole corpus, which +makes incremental update expensive: one edited file can change community membership. +Good for global sensemaking questions, structurally poor at "which four files must I +edit". + +## 8. AST / symbol / code-structure indexing (LSP, tree-sitter, ctags, Aider repo-map) + +**What it computes.** Exact symbol definitions/references; repo-map ranks files by +PageRank over the symbol graph. + +**Fundamental limitation — static structure is *incomplete*, not merely noisy.** It +sees only relations the language makes explicit. It cannot see: a schema and the +migration that must accompany it; a config key and its consumer; a feature flag and +its test; a protocol and its two independent implementations. My diagnostic quantifies +this: static edges cover 30.7% of the pairs, while 9.8% are reachable *only* by +history. Structural indexing is a high-precision, low-recall relation. + +## 9. Evolutionary coupling / MSR (Zimmermann's ROSE, Ying et al.) + +**What it computes.** Association rules over co-changed files mined from VCS history. + +**Fundamental limitation.** Cold start (new files have no history), drift (couplings +decay as the design changes), and — decisively for benchmark honesty — **it is +trivially leaky if evaluated without a temporal split**. It is also a *recommender* +for an already-known seed file, not a retriever from a natural-language question. It +answers "what else changes with X", never "what is X". + +That is precisely why it composes with §1: lexical retrieval is good at finding X from +a question, and bad at finding what accompanies X. + +## 10. Semantic caching of LLM calls + +**What it computes.** Reuse a cached answer when `sim(q, q_cached) > τ`. + +**Fundamental limitation — it is unsound by construction.** Validity of a cached +answer depends on whether the *world it was computed from* has changed, and query +similarity carries no information about that. Two identical questions asked before and +after an edit must get different answers; a semantic cache returns the stale one with +high confidence. There is no threshold that fixes this, because the failure is not in +the similarity estimate — it is that the cache key omits the dependency. + +This is the gap the memory plane of the proposed architecture targets. + +## 11. Context compression (LLMLingua, RECOMP, selective context) + +**What it computes.** Drop low-information tokens from an already-retrieved context. + +**Fundamental limitation — it operates strictly downstream of a bad selection.** +Compression can shrink what you retrieved; it cannot retrieve what you missed. If the +required file was never in `S`, no compressor recovers it. Compression improves the +constant factor on `tokens`, and does nothing for `recall`. + +## 12. KV-cache reuse / prefix caching + +**Limitation.** Reuse is keyed on *literal token-prefix identity*. Any reordering or +one-token edit invalidates everything downstream. It is a systems optimisation with +zero semantic model of what the cached computation depended on — the same blind spot +as §10, at a different layer. + +## 13. Memory-augmented agents (episodic / semantic / procedural stores) + +**Limitation — write-time amnesia about provenance.** Nearly all implementations store +*what was concluded* and not *what it was concluded from*. Consequently they cannot +invalidate: a memory is retired by recency heuristics or LLM-judged staleness, never +by the fact that its evidence changed. They also store natural-language conclusions, +which cannot be checked for consistency automatically. + +## 14. Multi-agent routing / mixture-of-agents / task graphs + +**Limitation.** Routing is normally learned over *task descriptions*, so two agents +working on overlapping code are not detected as overlapping. Coordination cost grows +with agent count because there is no shared, addressable substrate — agents exchange +prose, so identical intermediate work is redone with no way to notice. + +## 15. Learned / neural indexes (DSI, generative retrieval) + +**Limitation.** The corpus is baked into model weights, so incremental update means +retraining or a fragile patching scheme. For a codebase changing hourly this is +disqualifying. Also inherits pointwise independence. + +--- + +## The common root + +Reading down the list, four distinct families collapse into **one** structural +assumption: + +> **Retrieval = rank documents independently by similarity to the query, then take the +> top k.** + +Everything above is an optimisation of some part of that sentence: better `sim` +(§2,§4), faster `argmax` (§5), more robust aggregation (§3), a wider candidate net +(§6,§8,§9), or post-hoc cleanup (§11). + +Three consequences follow, and none of them are fixable inside the assumption: + +1. **Sets are never optimised.** The objective factorises over documents, so + "necessary together" is inexpressible. The top-k of a pointwise scorer is `k` + individually-plausible documents, which is not the same object as a sufficient set. +2. **Necessity is conflated with similarity.** The system answers "what looks like the + question", where the agent needs "what must be true for the task to be completed". + These coincide only for lookup questions. +3. **Reuse is unsound.** Nothing records what a conclusion depended on, so nothing can + be safely reused after the corpus changes. + +Stage 2 attacks these three directly. + +--- + +## Prior art most adjacent to the proposal (stated up front, not buried) + +Honesty requires naming these before claiming anything. The proposed architecture is +**not** the first system to use co-change or structure for code retrieval: + +- **Zimmermann et al., "Mining Version Histories to Guide Software Changes" (ROSE, + ICSE'04 / TSE'05)** — association-rule co-change recommendation. Directly prior art + for the evolutionary relation. Differences: ROSE is seeded by a file the developer + already opened, not by a natural-language query; it recommends *changes*, not + retrieval context; and it optimises confidence/support, not a token budget. +- **Ying et al., "Predicting Source Code Changes by Mining Change History" (TSE'04)** — + frequent-pattern mining, same family, same seeding assumption. +- **Aider's repo-map / PageRank over the symbol graph** — structural ranking of files + for LLM context. Query-independent global ranking; no conditional completion, no + history, no budgeted coverage objective. +- **CoCoMIC, RepoHyper, RepoFusion, GraphCodeBERT** — cross-file code completion using + dependency context. Closest in spirit on the structural side; targeted at *code + completion at a cursor*, where the seed is given by construction, rather than at + retrieval from a task description. +- **Build systems (Bazel, Nix, `ccache`) and incremental computation (Adapton, + self-adjusting computation)** — content-addressed action caches with dependency-keyed + invalidation. This is the direct ancestor of the memory plane; the transplant to + *agent reasoning* is the part I claim as new, and §8 of `novelty.md` scopes that + claim carefully. +- **Submodular / facility-location summarisation (Lin & Bilmes 2011), MMR** — budgeted + set selection in IR. Prior art for the selection stage's mathematics. MMR maximises + *dissimilarity*; the objective proposed here maximises *coverage of implicated + obligations*, which is a different function with a different optimum. + +The claim under test is therefore not "co-change is new" or "greedy coverage is new". +It is that **conditional set-completion, budget-separated from anchor ranking, is the +composition that makes structural and historical evidence pay off where injecting it +into fusion measurably does not** — a claim this repository's own shipped +`graph_source = False` makes falsifiable rather than rhetorical. diff --git a/research/selection.md b/research/selection.md new file mode 100644 index 0000000..b00d888 --- /dev/null +++ b/research/selection.md @@ -0,0 +1,95 @@ +# Stage 4 — Scoring, TOP-3, and composition + +Scores are 1–5. `Feasible` means *testable with evidence available in this +environment* (eight real repositories, git history, no LLM API, no GPU) — a +hypothesis that can only be validated by machinery I do not have scores low no matter +how good it is, and is recorded as follow-up rather than silently dropped. + +| # | Hypothesis | Novelty | Theory | Feasible | Scale | Δtokens | Δlatency | Quality | Multi-agent | Risk | +|---|---|---|---|---|---|---|---|---|---|---| +| H1 | Conditional set completion | 4 | 5 | **5** | 5 | 3 | 4 | **5** | 3 | 2 | +| H2 | Budgeted coverage selection | 3 | 5 | **5** | 5 | **5** | 4 | 4 | 3 | 2 | +| H3 | Evidence-keyed memoisation | 4 | 4 | **5** | 5 | **5** | **5** | 3 | **5** | 2 | +| H4 | Granularity law of survival | 4 | 4 | **5** | 4 | 4 | 4 | 2 | 4 | 1 | +| H5 | Entropy-minimising retrieval | 4 | **5** | 1 | 2 | 4 | 1 | 4 | 2 | 4 | +| H6 | ddmin oracle for minimal context | **5** | **5** | 1 | 1 | 5 | 1 | 5 | 2 | 3 | +| H7 | Trace prefetching | 4 | 4 | 1 | 4 | 2 | 5 | 3 | 3 | 3 | +| H8 | Obligation-level slices | 2 | 3 | 4 | 5 | **5** | 4 | 2 | 2 | 1 | +| H9 | Index the derivative | 4 | 3 | 2 | 3 | 2 | 3 | 3 | 2 | **5** | +| H10 | Self-organising index | 3 | 4 | 1 | 4 | 2 | 2 | 4 | 4 | 4 | +| H11 | Obligation routing | 4 | 4 | 2 | 4 | 3 | 3 | 2 | **5** | 3 | +| H12 | Negative-result atoms | 4 | 2 | 3 | 4 | 4 | 4 | 2 | **5** | 2 | +| H13 | Speculative closure | 2 | 2 | 3 | 3 | 1 | 2 | 1 | 2 | 3 | + +## TOP-3 + +**H1 — conditional set completion.** Highest expected quality gain, premise already +measured at 71.8% reachability, and it has a *falsifiable prior*: this repository +shipped `graph_source = False` after structural evidence failed inside fusion. If H1's +composition claim is wrong, the experiment will reproduce that failure and say so. + +**H2 — budgeted coverage selection.** The only candidate that attacks token cost at the +objective level rather than by post-hoc compression, and it is the natural consumer of +H1's output: once completion produces a *set* of candidates with overlapping coverage, +top-k is provably the wrong selector. + +**H3 — evidence-keyed memoisation** (with **H4** as its measurement). Orthogonal to +H1/H2 — it addresses reuse, not retrieval — and it is the only one of the thirteen +that changes a *correctness* property rather than a quality metric: unsound reuse goes +to zero by construction. H4 makes it quantitative and is nearly free to measure once +H3 exists. + +## Rejected, with reasons + +- **H6** (ddmin oracle) is the single most valuable idea in the list and I cannot run + it: it needs `O(|S| log|S|)` LLM-agent executions per query plus a per-repo test + harness. Recorded as the top follow-up. Its absence is also the main threat to + validity of everything below, because it means I inherit the proxy label "files the + commit touched" rather than true minimal sufficiency — stated again in `experiments.md`. +- **H5, H7, H10** need a live agent workload or LLM scorer. Deferred, not disproven. +- **H13** rejected on measurement: closure latency turns out to be ~1 ms (§ results), + so there is nothing to amortise. +- **H9** carries an unacceptable circularity risk against a commit-derived benchmark; + its safe fragment (co-change under strict temporal split) is absorbed into H1. +- **H8** is largely already implemented here (`retrieval/skeleton.py`); it enters as a + cost model, not a claim. + +## Composition + +H1, H2, H3 compose without conflict because they act on different objects: + +``` + query + │ + ▼ + ┌─────────────┐ H1: anchors are found by the existing high-precision + │ NUCLEATE │ lexical/symbol retriever — unchanged, and deliberately so + └─────────────┘ + │ A₀ + ▼ + ┌─────────────┐ H1: completion conditioned on A₀, NOT on the query; + │ ACCRETE │ typed relation union {edge, cochange, testlink, stem, dir} + └─────────────┘ + │ candidates with coverage sets + ▼ + ┌─────────────┐ H2: greedy coverage-per-token under budget B, + │ SELECT │ replacing top-k + MMR + └─────────────┘ + │ S + ▼ + ┌─────────────┐ H3/H4: conclusions computed over S are stored keyed by + │ MEMOISE │ H(content hashes of S); invalidation is exact + └─────────────┘ +``` + +The composed system is specified in `architecture.md` under the working name +**NUCLEUS** (*Necessity-driven Unified Closure over Lexical, Evolutionary and +Structural relations*). + +One property of this composition is worth stating before any measurement, because it +is what makes the design defensible against the prior negative result: **accretion +never displaces an anchor.** Completions are allocated their own budget and appended +behind the anchor head, so a completion can only take a slot that a *lower-ranked +baseline result* would have taken. The 2015-vintage failure mode of graph retrieval — +architectural neighbours evicting direct hits — is excluded by construction rather +than by tuning. Whether that is enough is Stage 6's problem. diff --git a/research/test_nucleus.py b/research/test_nucleus.py new file mode 100644 index 0000000..5eeb087 --- /dev/null +++ b/research/test_nucleus.py @@ -0,0 +1,129 @@ +"""Correctness tests for the NUCLEUS prototype. + +Run explicitly (not part of the product suite, which is scoped to `tests/`): + + python -m pytest research/test_nucleus.py --no-cov -q + +The first test is the load-bearing one. Every headline delta in +`research/experiments.md` is "NUCLEUS vs the shipped pipeline", which is only a valid +attribution if NUCLEUS with accretion disabled *is* the shipped pipeline. If that +equivalence ever breaks, the benchmark is measuring an unrelated reimplementation and +the numbers mean nothing. +""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +import pytest + +from codebase_index.retrieval.pipeline import search +from research.nucleus.relations import ( + CoChangeModel, RelationGraph, core_stem, is_testish, load_static_edges, stem_tokens, +) +from research.nucleus.search import NucleusParams, ObligationIndex, nucleus_search + +INDEX = Path(__file__).parent / "data" / "index" / "codebase-index.sqlite" + +QUERIES = [ + "redact secrets before output", + "incremental index update", + "graph edge confidence", + "token budget for snippets", + "fuzzy symbol matching", +] + + +requires_index = pytest.mark.skipif( + not INDEX.exists(), reason="run research/build_indexes.py first" +) + + +@pytest.fixture() +def conn(): + c = sqlite3.connect(INDEX) + c.row_factory = sqlite3.Row + yield c + c.close() + + +@requires_index +@pytest.mark.parametrize("q", QUERIES) +def test_accretion_off_reproduces_shipped_pipeline(conn, q): + """NUCLEUS(accrete=False) must equal codebase_index's own search(), result by result.""" + theirs = search(conn, q, mode="hybrid", limit=10, token_budget=1500, + no_fallback=True, explain=False) + ours = nucleus_search(conn, q, limit=10, token_budget=1500, + params=NucleusParams(accrete=False)) + + def shape(payload): + return [ + (r["path"], r["line_start"], r["line_end"], r["token_est"], + (r.get("snippet") or "")[:200]) + for r in payload["results"] + ] + + assert shape(ours) == shape(theirs) + assert ours["intent"] == theirs["intent"] + assert ours["confidence"] == theirs["confidence"] + + +@requires_index +def test_accretion_changes_the_page(conn): + """Sanity: with a relation graph present, accretion actually does something.""" + files = [r[0].replace("\\", "/") for r in conn.execute("SELECT path FROM files")] + graph = RelationGraph(files, static=load_static_edges(INDEX), + cochange=CoChangeModel(commits=[])) + obl = ObligationIndex(conn) + changed = 0 + for q in QUERIES: + off = nucleus_search(conn, q, limit=10, token_budget=1500, + params=NucleusParams(accrete=False)) + on = nucleus_search(conn, q, limit=10, token_budget=1500, graph=graph, + obligations=obl, params=NucleusParams()) + if [r["path"] for r in off["results"]] != [r["path"] for r in on["results"]]: + changed += 1 + assert changed > 0, "accretion never altered any page — the graph is not wired in" + + +@requires_index +def test_anchor_head_is_never_displaced(conn): + """The design's central safety property, asserted rather than asserted-in-prose. + + A completion may only take a slot a lower-ranked baseline result would have held. + If this fails, NUCLEUS has reintroduced exactly the failure mode that made this + repository ship `graph_source = False`. + """ + files = [r[0].replace("\\", "/") for r in conn.execute("SELECT path FROM files")] + graph = RelationGraph(files, static=load_static_edges(INDEX), + cochange=CoChangeModel(commits=[])) + obl = ObligationIndex(conn) + params = NucleusParams() + for q in QUERIES: + off = nucleus_search(conn, q, limit=10, token_budget=1500, + params=NucleusParams(accrete=False)) + on = nucleus_search(conn, q, limit=10, token_budget=1500, graph=graph, + obligations=obl, params=params) + head = params.anchor_head + assert [r["path"] for r in on["results"][:head]] == \ + [r["path"] for r in off["results"][:head]] + + +def test_cochange_model_refuses_lookahead(): + """The no-lookahead guarantee is a property of the data structure, not a habit.""" + m = CoChangeModel(commits=[(5, ["a.py", "b.py"]), (3, ["a.py", "c.py"])]) + m.advance_to(4) + assert m.confidence("a.py", "b.py") > 0 # commit at position 5 is older + assert m.confidence("a.py", "c.py") == 0.0 # position 3 is newer, must be unseen + with pytest.raises(ValueError): + m.advance_to(9) # rewinding would reveal the future + + +def test_relation_primitives(): + assert core_stem("tests/test_service.py") == "service" + assert core_stem("src/Service.java") == "service" + assert is_testish("tests/test_service.py") + assert not is_testish("src/service.py") + assert "service" in stem_tokens("src/user_service.py") + assert "user" in stem_tokens("src/UserService.java") From 3a79062f2e3174703baa11af0ce899ae5540bd70 Mon Sep 17 00:00:00 2001 From: denfry Date: Thu, 10 Sep 2026 09:35:36 +0300 Subject: [PATCH 03/18] research: 2.0 repository audit and 1.10.0 baselines --- research/2.0-audit.md | 175 ++++++++++++++++++ tests/eval/results/1.10.0-perf.json | 14 ++ .../eval/results/1.10.0-retrieval-8corpus.txt | 58 ++++++ 3 files changed, 247 insertions(+) create mode 100644 research/2.0-audit.md create mode 100644 tests/eval/results/1.10.0-perf.json create mode 100644 tests/eval/results/1.10.0-retrieval-8corpus.txt diff --git a/research/2.0-audit.md b/research/2.0-audit.md new file mode 100644 index 0000000..56ed1b7 --- /dev/null +++ b/research/2.0-audit.md @@ -0,0 +1,175 @@ +# codebase-index 2.0 — Phase A repository audit + +Audit date: 2026-09-10. Scope: the whole repository at `release/1.10.0` (committed as +`90afa71 release: v1.10.0` before any 2.0 work, so 2.0 has a clean, attributable base), +plus the untracked `research/` spike. No production behaviour was changed during this +phase. + +## 1. Current state + +| Item | Finding | +|---|---| +| Version in tree | `1.10.0` (`src/codebase_index/__init__.py`, plugin manifest, skill stamps, lock tag) | +| Last tag on `main` | `v1.9.0` (merge `b48ed14`); 1.10.0 was staged locally, never merged or tagged | +| Python | `>=3.11`; CI matrix Ubuntu/macOS/Windows × 3.11/3.12/3.13 | +| Runtime deps | typer, rich, pydantic, pathspec, tree-sitter (pinned), tree-sitter-language-pack (pinned) | +| Tests | 536 collected, all pass (4 platform skips); `ruff check src tests` clean; `mypy` clean (61 files) | +| Research spike | `research/` untracked; `research/data/` git-ignored (private corpora commit subjects + SQLite indexes) | +| Working tree anomaly | `tests/fixtures/sample_repo/node_modules/leftpad/index.js` was deleted (unstaged). It is the fixture that proves the dependency-dir gate; `test_discovery` passed vacuously without it. Restored from HEAD. | + +## 2. Architecture map + +``` +discovery/ (walker, ignore, classify) gates: ignore files, built-in denylist, + │ secret filenames, NUL sniff, size + ▼ +parsers/ (tree-sitter Tier A/B, line chunker) chunks (symbol bodies / 80-line windows / doc), + │ symbols, raw edges + ▼ +indexer/pipeline.py build_index / update_index (mtime+size fast path, sha256 confirm) + │ graph/builder.py resolves edges globally + ▼ +storage/ index.sqlite (WAL): files, symbols, chunks(+FTS5 triggers), edges, modules, meta, + optional vec_* tables. SCHEMA_VERSION = 3. + ▲ + │ reads +retrieval/pipeline.search: intent → retrievers(fts, symbol, path, [vector], [graph]) → RRF + (+file agreement) → rerank (priors, name co-occurrence) → dedup → per-file packing + → confidence → skeleton compactor → apply_budget (greedy, redact) → paginate + ▲ +service.py (shared by CLI and MCP: db/config resolution, search/diff-impact/architecture/stats) + ├── cli.py (Typer) ── skill wrappers (skill_template/scripts/cbx, cbx.ps1; bin/cbx for plugin) + └── mcp/server.py (FastMCP, stdio; every payload in {schema_version: 1, tool} envelope) +``` + +Freshness (`indexer/freshness.py`) is a single index-level signal: git fast path (HEAD equals +indexed head and a clean tree) or a full walk diffing `(mtime, size, sha256)`. + +## 3. Reusable components for 2.0 + +| Component | Why it matters for evidence memory | +|---|---| +| `service.py` | One place to integrate memory for both CLI and MCP, preventing surface drift. | +| `storage/db.py` `Database` | WAL + pragmas + version-guard pattern to reuse for a memory store. | +| `discovery.walker/classify/ignore` | Gates must be re-applied before any evidence is read or retained. | +| `retrieval/budget.apply_budget` | The exact point where a snippet is *delivered* — the natural evidence mint point. | +| `retrieval.pipeline.search(explain=…)` | Precedent for an opt-in, zero-cost-when-off extension of the payload. | +| `tests/eval/metrics.py` | Paired bootstrap CI + permutation p (seeded) — reused verbatim by any new benchmark. | +| `tests/eval/gen_queries.py` output | Each query carries its `commit`, which makes a real-history replay benchmark possible. | +| `tests/golden_utils.py` | Golden normalisation for CLI/MCP contract snapshots. | +| `research/nucleus/memory.py` | Survival measurement via `git diff --unified=0` hunks (validated H3/H4 method). | +| `scripts/sync_skill_copies.py` | Single source for skill text + version stamps across five committed copies. | + +## 4. Technical debt found (verified by reading the code) + +These are pre-existing and independent of memory; the ones that touch 2.0 correctness or +documentation honesty are called out. + +1. **No schema migration framework.** `Database._guard_version` tolerates older schemas on + open; `index`/`update` *delete the DB file* when `schema_version < SCHEMA_VERSION` + (`cli.py` `index`/`update`). Anything stored in `index.sqlite` is therefore destroyed by + a rebuild or a schema bump. **Direct constraint on where memory may live.** +2. **Two line-numbering models inside the indexer.** Chunkers slice with `str.splitlines()` + (`parsers/line_chunker.py`, `parsers/symbol_chunks.py`), tree-sitter symbol rows count + `\n`, doc chunks count `\n`. They agree except for files containing `\f`, `\v`, + `\x1c-\x1e`, `\x85`, `
`, `
`. Evidence hashing must follow the chunkers' + model so a snippet and its hashed span are the same bytes. +3. **Search trusts index content without checking the working tree.** Snippets come from + `chunks.content`; the only staleness signal is index-level. `_git_clean_at` reports + fresh when HEAD equals the indexed head and the tree is clean, which is wrong for an + index built from a dirty tree that was later reverted (by inspection). +4. **`index` and `update` hardcode the DB path** (`cli.py`), ignoring `CBX_DB_PATH`, while + every read path honours it via `service.resolve_db`. +5. **Documentation claims more security than is implemented.** + - Redaction: code implements private keys, AWS access keys, and assigned secrets + (`output/redact.py`). JWTs, bearer tokens, connection strings, and Slack tokens are + documented but not implemented. Markers are `<>`, docs say `«…»`. + - Secret filenames: code excludes `.env`, `.env.*`, `id_rsa`, `id_ed25519`, + `credentials.json`, `service-account.json`, `secrets.json`, `*.pem|key|p12|pfx`. + Docs list `*.crt`, `*.keystore`, `credentials*`, `secrets*`, `id_rsa*` (the prefix + forms would also exclude legitimate source such as `secrets.py`). + - Generated files are *indexed and demoted* (`is_generated`), not excluded; `*.lock`, + `*.pb.go`, `*_pb2.py` are not recognised. `vendor/` is not in the denylist. Binary + detection is a NUL sniff only. + - `doctor` implements 5 checks (cache ignored, hooks, freshness, symbol extraction, + graph coverage); the secret-leak scan, endpoint report, permissions check and + allowed-tools diff are documented but absent. +6. **Plugin wrapper whitelist is stale.** `bin/cbx` omits `architecture`, `diff-impact`, + `path`, `describe`, which the plugin skill tells the agent to run. +7. **`docs/DATABASE_SCHEMA.md` describes a schema that does not exist** (`language`, + `content_hash`, `chunk_index`, `summaries` table…); `docs/SCHEMA.md` is the accurate one. +8. **The commit-derived retrieval benchmark retrieves at HEAD**, i.e. after the change the + query describes. Symmetric across variants, so deltas are valid, but absolute scores are + optimistic. A replay benchmark can retrieve at the parent commit instead. + +## 5. Research artifacts and the constraints they impose + +| Result (`research/experiments.md`) | Constraint on 2.0 | +|---|---| +| Anchor-and-expand: MRR −0.0073 (p<0.001); recovery 0.0310 < eviction cost 0.0347 | No default graph/co-change expansion that spends rank slots. | +| Contract slices appended: useful@budget +0.0056 (p=0.030) for +6.4% tokens; useful/token 5.22e-4 vs 5.51e-4 | Not shipped as a retrieval feature; `hybrid13` page-matched control is mandatory for any future attempt. | +| LORO weight fit: in-sample +46%, held-out −0.0024, 1/8 folds | No fitted weights; conservative defaults only. | +| Span survival 0.795 / 0.521 / 0.309 at 1/10/20 commits; semantic-cache stale 20.5% / 47.9% / 69.1% | Reuse must be keyed on evidence identity and validated against current bytes, never on query similarity. | +| Span-keyed retains 1.36× file-keyed reuse at h=10 | Evidence granularity should be the delivered span, not the file. | +| 1738 atom reads → 1098 distinct; 42.0% of evidence tokens repeated | Upper bound for dedup when earlier evidence is still in the consumer's context. | +| Whole-conclusion reuse unmeasured; LSA is not a neural encoder; ground truth is a proxy | No claims about conclusion caching, embeddings, or agent success. | +| Prior art: invalidation contracts (2609.00243), PlanFence (2609.03340), Bazel/Nix | No mechanism-novelty claim; contribution is a local, tested implementation plus measurements. | + +The research measured atom *sharing*; it did not measure where the repeated tokens can +actually be withheld. A repeated atom saves tokens only if the consumer still holds the +earlier copy. That is the central design question for Phase B. + +## 6. Baselines measured for the 2.0 regression boundary + +### Retrieval (tests/eval/run_eval.py, 420 queries, 8 corpora, re-run today) + +Raw output: `tests/eval/results/1.10.0-retrieval-8corpus.txt`. + +| Metric | 1.9.0 (pinned) | 1.10.0 default | Δ | p | +|---|---|---|---|---| +| MRR | 0.560 | 0.580 | +0.0200 | 0.002 | +| recall@10 | 0.623 | 0.687 | +0.0631 | <0.001 | +| nDCG@10 | 0.522 | 0.554 | +0.0316 | <0.001 | +| useful@budget | 0.583 | 0.613 | +0.0296 | 0.011 | +| tokens/query | 1099 | 1084 | −16 | — | +| p50 / p95 ms | 48.6 / 95.0 | 53.0 / 102.3 | | | + +Absolute values are below the published 1.10.0 table (MRR 0.5955) because several sibling +corpora are live working trees that changed after 2026-09-02 (two have >100 uncommitted +files). The paired delta against 1.9.0 reproduces (+0.020 vs +0.019 published). **Any 2.0 +retrieval comparison must be run back-to-back against this pinned commit on the same trees.** + +### Performance (this repository, 311 files; `tests/eval/results/1.10.0-perf.json`) + +| Measure | 1.10.0 | +|---|---| +| Cold index | 4.32 s (5 636 chunks, 1 242 symbols) | +| Index DB size | 5.8 MB | +| No-op `update` | 0.139 s | +| One-file `update` | 0.251 s | +| Query latency (123 queries × 3, in-process) | p50 38.1 ms, p95 59.8 ms | + +## 7. 2.0 integration points + +- `retrieval/pipeline.search`: after budgeting/pagination, the delivered page and its + candidates are known — an opt-in hook here can validate and annotate without touching + ranking. +- `service.search_payload`: session resolution, memory store access, config gating. +- CLI `search`/`explain` options; MCP `search_code`/`explain_code` parameters. +- `stats`, `doctor`, MCP `index_stats`/`healthcheck`: additive diagnostics. +- Skill template + both wrapper whitelists (`skill_template/scripts/cbx{,.ps1}`, `bin/cbx`), + `tests/test_packaging.py` asserts the whitelist string. +- `Config`: a new section must stay out of `config_hash` (it is not indexing-relevant). + +## 8. Compatibility risks + +| Risk | Mitigation to design for | +|---|---| +| CLI/MCP goldens (`tests/golden/*.json`) freeze payload shape | Default packets must be byte-identical when no memory feature is engaged. | +| MCP `schema_version` 1 contract | Only additive fields; no removals or type changes. | +| Index schema bump forces a full rebuild for every 1.x user | Avoid touching index tables. | +| Storing new state in `index.sqlite` would be deleted by `index --rebuild` / `clean` | Store memory separately. | +| Concurrent `update` (long write txn) vs search-time writes | Separate WAL file, short transactions, busy timeout, degrade instead of fail. | +| Parallel agent tool calls against one DB | Idempotent upserts; never lose information on a race. | +| Windows: case-insensitive paths, CRLF checkouts, file locking | Canonical repo identity, CRLF-insensitive hashing, no long-held locks. | +| Skill auto-update rewrites installed copies on version change | Sync script and stamps must move together to 2.0.0. | diff --git a/tests/eval/results/1.10.0-perf.json b/tests/eval/results/1.10.0-perf.json new file mode 100644 index 0000000..fe26f24 --- /dev/null +++ b/tests/eval/results/1.10.0-perf.json @@ -0,0 +1,14 @@ +{ + "cold_index_s": 4.316, + "files": 311, + "chunks": 5636, + "symbols": 1242, + "db_bytes": 5820416, + "noop_update_s": 0.139, + "one_file_update_s": 0.251, + "one_file_update_indexed": 1, + "queries": 123, + "query_p50_ms": 38.14, + "query_p95_ms": 59.77, + "query_mean_ms": 39.28 +} \ No newline at end of file diff --git a/tests/eval/results/1.10.0-retrieval-8corpus.txt b/tests/eval/results/1.10.0-retrieval-8corpus.txt new file mode 100644 index 0000000..bf0b041 --- /dev/null +++ b/tests/eval/results/1.10.0-retrieval-8corpus.txt @@ -0,0 +1,58 @@ +corpus: codebase-index (../codebase-index) � 87 queries from research/data/codebase-index.yml +corpus: Civitas (../Civitas) � 64 queries from research/data/Civitas.yml +corpus: PoliternalSite (../PoliternalSite) � 120 queries from research/data/PoliternalSite.yml +corpus: PoliternalParkour (../PoliternalParkour) � 64 queries from research/data/PoliternalParkour.yml +corpus: TerraForge (../TerraForge) � 17 queries from research/data/TerraForge.yml +corpus: denfry.github.io (../denfry.github.io) � 39 queries from research/data/denfry.github.io.yml +corpus: DevGraph (../DevGraph) � 12 queries from research/data/DevGraph.yml +corpus: Windows-Cleaner-and-Optimizer-main (../Windows-Cleaner-and-Optimizer-main) � 17 queries from research/data/WinCleaner.yml +pooled: 420 queries across 8 corpora +building index for codebase-index (once, shared by all variants)... +building index for Civitas (once, shared by all variants)... +building index for PoliternalSite (once, shared by all variants)... +building index for PoliternalParkour (once, shared by all variants)... +building index for TerraForge (once, shared by all variants)... +building index for denfry.github.io (once, shared by all variants)... +building index for DevGraph (once, shared by all variants)... +building index for Windows-Cleaner-and-Optimizer-main (once, shared by all variants)... + +| label | recall@5 | recall@10 | MRR | oracle | eff | nDCG@10 | hit@3 | P@5 | MAP | useful@budget | tokens | dup% | p50_ms | p95_ms | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +| baseline (1.7.0) | 0.376 | 0.481 | 0.318 | 0.714 | 0.446 | 0.329 | 0.383 | 0.130 | 0.256 | 0.468 | 508.1 | 1.3 | 24.2 | 40.1 | +| previous release (1.9.0) | 0.589 (+0.214) | 0.623 (+0.142) | 0.560 (+0.242) | 0.900 (+0.186) | 0.622 (+0.177) | 0.522 (+0.193) | 0.655 (+0.271) | 0.190 (+0.059) | 0.452 (+0.196) | 0.583 (+0.115) | 1099.4 (+591.3) | 0.0 (-1.3) | 48.6 (+24.4) | 95.0 (+54.9) | +| default (all signals) | 0.610 (+0.235) | 0.687 (+0.205) | 0.580 (+0.262) | 0.900 (+0.186) | 0.645 (+0.199) | 0.554 (+0.225) | 0.667 (+0.283) | 0.197 (+0.067) | 0.475 (+0.219) | 0.613 (+0.144) | 1083.6 (+575.5) | 0.0 (-1.3) | 53.0 (+28.9) | 102.3 (+62.2) | + +paired comparison: default (all signals) vs previous release (1.9.0) (n=420) +| metric | delta | 95% CI | p | sig | +|---|---|---|---|---| +| recall@5 | +0.0210 | [+0.0111,+0.0329] | 0.000 | yes | +| recall@10 | +0.0631 | [+0.0429,+0.0861] | 0.000 | yes | +| MRR | +0.0200 | [+0.0076,+0.0326] | 0.002 | yes | +| nDCG@10 | +0.0316 | [+0.0213,+0.0425] | 0.000 | yes | +| hit@3 | +0.0119 | [+0.0000,+0.0262] | 0.175 | no | +| P@5 | +0.0075 | [+0.0043,+0.0113] | 0.000 | yes | +| MAP | +0.0230 | [+0.0127,+0.0348] | 0.000 | yes | +| useful@budget | +0.0296 | [+0.0056,+0.0538] | 0.011 | yes | +| oracle | +0.0000 | [+0.0000,+0.0000] | 1.000 | no | +| cand_recall | +0.0000 | [+0.0000,+0.0000] | 1.000 | no | + +Per-corpus (default config): +| label | recall@5 | recall@10 | MRR | oracle | eff | nDCG@10 | hit@3 | P@5 | MAP | useful@budget | tokens | dup% | p50_ms | p95_ms | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +| codebase-index | 0.548 | 0.634 | 0.649 | 0.931 | 0.697 | 0.552 | 0.736 | 0.226 | 0.459 | 0.617 | 1193.3 | 0.0 | 53.1 | 80.7 | +| Civitas | 0.546 | 0.633 | 0.572 | 0.938 | 0.610 | 0.529 | 0.656 | 0.175 | 0.462 | 0.548 | 921.5 | 0.0 | 55.0 | 137.8 | +| PoliternalSite | 0.560 | 0.631 | 0.548 | 0.875 | 0.626 | 0.516 | 0.625 | 0.184 | 0.441 | 0.570 | 1101.8 | 0.0 | 72.1 | 124.8 | +| PoliternalParkour | 0.695 | 0.809 | 0.488 | 0.875 | 0.557 | 0.561 | 0.609 | 0.147 | 0.479 | 0.680 | 931.9 | 0.0 | 47.8 | 112.0 | +| TerraForge | 0.706 | 0.740 | 0.669 | 1.000 | 0.669 | 0.646 | 0.824 | 0.282 | 0.571 | 0.549 | 933.8 | 0.0 | 43.3 | 114.9 | +| denfry.github.io | 0.656 | 0.720 | 0.577 | 0.795 | 0.725 | 0.574 | 0.667 | 0.191 | 0.505 | 0.720 | 1226.9 | 0.0 | 22.3 | 43.8 | +| DevGraph | 0.819 | 0.819 | 0.694 | 1.000 | 0.694 | 0.667 | 0.750 | 0.317 | 0.574 | 0.778 | 1260.8 | 0.0 | 33.6 | 52.7 | +| Windows-Cleaner-and-Optimizer-main | 0.863 | 0.863 | 0.675 | 0.941 | 0.718 | 0.677 | 0.647 | 0.261 | 0.583 | 0.578 | 1270.2 | 0.0 | 24.7 | 40.4 | + +Per-category MRR (default config): + bug 0.647 + change 0.529 + docs 0.576 + feature 0.643 + perf 1.000 + refactor 0.750 + test 0.494 From 57ae98084fd9343aae2416c55720e97b8146d051 Mon Sep 17 00:00:00 2001 From: denfry Date: Thu, 10 Sep 2026 09:38:32 +0300 Subject: [PATCH 04/18] docs: evidence memory design for 2.0 Phase B design, written before implementation: content-addressed evidence references validated against the working tree, explicit session-scoped withholding with unchanged page composition, a separate content-free memory.sqlite, and the sequential real-history benchmark that decides whether withholding is worth shipping. --- .../2026-09-10-evidence-memory-design.md | 365 ++++++++++++++++++ 1 file changed, 365 insertions(+) create mode 100644 docs/superpowers/specs/2026-09-10-evidence-memory-design.md diff --git a/docs/superpowers/specs/2026-09-10-evidence-memory-design.md b/docs/superpowers/specs/2026-09-10-evidence-memory-design.md new file mode 100644 index 0000000..baf97ce --- /dev/null +++ b/docs/superpowers/specs/2026-09-10-evidence-memory-design.md @@ -0,0 +1,365 @@ +# Evidence memory — 2.0 design + +Status: Phase B design, written before implementation. Inputs: `research/2.0-audit.md`, +`research/experiments.md`. Every decision below names the evidence or requirement it +rests on; alternatives that were considered and rejected are listed in §15. + +## 1. Goal + +Let a coding agent stop re-receiving repository evidence it already holds, **only while +that evidence is provably unchanged**, and tell it exactly which evidence it holds has +become stale. Make any piece of evidence citable by a short reference that any agent can +re-verify later. + +Non-goals for 2.0: storing agent conclusions or prose; semantic or query-keyed caching; +changing ranking; graph/co-change expansion; cross-repository memory; an LLM in the loop. + +## 2. The central observation + +The research measured that 42% of evidence tokens repeat across a workload. A repeated +atom can only be *withheld* from a consumer that still holds the earlier copy in its +context window. Across independent contexts (a new session, a subagent, a different +tool) the text must be sent again — no store can change that. So 2.0 separates two +things the research merged: + +| Value | Where it applies | Mechanism | +|---|---|---| +| Token reuse | Within one agent context | Session ledger: omit a snippet already delivered to this context, if its source bytes are unchanged | +| Validity | Everywhere, any time later | Content-addressed evidence references re-verified against the working tree | + +## 3. Concepts + +**Evidence atom.** A contiguous span of one file, identified by the exact bytes it held +when it was delivered: `(repo, path, span_sha256)`. Line numbers are a locator, not +identity. + +**Evidence reference.** A self-describing, verifiable string: +`:-@`, e.g. +`src/auth/service.py:42-81@3f9a2c1b7d4e8a90`. It can be stored anywhere (agent notes, a +PR description, another tool's memory) and verified without the memory database. + +**Session.** An opaque tag naming *one agent context*. Snippets are only ever withheld from +the session they were delivered to. Tags are supplied explicitly by the caller. + +**Delivery.** A snippet that was actually placed in a packet (inside the token budget) +and verified against the working tree at delivery time. + +**Verdict states.** + +| State | Valid? | Meaning | +|---|---|---| +| `valid` | yes | Bytes at the recorded locator are identical | +| `relocated` | yes | Identical bytes occur exactly once elsewhere in the same file | +| `changed` | no | The bytes no longer occur in the file | +| `ambiguous` | no | The bytes occur more than once in the file; identity cannot be established | +| `deleted` | no | The file no longer exists | +| `excluded` | no | The path is now rejected by the discovery gates (ignore/secret/binary/size) | +| `unreadable` | no | I/O error | + +Every uncertain case resolves to *invalid* (requirement: prefer false invalidation to +stale reuse). + +## 4. Identity and the line model + +The indexer reads text with `read_text(errors="ignore")` (universal newlines) and chunks +with `str.splitlines()` (audit §4.2). Hashing uses the same line model so that a snippet +and its hashed span are the same lines: + +``` +raw = file bytes +text = raw.decode("utf-8", "surrogateescape") # invalid bytes stay visible +text = text.replace("\r\n", "\n").replace("\r", "\n") # universal newlines +lines = text.splitlines() # chunker line model +span = "\n".join(lines[start-1:end]) +sha = sha256(span.encode("utf-8", "surrogateescape")) +``` + +Only line terminators are normalised (a CRLF checkout of identical code is identical +evidence). Whitespace, comments and case are **not** normalised: a strict hash cannot +confuse two different programs, and fuzzy similarity is never used as proof (§15). +`surrogateescape` means a change to an undecodable byte still changes the hash. + +Repository scope: `repo_id = sha256("repo:" + canonical root)`, where the canonical root is +the resolved absolute POSIX path, case-folded on Windows. Rows from another `repo_id` are +never read. Moving a checkout directory therefore starts a fresh scope (a false +invalidation, never a false reuse). + +## 5. Validation + +``` +validate(ref, root, gates, hint=None): + if gates reject path: return excluded # never open excluded files + if path missing: return deleted + lines = line_model(read(path)) + if sha(lines[start-1:end]) == ref.sha: return valid(start, end) + candidates = hint.first_line_sha ? {i : sha(lines[i]) == first_line_sha} + : bounded scan of all starts (≤ 200k line-steps) + matches = {i in candidates : sha(lines[i:i+n]) == ref.sha} + |matches| == 1 → relocated(i+1, i+n); > 1 → ambiguous; 0 → changed + (bounded scan exhausted → changed, conservatively) +``` + +A reference's hash prefix (64 bits) is compared as a prefix; stored rows compare the full +256-bit hash. + +**Cross-file moves are not reused.** `src/a.py → src/core/a.py` yields `deleted` for the +old reference. The path is part of what the agent was told (imports and module identity +change with it), and the new location is found by ordinary retrieval. Identical short +text in an unrelated file must never validate a reference (requirement 38). + +**Symbol evolution** follows from byte identity: a rename, signature change, body change or +doc change inside the span is `changed`; a split or merge changes the bytes and is +`changed`; code that only shifted inside its file is `relocated`. No attempt is made to +decide semantic equivalence. + +## 6. Where evidence is minted — and per-result verification + +`retrieval.pipeline.search` gains an optional `evidence` hook invoked after budgeting and +pagination with the delivered page and the candidates behind it. `None` (the default for +library callers) leaves the function byte-identical. + +For each result that carries a snippet, the hook reads the working-tree file (through the +gates), computes the span hash, and checks that the index content the snippet was built +from is present in the current span (equality for chunk results; containment for +signature/doc results). This closes audit finding §4.3 per result: + +- consistent → the snippet reflects current bytes; it is eligible to be a delivery; +- inconsistent → the result is marked `"stale": true` (index entry older than the file) and + is never recorded or withheld. + +With a fresh index nothing is added to the packet, so default output is unchanged. + +## 7. Reuse: session-scoped withholding + +With a session tag, for each verified result: + +``` +atom = (repo_id, path, span_sha) +known = ledger[session] has atom with (full == true or snippet_sha == sha(snippet)) +if known: snippet → null, result.reused = true, tokens_saved += token_est +else: deliver snippet; ledger[session] += (atom, snippet_sha, full) +``` + +- `full` is true when the delivered snippet text is the entire span. A skeleton or a bare + signature only covers part of the span, so it can only satisfy a later request for the + *same* snippet text. Query-dependent skeletons therefore never hide lines the agent did + not see. +- **Budget accounting is unchanged**: a withheld snippet still consumes its budget in + `apply_budget`, so the page, the ranks, which results carry snippets and + `recommended_reads` are identical to the no-memory packet. The only differences are + `snippet: null` + `reused: true` on withheld results and a `memory` block. Tokens saved + are therefore pure savings with identical coverage, which keeps attribution exact. + Reinvesting freed budget is a separate, later experiment. +- A session with an empty ledger produces the no-memory packet plus the `memory` block. + +**Invalidation notices.** On each session call, deliveries not yet reported invalid are +re-validated (using the stored first-line hash for relocation). Newly invalid ones are +listed once: + +```json +"memory": { + "session": "auth-fix", + "reused": 2, "tokens_saved": 412, + "invalidated": [{"ref": "src/auth/service.py:42-81@3f9a2c1b7d4e8a90", "state": "changed"}] +} +``` + +**Why sessions are explicit.** An MCP server process outlives `/clear` and survives context +compaction; a CLI process knows nothing about the caller; a time window would let two +parallel agents withhold evidence from each other; subagents have fresh contexts but may +inherit environment variables. None of these identify an agent context, so the caller +names it (`--session`, `CBX_SESSION`, MCP `session`). Documented rule: a session tag is +used by exactly one context and is replaced after `/clear`, compaction, or whenever the +agent cannot see earlier snippets. + +## 8. Claims depend on evidence; memory does not store claims + +A conclusion ("auth uses JWT and Redis") belongs to the agent. 2.0 makes its dependency +set checkable instead of caching it: the agent cites references next to its note, and +`verify` reports whether every dependency still holds (`all_valid`). The store keeps +atoms independent of sessions, so a later `claims → atoms` table can reference them +without redesign. No conclusion text, prompt, or reasoning is stored. + +## 9. Storage + +A separate SQLite file, `.claude/cache/codebase-index/memory.sqlite` (override +`CBX_MEMORY_PATH`; when only `CBX_DB_PATH` is set, next to that DB). + +Why not tables in `index.sqlite`: `index --rebuild`, schema-triggered rebuilds and `clean` +delete that file (audit §4.1), which would destroy memory on every rebuild (requirement +7.6); and search-time ledger writes would contend with `update`'s long write transaction +on the same WAL. A second file in the same cache directory reuses the existing SQLite +pattern with none of those couplings, and leaves the 1.x index schema untouched, so +upgrading requires no reindex. + +Schema v1 (content-free — no source text is ever stored): + +```sql +meta(key PRIMARY KEY, value) -- schema_version, last_gc_at +atoms(id PK, repo_id, path, span_sha, line_count, first_line_sha, + first_seen_at, last_state, last_checked_at, + UNIQUE(repo_id, path, span_sha)) +sessions(id PK, repo_id, tag_sha, created_at, last_used_at, + tokens_delivered, tokens_saved, reused, UNIQUE(repo_id, tag_sha)) +deliveries(session_id → sessions ON DELETE CASCADE, atom_id → atoms ON DELETE CASCADE, + snippet_sha, full, line_start, line_end, token_est, delivered_at, + invalid_state NULL, PRIMARY KEY(session_id, atom_id, snippet_sha)) +``` + +- Session tags are stored as `sha256(repo_id + tag)`; the tag itself is never persisted. +- **Migrations**: ordered `MIGRATIONS[v] → callable`, applied in one `BEGIN IMMEDIATE` + transaction; `user_version`-style guard in `meta`. A newer on-disk version disables + memory for that process (search still works, reported by `stats`/`doctor`), it never + downgrades. +- **Corruption**: `sqlite3.DatabaseError` on open → the file is renamed to + `memory.sqlite.corrupt-` (preserved, not deleted), a fresh store is + created, and `doctor` reports it. +- **Concurrency**: WAL, `busy_timeout` 2000 ms, short `BEGIN IMMEDIATE` writes, idempotent + upserts. A lock timeout degrades that call to "no withholding, no ledger write" — the + packet is still correct, only larger. +- **Bounded growth**: `memory.retention_days` (14) removes sessions not used within the + window; orphan atoms are removed; `memory.max_deliveries` (50 000) drops the least + recently used sessions beyond the cap. GC runs at most daily on a session write, and via + `memory gc`. GC only deletes rows; it cannot make evidence valid, because validity is + computed from the working tree on every check. + +## 10. Branches, worktrees, dirty trees + +Validity is byte equality with the **current working tree**, never a commit. Consequences: + +- Dirty, staged and unstaged edits are what is validated against; `HEAD` is irrelevant. +- `git checkout`/rebase: evidence whose bytes are identical on the new branch stays valid + (it is the same text); anything else is `changed`/`deleted`. +- Each worktree has its own cache directory, hence its own memory; sharing across worktrees + is not attempted. +- New files have no prior evidence; deleted files yield `deleted`; renames yield `deleted` + for the old path. + +## 11. Public surface + +| Surface | Addition | +|---|---| +| `search`, `explain` | `--session TAG` (env `CBX_SESSION`) | +| `verify` (new, read-only) | `codebase-index verify [REF ...] [--session TAG] [--strict] [--json]`: verdict per reference or per session delivery, `all_valid`; `--strict` exits 1 when anything is invalid | +| `memory gc`, `memory clear` (new, maintenance) | Explicit GC; delete one session or all memory (`--yes`) | +| `stats`, `doctor` | Additive `memory` block / `memory_store` finding | +| MCP | `session` parameter on `search_code`/`explain_code`; `verify_evidence(refs, session)`; memory block in `index_stats`/`healthcheck`. No GC/clear over MCP (same policy as `clean`). | +| Skill | `verify` allowed; `memory` not allowed (like `clean`) | + +MCP `schema_version` stays 1: every change is an additive field. No CLI command, flag or +JSON field is removed or retyped. + +Config: `memory.enabled` (true), `memory.retention_days` (14), `memory.max_deliveries` +(50 000); env `CBX_MEMORY=0` disables everything, giving 1.10.0 output byte-for-byte. Not +part of `config_hash`. + +## 12. Security + +- The store holds hashes, paths, line numbers, token counts and timestamps. It holds **no + source text**, so no secret can be persisted by it through any path. +- Gates (ignore files, built-in denylist, secret filenames, size, NUL sniff) run before any + working-tree read performed by validation; an excluded path is reported `excluded`, + never opened, and its rows are purged by GC. +- Snippet content still comes only from the index and still passes redaction; memory adds + no new content path to the agent. +- Session tags are hashed; prompts, queries and conclusions are not stored. +- Everything stays local; no new dependency, no network. + +## 13. Failure modes + +| Failure | Behaviour | +|---|---| +| Memory DB locked / read-only / corrupt / newer schema | Packet without withholding; reason in `stats`/`doctor`; corrupt file preserved | +| Index stale for a result | `stale: true`, snippet delivered, no ledger write | +| Packet truncated by the agent harness after a ledger write | Later call may withhold what was never seen. Mitigation: default budgets stay far below harness limits; documented rule to start a new session when output was truncated | +| Session tag reused by another context | Withholding from a context that lacks the text. Mitigation: documented one-context rule; tags are caller-chosen, never inferred | +| Hash prefix collision on a bare reference | 64-bit prefix; stored rows use full 256-bit hashes | +| Parallel calls in one session | Both may deliver (duplicate tokens); neither withholds unseen text | + +## 14. Guarantees and tests + +Invariants, each backed by a test: + +1. Same bytes → same `span_sha`, independent of line endings and position. +2. Changed bytes → never `valid`/`relocated` under the old reference. +3. Repository A rows are invisible to repository B. +4. Deleted or excluded files → never valid; excluded files never opened. +5. `CBX_MEMORY=0` → payload byte-identical to 1.10.0 for every query in the eval sets. +6. Memory enabled, no session, fresh index → byte-identical to (5). +7. Session with empty ledger → (5) plus a `memory` block; after re-filling withheld + snippets, identical to (5). +8. GC never changes a verdict; after GC the set of withheld snippets can only shrink. +9. Rebuilding the index with unchanged sources keeps every reference valid. +10. Excluded content (`.env`, keys, ignored paths) never reaches `memory.sqlite` + (byte scan of the DB file). +11. Deterministic stale-context scenario (§16.2): a query-keyed cache serves the stale fact; + evidence memory reports `changed` and re-delivers current text. + +Lifecycle tests use real git repositories: edit, insert-above (relocation), rename, delete, +move across files, branch switch, detached HEAD, worktree, dirty/staged/unstaged, new +file, rebase. Randomised property tests use a seeded `random.Random` (no new dependency). + +## 15. Rejected alternatives + +| Alternative | Reason | +|---|---| +| Query-similarity (semantic) cache | 20.5–69.1% stale on real history; the key omits the dependency | +| Store snippet text in memory | Duplicates the index, creates a second secret-retention surface, unnecessary: validation needs only hashes | +| Memory tables inside `index.sqlite` | Destroyed by rebuild/`clean`; write contention with `update` | +| Automatic session per MCP process / time window | Does not identify an agent context (`/clear`, compaction, parallel agents) | +| Normalised (whitespace/comment-insensitive) identity | Can equate different programs; similarity is never proof | +| Cross-file relocation by content | Path is part of the evidence; short identical text in unrelated files would validate | +| Whole-file atoms | 1.36× less surviving reuse at 10 commits (research H4) | +| Reinvest withheld budget into more results | Changes the page and confounds attribution; deferred to a separate experiment | +| Contract slices / accretion / co-change in the default packet | Negative slot economics and lower useful-per-token (research Exp. 1–7) | +| Emitting a reference on every result by default | ~5–9% packet tokens for a feature most calls do not use; references are available from `verify --session` | + +## 16. Benchmark plan (Phase D) + +### 16.1 Sequential real-history replay (`tests/eval/memory_eval.py`) + +For each corpus: a `git clone --shared --no-checkout` into a temp dir (source repositories +are never modified); queries ordered oldest-first by their commit; for task *i* the tree is +checked out at the **parent** of the task's commit, the index is updated incrementally, +and the retrieval call is made. Tasks are grouped into sessions of *K* consecutive tasks +(K ∈ {1, 5, 10, 25, all}). The repository really evolves between tasks. + +Arms, all on identical packets: + +| Arm | Description | +|---|---| +| A | Read the full files of every delivered result (the "just reread the file" baseline) | +| A-mem | A with file-hash memory: skip a file already read in the session and unchanged | +| B | 1.10.0 packets, no memory | +| S | Unsafe dedup: withhold any result whose `(path, start, end)` was delivered in the session, no validation (models "trust what you saw") | +| C | 2.0 validated session memory | + +The oracle is independent of the memory code: the harness keeps every snippet text it has +handed to each session, and a withheld snippet counts as **stale** unless the text B would +deliver now equals a text previously delivered to that session (or is contained in a +previously delivered full span). + +Metrics (defined in `docs/MEMORY.md`): evidence/token reuse rate, validated and stale reuse +rate, invalidation precision/recall, survival by commit distance, unique-atom ratio, +repeat-read reduction (A vs A-mem), useful@budget per arm (stale evidence counts as absent), +packet tokens (serialised JSON / 4, so notice and marker overhead is billed), latency. +Paired bootstrap CIs over tasks. + +### 16.2 Deterministic stale-context test + +T0 deliver evidence E proving fact X · T1 unrelated commit (E still `valid`, withheld) · T2 +edit E so X is false · T3 semantically similar query. Expected: a query-keyed cache returns +the T0 packet (stale X); memory reports `changed` for E and delivers the current text. + +### 16.3 Regression and performance + +The 8-corpus retrieval eval re-run back-to-back with the pinned 1.10.0 commit (identical +ranking expected by construction); latency p50/p95 with memory off, on without session, and +on with session; `verify` latency; GC time; `memory.sqlite` size after the replay. + +## 17. Decisions deferred to measurement + +- Whether withholding produces material savings at realistic session lengths (if not, 2.0 + ships verification only and says so — or is not released as 2.0). +- The primary session length reported (all K are published). +- Whether a notice's token cost ever outweighs the tokens it saves. From 7ef1f0b2e62647cbf12d8843c2a7a2d50a95d6f9 Mon Sep 17 00:00:00 2001 From: denfry Date: Thu, 10 Sep 2026 10:20:08 +0300 Subject: [PATCH 05/18] refactor(discovery): share indexing gates with later working-tree reads Evidence validation reads files long after they were indexed. Any path the walker refuses (ignore rules, dependency and build directories, secret filenames, size, NUL sniff) must be refused identically there, so both now go through one PathGate. The walker keeps its exact order and semantics; a parity test checks that the gate admits precisely the files a walk indexes. PathGate.read also re-gates the resolved on-disk path, so a symlink or a differently-cased path on a case-insensitive filesystem cannot reach a file the walker would never have indexed. --- src/codebase_index/discovery/gates.py | 117 +++++++++++++++++++++++++ src/codebase_index/discovery/walker.py | 23 ++--- 2 files changed, 124 insertions(+), 16 deletions(-) create mode 100644 src/codebase_index/discovery/gates.py diff --git a/src/codebase_index/discovery/gates.py b/src/codebase_index/discovery/gates.py new file mode 100644 index 0000000..878f88f --- /dev/null +++ b/src/codebase_index/discovery/gates.py @@ -0,0 +1,117 @@ +"""Discovery gates shared by indexing and by every later read of the working tree. + +A path the indexer refuses (ignored, dependency/build directory, secret filename, +oversized, binary) must be refused identically wherever else the tool reads files. +Keeping a single implementation is how that stays true: the walker and evidence +validation both go through `PathGate`. +""" + +from __future__ import annotations + +import stat as stat_mod +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +from ..config import Config +from . import classify +from .ignore import IgnoreMatcher + +BINARY_SNIFF_BYTES = 4096 + + +@dataclass(frozen=True) +class GateRead: + """Outcome of reading one repo-relative file through the gates. + + ``state`` is ``ok`` (``data`` holds the bytes), ``excluded`` (a gate refused it and the + file was not opened unless the refusal is content-based), ``deleted`` or ``unreadable``. + """ + + state: str + data: bytes = b"" + reason: str = "" + + +class PathGate: + def __init__(self, root: Path | str, config: Config) -> None: + self.root = Path(root).resolve() + self.max_file_bytes = config.max_file_bytes + self.matcher = IgnoreMatcher.from_root( + self.root, + ignore_files=config.ignore_files, + extra_ignore=config.extra_ignore, + ) + + # -- the walker's gates, in the walker's order ----------------------------------- + def dir_allowed(self, name: str, rel_dir: str) -> bool: + return not self.matcher.is_ignored_dir(name) and not self.matcher.is_ignored( + rel_dir + "/" + ) + + def name_rejection(self, rel: str) -> Optional[str]: + if self.matcher.is_ignored(rel): + return "ignored by an ignore file or the built-in denylist" + if classify.is_secret_filename(rel): + return "secret-like filename" + return None + + def content_rejection(self, size: int, head: bytes) -> Optional[str]: + if size > self.max_file_bytes: + return "larger than max_file_bytes" + if classify.looks_binary(head): + return "binary content" + return None + + # -- whole-path checks for reads that did not come from a walk ------------------- + def path_rejection(self, rel: str) -> Optional[str]: + """Every name-based gate for a repo-relative POSIX path, including its directories.""" + parts = rel.split("/") + for depth in range(1, len(parts)): + if not self.dir_allowed(parts[depth - 1], "/".join(parts[:depth])): + return "inside an ignored directory" + return self.name_rejection(rel) + + def read(self, rel: str) -> GateRead: + """Read an admitted file, or explain why it is not admitted. + + Name-based gates run before the file is touched. The resolved on-disk path is gated + again, so a symlink or (on case-insensitive filesystems) a differently-cased path + cannot reach a file the walker would never have indexed. + """ + reason = self.path_rejection(rel) + if reason: + return GateRead("excluded", reason=reason) + path = self.root / rel + try: + resolved = path.resolve() + except OSError as exc: + return GateRead("unreadable", reason=str(exc)) + try: + actual_rel = resolved.relative_to(self.root).as_posix() + except ValueError: + return GateRead("excluded", reason="resolves outside the repository") + if actual_rel != rel: + reason = self.path_rejection(actual_rel) + if reason: + return GateRead("excluded", reason=reason) + try: + st = path.stat() + except FileNotFoundError: + return GateRead("deleted", reason="file no longer exists") + except OSError as exc: + return GateRead("unreadable", reason=str(exc)) + if not stat_mod.S_ISREG(st.st_mode): + return GateRead("deleted", reason="no longer a regular file") + if st.st_size > self.max_file_bytes: + return GateRead("excluded", reason="larger than max_file_bytes") + try: + data = path.read_bytes() + except FileNotFoundError: + return GateRead("deleted", reason="file no longer exists") + except OSError as exc: + return GateRead("unreadable", reason=str(exc)) + reason = self.content_rejection(len(data), data[:BINARY_SNIFF_BYTES]) + if reason: + return GateRead("excluded", reason=reason) + return GateRead("ok", data=data) diff --git a/src/codebase_index/discovery/walker.py b/src/codebase_index/discovery/walker.py index 9c707a0..9a3327e 100644 --- a/src/codebase_index/discovery/walker.py +++ b/src/codebase_index/discovery/walker.py @@ -9,9 +9,7 @@ from ..config import Config from . import classify -from .ignore import IgnoreMatcher - -_BINARY_SNIFF_BYTES = 4096 +from .gates import BINARY_SNIFF_BYTES, PathGate @dataclass @@ -26,38 +24,31 @@ class Candidate: def walk(root: Path, config: Config) -> Iterator[Candidate]: root = Path(root).resolve() - matcher = IgnoreMatcher.from_root( - root, - ignore_files=config.ignore_files, - extra_ignore=config.extra_ignore, - ) + gate = PathGate(root, config) for dirpath, dirnames, filenames in os.walk(root): dirnames[:] = [ - d - for d in dirnames - if not matcher.is_ignored_dir(d) - and not matcher.is_ignored(_rel(root, Path(dirpath) / d) + "/") + d for d in dirnames if gate.dir_allowed(d, _rel(root, Path(dirpath) / d)) ] for fname in filenames: abs_path = Path(dirpath) / fname rel = _rel(root, abs_path) - if matcher.is_ignored(rel) or classify.is_secret_filename(rel): + if gate.name_rejection(rel): continue try: size = abs_path.stat().st_size except OSError: continue - if size > config.max_file_bytes: + if size > gate.max_file_bytes: continue try: with abs_path.open("rb") as fh: - head = fh.read(_BINARY_SNIFF_BYTES) + head = fh.read(BINARY_SNIFF_BYTES) except OSError: continue - if classify.looks_binary(head): + if gate.content_rejection(size, head): continue lang = classify.detect_language(rel) From 4db84f3ccb910a3e4724d59b337f8d861309c15a Mon Sep 17 00:00:00 2001 From: denfry Date: Thu, 10 Sep 2026 10:20:08 +0300 Subject: [PATCH 06/18] feat(memory): evidence identity, validation and a content-free store An evidence reference is path:start-end@hash, where the hash covers the exact bytes of the span under the indexer's own line model (universal newlines, splitlines). Only line terminators are normalised. Validity is byte identity with the working tree: valid, relocated (identical bytes occur exactly once elsewhere in the same file), or changed, ambiguous, deleted, excluded and unreadable, all of which are invalid. References are untrusted input, so absolute paths, drive letters, .. segments and NUL are rejected before any filesystem access. memory.sqlite is separate from index.sqlite because rebuilds and clean delete the index file, and so that ledger writes never queue behind an update transaction. It stores hashes, paths, line numbers, token counts and timestamps only. Session tags are stored hashed. Migrations run in single transactions, a newer schema is refused without touching the file, a corrupt file is moved aside rather than deleted, lock contention degrades instead of blocking, and GC only removes rows. --- src/codebase_index/memory/__init__.py | 5 + src/codebase_index/memory/identity.py | 181 +++++++++++ src/codebase_index/memory/store.py | 428 ++++++++++++++++++++++++++ src/codebase_index/memory/validate.py | 141 +++++++++ tests/test_memory_identity.py | 156 ++++++++++ tests/test_memory_store.py | 208 +++++++++++++ tests/test_memory_validate.py | 210 +++++++++++++ 7 files changed, 1329 insertions(+) create mode 100644 src/codebase_index/memory/__init__.py create mode 100644 src/codebase_index/memory/identity.py create mode 100644 src/codebase_index/memory/store.py create mode 100644 src/codebase_index/memory/validate.py create mode 100644 tests/test_memory_identity.py create mode 100644 tests/test_memory_store.py create mode 100644 tests/test_memory_validate.py diff --git a/src/codebase_index/memory/__init__.py b/src/codebase_index/memory/__init__.py new file mode 100644 index 0000000..b1400ce --- /dev/null +++ b/src/codebase_index/memory/__init__.py @@ -0,0 +1,5 @@ +"""Evidence memory: verifiable references to repository evidence and session reuse. + +See docs/MEMORY.md. Nothing in this package stores source text; identity is a hash of +the exact bytes a span held, and validity is always re-checked against the working tree. +""" diff --git a/src/codebase_index/memory/identity.py b/src/codebase_index/memory/identity.py new file mode 100644 index 0000000..ba97633 --- /dev/null +++ b/src/codebase_index/memory/identity.py @@ -0,0 +1,181 @@ +"""Evidence identity: the line model, span hashing, and verifiable references. + +Pure functions only — no filesystem, no database — so every identity property can be +tested exhaustively and cannot depend on I/O timing. + +The line model deliberately mirrors the indexer. It reads files with universal newlines +and slices chunks with ``str.splitlines()`` (parsers/line_chunker.py, +parsers/symbol_chunks.py), so a snippet and the span hashed for it are the same lines. +Two things differ on purpose: + +* bytes are decoded with ``surrogateescape`` instead of ``ignore``, so editing an + undecodable byte still changes the hash; +* only line terminators are normalised. Whitespace, comments and case are part of the + identity: a strict hash can never equate two different programs. +""" + +from __future__ import annotations + +import hashlib +import re +import sys +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Optional, Sequence + +REF_HASH_CHARS = 16 +"""Hex characters of the span hash shown in a reference (64 bits).""" + +MIN_REF_HASH_CHARS = 12 + +_REF_RE = re.compile(r"^(?P.+):(?P\d+)-(?P\d+)@(?P[0-9a-fA-F]+)$") +_DRIVE_RE = re.compile(r"^[A-Za-z]:") + + +def split_lines(raw: bytes) -> list[str]: + """Split file bytes into lines exactly as the chunkers number them.""" + text = raw.decode("utf-8", "surrogateescape") + if "\r" in text: + text = text.replace("\r\n", "\n").replace("\r", "\n") + return text.splitlines() + + +def _encode(text: str) -> bytes: + return text.encode("utf-8", "surrogateescape") + + +def sha_hex(text: str) -> str: + return hashlib.sha256(_encode(text)).hexdigest() + + +def span_text(lines: Sequence[str], line_start: int, line_end: int) -> Optional[str]: + """Text of 1-based inclusive lines, or None when the range is not inside the file.""" + if line_start < 1 or line_end < line_start or line_end > len(lines): + return None + return "\n".join(lines[line_start - 1 : line_end]) + + +def span_sha(lines: Sequence[str], line_start: int, line_end: int) -> Optional[str]: + text = span_text(lines, line_start, line_end) + return None if text is None else sha_hex(text) + + +def line_sha(line: str) -> str: + """Short hash of one line; used only to find relocation candidates, never as proof.""" + return sha_hex(line)[:REF_HASH_CHARS] + + +def visible_text(text: str) -> str: + """The text the indexer stored for these lines (undecodable bytes dropped).""" + return _encode(text).decode("utf-8", "ignore") + + +def content_matches(index_content: Optional[str], span: str) -> bool: + """Does text taken from the index still describe these working-tree lines? + + Chunk results store the whole span, so the common case is equality. Symbol and doc + results store an excerpt (a signature, a heading), which must occur inside the span. + Either way the snippet delivered to the agent is text that exists in the current + file at that locator, which is all an evidence atom asserts. + """ + if not index_content: + return False + visible = visible_text(span) + content = index_content.replace("\r\n", "\n") + return content == visible or content in visible + + +def is_full_span(index_content: Optional[str], span: str) -> bool: + """True when the index text is the entire span rather than an excerpt of it.""" + if not index_content: + return False + return index_content.replace("\r\n", "\n") == visible_text(span) + + +def normalize_rel_path(path: str) -> str: + """Canonical repo-relative POSIX path, or ValueError for anything that could escape. + + References arrive from agents, notes and other tools, so they are untrusted input: an + absolute path, a drive letter, a ``..`` segment or a NUL byte is rejected before any + filesystem access is attempted. + """ + if not path or "\x00" in path: + raise ValueError("evidence path is empty or contains NUL") + posix = path.replace("\\", "/") + if posix.startswith("/") or _DRIVE_RE.match(posix): + raise ValueError(f"evidence path must be repository-relative: {path!r}") + parts = [p for p in PurePosixPath(posix).parts if p not in ("", ".")] + if not parts or any(p == ".." for p in parts): + raise ValueError(f"evidence path must stay inside the repository: {path!r}") + return "/".join(parts) + + +@dataclass(frozen=True) +class EvidenceRef: + """``:-@`` — a citable, re-verifiable piece of evidence. + + The line range is the locator at the time the evidence was observed; identity is the + path plus the span hash. ``sha`` may be a prefix (from a printed reference) or a full + 256-bit digest (from the store). + """ + + path: str + line_start: int + line_end: int + sha: str + + @property + def line_count(self) -> int: + return self.line_end - self.line_start + 1 + + def matches(self, full_sha: Optional[str]) -> bool: + return bool(full_sha) and str(full_sha).startswith(self.sha) + + def __str__(self) -> str: + return f"{self.path}:{self.line_start}-{self.line_end}@{self.sha[:REF_HASH_CHARS]}" + + +def make_ref(path: str, line_start: int, line_end: int, full_sha: str) -> EvidenceRef: + return EvidenceRef(normalize_rel_path(path), int(line_start), int(line_end), full_sha.lower()) + + +def parse_ref(text: str) -> EvidenceRef: + """Parse a printed reference. Paths may contain ``:`` or ``@``; parsing is right-anchored.""" + match = _REF_RE.match(text.strip()) + if match is None: + raise ValueError(f"not an evidence reference (expected path:start-end@hash): {text!r}") + start, end = int(match["start"]), int(match["end"]) + if start < 1 or end < start: + raise ValueError(f"invalid line range in evidence reference: {text!r}") + sha = match["sha"].lower() + if not MIN_REF_HASH_CHARS <= len(sha) <= 64: + raise ValueError( + f"evidence hash must be {MIN_REF_HASH_CHARS}-64 hex characters: {text!r}" + ) + return EvidenceRef(normalize_rel_path(match["path"]), start, end, sha) + + +def repo_id_for(root: Path | str) -> str: + """Scope key for one checkout. Case-folded on Windows, where paths are case-insensitive.""" + canonical = Path(root).resolve().as_posix() + if sys.platform == "win32": + canonical = canonical.casefold() + return hashlib.sha256(f"repo:{canonical}".encode("utf-8")).hexdigest() + + +def session_key(repo_id: str, tag: str) -> str: + """Stored form of a caller's session tag; the tag itself is never persisted.""" + return hashlib.sha256(f"session:{repo_id}:{tag}".encode("utf-8")).hexdigest() + + +_SESSION_TAG_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$") + + +def validate_session_tag(tag: str) -> str: + tag = tag.strip() + if not _SESSION_TAG_RE.match(tag): + raise ValueError( + "session tag must be 1-128 characters of letters, digits, '.', '_', ':' or '-', " + "starting with a letter or digit" + ) + return tag diff --git a/src/codebase_index/memory/store.py b/src/codebase_index/memory/store.py new file mode 100644 index 0000000..95f7497 --- /dev/null +++ b/src/codebase_index/memory/store.py @@ -0,0 +1,428 @@ +"""Persistent evidence memory: ``memory.sqlite``, separate from the index. + +Separate because ``index --rebuild``, schema-triggered rebuilds and ``clean`` delete +``index.sqlite``, and because ledger writes at search time must not queue behind a long +``update`` transaction on the same WAL. + +The store is content-free: paths, line numbers, hashes, token counts and timestamps. +No source text, query, prompt or session tag is ever written. +""" + +from __future__ import annotations + +import sqlite3 +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Callable, Iterator, Optional, Sequence + +SCHEMA_VERSION = 1 +BUSY_TIMEOUT_MS = 2000 + + +class MemoryUnavailable(RuntimeError): + """Memory cannot serve this call. Retrieval continues without withholding anything.""" + + +def utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def iso(moment: datetime) -> str: + return moment.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +_V1 = ( + "CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)", + """CREATE TABLE IF NOT EXISTS atoms ( + id INTEGER PRIMARY KEY, + repo_id TEXT NOT NULL, + path TEXT NOT NULL, + span_sha TEXT NOT NULL, + line_count INTEGER NOT NULL, + first_line_sha TEXT NOT NULL, + first_seen_at TEXT NOT NULL, + UNIQUE (repo_id, path, span_sha) + )""", + """CREATE TABLE IF NOT EXISTS sessions ( + id INTEGER PRIMARY KEY, + repo_id TEXT NOT NULL, + tag_sha TEXT NOT NULL, + created_at TEXT NOT NULL, + last_used_at TEXT NOT NULL, + calls INTEGER NOT NULL DEFAULT 0, + tokens_delivered INTEGER NOT NULL DEFAULT 0, + tokens_saved INTEGER NOT NULL DEFAULT 0, + reused INTEGER NOT NULL DEFAULT 0, + invalidations INTEGER NOT NULL DEFAULT 0, + UNIQUE (repo_id, tag_sha) + )""", + """CREATE TABLE IF NOT EXISTS deliveries ( + session_id INTEGER NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, + atom_id INTEGER NOT NULL REFERENCES atoms(id) ON DELETE CASCADE, + snippet_sha TEXT NOT NULL, + full INTEGER NOT NULL, + line_start INTEGER NOT NULL, + line_end INTEGER NOT NULL, + token_est INTEGER NOT NULL, + delivered_at TEXT NOT NULL, + invalid_state TEXT, + PRIMARY KEY (session_id, atom_id, snippet_sha) + )""", + "CREATE INDEX IF NOT EXISTS idx_deliveries_atom ON deliveries(atom_id)", + "CREATE INDEX IF NOT EXISTS idx_sessions_used ON sessions(last_used_at)", +) + +# version -> statements that bring a store from version-1 to version. Applied in order, +# each inside one transaction together with the version bump. +MIGRATIONS: dict[int, Sequence[str]] = {1: _V1} + + +@dataclass(frozen=True) +class NewDelivery: + path: str + span_sha: str + line_count: int + first_line_sha: str + snippet_sha: str + full: bool + line_start: int + line_end: int + token_est: int + + +@dataclass(frozen=True) +class Delivery: + atom_id: int + path: str + span_sha: str + line_count: int + first_line_sha: str + line_start: int + line_end: int + + +class MemoryStore: + def __init__(self, conn: sqlite3.Connection, path: Path, recovered_from: Optional[str]): + self.conn = conn + self.path = path + self.recovered_from = recovered_from + + # -- lifecycle ------------------------------------------------------------------- + @classmethod + def open(cls, path: Path | str, *, migrations: Optional[dict[int, Sequence[str]]] = None, + schema_version: int = SCHEMA_VERSION, + busy_timeout_ms: Optional[int] = None) -> "MemoryStore": + """Open (creating or migrating) a store, or raise ``MemoryUnavailable``. + + A file SQLite cannot read as a database is moved aside to + ``memory.sqlite.corrupt-`` — preserved, never deleted — and a fresh + store is created. A lock or I/O failure is *not* corruption and is reported as + unavailable without touching the file. + """ + path = Path(path) + migrations = MIGRATIONS if migrations is None else migrations + if busy_timeout_ms is None: + busy_timeout_ms = BUSY_TIMEOUT_MS + try: + path.parent.mkdir(parents=True, exist_ok=True) + except OSError as exc: + raise MemoryUnavailable(f"cannot create memory directory: {exc}") from exc + recovered: Optional[str] = None + try: + conn = cls._connect_and_migrate(path, migrations, schema_version, busy_timeout_ms) + except sqlite3.OperationalError as exc: + raise MemoryUnavailable(f"memory store unavailable: {exc}") from exc + except sqlite3.DatabaseError: + recovered = _quarantine(path) + try: + conn = cls._connect_and_migrate(path, migrations, schema_version, busy_timeout_ms) + except sqlite3.Error as exc: + raise MemoryUnavailable(f"memory store unavailable after recovery: {exc}") from exc + return cls(conn, path, recovered) + + @staticmethod + def _connect_and_migrate(path: Path, migrations: dict[int, Sequence[str]], + schema_version: int, busy_timeout_ms: int) -> sqlite3.Connection: + conn = sqlite3.connect(path, isolation_level=None, timeout=busy_timeout_ms / 1000) + try: + conn.execute(f"PRAGMA busy_timeout = {int(busy_timeout_ms)}") + conn.execute("PRAGMA journal_mode = WAL") + conn.execute("PRAGMA synchronous = NORMAL") + conn.execute("PRAGMA foreign_keys = ON") + current = _stored_version(conn) + if current > schema_version: + raise MemoryUnavailable( + f"memory store schema {current} is newer than this version supports " + f"({schema_version}); upgrade codebase-index or run `memory clear`" + ) + for version in range(current + 1, schema_version + 1): + with _transaction(conn): + for statement in migrations[version]: + conn.execute(statement) + conn.execute( + "INSERT INTO meta(key, value) VALUES ('schema_version', ?) " + "ON CONFLICT(key) DO UPDATE SET value = excluded.value", + (str(version),), + ) + except BaseException: + conn.close() + raise + return conn + + def close(self) -> None: + self.conn.close() + + def __enter__(self) -> "MemoryStore": + return self + + def __exit__(self, *exc: object) -> None: + self.close() + + @property + def schema_version(self) -> int: + return _stored_version(self.conn) + + @contextmanager + def _write(self) -> Iterator[sqlite3.Connection]: + try: + with _transaction(self.conn): + yield self.conn + except sqlite3.OperationalError as exc: + raise MemoryUnavailable(f"memory store busy: {exc}") from exc + + # -- sessions -------------------------------------------------------------------- + def touch_session(self, repo_id: str, tag_sha: str, *, now: datetime) -> int: + stamp = iso(now) + with self._write() as conn: + conn.execute( + "INSERT INTO sessions(repo_id, tag_sha, created_at, last_used_at, calls) " + "VALUES (?, ?, ?, ?, 1) ON CONFLICT(repo_id, tag_sha) DO UPDATE SET " + "last_used_at = excluded.last_used_at, calls = calls + 1", + (repo_id, tag_sha, stamp, stamp), + ) + row = conn.execute( + "SELECT id FROM sessions WHERE repo_id = ? AND tag_sha = ?", (repo_id, tag_sha) + ).fetchone() + return int(row[0]) + + def find_session(self, repo_id: str, tag_sha: str) -> Optional[int]: + row = self.conn.execute( + "SELECT id FROM sessions WHERE repo_id = ? AND tag_sha = ?", (repo_id, tag_sha) + ).fetchone() + return int(row[0]) if row else None + + def is_known(self, session_id: int, path: str, span_sha: str, snippet_sha: str) -> bool: + """Was exactly this text — or the whole unchanged span — delivered to this session? + + Deliveries later found invalid never count, even if the bytes come back: the + agent was told that evidence changed, so it gets the text again. + """ + row = self.conn.execute( + "SELECT 1 FROM deliveries d JOIN atoms a ON a.id = d.atom_id " + "WHERE d.session_id = ? AND a.path = ? AND a.span_sha = ? " + "AND d.invalid_state IS NULL AND (d.full = 1 OR d.snippet_sha = ?) LIMIT 1", + (session_id, path, span_sha, snippet_sha), + ).fetchone() + return row is not None + + def pending(self, session_id: int) -> list[Delivery]: + """Distinct evidence delivered to this session and not yet reported invalid.""" + rows = self.conn.execute( + "SELECT a.id, a.path, a.span_sha, a.line_count, a.first_line_sha, " + " MIN(d.line_start), MIN(d.line_end) " + "FROM deliveries d JOIN atoms a ON a.id = d.atom_id " + "WHERE d.session_id = ? AND d.invalid_state IS NULL " + "GROUP BY a.id ORDER BY a.path, MIN(d.line_start)", + (session_id,), + ).fetchall() + return [Delivery(int(r[0]), r[1], r[2], int(r[3]), r[4], int(r[5]), int(r[6])) + for r in rows] + + def record(self, repo_id: str, session_id: int, items: Sequence[NewDelivery], *, + now: datetime) -> None: + if not items: + return + stamp = iso(now) + with self._write() as conn: + for item in items: + conn.execute( + "INSERT INTO atoms(repo_id, path, span_sha, line_count, first_line_sha, " + "first_seen_at) VALUES (?, ?, ?, ?, ?, ?) " + "ON CONFLICT(repo_id, path, span_sha) DO NOTHING", + (repo_id, item.path, item.span_sha, item.line_count, + item.first_line_sha, stamp), + ) + atom_id = conn.execute( + "SELECT id FROM atoms WHERE repo_id = ? AND path = ? AND span_sha = ?", + (repo_id, item.path, item.span_sha), + ).fetchone()[0] + conn.execute( + "INSERT INTO deliveries(session_id, atom_id, snippet_sha, full, line_start, " + "line_end, token_est, delivered_at, invalid_state) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL) " + "ON CONFLICT(session_id, atom_id, snippet_sha) DO UPDATE SET " + "full = MAX(full, excluded.full), line_start = excluded.line_start, " + "line_end = excluded.line_end, token_est = excluded.token_est, " + "delivered_at = excluded.delivered_at, invalid_state = NULL", + (session_id, atom_id, item.snippet_sha, int(item.full), item.line_start, + item.line_end, item.token_est, stamp), + ) + + def mark_invalid(self, session_id: int, states: Sequence[tuple[int, str]]) -> None: + if not states: + return + with self._write() as conn: + conn.executemany( + "UPDATE deliveries SET invalid_state = ? WHERE session_id = ? AND atom_id = ?", + [(state, session_id, atom_id) for atom_id, state in states], + ) + + def add_counters(self, session_id: int, *, tokens_delivered: int, tokens_saved: int, + reused: int, invalidations: int) -> None: + with self._write() as conn: + conn.execute( + "UPDATE sessions SET tokens_delivered = tokens_delivered + ?, " + "tokens_saved = tokens_saved + ?, reused = reused + ?, " + "invalidations = invalidations + ? WHERE id = ?", + (tokens_delivered, tokens_saved, reused, invalidations, session_id), + ) + + def first_line_hint(self, repo_id: str, path: str, sha_prefix: str) -> Optional[str]: + """Relocation hint for a printed reference, when this store has seen the evidence.""" + rows = self.conn.execute( + "SELECT first_line_sha FROM atoms WHERE repo_id = ? AND path = ? " + "AND span_sha >= ? AND span_sha < ? LIMIT 2", + (repo_id, path, sha_prefix, sha_prefix + "g"), + ).fetchall() + return rows[0][0] if len(rows) == 1 else None + + # -- observability and maintenance ----------------------------------------------- + def stats(self, repo_id: str) -> dict: + one = self.conn.execute + sessions = one( + "SELECT COUNT(*), COALESCE(SUM(tokens_delivered),0), COALESCE(SUM(tokens_saved),0), " + "COALESCE(SUM(reused),0), COALESCE(SUM(invalidations),0), MAX(last_used_at) " + "FROM sessions WHERE repo_id = ?", (repo_id,), + ).fetchone() + atoms = one("SELECT COUNT(*) FROM atoms WHERE repo_id = ?", (repo_id,)).fetchone()[0] + deliveries = one( + "SELECT COUNT(*) FROM deliveries d JOIN sessions s ON s.id = d.session_id " + "WHERE s.repo_id = ?", (repo_id,), + ).fetchone()[0] + last_gc = one("SELECT value FROM meta WHERE key = 'last_gc_at'").fetchone() + return { + "schema_version": self.schema_version, + "sessions": int(sessions[0]), + "atoms": int(atoms), + "deliveries": int(deliveries), + "tokens_delivered": int(sessions[1]), + "tokens_saved": int(sessions[2]), + "reused": int(sessions[3]), + "invalidations": int(sessions[4]), + "last_used_at": sessions[5], + "last_gc_at": last_gc[0] if last_gc else None, + "bytes": _file_bytes(self.path), + } + + def gc_due(self, *, now: datetime, every: timedelta = timedelta(days=1)) -> bool: + row = self.conn.execute("SELECT value FROM meta WHERE key = 'last_gc_at'").fetchone() + return row is None or row[0] < iso(now - every) + + def gc(self, *, now: datetime, retention_days: int, max_deliveries: int) -> dict: + """Delete expired sessions, sessions beyond the size cap, and orphaned atoms. + + Only rows are removed. Validity is recomputed from the working tree on every + check, so GC can make memory withhold less, never make it withhold wrongly. + """ + cutoff = iso(now - timedelta(days=max(0, retention_days))) + with self._write() as conn: + expired = conn.execute( + "DELETE FROM sessions WHERE last_used_at < ?", (cutoff,) + ).rowcount + total = conn.execute("SELECT COUNT(*) FROM deliveries").fetchone()[0] + capped = 0 + if total > max_deliveries: + for session_id, count in conn.execute( + "SELECT s.id, COUNT(d.atom_id) FROM sessions s " + "LEFT JOIN deliveries d ON d.session_id = s.id " + "GROUP BY s.id ORDER BY s.last_used_at, s.id" + ).fetchall(): + if total <= max_deliveries: + break + conn.execute("DELETE FROM sessions WHERE id = ?", (session_id,)) + total -= count + capped += 1 + orphans = conn.execute( + "DELETE FROM atoms WHERE id NOT IN (SELECT DISTINCT atom_id FROM deliveries)" + ).rowcount + conn.execute( + "INSERT INTO meta(key, value) VALUES ('last_gc_at', ?) " + "ON CONFLICT(key) DO UPDATE SET value = excluded.value", + (iso(now),), + ) + return {"expired_sessions": max(0, expired), "capped_sessions": capped, + "orphan_atoms": max(0, orphans)} + + def clear(self, repo_id: str, tag_sha: Optional[str] = None) -> int: + with self._write() as conn: + if tag_sha is None: + removed = conn.execute( + "DELETE FROM sessions WHERE repo_id = ?", (repo_id,) + ).rowcount + conn.execute("DELETE FROM atoms WHERE repo_id = ?", (repo_id,)) + else: + removed = conn.execute( + "DELETE FROM sessions WHERE repo_id = ? AND tag_sha = ?", (repo_id, tag_sha) + ).rowcount + conn.execute( + "DELETE FROM atoms WHERE id NOT IN (SELECT DISTINCT atom_id FROM deliveries)" + ) + return max(0, removed) + + def vacuum(self) -> None: + try: + self.conn.execute("VACUUM") + except sqlite3.OperationalError as exc: + raise MemoryUnavailable(f"memory store busy: {exc}") from exc + + +@contextmanager +def _transaction(conn: sqlite3.Connection) -> Iterator[None]: + conn.execute("BEGIN IMMEDIATE") + try: + yield + except BaseException: + conn.execute("ROLLBACK") + raise + conn.execute("COMMIT") + + +def _stored_version(conn: sqlite3.Connection) -> int: + has_meta = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'meta'" + ).fetchone() + if not has_meta: + return 0 + row = conn.execute("SELECT value FROM meta WHERE key = 'schema_version'").fetchone() + return int(row[0]) if row else 0 + + +def _file_bytes(path: Path) -> int: + return sum( + p.stat().st_size + for p in (path, path.with_name(path.name + "-wal"), path.with_name(path.name + "-shm")) + if p.exists() + ) + + +def _quarantine(path: Path, clock: Callable[[], datetime] = utc_now) -> str: + """Move an unreadable store (and its WAL sidecars) aside; return the new name.""" + suffix = clock().strftime("%Y%m%dT%H%M%SZ") + target = path.with_name(f"{path.name}.corrupt-{suffix}") + for extra in ("", "-wal", "-shm"): + src = path.with_name(path.name + extra) + if src.exists(): + src.replace(target.with_name(target.name + extra)) + return target.name diff --git a/src/codebase_index/memory/validate.py b/src/codebase_index/memory/validate.py new file mode 100644 index 0000000..e8363a4 --- /dev/null +++ b/src/codebase_index/memory/validate.py @@ -0,0 +1,141 @@ +"""Validate evidence references against the current working tree. + +Validity is byte identity with the working tree — never a commit, a timestamp, or a +similarity score. Every uncertain outcome is reported as invalid: a false invalidation +costs the agent one re-read, a false validation costs it a wrong answer. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional + +from ..discovery.gates import PathGate +from . import identity as ident + +VALID_STATES = frozenset({"valid", "relocated"}) + +MAX_SCAN_STEPS = 200_000 +"""Upper bound on line-slices hashed while searching a file for moved content.""" + + +@dataclass(frozen=True) +class Verdict: + ref: ident.EvidenceRef + state: str + line_start: Optional[int] = None + line_end: Optional[int] = None + reason: str = "" + + @property + def valid(self) -> bool: + return self.state in VALID_STATES + + def as_dict(self) -> dict: + return { + "ref": str(self.ref), + "state": self.state, + "valid": self.valid, + "line_start": self.line_start, + "line_end": self.line_end, + "reason": self.reason, + } + + +class FileView: + """Lines of one working-tree file; per-line hashes are computed only if needed.""" + + def __init__(self, lines: list[str]) -> None: + self.lines = lines + self._line_shas: Optional[list[str]] = None + + def line_shas(self) -> list[str]: + if self._line_shas is None: + self._line_shas = [ident.line_sha(line) for line in self.lines] + return self._line_shas + + def sha(self, line_start: int, line_end: int) -> Optional[str]: + return ident.span_sha(self.lines, line_start, line_end) + + +def locate( + view: FileView, + sha: str, + line_count: int, + *, + first_line_sha: Optional[str] = None, + limit: int = 2, + max_steps: Optional[int] = None, +) -> Optional[list[int]]: + """1-based start lines where a span with this hash occurs (at most ``limit``). + + Returns ``None`` when the bounded search gave up, which callers must treat as "not + found". With ``first_line_sha`` only lines that hash like the span's first line are + tried, which makes relocation cheap for stored evidence. + """ + if max_steps is None: + max_steps = MAX_SCAN_STEPS + total = len(view.lines) + if line_count < 1 or line_count > total: + return [] + starts: list[int] | range = range(total - line_count + 1) + if first_line_sha is not None: + shas = view.line_shas() + starts = [i for i in starts if shas[i] == first_line_sha] + if len(starts) * line_count > max_steps: + return None + found: list[int] = [] + for i in starts: + if ident.sha_hex("\n".join(view.lines[i : i + line_count])).startswith(sha): + found.append(i + 1) + if len(found) >= limit: + break + return found + + +class WorkingTree: + """Gated, cached file access for one validation batch (one CLI/MCP call).""" + + def __init__(self, gate: PathGate) -> None: + self.gate = gate + self._views: dict[str, tuple[Optional[FileView], str, str]] = {} + + def view(self, rel: str) -> tuple[Optional[FileView], str, str]: + cached = self._views.get(rel) + if cached is None: + result = self.gate.read(rel) + if result.state == "ok": + cached = (FileView(ident.split_lines(result.data)), "ok", "") + else: + cached = (None, result.state, result.reason) + self._views[rel] = cached + return cached + + +def validate( + ref: ident.EvidenceRef, + tree: WorkingTree, + *, + first_line_sha: Optional[str] = None, +) -> Verdict: + view, state, reason = tree.view(ref.path) + if view is None: + return Verdict(ref, state, reason=reason) + if ref.matches(view.sha(ref.line_start, ref.line_end)): + return Verdict(ref, "valid", ref.line_start, ref.line_end, "unchanged") + starts = locate(view, ref.sha, ref.line_count, first_line_sha=first_line_sha) + if starts is None: + return Verdict( + ref, "changed", + reason="differs at the recorded lines; file too large to search for moved content", + ) + if len(starts) == 1: + start = starts[0] + end = start + ref.line_count - 1 + return Verdict(ref, "relocated", start, end, f"identical content now at lines {start}-{end}") + if len(starts) > 1: + return Verdict( + ref, "ambiguous", + reason="identical content occurs more than once in the file; cannot identify it", + ) + return Verdict(ref, "changed", reason="content no longer occurs in the file") diff --git a/tests/test_memory_identity.py b/tests/test_memory_identity.py new file mode 100644 index 0000000..4425828 --- /dev/null +++ b/tests/test_memory_identity.py @@ -0,0 +1,156 @@ +"""Evidence identity: line model, hashing, references. Pure functions, so tested exhaustively.""" + +from __future__ import annotations + +import random +import sys + +import pytest + +from codebase_index.memory import identity as ident +from codebase_index.parsers.line_chunker import chunk_text + + +def test_crlf_and_lf_checkouts_are_the_same_evidence(): + lf = b"def f():\n return 1\n" + crlf = b"def f():\r\n return 1\r\n" + assert ident.split_lines(lf) == ident.split_lines(crlf) + assert ident.span_sha(ident.split_lines(lf), 1, 2) == ident.span_sha(ident.split_lines(crlf), 1, 2) + + +def test_line_model_matches_the_chunker_including_form_feeds(): + """Chunk content and the hashed span must be the same lines, or verification lies.""" + raw = b"a = 1\n\x0cb = 2\nc = 3\r\nd = 4\n" + text = raw.decode("utf-8").replace("\r\n", "\n") + chunk = chunk_text(text, window_lines=80, overlap_lines=0)[0] + lines = ident.split_lines(raw) + assert ident.span_text(lines, chunk.line_start, chunk.line_end) == chunk.content + + +def test_whitespace_and_comments_are_part_of_identity(): + base = ident.split_lines(b"x = 1\n") + for variant in (b"x = 1\n", b"x = 1 # note\n", b"X = 1\n"): + assert ident.span_sha(ident.split_lines(variant), 1, 1) != ident.span_sha(base, 1, 1) + + +def test_undecodable_bytes_still_change_the_hash(): + a = ident.split_lines(b"blob = '\xff'\n") + b = ident.split_lines(b"blob = '\xfe'\n") + assert ident.span_sha(a, 1, 1) != ident.span_sha(b, 1, 1) + # ...while the indexer-visible text (bytes dropped) is identical for both. + assert ident.visible_text(a[0]) == ident.visible_text(b[0]) + + +def test_span_identity_is_independent_of_position(): + body = [b"def g(x):", b" return x * 2"] + top = ident.split_lines(b"\n".join(body) + b"\n") + shifted = ident.split_lines(b"import os\n\n" + b"\n".join(body) + b"\n") + assert ident.span_sha(top, 1, 2) == ident.span_sha(shifted, 3, 4) + + +@pytest.mark.parametrize("start,end", [(0, 1), (2, 1), (1, 99)]) +def test_out_of_range_spans_have_no_identity(start, end): + assert ident.span_sha(["a", "b"], start, end) is None + + +def test_reference_round_trip_and_prefix_matching(): + lines = ident.split_lines(b"one\ntwo\nthree\n") + full = ident.span_sha(lines, 2, 3) + ref = ident.make_ref("src/mod.py", 2, 3, full) + printed = str(ref) + assert printed == f"src/mod.py:2-3@{full[:ident.REF_HASH_CHARS]}" + parsed = ident.parse_ref(printed) + assert parsed.path == "src/mod.py" and (parsed.line_start, parsed.line_end) == (2, 3) + assert parsed.matches(full) + assert not parsed.matches(ident.span_sha(lines, 1, 2)) + + +def test_reference_parsing_is_right_anchored_for_odd_paths(): + ref = ident.parse_ref("docs/a:b@c.md:10-12@0123456789abcdef") + assert ref.path == "docs/a:b@c.md" and ref.line_start == 10 and ref.line_end == 12 + + +def test_windows_separators_normalise(): + assert ident.parse_ref("src\\pkg\\m.py:1-2@0123456789ab").path == "src/pkg/m.py" + + +@pytest.mark.parametrize( + "text", + [ + "src/m.py:1-2", # no hash + "src/m.py:3-2@0123456789abcdef", # inverted range + "src/m.py:0-2@0123456789abcdef", # zero line + "src/m.py:1-2@0123456789", # hash too short + "src/m.py:1-2@zzzzzzzzzzzzzzzz", # not hex + "/etc/passwd:1-1@0123456789abcdef", # absolute + "C:/Windows/win.ini:1-1@0123456789abcdef", + "../outside.py:1-1@0123456789abcdef", + "src/../../outside.py:1-1@0123456789abcdef", + "src/m\x00.py:1-1@0123456789abcdef", + ], +) +def test_malformed_or_escaping_references_are_rejected(text): + with pytest.raises(ValueError): + ident.parse_ref(text) + + +def test_content_matches_equality_and_excerpts(): + span = "def f(a, b):\n return a + b" + assert ident.content_matches(span, span) + assert ident.is_full_span(span, span) + assert ident.content_matches("def f(a, b):", span) + assert not ident.is_full_span("def f(a, b):", span) + assert not ident.content_matches("def f(a):", span) + assert not ident.content_matches(None, span) + assert not ident.content_matches("", span) + assert ident.content_matches(span.replace("\n", "\r\n"), span) + + +def test_repo_ids_differ_per_root_and_fold_case_on_windows(tmp_path): + a, b = tmp_path / "a", tmp_path / "b" + a.mkdir() + b.mkdir() + assert ident.repo_id_for(a) != ident.repo_id_for(b) + assert ident.repo_id_for(a) == ident.repo_id_for(str(a)) + if sys.platform == "win32": + assert ident.repo_id_for(str(a).upper()) == ident.repo_id_for(str(a).lower()) + + +def test_session_keys_are_scoped_to_repository_and_hide_the_tag(): + key_a = ident.session_key("repoA", "auth-fix") + assert key_a != ident.session_key("repoB", "auth-fix") + assert key_a != ident.session_key("repoA", "auth-fix-2") + assert "auth" not in key_a + + +@pytest.mark.parametrize("tag", ["", " ", "-lead", "a b", "x" * 129, "emoji-😀", "a/b"]) +def test_invalid_session_tags(tag): + with pytest.raises(ValueError): + ident.validate_session_tag(tag) + + +def test_valid_session_tags(): + assert ident.validate_session_tag(" auth-fix.2026:09_10 ") == "auth-fix.2026:09_10" + + +def test_randomised_edits_change_identity_iff_span_bytes_change(): + """Property: for any edit, the span hash is unchanged exactly when the span text is.""" + rng = random.Random(20260910) + vocab = ["x = 1", "return x", "", " pass", "def f():", "}", "# c", "y = 'é'"] + for _ in range(400): + lines = [rng.choice(vocab) for _ in range(rng.randint(3, 25))] + start = rng.randint(1, len(lines)) + end = rng.randint(start, len(lines)) + before = ident.span_sha(lines, start, end) + edited = list(lines) + op = rng.choice(["replace", "insert", "delete", "noop"]) + pos = rng.randrange(len(edited)) + if op == "replace": + edited[pos] = rng.choice(vocab) + elif op == "insert": + edited.insert(pos, rng.choice(vocab)) + elif op == "delete" and len(edited) > 1: + del edited[pos] + after = ident.span_sha(edited, start, end) + same_text = ident.span_text(edited, start, end) == ident.span_text(lines, start, end) + assert (after == before) == same_text diff --git a/tests/test_memory_store.py b/tests/test_memory_store.py new file mode 100644 index 0000000..e77d289 --- /dev/null +++ b/tests/test_memory_store.py @@ -0,0 +1,208 @@ +"""memory.sqlite: schema/migrations, ledger semantics, scoping, GC, recovery, locking.""" + +from __future__ import annotations + +import sqlite3 +from datetime import datetime, timedelta, timezone + +import pytest + +from codebase_index.memory.store import ( + SCHEMA_VERSION, + MemoryStore, + MemoryUnavailable, + NewDelivery, +) + +NOW = datetime(2026, 9, 10, 12, 0, tzinfo=timezone.utc) + + +def _item(path="src/a.py", span="s" * 64, snippet="p" * 64, full=True, start=1, end=5, tokens=40): + return NewDelivery(path=path, span_sha=span, line_count=end - start + 1, + first_line_sha="f" * 16, snippet_sha=snippet, full=full, + line_start=start, line_end=end, token_est=tokens) + + +@pytest.fixture +def store(tmp_path): + with MemoryStore.open(tmp_path / "memory.sqlite") as s: + yield s + + +def test_open_creates_current_schema_and_is_idempotent(tmp_path): + path = tmp_path / "memory.sqlite" + with MemoryStore.open(path) as s: + assert s.schema_version == SCHEMA_VERSION + with MemoryStore.open(path) as s: + assert s.schema_version == SCHEMA_VERSION + assert s.recovered_from is None + + +def test_migrations_run_in_order_inside_transactions(tmp_path): + path = tmp_path / "memory.sqlite" + with MemoryStore.open(path): + pass + from codebase_index.memory.store import MIGRATIONS + + future = {**MIGRATIONS, 2: ("ALTER TABLE atoms ADD COLUMN note TEXT",)} + with MemoryStore.open(path, migrations=future, schema_version=2) as s: + assert s.schema_version == 2 + cols = [r[1] for r in s.conn.execute("PRAGMA table_info(atoms)")] + assert "note" in cols + + broken = {**MIGRATIONS, 2: ("ALTER TABLE nope ADD COLUMN x TEXT",)} + other = tmp_path / "other.sqlite" + with MemoryStore.open(other): + pass + with pytest.raises(MemoryUnavailable): + MemoryStore.open(other, migrations=broken, schema_version=2) + with MemoryStore.open(other) as s: + assert s.schema_version == 1 # failed migration rolled back, nothing half-applied + + +def test_newer_schema_is_refused_without_touching_the_file(tmp_path): + path = tmp_path / "memory.sqlite" + with MemoryStore.open(path) as s: + s.conn.execute("UPDATE meta SET value = '99' WHERE key = 'schema_version'") + before = path.read_bytes() + with pytest.raises(MemoryUnavailable, match="newer"): + MemoryStore.open(path) + assert path.read_bytes() == before + + +def test_corrupt_store_is_preserved_and_replaced(tmp_path): + path = tmp_path / "memory.sqlite" + path.write_bytes(b"this is not a sqlite database" * 100) + with MemoryStore.open(path) as s: + assert s.recovered_from is not None + assert (tmp_path / s.recovered_from).read_bytes().startswith(b"this is not") + assert s.schema_version == SCHEMA_VERSION + + +def test_known_requires_full_span_or_identical_snippet(store): + sid = store.touch_session("repo", "tag", now=NOW) + store.record("repo", sid, [_item(full=False, snippet="a" * 64)], now=NOW) + assert store.is_known(sid, "src/a.py", "s" * 64, "a" * 64) + assert not store.is_known(sid, "src/a.py", "s" * 64, "b" * 64) # different skeleton + store.record("repo", sid, [_item(full=True, snippet="c" * 64)], now=NOW) + assert store.is_known(sid, "src/a.py", "s" * 64, "b" * 64) # whole span delivered + assert not store.is_known(sid, "src/a.py", "t" * 64, "c" * 64) # other bytes + + +def test_invalidated_delivery_is_not_known_until_redelivered(store): + sid = store.touch_session("repo", "tag", now=NOW) + store.record("repo", sid, [_item()], now=NOW) + atom_id = store.pending(sid)[0].atom_id + store.mark_invalid(sid, [(atom_id, "changed")]) + assert not store.is_known(sid, "src/a.py", "s" * 64, "p" * 64) + assert store.pending(sid) == [] + store.record("repo", sid, [_item()], now=NOW) + assert store.is_known(sid, "src/a.py", "s" * 64, "p" * 64) + + +def test_sessions_and_repositories_are_isolated(store): + a = store.touch_session("repoA", "tag", now=NOW) + b = store.touch_session("repoB", "tag", now=NOW) + other = store.touch_session("repoA", "tag2", now=NOW) + store.record("repoA", a, [_item()], now=NOW) + assert not store.is_known(b, "src/a.py", "s" * 64, "p" * 64) + assert not store.is_known(other, "src/a.py", "s" * 64, "p" * 64) + assert store.find_session("repoB", "missing") is None + assert store.stats("repoA")["deliveries"] == 1 + assert store.stats("repoB")["deliveries"] == 0 + + +def test_counters_and_stats(store): + sid = store.touch_session("repo", "tag", now=NOW) + store.touch_session("repo", "tag", now=NOW) + store.record("repo", sid, [_item(), _item(path="src/b.py")], now=NOW) + store.add_counters(sid, tokens_delivered=80, tokens_saved=40, reused=1, invalidations=2) + stats = store.stats("repo") + assert stats["sessions"] == 1 and stats["atoms"] == 2 and stats["deliveries"] == 2 + assert (stats["tokens_delivered"], stats["tokens_saved"]) == (80, 40) + assert (stats["reused"], stats["invalidations"]) == (1, 2) + assert stats["bytes"] > 0 + + +def test_gc_expires_old_sessions_and_orphan_atoms_but_keeps_recent(store): + old = store.touch_session("repo", "old", now=NOW - timedelta(days=30)) + new = store.touch_session("repo", "new", now=NOW) + store.record("repo", old, [_item(path="src/old.py")], now=NOW - timedelta(days=30)) + store.record("repo", new, [_item(path="src/new.py")], now=NOW) + result = store.gc(now=NOW, retention_days=14, max_deliveries=1000) + assert result == {"expired_sessions": 1, "capped_sessions": 0, "orphan_atoms": 1} + assert store.is_known(new, "src/new.py", "s" * 64, "p" * 64) + assert store.stats("repo")["atoms"] == 1 + assert not store.gc_due(now=NOW) + + +def test_gc_caps_total_deliveries_by_least_recent_use(store): + for day in range(5): + sid = store.touch_session("repo", f"s{day}", now=NOW - timedelta(days=4 - day)) + store.record("repo", sid, [_item(path=f"src/{day}_{i}.py") for i in range(3)], + now=NOW) + result = store.gc(now=NOW, retention_days=365, max_deliveries=7) + assert result["capped_sessions"] == 3 + assert store.stats("repo")["deliveries"] == 6 + assert store.find_session("repo", "s4") is not None + + +def test_gc_only_shrinks_what_is_known(store): + """GC never adds knowledge: every key known after GC was known before it.""" + keys = [] + for day in range(4): + sid = store.touch_session("repo", f"s{day}", now=NOW - timedelta(days=day * 10)) + store.record("repo", sid, [_item(path=f"src/{day}.py")], now=NOW) + keys.append((sid, f"src/{day}.py")) + before = {k for k in keys if store.is_known(k[0], k[1], "s" * 64, "p" * 64)} + store.gc(now=NOW, retention_days=15, max_deliveries=2) + after = {k for k in keys if store.is_known(k[0], k[1], "s" * 64, "p" * 64)} + assert after <= before and len(after) < len(before) + + +def test_clear_one_session_or_whole_repository(store): + a = store.touch_session("repo", "a", now=NOW) + b = store.touch_session("repo", "b", now=NOW) + elsewhere = store.touch_session("other", "a", now=NOW) + for sid, repo in ((a, "repo"), (b, "repo"), (elsewhere, "other")): + store.record(repo, sid, [_item()], now=NOW) + assert store.clear("repo", "a") == 1 + assert store.find_session("repo", "b") is not None + assert store.clear("repo") == 1 + assert store.stats("repo")["atoms"] == 0 + assert store.stats("other")["deliveries"] == 1 + + +def test_first_line_hint_by_reference_prefix(store): + sid = store.touch_session("repo", "tag", now=NOW) + store.record("repo", sid, [_item(span="abcdef0123456789" + "0" * 48)], now=NOW) + assert store.first_line_hint("repo", "src/a.py", "abcdef0123456789") == "f" * 16 + assert store.first_line_hint("repo", "src/a.py", "ffff000011112222") is None + assert store.first_line_hint("other", "src/a.py", "abcdef0123456789") is None + + +def test_lock_contention_degrades_instead_of_blocking_forever(tmp_path): + path = tmp_path / "memory.sqlite" + with MemoryStore.open(path): + pass + holder = sqlite3.connect(path, isolation_level=None) + holder.execute("BEGIN IMMEDIATE") + try: + with MemoryStore.open(path, busy_timeout_ms=50) as s: + with pytest.raises(MemoryUnavailable, match="busy"): + s.touch_session("repo", "tag", now=NOW) + finally: + holder.execute("ROLLBACK") + holder.close() + + +def test_store_contains_no_source_text(tmp_path): + path = tmp_path / "memory.sqlite" + secret_line = "AWS_SECRET_ACCESS_KEY = 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'" + with MemoryStore.open(path) as s: + sid = s.touch_session("repo", "tag-sha", now=NOW) + s.record("repo", sid, [_item()], now=NOW) + s.conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") + blob = b"".join(p.read_bytes() for p in tmp_path.iterdir() if p.is_file()) + assert secret_line.encode() not in blob + assert b"wJalrXUtnFEMI" not in blob diff --git a/tests/test_memory_validate.py b/tests/test_memory_validate.py new file mode 100644 index 0000000..0e99cdc --- /dev/null +++ b/tests/test_memory_validate.py @@ -0,0 +1,210 @@ +"""Working-tree validation: verdict states, relocation, and gate parity with the indexer.""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +import pytest + +from codebase_index.config import Config +from codebase_index.discovery.gates import PathGate +from codebase_index.discovery.walker import walk +from codebase_index.memory import identity as ident +from codebase_index.memory.validate import FileView, WorkingTree, locate, validate + +BODY = "def total(items):\n return sum(i.price for i in items)\n" + + +def _repo(tmp_path: Path, files: dict[str, str | bytes], **cfg) -> tuple[Path, WorkingTree]: + root = tmp_path / "repo" + for rel, content in files.items(): + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + if isinstance(content, bytes): + path.write_bytes(content) + else: + path.write_text(content, encoding="utf-8", newline="") + root.mkdir(exist_ok=True) + config = Config(**cfg) + config.root = str(root) + return root, WorkingTree(PathGate(root, config)) + + +def _ref(root: Path, rel: str, start: int, end: int) -> ident.EvidenceRef: + lines = ident.split_lines((root / rel).read_bytes()) + return ident.make_ref(rel, start, end, ident.span_sha(lines, start, end)) + + +def _fresh(root: Path, config_kwargs: dict | None = None) -> WorkingTree: + config = Config(**(config_kwargs or {})) + config.root = str(root) + return WorkingTree(PathGate(root, config)) + + +def test_unchanged_evidence_is_valid(tmp_path): + root, tree = _repo(tmp_path, {"src/cart.py": "import os\n\n" + BODY}) + ref = _ref(root, "src/cart.py", 3, 4) + verdict = validate(ref, tree) + assert verdict.state == "valid" and verdict.valid + assert (verdict.line_start, verdict.line_end) == (3, 4) + + +def test_edit_elsewhere_in_the_file_keeps_the_span_valid(tmp_path): + root, _ = _repo(tmp_path, {"src/cart.py": "import os\n\n" + BODY + "\nX = 1\n"}) + ref = _ref(root, "src/cart.py", 3, 4) + (root / "src/cart.py").write_text("import os\n\n" + BODY + "\nX = 2\n", encoding="utf-8") + assert validate(ref, _fresh(root)).state == "valid" + + +@pytest.mark.parametrize("with_hint", [True, False]) +def test_insertion_above_relocates(tmp_path, with_hint): + root, _ = _repo(tmp_path, {"src/cart.py": "import os\n\n" + BODY}) + ref = _ref(root, "src/cart.py", 3, 4) + hint = ident.line_sha("def total(items):") if with_hint else None + (root / "src/cart.py").write_text("import os\nimport sys\n\n\n" + BODY, encoding="utf-8") + verdict = validate(ref, _fresh(root), first_line_sha=hint) + assert verdict.state == "relocated" and verdict.valid + assert (verdict.line_start, verdict.line_end) == (5, 6) + + +def test_body_change_invalidates(tmp_path): + root, _ = _repo(tmp_path, {"src/cart.py": BODY}) + ref = _ref(root, "src/cart.py", 1, 2) + (root / "src/cart.py").write_text(BODY.replace("sum(", "max("), encoding="utf-8") + verdict = validate(ref, _fresh(root)) + assert verdict.state == "changed" and not verdict.valid + + +def test_whitespace_only_change_invalidates(tmp_path): + root, _ = _repo(tmp_path, {"src/cart.py": BODY}) + ref = _ref(root, "src/cart.py", 1, 2) + (root / "src/cart.py").write_text(BODY.replace(" return", " return"), encoding="utf-8") + assert validate(ref, _fresh(root)).state == "changed" + + +def test_crlf_conversion_is_not_a_change(tmp_path): + root, _ = _repo(tmp_path, {"src/cart.py": BODY}) + ref = _ref(root, "src/cart.py", 1, 2) + (root / "src/cart.py").write_bytes(BODY.replace("\n", "\r\n").encode()) + assert validate(ref, _fresh(root)).state == "valid" + + +def test_duplicated_content_is_ambiguous_not_valid(tmp_path): + root, _ = _repo(tmp_path, {"src/cart.py": "# a\n" + BODY}) + ref = _ref(root, "src/cart.py", 2, 3) + (root / "src/cart.py").write_text("# b\n\n" + BODY + BODY, encoding="utf-8") + assert validate(ref, _fresh(root)).state == "ambiguous" + + +def test_deleted_file(tmp_path): + root, _ = _repo(tmp_path, {"src/cart.py": BODY}) + ref = _ref(root, "src/cart.py", 1, 2) + (root / "src/cart.py").unlink() + assert validate(ref, _fresh(root)).state == "deleted" + + +def test_move_to_another_file_is_not_reused(tmp_path): + root, _ = _repo(tmp_path, {"src/cart.py": BODY}) + ref = _ref(root, "src/cart.py", 1, 2) + (root / "src/core").mkdir() + (root / "src/cart.py").replace(root / "src/core/cart.py") + assert validate(ref, _fresh(root)).state == "deleted" + + +def test_range_past_end_of_shortened_file_is_changed(tmp_path): + root, _ = _repo(tmp_path, {"src/cart.py": BODY + "x = 1\n"}) + ref = _ref(root, "src/cart.py", 2, 3) + (root / "src/cart.py").write_text("y = 2\n", encoding="utf-8") + assert validate(ref, _fresh(root)).state == "changed" + + +@pytest.mark.parametrize( + "rel,files,cfg", + [ + (".env", {".env": "API_KEY=abc\n"}, {}), + ("config/prod.pem", {"config/prod.pem": "-----BEGIN KEY-----\n"}, {}), + ("node_modules/pkg/index.js", {"node_modules/pkg/index.js": "x\n"}, {}), + ("build/out.py", {"build/out.py": "x\n"}, {}), + ("private/notes.py", {"private/notes.py": "x\n", ".gitignore": "private/\n"}, {}), + ("big.py", {"big.py": "x = 1\n" * 50}, {"max_file_bytes": 64}), + ("blob.py", {"blob.py": b"x = 1\n\x00\x01"}, {}), + ], +) +def test_gated_paths_are_excluded(tmp_path, rel, files, cfg): + root, _ = _repo(tmp_path, files) + ref = ident.EvidenceRef(rel, 1, 1, "0" * 16) + verdict = validate(ref, _fresh(root, cfg)) + assert verdict.state == "excluded" and not verdict.valid + + +def test_name_gated_files_are_never_opened(tmp_path, monkeypatch): + root, _ = _repo(tmp_path, {".env": "API_KEY=abc\n", "node_modules/a.js": "x\n"}) + opened: list[str] = [] + real = Path.read_bytes + + def spy(self): + opened.append(self.name) + return real(self) + + monkeypatch.setattr(Path, "read_bytes", spy) + tree = _fresh(root) + for rel in (".env", "node_modules/a.js"): + assert validate(ident.EvidenceRef(rel, 1, 1, "0" * 16), tree).state == "excluded" + assert opened == [] + + +@pytest.mark.skipif(sys.platform != "win32", reason="case-insensitive filesystem check") +def test_case_variant_cannot_bypass_directory_gate(tmp_path): + root, _ = _repo(tmp_path, {"node_modules/pkg/a.js": "module.exports = 1\n"}) + ref = ident.EvidenceRef("NODE_MODULES/pkg/a.js", 1, 1, "0" * 16) + assert validate(ref, _fresh(root)).state == "excluded" + + +def test_symlink_outside_repository_is_excluded(tmp_path): + outside = tmp_path / "outside.py" + outside.write_text("SECRET = 1\n", encoding="utf-8") + root, _ = _repo(tmp_path, {"src/keep.py": "x = 1\n"}) + try: + os.symlink(outside, root / "src" / "link.py") + except (OSError, NotImplementedError): + pytest.skip("symlinks unavailable") + ref = ident.EvidenceRef("src/link.py", 1, 1, "0" * 16) + verdict = validate(ref, _fresh(root)) + assert verdict.state == "excluded" and "outside" in verdict.reason + + +def test_gate_admits_exactly_what_the_walker_indexes(sample_repo): + """Security parity: validation can never read a file the indexer would refuse.""" + config = Config() + config.root = str(sample_repo) + walked = {c.rel_path for c in walk(sample_repo, config)} + gate = PathGate(sample_repo, config) + admitted = set() + for path in Path(sample_repo).rglob("*"): + if path.is_file(): + rel = path.relative_to(sample_repo).as_posix() + if gate.read(rel).state == "ok": + admitted.add(rel) + assert admitted == walked + assert "node_modules/leftpad/index.js" not in admitted + + +def test_locate_is_bounded_and_limited(): + view = FileView(["same"] * 50) + sha = ident.span_sha(view.lines, 1, 2) + assert locate(view, sha, 2, limit=2) == [1, 2] + assert locate(view, sha, 2, max_steps=10) is None + assert locate(view, sha, 99) == [] + + +def test_bounded_search_failure_is_reported_invalid(tmp_path, monkeypatch): + import codebase_index.memory.validate as mod + + root, _ = _repo(tmp_path, {"src/cart.py": "import os\n\n" + BODY}) + ref = _ref(root, "src/cart.py", 3, 4) + (root / "src/cart.py").write_text("import os\nimport sys\n\n\n" + BODY, encoding="utf-8") + monkeypatch.setattr(mod, "MAX_SCAN_STEPS", 1) + verdict = mod.validate(ref, _fresh(root)) + assert verdict.state == "changed" and "too large" in verdict.reason From 01f98b5f61f96345e80d4418982428d5f2601fd7 Mon Sep 17 00:00:00 2001 From: denfry Date: Thu, 10 Sep 2026 10:20:09 +0300 Subject: [PATCH 07/18] feat(retrieval): verify delivered snippets and reuse them within a session search() gains an optional evidence hook that runs after ranking, budgeting and pagination are final, so memory cannot change which results are returned or which carry snippets. Without a session the packet is unchanged except for stale: true on a result whose index text no longer matches the working tree. With an explicit session tag, a snippet the session already received from byte-identical source is replaced by snippet: null, reused: true (a skeleton is only withheld when the same skeleton or the whole span was delivered), and evidence the session received earlier that has since changed is reported once under memory.invalidated. Budget accounting is untouched: filling withheld snippets back in reproduces the no-memory packet exactly, which the tests assert. CBX_MEMORY=0 or memory.enabled=false restores 1.10.0 output. Sessions are never inferred from a process, an environment variable or a time window, because none of those identify what an agent still holds in context. --- src/codebase_index/config.py | 9 + src/codebase_index/memory/session.py | 197 ++++++++++++++++++ src/codebase_index/retrieval/pipeline.py | 8 +- src/codebase_index/service.py | 86 ++++++-- tests/test_memory_session.py | 250 +++++++++++++++++++++++ 5 files changed, 533 insertions(+), 17 deletions(-) create mode 100644 src/codebase_index/memory/session.py create mode 100644 tests/test_memory_session.py diff --git a/src/codebase_index/config.py b/src/codebase_index/config.py index 217cad7..72a346d 100644 --- a/src/codebase_index/config.py +++ b/src/codebase_index/config.py @@ -43,6 +43,14 @@ class GraphConfig(BaseModel): node_cap: int = 40 +class MemoryConfig(BaseModel): + """Evidence memory (docs/MEMORY.md). Not indexing-relevant, so not in config_hash.""" + + enabled: bool = True + retention_days: int = 14 + max_deliveries: int = 50_000 + + class Config(BaseModel): root: str = "." languages: Union[Literal["auto"], list[str]] = "auto" @@ -53,6 +61,7 @@ class Config(BaseModel): retrieval: RetrievalConfig = RetrievalConfig() embeddings: EmbeddingsConfig = EmbeddingsConfig() graph: GraphConfig = GraphConfig() + memory: MemoryConfig = MemoryConfig() redaction: dict = {"enabled": True} def config_hash(self) -> str: diff --git a/src/codebase_index/memory/session.py b/src/codebase_index/memory/session.py new file mode 100644 index 0000000..17a59eb --- /dev/null +++ b/src/codebase_index/memory/session.py @@ -0,0 +1,197 @@ +"""Evidence at retrieval time: verify each delivered snippet, reuse it within a session. + +`retrieval.pipeline.search` calls the processor after ranking, budgeting and pagination +are final (docs/MEMORY.md). It never adds, removes or reorders a result, and it never +changes which results carry a snippet. It can only: + +* mark a result ``stale`` when the index text it came from no longer matches the file; +* in a session, replace a snippet that session already received from byte-identical + source with ``snippet: null, reused: true``; +* in a session, list evidence the session received earlier that has since changed. + +A session tag names exactly one agent context. It is always supplied by the caller — +never inferred from a process, an environment variable or a time window — because none +of those identify what an agent still holds in its context window. +""" + +from __future__ import annotations + +import sqlite3 +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Any, Iterator, Optional, Sequence + +from ..config import Config +from ..discovery.gates import PathGate +from . import identity as ident +from .store import MemoryStore, MemoryUnavailable, NewDelivery, utc_now +from .validate import WorkingTree, validate + + +@dataclass +class Session: + tag: str + repo_id: str + store: Optional[MemoryStore] = None + session_id: Optional[int] = None + unavailable: Optional[str] = None + + @property + def usable(self) -> bool: + return self.store is not None and self.session_id is not None and not self.unavailable + + +class EvidenceProcessor: + def __init__(self, *, root: Path, config: Config, now: datetime, + session: Optional[Session] = None) -> None: + self.tree = WorkingTree(PathGate(root, config)) + self.config = config + self.now = now + self.session = session + + def __call__(self, payload: dict, candidates: Sequence[Any]) -> None: + session = self.session + notices = self._notices(session) if session is not None and session.usable else [] + reused = tokens_saved = tokens_delivered = 0 + fresh: list[NewDelivery] = [] + + for result, candidate in zip(payload.get("results", []), candidates): + snippet = result.get("snippet") + if not snippet: + continue # nothing was delivered for this result + observed = self._observe(result, candidate) + if observed is None: + result["stale"] = True + continue + if session is None or not session.usable: + continue + rel, span, span_sha = observed + tokens = int(result.get("token_est") or 0) + snippet_sha = ident.sha_hex(snippet) + if self._known(session, rel, span_sha, snippet_sha): + result["snippet"] = None + result["reused"] = True + reused += 1 + tokens_saved += tokens + continue + fresh.append( + NewDelivery( + path=rel, + span_sha=span_sha, + line_count=int(result["line_end"]) - int(result["line_start"]) + 1, + first_line_sha=ident.line_sha(span.split("\n", 1)[0]), + snippet_sha=snippet_sha, + full=not result.get("skeletonized") + and ident.is_full_span(getattr(candidate, "content", None), span), + line_start=int(result["line_start"]), + line_end=int(result["line_end"]), + token_est=tokens, + ) + ) + tokens_delivered += tokens + + if session is not None: + payload["memory"] = self._finish( + session, fresh, notices, + reused=reused, tokens_saved=tokens_saved, tokens_delivered=tokens_delivered, + ) + + def _observe(self, result: dict, candidate: Any) -> Optional[tuple[str, str, str]]: + """(path, span text, span sha) when the delivered snippet matches the working tree.""" + try: + rel = ident.normalize_rel_path(str(result["path"])) + except ValueError: + return None + view, _state, _reason = self.tree.view(rel) + if view is None: + return None + span = ident.span_text(view.lines, int(result["line_start"]), int(result["line_end"])) + if span is None or not ident.content_matches(getattr(candidate, "content", None), span): + return None + return rel, span, ident.sha_hex(span) + + def _known(self, session: Session, rel: str, span_sha: str, snippet_sha: str) -> bool: + assert session.store is not None and session.session_id is not None + try: + return session.store.is_known(session.session_id, rel, span_sha, snippet_sha) + except sqlite3.Error as exc: + session.unavailable = f"memory store unavailable: {exc}" + return False + + def _notices(self, session: Session) -> list[dict]: + assert session.store is not None and session.session_id is not None + try: + pending = session.store.pending(session.session_id) + except sqlite3.Error as exc: + session.unavailable = f"memory store unavailable: {exc}" + return [] + notices: list[dict] = [] + states: list[tuple[int, str]] = [] + for delivery in pending: + ref = ident.EvidenceRef(delivery.path, delivery.line_start, delivery.line_end, + delivery.span_sha) + verdict = validate(ref, self.tree, first_line_sha=delivery.first_line_sha) + if not verdict.valid: + states.append((delivery.atom_id, verdict.state)) + notices.append({"ref": str(ref), "state": verdict.state}) + if states: + try: + session.store.mark_invalid(session.session_id, states) + except (MemoryUnavailable, sqlite3.Error) as exc: + session.unavailable = str(exc) + return notices + + def _finish(self, session: Session, fresh: list[NewDelivery], notices: list[dict], *, + reused: int, tokens_saved: int, tokens_delivered: int) -> dict: + block: dict = { + "session": session.tag, + "reused": reused, + "tokens_saved": tokens_saved, + "invalidated": notices, + } + if not session.usable: + block["available"] = False + block["reason"] = session.unavailable or "memory store unavailable" + return block + store = session.store + assert store is not None and session.session_id is not None + try: + store.record(session.repo_id, session.session_id, fresh, now=self.now) + store.add_counters( + session.session_id, tokens_delivered=tokens_delivered, + tokens_saved=tokens_saved, reused=reused, invalidations=len(notices), + ) + if store.gc_due(now=self.now): + store.gc(now=self.now, retention_days=self.config.memory.retention_days, + max_deliveries=self.config.memory.max_deliveries) + except (MemoryUnavailable, sqlite3.Error) as exc: + # The packet is already correct; only future withholding is affected. + block["degraded"] = str(exc) + return block + + +@contextmanager +def open_evidence(*, root: Path, config: Config, memory_path: Path, tag: Optional[str], + now: Optional[datetime] = None) -> Iterator[EvidenceProcessor]: + """Processor for one retrieval call; opens the store only when a session is named.""" + now = now or utc_now() + session: Optional[Session] = None + store: Optional[MemoryStore] = None + if tag is not None: + repo_id = ident.repo_id_for(root) + session = Session(tag=tag, repo_id=repo_id) + try: + store = MemoryStore.open(memory_path) + session.store = store + session.session_id = store.touch_session( + repo_id, ident.session_key(repo_id, tag), now=now + ) + except (MemoryUnavailable, sqlite3.Error) as exc: + session.unavailable = str(exc) + try: + yield EvidenceProcessor(root=Path(root), config=config, now=now, session=session) + finally: + if store is not None: + store.close() diff --git a/src/codebase_index/retrieval/pipeline.py b/src/codebase_index/retrieval/pipeline.py index 9d5e3dd..2cac80e 100644 --- a/src/codebase_index/retrieval/pipeline.py +++ b/src/codebase_index/retrieval/pipeline.py @@ -9,7 +9,7 @@ import re import sqlite3 from pathlib import Path -from typing import Optional +from typing import Callable, Optional from ..config import Config from ..indexer.freshness import compute_freshness @@ -181,6 +181,7 @@ def search( compact: bool = True, compact_min_reduction: float = 0.25, explain: bool = False, + evidence: Optional[Callable[[dict, list], None]] = None, ) -> dict: tuning = tuning or DEFAULT_TUNING plan = detect_intent(query) @@ -299,4 +300,9 @@ def search( for rank, c in enumerate(ranked, start=1) ], } + if evidence is not None: + # Evidence memory (memory/session.py) sees the delivered page and the candidates + # behind it only after ranking, budgeting and pagination are final, so it cannot + # change which results are returned or which carry snippets. + evidence(payload, ranked[offset:offset + limit]) return payload diff --git a/src/codebase_index/service.py b/src/codebase_index/service.py index 1857221..464d042 100644 --- a/src/codebase_index/service.py +++ b/src/codebase_index/service.py @@ -12,8 +12,9 @@ import os import sqlite3 import subprocess +from contextlib import contextmanager from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Optional, Union +from typing import TYPE_CHECKING, Any, Callable, Iterator, Optional, Union if TYPE_CHECKING: from .config import Config @@ -77,33 +78,86 @@ def search_payload( no_fallback: bool = False, backend: Any = None, raw: bool = False, + session: Optional[str] = None, ) -> dict: """One search session: open the DB (vector-enabled when the backend is live), run retrieval, return the payload dict both surfaces serialize. ``raw`` forces full snippets; otherwise snippets are skeletonized when - ``cfg.retrieval.compact_snippets`` is on (the default).""" + ``cfg.retrieval.compact_snippets`` is on (the default). + + ``session`` names one agent context for evidence reuse (docs/MEMORY.md); + ``ValueError`` for a malformed tag. Without it the packet is the plain + retrieval packet, plus ``stale`` on any result whose index text no longer + matches the working tree.""" from .retrieval.pipeline import search as run_search from .storage.db import Database compact = cfg.retrieval.compact_snippets and not raw + tag = session_tag(session) + enabled = memory_enabled(cfg) with Database(db_path) as db: if backend is not None and getattr(backend, "enabled", False): db.enable_vectors() - return run_search( - db.conn, - query, - mode=mode, - limit=limit, - offset=offset, - token_budget=token_budget, - no_fallback=no_fallback, - backend=backend, - root=Path(cfg.root), - config=cfg, - compact=compact, - compact_min_reduction=cfg.retrieval.compact_min_reduction, - ) + with _evidence(cfg, tag, enabled) as evidence: + payload = run_search( + db.conn, + query, + mode=mode, + limit=limit, + offset=offset, + token_budget=token_budget, + no_fallback=no_fallback, + backend=backend, + root=Path(cfg.root), + config=cfg, + compact=compact, + compact_min_reduction=cfg.retrieval.compact_min_reduction, + evidence=evidence, + ) + if tag is not None and not enabled: + payload["memory"] = {"session": tag, "available": False, "reason": "memory is disabled"} + return payload + + +def memory_enabled(cfg: "Config") -> bool: + """Evidence memory is on unless the config disables it or ``CBX_MEMORY=0``.""" + if os.environ.get("CBX_MEMORY", "").strip() == "0": + return False + return bool(cfg.memory.enabled) + + +def memory_path_for(cfg: "Config") -> Path: + """``CBX_MEMORY_PATH``, else next to a ``CBX_DB_PATH`` override, else the cache dir.""" + override = os.environ.get("CBX_MEMORY_PATH") + if override: + return Path(override) + db_override = os.environ.get("CBX_DB_PATH") + if db_override: + return Path(db_override).with_name("memory.sqlite") + return cache_dir_for(cfg) / "memory.sqlite" + + +def session_tag(session: Optional[str]) -> Optional[str]: + """Validated session tag, or None. Sessions are only ever named explicitly.""" + if session is None or not session.strip(): + return None + from .memory.identity import validate_session_tag + + return validate_session_tag(session) + + +@contextmanager +def _evidence(cfg: "Config", tag: Optional[str], enabled: bool) -> Iterator[Any]: + if not enabled: + yield None + return + from .memory.session import open_evidence + + with open_evidence( + root=Path(cfg.root), config=cfg, memory_path=memory_path_for(cfg), tag=tag + ) as processor: + yield processor def diff_impact_payload( diff --git a/tests/test_memory_session.py b/tests/test_memory_session.py new file mode 100644 index 0000000..19b1efe --- /dev/null +++ b/tests/test_memory_session.py @@ -0,0 +1,250 @@ +"""End-to-end evidence memory through the service layer: retrieve -> persist -> reuse -> +edit -> invalidate. The load-bearing assertions are equivalences with the no-memory +packet, because they are what make every measured saving attributable to memory alone. +""" + +from __future__ import annotations + +import copy +import sqlite3 + +import pytest + +from codebase_index.config import Config +from codebase_index.indexer.pipeline import build_index, update_index +from codebase_index.retrieval.pipeline import search +from codebase_index.service import search_payload +from codebase_index.storage.db import Database + +INVOICE = '''def compute_invoice_total(lines, tax_rate): + """Sum invoice line amounts and apply the tax rate.""" + subtotal = sum(line.amount for line in lines) + return round(subtotal * (1 + tax_rate), 2) + + +def format_invoice_number(prefix, sequence): + """Render an invoice number such as INV-000042.""" + return f"{prefix}-{sequence:06d}" +''' + +REFUND = '''def issue_refund(invoice, amount): + """Refund part of an invoice total.""" + if amount > invoice.total: + raise ValueError("refund exceeds invoice total") + return invoice.total - amount +''' + +QUERY = "compute invoice total" + + +@pytest.fixture +def repo(tmp_path, monkeypatch): + root = tmp_path / "repo" + (root / "src" / "billing").mkdir(parents=True) + (root / "src" / "billing" / "invoice.py").write_text(INVOICE, encoding="utf-8") + (root / "src" / "billing" / "refund.py").write_text(REFUND, encoding="utf-8") + monkeypatch.setenv("CBX_MEMORY_PATH", str(tmp_path / "memory.sqlite")) + monkeypatch.delenv("CBX_MEMORY", raising=False) + monkeypatch.delenv("CBX_DB_PATH", raising=False) + cfg = Config() + cfg.root = str(root) + db_path = tmp_path / "index.sqlite" + with Database(db_path) as db: + build_index(cfg, db, root=root) + return root, cfg, db_path + + +def _run(repo, query=QUERY, session=None): + _root, cfg, db_path = repo + return search_payload(db_path, cfg, query, mode="hybrid", limit=10, token_budget=1500, + no_fallback=False, session=session) + + +def _baseline(repo, monkeypatch, query=QUERY): + monkeypatch.setenv("CBX_MEMORY", "0") + try: + return _run(repo, query) + finally: + monkeypatch.delenv("CBX_MEMORY") + + +def _update(repo): + root, cfg, db_path = repo + with Database(db_path) as db: + update_index(cfg, db, root=root) + + +def _notice_states(payload): + return {n["ref"].rsplit("@", 1)[0].rsplit(":", 1)[0]: n["state"] + for n in payload["memory"]["invalidated"]} + + +def test_memory_disabled_is_identical_to_the_retrieval_pipeline(repo, monkeypatch): + root, cfg, db_path = repo + monkeypatch.setenv("CBX_MEMORY", "0") + via_service = _run(repo) + with Database(db_path) as db: + direct = search(db.conn, QUERY, mode="hybrid", limit=10, token_budget=1500, + no_fallback=False, root=root, config=cfg, compact=True, + compact_min_reduction=0.25) + assert via_service == direct + + +def test_enabled_without_session_on_a_fresh_index_is_identical(repo, monkeypatch): + assert _run(repo) == _baseline(repo, monkeypatch) + + +def test_first_session_call_is_the_baseline_plus_a_memory_block(repo, monkeypatch): + first = _run(repo, session="t1") + assert first.pop("memory") == {"session": "t1", "reused": 0, "tokens_saved": 0, + "invalidated": []} + assert first == _baseline(repo, monkeypatch) + + +def test_repeat_call_withholds_exactly_what_the_session_already_holds(repo, monkeypatch): + first = _run(repo, session="t1") + second = _run(repo, session="t1") + delivered = [r for r in first["results"] if r["snippet"]] + assert delivered + assert [r.get("reused", False) for r in second["results"]] == [ + bool(r["snippet"]) for r in first["results"] + ] + assert second["memory"]["reused"] == len(delivered) + assert second["memory"]["tokens_saved"] == sum(r["token_est"] for r in delivered) + + restored = copy.deepcopy(second) + restored.pop("memory") + for result, original in zip(restored["results"], first["results"]): + if result.pop("reused", False): + result["snippet"] = original["snippet"] + assert restored == _baseline(repo, monkeypatch) + + +def test_another_session_receives_everything(repo, monkeypatch): + _run(repo, session="t1") + other = _run(repo, session="t2") + assert other.pop("memory")["reused"] == 0 + assert other == _baseline(repo, monkeypatch) + + +def test_changed_evidence_is_reported_once_and_redelivered(repo): + root, _, _ = repo + first = _run(repo, session="t1") + body = next(r for r in first["results"] if r["path"] == "src/billing/refund.py") + assert "amount > invoice.total" in body["snippet"] + (root / "src/billing/refund.py").write_text(REFUND.replace(">", ">="), encoding="utf-8") + _update(repo) + second = _run(repo, session="t1") + assert _notice_states(second).get("src/billing/refund.py") == "changed" + redelivered = next(r for r in second["results"] if r["path"] == "src/billing/refund.py") + assert "amount >= invoice.total" in redelivered["snippet"] + assert not redelivered.get("reused") + # Unchanged evidence in the same packet is still withheld... + assert any(r.get("reused") for r in second["results"] + if r["path"] == "src/billing/invoice.py") + # ...and each invalidation is reported exactly once. + assert _run(repo, session="t1")["memory"]["invalidated"] == [] + + +def test_unindexed_edit_marks_results_stale_and_never_withholds_them(repo): + """The index still holds the old body; the snippet is flagged, never trusted or hidden.""" + root, _, _ = repo + _run(repo, session="t1") + (root / "src/billing/refund.py").write_text(REFUND.replace(">", ">="), encoding="utf-8") + second = _run(repo, session="t1") + stale = [r for r in second["results"] if r.get("stale")] + assert [r["path"] for r in stale] == ["src/billing/refund.py"] + assert "amount > invoice.total" in stale[0]["snippet"] + assert not stale[0].get("reused") + assert _notice_states(second).get("src/billing/refund.py") == "changed" + + +def test_excerpt_that_still_holds_is_not_stale(repo): + """A signature snippet stays accurate when only the body changes, so it is not flagged; + the span changed, so it is delivered again rather than withheld.""" + root, _, _ = repo + first = _run(repo, session="t1") + signature = next(r for r in first["results"] if r["path"] == "src/billing/invoice.py") + assert signature["snippet"] == "def compute_invoice_total(lines, tax_rate):" + (root / "src/billing/invoice.py").write_text(INVOICE.replace("round(", "floor("), + encoding="utf-8") + second = _run(repo, session="t1") + again = next(r for r in second["results"] if r["path"] == "src/billing/invoice.py") + assert not again.get("stale") and not again.get("reused") and again["snippet"] + + +def test_code_that_only_moved_within_its_file_is_still_reused(repo): + root, _, _ = repo + first = _run(repo, session="t1") + target = next(r for r in first["results"] if r["path"] == "src/billing/invoice.py" + and r["snippet"] and "def compute_invoice_total" in r["snippet"]) + (root / "src/billing/invoice.py").write_text("import math\nimport decimal\n\n\n" + INVOICE, + encoding="utf-8") + _update(repo) + second = _run(repo, session="t1") + moved = [r for r in second["results"] if r["path"] == "src/billing/invoice.py" + and (r["line_start"], r["line_end"]) == (target["line_start"] + 4, + target["line_end"] + 4)] + assert moved and moved[0].get("reused") is True and moved[0]["snippet"] is None + assert second["memory"]["invalidated"] == [] + + +def test_deleted_file_evidence_is_reported(repo): + root, _, _ = repo + first = _run(repo, "issue refund exceeds invoice total", session="t1") + assert any(r["path"] == "src/billing/refund.py" and r["snippet"] for r in first["results"]) + (root / "src/billing/refund.py").unlink() + _update(repo) + assert _notice_states(_run(repo, session="t1")).get("src/billing/refund.py") == "deleted" + + +def test_locked_memory_store_degrades_to_the_full_packet(repo, monkeypatch, tmp_path): + import codebase_index.memory.store as store_mod + + monkeypatch.setattr(store_mod, "BUSY_TIMEOUT_MS", 50) + _run(repo, session="t1") + holder = sqlite3.connect(tmp_path / "memory.sqlite", isolation_level=None) + holder.execute("BEGIN IMMEDIATE") + try: + second = _run(repo, session="t1") + finally: + holder.execute("ROLLBACK") + holder.close() + block = second.pop("memory") + assert block["available"] is False and block["reused"] == 0 + assert second == _baseline(repo, monkeypatch) + + +def test_session_with_memory_disabled_says_so(repo, monkeypatch): + monkeypatch.setenv("CBX_MEMORY", "0") + assert _run(repo, session="t1")["memory"] == { + "session": "t1", "available": False, "reason": "memory is disabled"} + + +def test_malformed_session_tag_is_rejected(repo): + with pytest.raises(ValueError): + _run(repo, session="not a tag!") + + +def test_session_survives_an_index_rebuild_with_unchanged_sources(repo): + root, cfg, db_path = repo + first = _run(repo, session="t1") + for suffix in ("", "-wal", "-shm"): + sidecar = db_path.with_name(db_path.name + suffix) + if sidecar.exists(): + sidecar.unlink() + with Database(db_path) as db: + build_index(cfg, db, root=root) + second = _run(repo, session="t1") + assert second["memory"]["reused"] == sum(1 for r in first["results"] if r["snippet"]) + + +def test_memory_store_never_contains_delivered_text_or_the_tag(repo, tmp_path): + first = _run(repo, session="privacy-canary-tag") + _run(repo, session="privacy-canary-tag") + blob = b"".join(p.read_bytes() for p in tmp_path.glob("memory.sqlite*")) + assert b"privacy-canary-tag" not in blob + for result in first["results"]: + for line in (result["snippet"] or "").splitlines(): + if len(line.strip()) >= 16: + assert line.strip().encode() not in blob From d049dcd7841ab3d329c1f8f022f1f17069bff00e Mon Sep 17 00:00:00 2001 From: denfry Date: Thu, 10 Sep 2026 10:50:11 +0300 Subject: [PATCH 08/18] feat(cli,mcp): verify evidence, session flags and memory maintenance - search/explain --session TAG and the MCP session parameter name one agent context for evidence reuse; malformed tags are rejected before any work. - verify [REF ...] [--session TAG] [--strict] and MCP verify_evidence re-check evidence against the working tree. Read-only and index-free; references are untrusted input. all_valid is false for an empty or unknown set. - memory gc and memory clear (with confirmation) are CLI-only maintenance, like clean: neither the skill wrappers nor MCP expose them. - stats, index_stats and healthcheck gain an additive memory block (counts, schema, size; no paths or content). doctor probes memory.sqlite read-only and reports corruption or a newer schema without repairing anything. - Markdown marks withheld snippets, stale results and invalidated evidence. MCP schema_version stays 1: every payload change is an added field. The three affected goldens change only by that block. --- src/codebase_index/cli.py | 141 +++++++++++++++++- src/codebase_index/doctor.py | 38 +++++ src/codebase_index/mcp/server.py | 77 ++++++++-- src/codebase_index/memory/store.py | 9 +- src/codebase_index/output/markdown.py | 54 ++++++- src/codebase_index/service.py | 135 +++++++++++++++++- tests/golden/mcp_healthcheck.json | 4 + tests/golden/mcp_index_stats.json | 4 + tests/golden/stats.json | 4 + tests/test_mcp_server.py | 1 + tests/test_memory_surfaces.py | 197 ++++++++++++++++++++++++++ 11 files changed, 643 insertions(+), 21 deletions(-) create mode 100644 tests/test_memory_surfaces.py diff --git a/src/codebase_index/cli.py b/src/codebase_index/cli.py index ffd216f..cbb9757 100644 --- a/src/codebase_index/cli.py +++ b/src/codebase_index/cli.py @@ -76,6 +76,32 @@ def _open_in_browser(path: Path) -> None: subprocess.Popen(["xdg-open", uri]) +def _checked_session(session: Optional[str]) -> Optional[str]: + """Validate a --session tag before any work is done; exit 2 when malformed.""" + from .service import session_tag + + try: + return session_tag(session) + except ValueError as exc: + typer.echo(f"[codebase-index] {exc}", err=True) + raise typer.Exit(code=2) + + +def _memory_line(memory: dict) -> str: + if not memory.get("enabled", True): + return "memory: disabled" + if not memory.get("exists"): + return "memory: empty (starts with the first --session search)" + if memory.get("available") is False: + return f"memory: unavailable — {memory.get('reason')}" + return ( + f"memory: {memory.get('atoms', 0)} evidence atoms · {memory.get('sessions', 0)} " + f"sessions · {memory.get('tokens_saved', 0)} tokens not resent · " + f"{memory.get('invalidations', 0)} invalidations · " + f"{memory.get('bytes', 0) // 1024} KB" + ) + + def _resolve_backend_for_search(ctx: "typer.Context"): """Embedding backend for query-time vector search (see service.search_backend).""" from .config import load @@ -383,6 +409,11 @@ def search( False, "--raw", help="Disable snippet skeletonization; return full raw snippets.", ), + session: Optional[str] = typer.Option( + None, "--session", + help="Tag naming ONE agent context: evidence it already received and that is " + "unchanged is not resent; changes to it are reported (docs/MEMORY.md).", + ), json_out: bool = typer.Option(False, "--json", help="Emit machine-readable JSON."), ) -> None: """Hybrid ranked search; returns compact results + recommended_reads.""" @@ -390,6 +421,7 @@ def search( from .output import markdown as md_renderer from .service import search_payload + session = _checked_session(session) if offset < 0: typer.echo("[codebase-index] --offset must be >= 0.") raise typer.Exit(code=2) @@ -408,6 +440,7 @@ def search( payload = search_payload( db_path, cfg, query, mode=mode, limit=limit, offset=offset, token_budget=token_budget, no_fallback=no_fallback, backend=backend, raw=raw, + session=session, ) want_json = json_out or (ctx.obj and ctx.obj.get("json")) @@ -523,6 +556,11 @@ def explain( False, "--raw", help="Disable snippet skeletonization; return full raw snippets.", ), + session: Optional[str] = typer.Option( + None, "--session", + help="Tag naming ONE agent context: evidence it already received and that is " + "unchanged is not resent; changes to it are reported (docs/MEMORY.md).", + ), json_out: bool = typer.Option(False, "--json", help="Emit machine-readable JSON."), ) -> None: """Intent-aware bundle for 'how does X work' / overview questions.""" @@ -530,12 +568,14 @@ def explain( from .output import markdown as md_renderer from .service import normalize_explain_query, search_payload + session = _checked_session(session) backend = _resolve_backend_for_search(ctx) db_path, cfg = _ensure_index(ctx) payload = search_payload( db_path, cfg, normalize_explain_query(query), mode="hybrid", limit=10, token_budget=token_budget, no_fallback=False, backend=backend, raw=raw, + session=session, ) want_json = json_out or (ctx.obj and ctx.obj.get("json")) @@ -603,6 +643,104 @@ def describe( typer.echo(json_renderer.render(payload) if is_json else md_renderer.render_describe(payload)) +@app.command("verify") +def verify( + ctx: typer.Context, + refs: Optional[list[str]] = typer.Argument( + None, help="Evidence references: path:start-end@hash (as printed by memory)." + ), + session: Optional[str] = typer.Option( + None, "--session", help="Also verify everything this session was given." + ), + strict: bool = typer.Option( + False, "--strict", help="Exit 1 unless every piece of evidence is still valid." + ), + json_flag: bool = typer.Option(False, "--json", help="Emit machine-readable JSON."), +) -> None: + """Is this evidence still true? Checks references against the working tree (read-only).""" + import json as _json + + from .output import markdown as md_renderer + from .service import resolve_db, verify_payload + + session = _checked_session(session) + if not refs and session is None: + typer.echo("[codebase-index] pass evidence references, --session, or both.", err=True) + raise typer.Exit(code=2) + _db_path, cfg = resolve_db(ctx.obj.get("root") if ctx.obj else None) + payload = verify_payload(cfg, refs or [], session) + is_json = json_flag or bool(ctx.obj and ctx.obj.get("json")) + typer.echo(_json.dumps(payload) if is_json else md_renderer.render_verify(payload)) + if strict and not payload["all_valid"]: + raise typer.Exit(code=1) + + +memory_app = typer.Typer( + help="Evidence memory maintenance. Deliberately not exposed to agents or MCP.", + no_args_is_help=True, +) +app.add_typer(memory_app, name="memory") + + +@memory_app.command("gc") +def memory_gc( + ctx: typer.Context, + json_flag: bool = typer.Option(False, "--json", help="Emit machine-readable JSON."), +) -> None: + """Drop expired sessions and orphaned evidence, then compact memory.sqlite.""" + import json as _json + + from .memory.store import MemoryUnavailable + from .service import memory_gc_payload, resolve_db + + _db_path, cfg = resolve_db(ctx.obj.get("root") if ctx.obj else None) + try: + payload = memory_gc_payload(cfg) + except MemoryUnavailable as exc: + typer.echo(f"[codebase-index] {exc}", err=True) + raise typer.Exit(code=1) + if json_flag or bool(ctx.obj and ctx.obj.get("json")): + typer.echo(_json.dumps(payload)) + elif not payload.get("exists"): + typer.echo("No evidence memory to collect.") + else: + typer.echo( + f"Removed {payload['expired_sessions']} expired and {payload['capped_sessions']} " + f"over-limit session(s), {payload['orphan_atoms']} orphaned atom(s). " + f"{payload['sessions']} session(s) and {payload['atoms']} atom(s) remain." + ) + + +@memory_app.command("clear") +def memory_clear( + ctx: typer.Context, + session: Optional[str] = typer.Option(None, "--session", help="Forget only this session."), + yes: bool = typer.Option(False, "--yes", help="Skip the confirmation prompt."), + json_flag: bool = typer.Option(False, "--json", help="Emit machine-readable JSON."), +) -> None: + """Forget one session, or all evidence memory for this repository. Source is untouched.""" + import json as _json + + from .memory.store import MemoryUnavailable + from .service import memory_clear_payload, resolve_db + + session = _checked_session(session) + is_json = json_flag or bool(ctx.obj and ctx.obj.get("json")) + _db_path, cfg = resolve_db(ctx.obj.get("root") if ctx.obj else None) + if not yes and not is_json and sys.stdin.isatty(): + what = f"session '{session}'" if session else "all evidence memory for this repository" + typer.confirm(f"Forget {what}?", abort=True) + try: + payload = memory_clear_payload(cfg, session) + except MemoryUnavailable as exc: + typer.echo(f"[codebase-index] {exc}", err=True) + raise typer.Exit(code=1) + if is_json: + typer.echo(_json.dumps(payload)) + else: + typer.echo(f"Forgot {payload['removed_sessions']} session(s).") + + @app.command("graph") def graph_view( ctx: typer.Context, @@ -687,7 +825,7 @@ def stats( raise typer.Exit(code=0) with Database(db_path) as db: - payload = stats_payload(db.conn) + payload = stats_payload(db.conn, cfg=_cfg) if is_json: typer.echo(_json.dumps(payload)) @@ -700,6 +838,7 @@ def stats( flag = " ⚠ 0 symbols" if (r["symbols"] or 0) == 0 and r["files"] >= 3 else "" tier = " · partial graph (Tier-B)" if r["graph"] == "partial" else "" typer.echo(f" {r['lang']}: {r['files']} files, {r['symbols']} symbols{flag}{tier}") + typer.echo(_memory_line(payload.get("memory") or {})) @app.command() diff --git a/src/codebase_index/doctor.py b/src/codebase_index/doctor.py index 0770cae..c4fcadd 100644 --- a/src/codebase_index/doctor.py +++ b/src/codebase_index/doctor.py @@ -7,6 +7,7 @@ from __future__ import annotations +import sqlite3 from dataclasses import dataclass from pathlib import Path from typing import Literal @@ -126,9 +127,46 @@ def run_doctor(root: Path, config: Config) -> list[Finding]: ) ) + findings.append(_memory_finding(config)) return findings +def _memory_finding(config: Config) -> Finding: + """Read-only probe of memory.sqlite. Doctor reports; it never repairs or migrates.""" + from .memory.store import SCHEMA_VERSION + from .service import memory_enabled, memory_path_for + + if not memory_enabled(config): + return Finding("memory_store", True, "info", "evidence memory is disabled") + path = memory_path_for(config) + quarantined = [p for p in path.parent.glob(f"{path.name}.corrupt-*") + if not p.name.endswith(("-wal", "-shm"))] + kept = f"; {len(quarantined)} quarantined corrupt store(s) kept beside it" if quarantined else "" + if not path.exists(): + return Finding("memory_store", True, "info", + f"no evidence memory yet (created on first --session use){kept}") + try: + conn = sqlite3.connect(f"file:{path.as_posix()}?mode=ro", uri=True) + try: + check = conn.execute("PRAGMA quick_check").fetchone()[0] + row = conn.execute("SELECT value FROM meta WHERE key = 'schema_version'").fetchone() + finally: + conn.close() + except sqlite3.Error as exc: + return Finding("memory_store", False, "medium", + f"memory store unreadable ({exc}); it is moved aside and recreated " + f"on next use{kept}") + version = int(row[0]) if row else 0 + if check != "ok": + return Finding("memory_store", False, "medium", + f"memory store integrity check failed: {check}{kept}") + if version > SCHEMA_VERSION: + return Finding("memory_store", False, "medium", + f"memory store schema {version} is newer than supported " + f"{SCHEMA_VERSION}; memory is off until codebase-index is upgraded{kept}") + return Finding("memory_store", True, "info", f"memory store healthy (schema {version}){kept}") + + # Threshold above which a tree-sitter language with zero symbols is treated as broken rather # than just a tiny/empty repo. _ZERO_SYMBOL_FILE_THRESHOLD = 3 diff --git a/src/codebase_index/mcp/server.py b/src/codebase_index/mcp/server.py index 6540297..10c1219 100644 --- a/src/codebase_index/mcp/server.py +++ b/src/codebase_index/mcp/server.py @@ -42,7 +42,11 @@ "Local codebase index. Use search_code for general queries, find_symbol for exact " "symbol lookups, find_refs to find callers/usages, impact_of for blast-radius analysis, " "explain_code for architecture/how-it-works questions, and architecture_overview to map " - "the codebase's modules, god nodes, and surprising connections before diving in." + "the codebase's modules, god nodes, and surprising connections before diving in. " + "Pass the same `session` tag on search_code/explain_code calls made from one context " + "so unchanged evidence you already received is not resent; use a new tag after your " + "context is cleared or compacted. verify_evidence checks whether evidence you hold or " + "cite is still true in the working tree." ), ) @@ -114,6 +118,9 @@ def healthcheck() -> str: "path": str(db_path), **compute_freshness(db.conn, Path(cfg.root), cfg).model_dump(), } + from ..service import memory_status_payload + + payload["memory"] = memory_status_payload(cfg) return _emit("healthcheck", payload) @@ -125,6 +132,7 @@ def search_code( token_budget: int = 1500, offset: int = 0, raw: bool = False, + session: Optional[str] = None, ) -> str: """Hybrid search over the codebase index. @@ -142,6 +150,11 @@ def search_code( offset: Result offset for pagination. Pass ``next_offset`` from a previous response to fetch the next page. raw: If true, return full raw snippets instead of skeletons. + session: Optional tag naming this one agent context. A result whose snippet this + session already received from unchanged source comes back with + ``snippet: null, reused: true``; evidence it received that has since + changed is listed under ``memory.invalidated``. Never share a tag with + another context; use a new tag after the context is cleared or compacted. """ db_path, cfg = _resolve_db() if not db_path.exists(): @@ -149,11 +162,14 @@ def search_code( from ..service import search_payload - payload = search_payload( - db_path, cfg, query, mode=mode, limit=limit, offset=offset, - token_budget=token_budget, no_fallback=False, backend=_search_backend(cfg), - raw=raw, - ) + try: + payload = search_payload( + db_path, cfg, query, mode=mode, limit=limit, offset=offset, + token_budget=token_budget, no_fallback=False, backend=_search_backend(cfg), + raw=raw, session=session, + ) + except ValueError as exc: + payload = {"error": str(exc)} return _emit("search_code", payload) @@ -273,6 +289,7 @@ def explain_code( token_budget: int = 2200, offset: int = 0, raw: bool = False, + session: Optional[str] = None, ) -> str: """Intent-aware retrieval for architecture / how-does-X-work questions. @@ -285,6 +302,7 @@ def explain_code( offset: Result offset for pagination. Pass ``next_offset`` from a previous response to fetch the next page. raw: If true, return full raw snippets instead of skeletons. + session: Optional tag naming this one agent context (see search_code). """ db_path, cfg = _resolve_db() if not db_path.exists(): @@ -292,14 +310,45 @@ def explain_code( from ..service import normalize_explain_query, search_payload - payload = search_payload( - db_path, cfg, normalize_explain_query(query), mode="hybrid", limit=10, - offset=offset, token_budget=token_budget, no_fallback=False, - backend=_search_backend(cfg), raw=raw, - ) + try: + payload = search_payload( + db_path, cfg, normalize_explain_query(query), mode="hybrid", limit=10, + offset=offset, token_budget=token_budget, no_fallback=False, + backend=_search_backend(cfg), raw=raw, session=session, + ) + except ValueError as exc: + payload = {"error": str(exc)} return _emit("explain_code", payload) +@_tool() +def verify_evidence(refs: Optional[list[str]] = None, session: Optional[str] = None) -> str: + """Check whether evidence is still true in the current working tree (read-only). + + Use before relying on something read earlier — especially after edits, a branch + switch, or when resuming from notes that cite evidence. Works without an index. + + Args: + refs: Evidence references ``path:start-end@hash``, e.g. from ``memory.invalidated`` + or from notes that cite evidence. + session: Also verify everything this session tag was given. + + Each verdict has ``state``: valid, relocated (identical content moved within the + file) — both still true — or changed, ambiguous, deleted, excluded, unreadable. + ``all_valid`` is true only when every checked piece of evidence holds. + """ + _, cfg = _resolve_db() + from ..service import verify_payload + + if not refs and not session: + return _emit("verify_evidence", {"error": "pass refs, session, or both"}) + try: + payload = verify_payload(cfg, refs or [], session) + except ValueError as exc: + payload = {"error": str(exc)} + return _emit("verify_evidence", payload) + + @_tool() def architecture_overview() -> str: """High-level map of the codebase from the cached graph analytics. @@ -367,8 +416,8 @@ def describe_symbol(symbol: str) -> str: @_tool() def index_stats() -> str: - """Return index freshness, file count, symbol count, and per-language coverage.""" - db_path, _ = _resolve_db() + """Return index freshness, file count, symbol count, per-language coverage, and memory.""" + db_path, cfg = _resolve_db() if not db_path.exists(): return _emit("index_stats", {"exists": False, "error": "No index found."}) @@ -376,7 +425,7 @@ def index_stats() -> str: from ..storage.db import Database with Database(db_path) as db: - payload = stats_payload(db.conn) + payload = stats_payload(db.conn, cfg=cfg) return _emit("index_stats", payload) diff --git a/src/codebase_index/memory/store.py b/src/codebase_index/memory/store.py index 95f7497..aa81aa7 100644 --- a/src/codebase_index/memory/store.py +++ b/src/codebase_index/memory/store.py @@ -229,11 +229,18 @@ def is_known(self, session_id: int, path: str, span_sha: str, snippet_sha: str) def pending(self, session_id: int) -> list[Delivery]: """Distinct evidence delivered to this session and not yet reported invalid.""" + return self._deliveries(session_id, "AND d.invalid_state IS NULL") + + def delivered(self, session_id: int) -> list[Delivery]: + """Every distinct piece of evidence delivered to this session, valid or not.""" + return self._deliveries(session_id, "") + + def _deliveries(self, session_id: int, condition: str) -> list[Delivery]: rows = self.conn.execute( "SELECT a.id, a.path, a.span_sha, a.line_count, a.first_line_sha, " " MIN(d.line_start), MIN(d.line_end) " "FROM deliveries d JOIN atoms a ON a.id = d.atom_id " - "WHERE d.session_id = ? AND d.invalid_state IS NULL " + f"WHERE d.session_id = ? {condition} " "GROUP BY a.id ORDER BY a.path, MIN(d.line_start)", (session_id,), ).fetchall() diff --git a/src/codebase_index/output/markdown.py b/src/codebase_index/output/markdown.py index 2994e1e..6914e34 100644 --- a/src/codebase_index/output/markdown.py +++ b/src/codebase_index/output/markdown.py @@ -30,8 +30,12 @@ def _render_dict(payload: dict) -> str: ) lines.append("") for r in payload["results"]: - if r.get("snippet"): - lines.append(f"`{r['path']}:{r['line_start']}-{r['line_end']}`") + location = f"`{r['path']}:{r['line_start']}-{r['line_end']}`" + if r.get("reused"): + lines.append(f"{location} — unchanged; already delivered in this session") + elif r.get("snippet"): + stale = " — index is older than the file; read the range" if r.get("stale") else "" + lines.append(f"{location}{stale}") lines.append("```") lines.append(r["snippet"]) lines.append("```") @@ -41,6 +45,9 @@ def _render_dict(payload: dict) -> str: for rr in payload["recommended_reads"]: lines.append(f"- `{rr['path']}:{rr['line_start']}-{rr['line_end']}`") + if payload.get("memory"): + lines.extend(_render_memory(payload["memory"])) + fb = payload.get("fallback_suggestions", {}).get("ripgrep") if fb: lines.append("\n**Fallback (low confidence) — try:**") @@ -58,6 +65,49 @@ def _render_dict(payload: dict) -> str: return "\n".join(lines) +def _render_memory(memory: dict) -> list[str]: + session = memory.get("session") + if memory.get("available") is False: + return [f"\n**Memory** (session `{session}`): unavailable — {memory.get('reason')}; " + "nothing was withheld."] + out = [f"\n**Memory** (session `{session}`): {memory.get('reused', 0)} snippet(s) not " + f"resent ({memory.get('tokens_saved', 0)} tokens)."] + invalidated = memory.get("invalidated") or [] + if invalidated: + out.append("**Changed since this session received it — do not rely on the earlier text:**") + out.extend(f"- `{n['ref']}` — {n['state']}" for n in invalidated) + return out + + +def render_verify(payload: dict) -> str: + """Render evidence verdicts: one row per reference, state first.""" + if payload.get("error"): + return f"_{payload['error']}_\n" + verdict = ("all evidence is still valid" if payload.get("all_valid") + else "some evidence is not valid") + lines = [f"**verify:** {verdict}"] + session = payload.get("session") + if session: + name = f"session `{session['session']}`" + if session.get("available") is False: + lines.append(f"{name}: unavailable — {session.get('reason')}") + elif not session.get("found"): + lines.append(f"{name}: no evidence recorded") + else: + lines.append(f"{name}: {session.get('evidence', 0)} piece(s) of evidence") + lines.append("") + rows = payload.get("evidence", []) + if rows: + lines.append("| state | evidence | lines now | reason |") + lines.append("|---|---|---|---|") + for v in rows: + now = f"{v['line_start']}-{v['line_end']}" if v.get("line_start") else "—" + lines.append(f"| {v['state'].upper()} | `{v['ref']}` | {now} | {v.get('reason', '')} |") + for err in payload.get("errors", []): + lines.append(f"- not a valid reference `{err['ref']}`: {err['error']}") + return "\n".join(lines).rstrip() + "\n" + + def _render_search_response(resp: SearchResponse) -> str: lines: list[str] = [] freshness = "fresh" if not resp.index.stale else "STALE" diff --git a/src/codebase_index/service.py b/src/codebase_index/service.py index 464d042..578a028 100644 --- a/src/codebase_index/service.py +++ b/src/codebase_index/service.py @@ -14,7 +14,7 @@ import subprocess from contextlib import contextmanager from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Iterator, Optional, Union +from typing import TYPE_CHECKING, Any, Callable, Iterator, Optional, Sequence, Union if TYPE_CHECKING: from .config import Config @@ -322,7 +322,133 @@ def architecture_payload(db_path: Path, cfg: "Config") -> dict[str, Any]: return {"exists": True, "available": True, "index": fresh.model_dump(), **summary} -def stats_payload(conn: sqlite3.Connection) -> dict[str, Any]: +def verify_payload( + cfg: "Config", refs: Sequence[str], session: Optional[str] = None +) -> dict[str, Any]: + """Re-check evidence references and/or everything one session was given. + + Read-only: nothing is recorded, so it is safe to run at any time and from any agent. + References are untrusted input and are validated before any file is read. + """ + from .discovery.gates import PathGate + from .memory import identity as ident + from .memory.store import MemoryStore, MemoryUnavailable + from .memory.validate import WorkingTree, validate + + tag = session_tag(session) + root = Path(cfg.root) + repo_id = ident.repo_id_for(root) + tree = WorkingTree(PathGate(root, cfg)) + enabled = memory_enabled(cfg) + path = memory_path_for(cfg) + store: Optional[MemoryStore] = None + store_problem: Optional[str] = None + if enabled and path.exists(): + try: + store = MemoryStore.open(path) + except MemoryUnavailable as exc: + store_problem = str(exc) + + evidence: list[dict] = [] + errors: list[dict] = [] + session_block: Optional[dict[str, Any]] = None + try: + for text in refs: + try: + ref = ident.parse_ref(text) + except ValueError as exc: + errors.append({"ref": text, "error": str(exc)}) + continue + hint = store.first_line_hint(repo_id, ref.path, ref.sha) if store else None + evidence.append(validate(ref, tree, first_line_sha=hint).as_dict()) + if tag is not None: + session_block = {"session": tag} + session_id = None + if not enabled: + session_block.update(available=False, reason="memory is disabled") + elif store_problem is not None: + session_block.update(available=False, reason=store_problem) + elif store is not None: + session_id = store.find_session(repo_id, ident.session_key(repo_id, tag)) + session_block["found"] = session_id is not None + if store is not None and session_id is not None: + delivered = store.delivered(session_id) + session_block["evidence"] = len(delivered) + for item in delivered: + ref = ident.EvidenceRef(item.path, item.line_start, item.line_end, + item.span_sha) + evidence.append( + validate(ref, tree, first_line_sha=item.first_line_sha).as_dict()) + finally: + if store is not None: + store.close() + + summary: dict[str, int] = {} + for verdict in evidence: + summary[verdict["state"]] = summary.get(verdict["state"], 0) + 1 + payload: dict[str, Any] = { + "all_valid": bool(evidence) and not errors and all(v["valid"] for v in evidence), + "summary": summary, + } + if session_block is not None: + payload["session"] = session_block + payload["evidence"] = evidence + payload["errors"] = errors + return payload + + +def memory_status_payload(cfg: "Config") -> dict[str, Any]: + """Evidence-memory health and counters for this repository (no paths, no content).""" + from .memory.identity import repo_id_for + from .memory.store import MemoryStore, MemoryUnavailable + + path = memory_path_for(cfg) + block: dict[str, Any] = {"enabled": memory_enabled(cfg), "exists": path.exists()} + if not block["exists"]: + return block + try: + with MemoryStore.open(path) as store: + block.update(store.stats(repo_id_for(Path(cfg.root)))) + if store.recovered_from: + block["recovered_from"] = store.recovered_from + except MemoryUnavailable as exc: + block.update(available=False, reason=str(exc)) + return block + + +def memory_gc_payload(cfg: "Config") -> dict[str, Any]: + """Apply retention and size limits, drop orphan atoms, and compact the store.""" + from .memory.identity import repo_id_for + from .memory.store import MemoryStore, utc_now + + path = memory_path_for(cfg) + if not path.exists(): + return {"exists": False} + with MemoryStore.open(path) as store: + removed = store.gc(now=utc_now(), retention_days=cfg.memory.retention_days, + max_deliveries=cfg.memory.max_deliveries) + store.vacuum() + return {"exists": True, **removed, **store.stats(repo_id_for(Path(cfg.root)))} + + +def memory_clear_payload(cfg: "Config", session: Optional[str] = None) -> dict[str, Any]: + """Forget one session, or all evidence memory for this repository.""" + from .memory import identity as ident + from .memory.store import MemoryStore + + tag = session_tag(session) + path = memory_path_for(cfg) + if not path.exists(): + return {"exists": False, "removed_sessions": 0, "session": tag} + repo_id = ident.repo_id_for(Path(cfg.root)) + with MemoryStore.open(path) as store: + removed = store.clear(repo_id, ident.session_key(repo_id, tag) if tag else None) + if tag is None: + store.vacuum() + return {"exists": True, "removed_sessions": removed, "session": tag} + + +def stats_payload(conn: sqlite3.Connection, cfg: Optional["Config"] = None) -> dict[str, Any]: """Index size, freshness, and per-language coverage with the graph tier.""" from .parsers.languages import has_full_graph from .storage import repo @@ -338,7 +464,7 @@ def stats_payload(conn: sqlite3.Connection) -> dict[str, Any]: } for r in repo.treesitter_coverage(conn) ] - return { + payload: dict[str, Any] = { "files": repo.count_files(conn), "symbols": repo.count_symbols(conn), "built_at": repo.get_meta(conn, "built_at"), @@ -346,3 +472,6 @@ def stats_payload(conn: sqlite3.Connection) -> dict[str, Any]: "treesitter_coverage": coverage, "exists": True, } + if cfg is not None: + payload["memory"] = memory_status_payload(cfg) + return payload diff --git a/tests/golden/mcp_healthcheck.json b/tests/golden/mcp_healthcheck.json index 565a111..e20798a 100644 --- a/tests/golden/mcp_healthcheck.json +++ b/tests/golden/mcp_healthcheck.json @@ -7,6 +7,10 @@ "path": ".claude/cache/codebase-index/index.sqlite", "stale": false }, + "memory": { + "enabled": true, + "exists": false + }, "package_version": "", "root": "", "schema_version": 1, diff --git a/tests/golden/mcp_index_stats.json b/tests/golden/mcp_index_stats.json index d8dcf27..8e6c7c6 100644 --- a/tests/golden/mcp_index_stats.json +++ b/tests/golden/mcp_index_stats.json @@ -3,6 +3,10 @@ "exists": true, "files": 6, "head_commit": "", + "memory": { + "enabled": true, + "exists": false + }, "schema_version": 1, "symbols": 7, "tool": "index_stats", diff --git a/tests/golden/stats.json b/tests/golden/stats.json index bf445b5..c89a6c0 100644 --- a/tests/golden/stats.json +++ b/tests/golden/stats.json @@ -3,6 +3,10 @@ "exists": true, "files": 6, "head_commit": "", + "memory": { + "enabled": true, + "exists": false + }, "symbols": 7, "treesitter_coverage": [ { diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index fd47394..b4af1ca 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -41,6 +41,7 @@ def test_mcp_server_has_expected_tools(): "path_between", "describe_symbol", "index_stats", + "verify_evidence", } diff --git a/tests/test_memory_surfaces.py b/tests/test_memory_surfaces.py new file mode 100644 index 0000000..f79b3c2 --- /dev/null +++ b/tests/test_memory_surfaces.py @@ -0,0 +1,197 @@ +"""Evidence memory through every public surface: CLI, MCP, Markdown, stats and doctor.""" + +from __future__ import annotations + +import json +import os +import sqlite3 +from pathlib import Path +from unittest.mock import patch + +import pytest +from typer.testing import CliRunner + +from codebase_index.cli import app +from codebase_index.config import Config +from codebase_index.doctor import run_doctor +from codebase_index.output.markdown import render, render_verify + +runner = CliRunner() + +REFUND = '''def issue_refund(invoice, amount): + """Refund part of an invoice total.""" + if amount > invoice.total: + raise ValueError("refund exceeds invoice total") + return invoice.total - amount +''' +QUERY = "refund exceeds invoice total" + + +@pytest.fixture +def indexed(tmp_path, monkeypatch): + for name in ("CBX_DB_PATH", "CBX_MEMORY_PATH", "CBX_MEMORY", "CBX_ROOT"): + monkeypatch.delenv(name, raising=False) + root = tmp_path / "repo" + (root / "billing").mkdir(parents=True) + (root / "billing" / "refund.py").write_text(REFUND, encoding="utf-8") + assert runner.invoke(app, ["--root", str(root), "index"]).exit_code == 0 + return root + + +def _cli(root: Path, *args: str): + return runner.invoke(app, ["--root", str(root), *args]) + + +def _json(result) -> dict: + assert result.exit_code == 0, result.output + return json.loads(result.output) + + +def test_cli_session_search_reuses_and_verify_reports_changes(indexed): + first = _json(_cli(indexed, "search", QUERY, "--session", "cli-1", "--json")) + assert first["memory"]["reused"] == 0 + second = _json(_cli(indexed, "search", QUERY, "--session", "cli-1", "--json")) + assert second["memory"]["reused"] >= 1 + + verdict = _json(_cli(indexed, "verify", "--session", "cli-1", "--json")) + assert verdict["all_valid"] is True and verdict["session"]["found"] is True + + (indexed / "billing" / "refund.py").write_text(REFUND.replace(">", ">="), encoding="utf-8") + changed = _json(_cli(indexed, "verify", "--session", "cli-1", "--json")) + assert changed["all_valid"] is False + assert "changed" in changed["summary"] + ref = next(v["ref"] for v in changed["evidence"] if v["state"] == "changed") + + strict = _cli(indexed, "verify", ref, "--strict", "--json") + assert strict.exit_code == 1 + assert json.loads(strict.output)["evidence"][0]["state"] == "changed" + + +def test_cli_verify_arguments_and_session_tags_are_validated(indexed): + assert _cli(indexed, "verify").exit_code == 2 + assert _cli(indexed, "search", QUERY, "--session", "bad tag").exit_code == 2 + assert _cli(indexed, "explain", QUERY, "--session", "bad/tag").exit_code == 2 + bad = _json(_cli(indexed, "verify", "../../etc/passwd:1-2@0123456789abcdef", "--json")) + assert bad["all_valid"] is False and bad["errors"] + + +def test_cli_verify_of_unknown_session_is_not_all_valid(indexed): + payload = _json(_cli(indexed, "verify", "--session", "never-used", "--json")) + assert payload["all_valid"] is False and payload["session"]["found"] is False + + +def test_cli_stats_memory_block_and_human_line(indexed): + assert _json(_cli(indexed, "stats", "--json"))["memory"] == {"enabled": True, "exists": False} + _json(_cli(indexed, "search", QUERY, "--session", "s", "--json")) + _json(_cli(indexed, "search", QUERY, "--session", "s", "--json")) + memory = _json(_cli(indexed, "stats", "--json"))["memory"] + assert memory["sessions"] == 1 and memory["reused"] >= 1 and memory["tokens_saved"] > 0 + human = _cli(indexed, "stats") + assert "tokens not resent" in human.output + + +def test_cli_memory_gc_and_clear(indexed): + _json(_cli(indexed, "search", QUERY, "--session", "a", "--json")) + _json(_cli(indexed, "search", QUERY, "--session", "b", "--json")) + gc = _json(_cli(indexed, "memory", "gc", "--json")) + assert gc["exists"] and gc["sessions"] == 2 + assert _json(_cli(indexed, "memory", "clear", "--session", "a", "--yes", "--json"))[ + "removed_sessions"] == 1 + assert _json(_cli(indexed, "memory", "clear", "--yes", "--json"))["removed_sessions"] == 1 + assert _json(_cli(indexed, "stats", "--json"))["memory"]["sessions"] == 0 + + +def test_cli_memory_disabled_by_environment(indexed, monkeypatch): + monkeypatch.setenv("CBX_MEMORY", "0") + payload = _json(_cli(indexed, "search", QUERY, "--session", "s", "--json")) + assert payload["memory"]["available"] is False + assert not (indexed / ".claude" / "cache" / "codebase-index" / "memory.sqlite").exists() + + +def test_markdown_marks_reused_stale_and_invalidated(): + payload = { + "query": "q", "intent": "keyword", "confidence": "high", + "results": [ + {"rank": 1, "path": "a.py", "line_start": 1, "line_end": 3, "reason": "x", + "snippet": None, "reused": True}, + {"rank": 2, "path": "b.py", "line_start": 4, "line_end": 9, "reason": "y", + "snippet": "def b(): pass", "stale": True}, + ], + "recommended_reads": [], + "fallback_suggestions": {}, + "memory": {"session": "s", "reused": 1, "tokens_saved": 12, + "invalidated": [{"ref": "c.py:1-2@0123456789abcdef", "state": "changed"}]}, + } + text = render(payload) + assert "already delivered in this session" in text + assert "index is older than the file" in text + assert "c.py:1-2@0123456789abcdef` — changed" in text + + verdicts = render_verify({ + "all_valid": False, "summary": {"changed": 1}, + "session": {"session": "s", "found": True, "evidence": 1}, + "evidence": [{"ref": "c.py:1-2@0123456789abcdef", "state": "changed", "valid": False, + "line_start": None, "line_end": None, "reason": "content no longer occurs"}], + "errors": [{"ref": "nope", "error": "not an evidence reference"}], + }) + assert "CHANGED" in verdicts and "some evidence is not valid" in verdicts and "nope" in verdicts + + +try: + from codebase_index.mcp import server as mcp_server + MCP_AVAILABLE = True +except ImportError: # pragma: no cover + MCP_AVAILABLE = False + + +@pytest.mark.skipif(not MCP_AVAILABLE, reason="mcp extra not installed") +def test_mcp_session_reuse_and_verify_evidence(indexed): + db_path = indexed / ".claude" / "cache" / "codebase-index" / "index.sqlite" + env = {"CBX_ROOT": str(indexed), "CBX_DB_PATH": str(db_path)} + with patch.dict(os.environ, env, clear=False): + json.loads(mcp_server.search_code(query=QUERY, session="mcp-1")) + again = json.loads(mcp_server.search_code(query=QUERY, session="mcp-1")) + assert again["schema_version"] == 1 and again["memory"]["reused"] >= 1 + + checked = json.loads(mcp_server.verify_evidence(session="mcp-1")) + assert checked["tool"] == "verify_evidence" and checked["all_valid"] is True + ref = checked["evidence"][0]["ref"] + by_ref = json.loads(mcp_server.verify_evidence(refs=[ref, "garbage"])) + assert by_ref["evidence"][0]["valid"] is True and by_ref["errors"] + + assert "error" in json.loads(mcp_server.verify_evidence()) + assert "error" in json.loads(mcp_server.search_code(query=QUERY, session="a b")) + assert "error" in json.loads(mcp_server.explain_code(query=QUERY, session="a b")) + health = json.loads(mcp_server.healthcheck()) + assert health["memory"]["exists"] is True + + +def _memory_path(root: Path) -> Path: + return root / ".claude" / "cache" / "codebase-index" / "memory.sqlite" + + +def _doctor_memory(root: Path): + cfg = Config() + cfg.root = str(root) + return next(f for f in run_doctor(root, cfg) if f.id == "memory_store") + + +def test_doctor_reports_memory_health_without_repairing(indexed): + assert _doctor_memory(indexed).ok + _json(_cli(indexed, "search", QUERY, "--session", "s", "--json")) + healthy = _doctor_memory(indexed) + assert healthy.ok and "healthy" in healthy.detail + + path = _memory_path(indexed) + conn = sqlite3.connect(path) + conn.execute("UPDATE meta SET value = '99' WHERE key = 'schema_version'") + conn.commit() + conn.close() + newer = _doctor_memory(indexed) + assert not newer.ok and "newer" in newer.detail + + for sidecar in path.parent.glob("memory.sqlite*"): + sidecar.unlink() + path.write_bytes(b"not a database" * 64) + broken = _doctor_memory(indexed) + assert not broken.ok and path.read_bytes().startswith(b"not a database") # untouched From ed44049134faf84fc16aa861d04015a699626a06 Mon Sep 17 00:00:00 2001 From: denfry Date: Thu, 10 Sep 2026 10:50:11 +0300 Subject: [PATCH 09/18] feat(skill): allow read-only verify in agent wrappers verify joins the wrapper whitelists; memory (gc/clear) stays excluded together with clean, init and watch, and the packaging test asserts it. The plugin wrappers in bin/ had drifted behind the skill template and were also missing architecture, diff-impact, path and describe, which the plugin skill tells agents to run. --- .claude/skills/codebase-index/scripts/cbx | 2 +- .claude/skills/codebase-index/scripts/cbx.ps1 | 2 +- .codex/skills/codebase-index/scripts/cbx | 2 +- .codex/skills/codebase-index/scripts/cbx.ps1 | 2 +- .opencode/skills/codebase-index/scripts/cbx | 2 +- .opencode/skills/codebase-index/scripts/cbx.ps1 | 2 +- bin/cbx | 2 +- bin/cbx.ps1 | 2 +- skill/scripts/cbx | 2 +- skill/scripts/cbx.ps1 | 2 +- src/codebase_index/skill_template/scripts/cbx | 2 +- src/codebase_index/skill_template/scripts/cbx.ps1 | 2 +- tests/test_packaging.py | 4 ++-- 13 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.claude/skills/codebase-index/scripts/cbx b/.claude/skills/codebase-index/scripts/cbx index 94a9114..18c3ed0 100644 --- a/.claude/skills/codebase-index/scripts/cbx +++ b/.claude/skills/codebase-index/scripts/cbx @@ -4,7 +4,7 @@ # - Whitelists subcommands so the skill can never invoke destructive ones (clean/init/watch). set -euo pipefail -ALLOWED="search explain architecture symbol refs impact diff-impact path describe graph stats doctor update index" +ALLOWED="search explain architecture symbol refs impact diff-impact path describe verify graph stats doctor update index" sub="${1:-}" case " $ALLOWED " in diff --git a/.claude/skills/codebase-index/scripts/cbx.ps1 b/.claude/skills/codebase-index/scripts/cbx.ps1 index eee56ae..6419e37 100644 --- a/.claude/skills/codebase-index/scripts/cbx.ps1 +++ b/.claude/skills/codebase-index/scripts/cbx.ps1 @@ -10,7 +10,7 @@ param( $ErrorActionPreference = "Stop" $allowed = @( "search", "explain", "architecture", "symbol", "refs", "impact", "diff-impact", - "path", "describe", "graph", "stats", "doctor", "update", "index" + "path", "describe", "verify", "graph", "stats", "doctor", "update", "index" ) if ($allowed -notcontains $Subcommand) { diff --git a/.codex/skills/codebase-index/scripts/cbx b/.codex/skills/codebase-index/scripts/cbx index 94a9114..18c3ed0 100644 --- a/.codex/skills/codebase-index/scripts/cbx +++ b/.codex/skills/codebase-index/scripts/cbx @@ -4,7 +4,7 @@ # - Whitelists subcommands so the skill can never invoke destructive ones (clean/init/watch). set -euo pipefail -ALLOWED="search explain architecture symbol refs impact diff-impact path describe graph stats doctor update index" +ALLOWED="search explain architecture symbol refs impact diff-impact path describe verify graph stats doctor update index" sub="${1:-}" case " $ALLOWED " in diff --git a/.codex/skills/codebase-index/scripts/cbx.ps1 b/.codex/skills/codebase-index/scripts/cbx.ps1 index eee56ae..6419e37 100644 --- a/.codex/skills/codebase-index/scripts/cbx.ps1 +++ b/.codex/skills/codebase-index/scripts/cbx.ps1 @@ -10,7 +10,7 @@ param( $ErrorActionPreference = "Stop" $allowed = @( "search", "explain", "architecture", "symbol", "refs", "impact", "diff-impact", - "path", "describe", "graph", "stats", "doctor", "update", "index" + "path", "describe", "verify", "graph", "stats", "doctor", "update", "index" ) if ($allowed -notcontains $Subcommand) { diff --git a/.opencode/skills/codebase-index/scripts/cbx b/.opencode/skills/codebase-index/scripts/cbx index 94a9114..18c3ed0 100644 --- a/.opencode/skills/codebase-index/scripts/cbx +++ b/.opencode/skills/codebase-index/scripts/cbx @@ -4,7 +4,7 @@ # - Whitelists subcommands so the skill can never invoke destructive ones (clean/init/watch). set -euo pipefail -ALLOWED="search explain architecture symbol refs impact diff-impact path describe graph stats doctor update index" +ALLOWED="search explain architecture symbol refs impact diff-impact path describe verify graph stats doctor update index" sub="${1:-}" case " $ALLOWED " in diff --git a/.opencode/skills/codebase-index/scripts/cbx.ps1 b/.opencode/skills/codebase-index/scripts/cbx.ps1 index eee56ae..6419e37 100644 --- a/.opencode/skills/codebase-index/scripts/cbx.ps1 +++ b/.opencode/skills/codebase-index/scripts/cbx.ps1 @@ -10,7 +10,7 @@ param( $ErrorActionPreference = "Stop" $allowed = @( "search", "explain", "architecture", "symbol", "refs", "impact", "diff-impact", - "path", "describe", "graph", "stats", "doctor", "update", "index" + "path", "describe", "verify", "graph", "stats", "doctor", "update", "index" ) if ($allowed -notcontains $Subcommand) { diff --git a/bin/cbx b/bin/cbx index cfe60b7..fd4af7b 100644 --- a/bin/cbx +++ b/bin/cbx @@ -3,7 +3,7 @@ # from the venv provisioned by scripts/bootstrap.sh (located via the .venv-path pointer). set -euo pipefail -ALLOWED="search explain symbol refs impact graph stats doctor update index" +ALLOWED="search explain architecture symbol refs impact diff-impact path describe verify graph stats doctor update index" sub="${1:-}" case " $ALLOWED " in *" ${sub} "*) : ;; diff --git a/bin/cbx.ps1 b/bin/cbx.ps1 index 85face7..3d9a8ba 100644 --- a/bin/cbx.ps1 +++ b/bin/cbx.ps1 @@ -5,7 +5,7 @@ param( [Parameter(ValueFromRemainingArguments = $true)] [string[]]$Rest ) $ErrorActionPreference = "Stop" -$allowed = @("search", "explain", "symbol", "refs", "impact", "graph", "stats", "doctor", "update", "index") +$allowed = @("search", "explain", "architecture", "symbol", "refs", "impact", "diff-impact", "path", "describe", "verify", "graph", "stats", "doctor", "update", "index") if ($allowed -notcontains $Subcommand) { Write-Error "cbx: refusing subcommand '$Subcommand'. Allowed: $($allowed -join ', ')" exit 2 diff --git a/skill/scripts/cbx b/skill/scripts/cbx index 94a9114..18c3ed0 100644 --- a/skill/scripts/cbx +++ b/skill/scripts/cbx @@ -4,7 +4,7 @@ # - Whitelists subcommands so the skill can never invoke destructive ones (clean/init/watch). set -euo pipefail -ALLOWED="search explain architecture symbol refs impact diff-impact path describe graph stats doctor update index" +ALLOWED="search explain architecture symbol refs impact diff-impact path describe verify graph stats doctor update index" sub="${1:-}" case " $ALLOWED " in diff --git a/skill/scripts/cbx.ps1 b/skill/scripts/cbx.ps1 index eee56ae..6419e37 100644 --- a/skill/scripts/cbx.ps1 +++ b/skill/scripts/cbx.ps1 @@ -10,7 +10,7 @@ param( $ErrorActionPreference = "Stop" $allowed = @( "search", "explain", "architecture", "symbol", "refs", "impact", "diff-impact", - "path", "describe", "graph", "stats", "doctor", "update", "index" + "path", "describe", "verify", "graph", "stats", "doctor", "update", "index" ) if ($allowed -notcontains $Subcommand) { diff --git a/src/codebase_index/skill_template/scripts/cbx b/src/codebase_index/skill_template/scripts/cbx index 94a9114..18c3ed0 100644 --- a/src/codebase_index/skill_template/scripts/cbx +++ b/src/codebase_index/skill_template/scripts/cbx @@ -4,7 +4,7 @@ # - Whitelists subcommands so the skill can never invoke destructive ones (clean/init/watch). set -euo pipefail -ALLOWED="search explain architecture symbol refs impact diff-impact path describe graph stats doctor update index" +ALLOWED="search explain architecture symbol refs impact diff-impact path describe verify graph stats doctor update index" sub="${1:-}" case " $ALLOWED " in diff --git a/src/codebase_index/skill_template/scripts/cbx.ps1 b/src/codebase_index/skill_template/scripts/cbx.ps1 index eee56ae..6419e37 100644 --- a/src/codebase_index/skill_template/scripts/cbx.ps1 +++ b/src/codebase_index/skill_template/scripts/cbx.ps1 @@ -10,7 +10,7 @@ param( $ErrorActionPreference = "Stop" $allowed = @( "search", "explain", "architecture", "symbol", "refs", "impact", "diff-impact", - "path", "describe", "graph", "stats", "doctor", "update", "index" + "path", "describe", "verify", "graph", "stats", "doctor", "update", "index" ) if ($allowed -notcontains $Subcommand) { diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 5297c2e..118c1b4 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -29,9 +29,9 @@ def test_packaged_cbx_whitelists_safe_subcommands_only(): cbx = (_template() / "scripts" / "cbx").read_text(encoding="utf-8") assert ( 'ALLOWED="search explain architecture symbol refs impact diff-impact path describe ' - 'graph stats doctor update index"' + 'verify graph stats doctor update index"' ) in cbx - for forbidden in ("clean", "init", "watch"): + for forbidden in ("clean", "init", "watch", "memory"): assert f" {forbidden} " not in f' {cbx.split("ALLOWED=")[1].splitlines()[0]} ' From 394db3bd0cc3cc50ed4a465b72af3c25dead5471 Mon Sep 17 00:00:00 2001 From: denfry Date: Thu, 10 Sep 2026 10:50:11 +0300 Subject: [PATCH 10/18] test(memory): repository lifecycle, security and stale-context suites Real git drives every case: unrelated commits, branch switch and return, detached HEAD, worktrees as separate scopes, dirty/staged/unstaged edits, renames, deletions, new files, rebase and CRLF checkouts. Excluded content (.env, secrets.json, node_modules, ignored paths, a file ignored after it was indexed) never reaches memory.sqlite and verifies as excluded. The signature test replays T0 learn a fact, T1 unrelated commit, T2 the fact changes, T3 a similar question: a query-keyed cache serves the stale fact, evidence memory reports the change and delivers the current text. --- tests/test_memory_lifecycle.py | 232 +++++++++++++++++++++++++++++ tests/test_memory_stale_context.py | 124 +++++++++++++++ 2 files changed, 356 insertions(+) create mode 100644 tests/test_memory_lifecycle.py create mode 100644 tests/test_memory_stale_context.py diff --git a/tests/test_memory_lifecycle.py b/tests/test_memory_lifecycle.py new file mode 100644 index 0000000..2af1f35 --- /dev/null +++ b/tests/test_memory_lifecycle.py @@ -0,0 +1,232 @@ +"""Evidence validity across real repository lifecycles, driven through real git. + +Validity is byte identity with the working tree, so every git operation is just another +way of changing (or restoring) bytes. These tests pin that down for the operations agents +actually perform, plus the security property that excluded content never reaches memory. +""" + +from __future__ import annotations + +import subprocess +from dataclasses import dataclass +from pathlib import Path + +import pytest + +from codebase_index.config import Config +from codebase_index.indexer.pipeline import build_index, update_index +from codebase_index.service import search_payload, verify_payload +from codebase_index.storage.db import Database + +REFUND = '''def issue_refund(invoice, amount): + """Refund part of an invoice total.""" + if amount > invoice.total: + raise ValueError("refund exceeds invoice total") + return invoice.total - amount +''' +INVOICE = '''def compute_invoice_total(lines, tax_rate): + """Sum invoice line amounts and apply the tax rate.""" + subtotal = sum(line.amount for line in lines) + return round(subtotal * (1 + tax_rate), 2) +''' +TAX = '''def tax_rate_for(region): + """Look up the sales tax rate for a region.""" + return {"eu": 0.2, "us": 0.07}.get(region, 0.0) +''' +QUERY = "refund exceeds invoice total" + + +def _git(root: Path, *args: str) -> str: + return subprocess.run( + ["git", "-c", "user.name=t", "-c", "user.email=t@example.com", + "-c", "core.autocrlf=false", "-C", str(root), *args], + check=True, capture_output=True, text=True, + ).stdout.strip() + + +@dataclass +class Repo: + root: Path + cfg: Config + db: Path + + def write(self, rel: str, text: str) -> None: + path = self.root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(text.encode("utf-8")) + + def commit(self, message: str) -> None: + _git(self.root, "add", "-A") + _git(self.root, "commit", "-q", "-m", message) + + def update(self) -> None: + with Database(self.db) as db: + update_index(self.cfg, db, root=self.root) + + def search(self, query: str = QUERY, session: str = "s1") -> dict: + return search_payload(self.db, self.cfg, query, mode="hybrid", limit=10, + token_budget=1500, no_fallback=False, session=session) + + def states(self, session: str = "s1") -> dict[str, str]: + payload = verify_payload(self.cfg, [], session) + return {v["ref"].rsplit("@", 1)[0].rsplit(":", 1)[0]: v["state"] + for v in payload["evidence"]} + + +def _make_repo(tmp_path: Path, monkeypatch, name: str = "repo") -> Repo: + for var in ("CBX_DB_PATH", "CBX_MEMORY", "CBX_ROOT"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("CBX_MEMORY_PATH", str(tmp_path / "memory.sqlite")) + root = tmp_path / name + root.mkdir() + _git(root, "init", "-q", "-b", "main") + repo = Repo(root, Config(root=str(root)), tmp_path / f"{name}.sqlite") + repo.cfg.root = str(root) + repo.write("billing/refund.py", REFUND) + repo.write("billing/invoice.py", INVOICE) + repo.write("billing/tax.py", TAX) + repo.commit("initial") + with Database(repo.db) as db: + build_index(repo.cfg, db, root=root) + return repo + + +@pytest.fixture +def repo(tmp_path, monkeypatch) -> Repo: + return _make_repo(tmp_path, monkeypatch) + + +def _delivered_paths(payload: dict) -> set[str]: + return {r["path"] for r in payload["results"] if r["snippet"]} + + +def test_unrelated_commit_keeps_evidence_valid_and_reused(repo): + delivered = _delivered_paths(repo.search()) + assert "billing/refund.py" in delivered + repo.write("billing/tax.py", TAX.replace("0.07", "0.08")) + repo.commit("adjust us tax") + repo.update() + assert set(repo.states().values()) == {"valid"} + again = repo.search() + assert again["memory"]["invalidated"] == [] + assert any(r.get("reused") for r in again["results"] if r["path"] == "billing/refund.py") + + +def test_branch_switch_invalidates_what_differs_and_restores_on_return(repo): + repo.search() + _git(repo.root, "checkout", "-q", "-b", "feature") + repo.write("billing/refund.py", REFUND.replace(">", ">=")) + repo.commit("tighten refund") + assert repo.states()["billing/refund.py"] == "changed" + _git(repo.root, "checkout", "-q", "main") + assert repo.states()["billing/refund.py"] == "valid" # identical bytes are identical evidence + + +def test_detached_head_is_validated_against_its_own_tree(repo): + first = _git(repo.root, "rev-parse", "HEAD") + repo.write("billing/refund.py", REFUND.replace("amount):", "amount, note=None):")) + repo.commit("add note") + repo.update() + repo.search() # evidence from the new commit + _git(repo.root, "checkout", "-q", "--detach", first) + assert repo.states()["billing/refund.py"] == "changed" + + +def test_worktree_is_a_separate_memory_scope(repo, tmp_path): + payload = repo.search() + ref = next(v["ref"] for v in verify_payload(repo.cfg, [], "s1")["evidence"]) + other = tmp_path / "wt" + _git(repo.root, "worktree", "add", "-q", "--detach", str(other), "HEAD") + cfg = Config() + cfg.root = str(other) + scoped = verify_payload(cfg, [ref], "s1") + assert scoped["session"]["found"] is False # sessions never leak across checkouts + assert scoped["evidence"][0]["state"] == "valid" # but the bytes there are the same + assert payload["memory"]["reused"] == 0 + + +def test_dirty_staged_and_unstaged_changes_are_what_is_validated(repo): + repo.search() + changed = REFUND.replace("refund exceeds", "refund is larger than") + repo.write("billing/refund.py", changed) + assert repo.states()["billing/refund.py"] == "changed" # unstaged edit + _git(repo.root, "add", "billing/refund.py") + assert repo.states()["billing/refund.py"] == "changed" # staged, same bytes + repo.write("billing/refund.py", REFUND) + assert repo.states()["billing/refund.py"] == "valid" # worktree wins over index + _git(repo.root, "reset", "-q", "billing/refund.py") + assert repo.states()["billing/refund.py"] == "valid" + + +def test_rename_and_delete_invalidate_the_old_paths(repo): + repo.search("refund invoice total tax rate region") + held = repo.states() + assert {"billing/refund.py", "billing/tax.py"} <= set(held) + _git(repo.root, "mv", "billing/refund.py", "billing/refunds.py") + (repo.root / "billing" / "tax.py").unlink() + states = repo.states() + assert states["billing/refund.py"] == "deleted" + assert states["billing/tax.py"] == "deleted" + + +def test_new_file_evidence_is_delivered_before_it_can_be_reused(repo): + repo.write("billing/credit.py", REFUND.replace("issue_refund", "issue_store_credit")) + repo.update() + first = repo.search("issue store credit refund exceeds invoice total") + hit = next(r for r in first["results"] if r["path"] == "billing/credit.py") + assert hit["snippet"] and not hit.get("reused") + + +def test_rebase_evidence_follows_bytes_not_history(repo): + repo.search("refund invoice total tax rate region") + _git(repo.root, "checkout", "-q", "-b", "topic") + repo.write("billing/invoice.py", INVOICE.replace("round(", "abs(round("). replace(", 2)", ", 2))")) + repo.commit("topic change to invoice") + _git(repo.root, "checkout", "-q", "main") + repo.write("billing/tax.py", TAX.replace("0.2", "0.21")) + repo.commit("main change to tax") + _git(repo.root, "checkout", "-q", "topic") + _git(repo.root, "rebase", "-q", "main") + states = repo.states() + assert states["billing/refund.py"] == "valid" + assert states["billing/invoice.py"] == "changed" + assert states.get("billing/tax.py", "changed") == "changed" + + +def test_crlf_checkout_is_not_a_change(repo): + repo.search() + (repo.root / "billing" / "refund.py").write_bytes(REFUND.replace("\n", "\r\n").encode()) + assert repo.states()["billing/refund.py"] == "valid" + + +CANARY = "CANARY_7f3a9e_do_not_persist" + + +def test_excluded_content_never_reaches_memory_or_verification(tmp_path, monkeypatch): + repo = _make_repo(tmp_path, monkeypatch) + repo.write(".env", f"API_KEY={CANARY}\n") + repo.write("config/secrets.json", f'{{"token": "{CANARY}"}}\n') + repo.write("node_modules/pkg/index.js", f"module.exports = '{CANARY}';\n") + repo.write("private/notes.py", f"NOTE = '{CANARY}'\n") + repo.write(".gitignore", "private/\n") + repo.write("billing/audit.py", f"def audit_refund():\n return '{CANARY} refund exceeds'\n") + repo.update() + + payload = repo.search(f"{CANARY} refund exceeds invoice total") + assert all(not r["path"].startswith((".env", "config/", "node_modules/", "private/")) + for r in payload["results"]) + # The file is indexed today; ignoring it afterwards must stop memory from using it. + assert any(r["path"] == "billing/audit.py" for r in payload["results"]) + repo.write(".gitignore", "private/\nbilling/audit.py\n") + later = repo.search(f"{CANARY} refund exceeds invoice total") + audit = [r for r in later["results"] if r["path"] == "billing/audit.py"] + assert audit and all(r.get("stale") for r in audit) + + refs = [f"{p}:1-1@0123456789abcdef" for p in + (".env", "config/secrets.json", "node_modules/pkg/index.js", "private/notes.py", + "billing/audit.py")] + verdicts = verify_payload(repo.cfg, refs)["evidence"] + assert [v["state"] for v in verdicts] == ["excluded"] * 5 + + blob = b"".join(p.read_bytes() for p in tmp_path.glob("memory.sqlite*")) + assert CANARY.encode() not in blob diff --git a/tests/test_memory_stale_context.py b/tests/test_memory_stale_context.py new file mode 100644 index 0000000..2e28b3d --- /dev/null +++ b/tests/test_memory_stale_context.py @@ -0,0 +1,124 @@ +"""Signature test for 2.0: a query-keyed cache serves a stale fact; evidence memory cannot. + + T0 an agent learns fact X (sessions last 3600 s) from evidence E + T1 an unrelated commit lands -> E is still true: both reuse it + T2 E changes (sessions now last 900 s) + T3 a semantically similar question is asked + query-keyed cache -> similar question, cache hit, serves X = 3600 (stale) + evidence memory -> E's bytes changed: reports E invalid, delivers 900 + +Deterministic: real git history, a token-overlap similarity for the cache, no model. +""" + +from __future__ import annotations + +import re +import subprocess +from pathlib import Path + +from codebase_index.config import Config +from codebase_index.indexer.pipeline import build_index, update_index +from codebase_index.service import search_payload, verify_payload +from codebase_index.storage.db import Database + +POLICY = '''"""Session lifetime policy.""" + +SESSION_TTL_SECONDS = 3600 # idle login session lifetime in seconds +''' +BILLING = "def invoice_total(lines):\n return sum(lines)\n" + + +class QueryKeyedCache: + """What a semantic cache does: reuse an answer when the question looks similar.""" + + def __init__(self, threshold: float = 0.5) -> None: + self.threshold = threshold + self.entries: list[tuple[set[str], dict]] = [] + + @staticmethod + def _words(query: str) -> set[str]: + return set(re.findall(r"[a-z0-9]+", query.lower())) + + def get(self, query: str): + words = self._words(query) + for key, packet in self.entries: + if len(words & key) / len(words | key) >= self.threshold: + return packet + return None + + def put(self, query: str, packet: dict) -> None: + self.entries.append((self._words(query), packet)) + + +def _git(root: Path, *args: str) -> None: + subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@example.com", + "-c", "core.autocrlf=false", "-C", str(root), *args], + check=True, capture_output=True) + + +def _policy_text(packet: dict) -> str: + return "\n".join(r["snippet"] or "" for r in packet["results"] + if r["path"] == "auth/policy.py") + + +def test_query_keyed_cache_goes_stale_where_evidence_memory_does_not(tmp_path, monkeypatch): + for var in ("CBX_DB_PATH", "CBX_MEMORY", "CBX_ROOT"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("CBX_MEMORY_PATH", str(tmp_path / "memory.sqlite")) + root = tmp_path / "repo" + (root / "auth").mkdir(parents=True) + (root / "billing").mkdir() + (root / "auth" / "policy.py").write_bytes(POLICY.encode()) + (root / "billing" / "invoice.py").write_bytes(BILLING.encode()) + _git(root, "init", "-q") + _git(root, "add", "-A") + _git(root, "commit", "-q", "-m", "initial") + cfg = Config() + cfg.root = str(root) + db_path = tmp_path / "index.sqlite" + with Database(db_path) as db: + build_index(cfg, db, root=root) + + def update() -> None: + with Database(db_path) as db: + update_index(cfg, db, root=root) + + def ask(query: str) -> dict: + return search_payload(db_path, cfg, query, mode="hybrid", limit=10, token_budget=1500, + no_fallback=False, session="agent") + + cache = QueryKeyedCache() + + # T0 — learn X from E + t0_query = "session ttl seconds lifetime" + t0 = ask(t0_query) + assert "3600" in _policy_text(t0) + cache.put(t0_query, t0) + + # T1 — unrelated commit: E still holds, so reuse is correct for both designs + (root / "billing" / "invoice.py").write_bytes(BILLING.replace("sum(lines)", "sum(lines, 0)").encode()) + _git(root, "commit", "-q", "-am", "unrelated billing change") + update() + t1 = ask(t0_query) + assert t1["memory"]["invalidated"] == [] + assert any(r.get("reused") for r in t1["results"] if r["path"] == "auth/policy.py") + assert cache.get(t0_query) is t0 + + # T2 — E changes: the fact is no longer true + (root / "auth" / "policy.py").write_bytes(POLICY.replace("3600", "900").encode()) + _git(root, "commit", "-q", "-am", "shorten sessions") + update() + + # T3 — a semantically similar question + t3_query = "session ttl lifetime in seconds" + cached = cache.get(t3_query) + assert cached is not None, "the cache must consider these questions similar" + assert "3600" in _policy_text(cached) # the cache serves the stale fact + assert "900" in (root / "auth" / "policy.py").read_text() + + t3 = ask(t3_query) + assert "900" in _policy_text(t3) and "3600" not in _policy_text(t3) + assert not any(r.get("reused") for r in t3["results"] if r["path"] == "auth/policy.py") + stale_refs = [n for n in t3["memory"]["invalidated"] if n["ref"].startswith("auth/policy.py:")] + assert [n["state"] for n in stale_refs] == ["changed"] + assert verify_payload(cfg, [stale_refs[0]["ref"]])["evidence"][0]["state"] == "changed" From 7c006727e7912f9cb9a628b8caea12cdd7799526 Mon Sep 17 00:00:00 2001 From: denfry Date: Thu, 10 Sep 2026 11:06:22 +0300 Subject: [PATCH 11/18] fix(memory): tell stale index text from derived index text The benchmark replay flagged results as stale right after the index was updated. Every sampled case was index text that is derived rather than copied: config-key chunks ("config key: scripts.build = ...") stored against a "{" line, and markdown section summaries ("### Heading: body"). No byte comparison can match those, so they were false alarms. A mismatch between index text and the working tree now means stale only when the file's current sha256 differs from files.sha256, the fingerprint the index was built from. When the index is current for the file, the excerpt is derived text: it is neither flagged, recorded, nor withheld. --- src/codebase_index/memory/session.py | 59 +++++++++++++++++++++------ src/codebase_index/memory/validate.py | 15 +++++-- src/codebase_index/service.py | 8 ++-- tests/test_memory_session.py | 43 +++++++++++++++++++ 4 files changed, 106 insertions(+), 19 deletions(-) diff --git a/src/codebase_index/memory/session.py b/src/codebase_index/memory/session.py index 17a59eb..7691d2a 100644 --- a/src/codebase_index/memory/session.py +++ b/src/codebase_index/memory/session.py @@ -21,7 +21,7 @@ from dataclasses import dataclass from datetime import datetime from pathlib import Path -from typing import Any, Iterator, Optional, Sequence +from typing import Any, Callable, Iterator, Optional, Sequence, Union from ..config import Config from ..discovery.gates import PathGate @@ -43,13 +43,34 @@ def usable(self) -> bool: return self.store is not None and self.session_id is not None and not self.unavailable +IndexSha = Callable[[str], Optional[str]] +_STALE = "stale" + + +def index_sha_lookup(conn: sqlite3.Connection) -> IndexSha: + """``files.sha256`` by path, cached for one call: the fingerprint the index was built from.""" + from ..storage import repo + + cache: dict[str, Optional[str]] = {} + + def lookup(rel: str) -> Optional[str]: + if rel not in cache: + row = repo.get_file(conn, rel) + cache[rel] = row["sha256"] if row else None + return cache[rel] + + return lookup + + class EvidenceProcessor: def __init__(self, *, root: Path, config: Config, now: datetime, - session: Optional[Session] = None) -> None: + session: Optional[Session] = None, + index_sha: Optional[IndexSha] = None) -> None: self.tree = WorkingTree(PathGate(root, config)) self.config = config self.now = now self.session = session + self.index_sha = index_sha def __call__(self, payload: dict, candidates: Sequence[Any]) -> None: session = self.session @@ -62,11 +83,11 @@ def __call__(self, payload: dict, candidates: Sequence[Any]) -> None: if not snippet: continue # nothing was delivered for this result observed = self._observe(result, candidate) - if observed is None: + if isinstance(observed, str): # _STALE result["stale"] = True continue - if session is None or not session.usable: - continue + if observed is None or session is None or not session.usable: + continue # derived index text is accurate but not byte-verifiable: never withheld rel, span, span_sha = observed tokens = int(result.get("token_est") or 0) snippet_sha = ident.sha_hex(snippet) @@ -98,19 +119,27 @@ def __call__(self, payload: dict, candidates: Sequence[Any]) -> None: reused=reused, tokens_saved=tokens_saved, tokens_delivered=tokens_delivered, ) - def _observe(self, result: dict, candidate: Any) -> Optional[tuple[str, str, str]]: - """(path, span text, span sha) when the delivered snippet matches the working tree.""" + def _observe(self, result: dict, candidate: Any) -> Union[tuple[str, str, str], str, None]: + """Classify one delivered snippet against the working tree. + + * ``(path, span text, span sha)`` — the snippet is text these exact lines hold now; + * ``"stale"`` — the file is gone, excluded, or its bytes differ from what was indexed; + * ``None`` — the index is current for this file but its text was derived (config-key + and section summaries), so it cannot be checked byte-for-byte. + """ try: rel = ident.normalize_rel_path(str(result["path"])) except ValueError: - return None + return _STALE view, _state, _reason = self.tree.view(rel) if view is None: - return None + return _STALE span = ident.span_text(view.lines, int(result["line_start"]), int(result["line_end"])) - if span is None or not ident.content_matches(getattr(candidate, "content", None), span): + if span is not None and ident.content_matches(getattr(candidate, "content", None), span): + return rel, span, ident.sha_hex(span) + if self.index_sha is None: return None - return rel, span, ident.sha_hex(span) + return None if self.index_sha(rel) == view.sha256 else _STALE def _known(self, session: Session, rel: str, span_sha: str, snippet_sha: str) -> bool: assert session.store is not None and session.session_id is not None @@ -174,7 +203,8 @@ def _finish(self, session: Session, fresh: list[NewDelivery], notices: list[dict @contextmanager def open_evidence(*, root: Path, config: Config, memory_path: Path, tag: Optional[str], - now: Optional[datetime] = None) -> Iterator[EvidenceProcessor]: + now: Optional[datetime] = None, + index_conn: Optional[sqlite3.Connection] = None) -> Iterator[EvidenceProcessor]: """Processor for one retrieval call; opens the store only when a session is named.""" now = now or utc_now() session: Optional[Session] = None @@ -191,7 +221,10 @@ def open_evidence(*, root: Path, config: Config, memory_path: Path, tag: Optiona except (MemoryUnavailable, sqlite3.Error) as exc: session.unavailable = str(exc) try: - yield EvidenceProcessor(root=Path(root), config=config, now=now, session=session) + yield EvidenceProcessor( + root=Path(root), config=config, now=now, session=session, + index_sha=index_sha_lookup(index_conn) if index_conn is not None else None, + ) finally: if store is not None: store.close() diff --git a/src/codebase_index/memory/validate.py b/src/codebase_index/memory/validate.py index e8363a4..9b5a10f 100644 --- a/src/codebase_index/memory/validate.py +++ b/src/codebase_index/memory/validate.py @@ -7,6 +7,7 @@ from __future__ import annotations +import hashlib from dataclasses import dataclass from typing import Optional @@ -43,10 +44,16 @@ def as_dict(self) -> dict: class FileView: - """Lines of one working-tree file; per-line hashes are computed only if needed.""" + """Lines of one working-tree file; per-line hashes are computed only if needed. - def __init__(self, lines: list[str]) -> None: + ``sha256`` is the digest of the raw bytes — the same fingerprint the indexer stores in + ``files.sha256`` — so a caller can tell "the index is older than this file" apart from + "the index text was derived rather than copied". + """ + + def __init__(self, lines: list[str], sha256: Optional[str] = None) -> None: self.lines = lines + self.sha256 = sha256 self._line_shas: Optional[list[str]] = None def line_shas(self) -> list[str]: @@ -105,7 +112,9 @@ def view(self, rel: str) -> tuple[Optional[FileView], str, str]: if cached is None: result = self.gate.read(rel) if result.state == "ok": - cached = (FileView(ident.split_lines(result.data)), "ok", "") + view = FileView(ident.split_lines(result.data), + hashlib.sha256(result.data).hexdigest()) + cached = (view, "ok", "") else: cached = (None, result.state, result.reason) self._views[rel] = cached diff --git a/src/codebase_index/service.py b/src/codebase_index/service.py index 578a028..d52521b 100644 --- a/src/codebase_index/service.py +++ b/src/codebase_index/service.py @@ -99,7 +99,7 @@ def search_payload( with Database(db_path) as db: if backend is not None and getattr(backend, "enabled", False): db.enable_vectors() - with _evidence(cfg, tag, enabled) as evidence: + with _evidence(cfg, tag, enabled, db.conn) as evidence: payload = run_search( db.conn, query, @@ -148,14 +148,16 @@ def session_tag(session: Optional[str]) -> Optional[str]: @contextmanager -def _evidence(cfg: "Config", tag: Optional[str], enabled: bool) -> Iterator[Any]: +def _evidence(cfg: "Config", tag: Optional[str], enabled: bool, + conn: sqlite3.Connection) -> Iterator[Any]: if not enabled: yield None return from .memory.session import open_evidence with open_evidence( - root=Path(cfg.root), config=cfg, memory_path=memory_path_for(cfg), tag=tag + root=Path(cfg.root), config=cfg, memory_path=memory_path_for(cfg), tag=tag, + index_conn=conn, ) as processor: yield processor diff --git a/tests/test_memory_session.py b/tests/test_memory_session.py index 19b1efe..1f57de4 100644 --- a/tests/test_memory_session.py +++ b/tests/test_memory_session.py @@ -173,6 +173,49 @@ def test_excerpt_that_still_holds_is_not_stale(repo): assert not again.get("stale") and not again.get("reused") and again["snippet"] +def test_derived_index_text_is_flagged_only_when_the_file_really_changed(repo, tmp_path): + """Config-key and section summaries are derived, not copied from the file, so they are + not byte-verifiable: never recorded or withheld, and flagged stale only when the file's + bytes differ from what was indexed. (Ranking decides whether such a chunk surfaces, so + this drives the processor with the exact chunk the index stored.)""" + from datetime import datetime, timezone + from types import SimpleNamespace + + from codebase_index.memory import identity as ident + from codebase_index.memory.session import EvidenceProcessor, Session, index_sha_lookup + from codebase_index.memory.store import MemoryStore + + root, cfg, db_path = repo + (root / "config.json").write_text('{"refund": {"window_days": 30}}\n', encoding="utf-8") + _update(repo) + derived = "config key: refund.window_days = 30" + with Database(db_path) as db: + stored = [r[0] for r in db.conn.execute( + "SELECT c.content FROM chunks c JOIN files f ON f.id = c.file_id " + "WHERE f.path = 'config.json' AND c.kind = 'doc'")] + assert stored == [derived] + + def process(store: MemoryStore, session_id: int) -> dict: + result = {"rank": 1, "path": "config.json", "line_start": 1, "line_end": 1, + "snippet": derived, "token_est": 9, "skeletonized": False} + repo_id = ident.repo_id_for(root) + with Database(db_path) as db: + EvidenceProcessor( + root=root, config=cfg, now=datetime.now(timezone.utc), + session=Session(tag="t", repo_id=repo_id, store=store, session_id=session_id), + index_sha=index_sha_lookup(db.conn), + )({"results": [result]}, [SimpleNamespace(content=derived)]) + return result + + with MemoryStore.open(tmp_path / "derived.sqlite") as store: + session_id = store.touch_session("r", "t", now=datetime.now(timezone.utc)) + fresh = process(store, session_id) + assert "stale" not in fresh and fresh["snippet"] == derived and "reused" not in fresh + assert store.pending(session_id) == [] # never recorded + (root / "config.json").write_text('{"refund": {"window_days": 14}}\n', encoding="utf-8") + assert process(store, session_id).get("stale") is True + + def test_code_that_only_moved_within_its_file_is_still_reused(repo): root, _, _ = repo first = _run(repo, session="t1") From 53511c8da00822ef33c5d595e078fb2354175581 Mon Sep 17 00:00:00 2001 From: denfry Date: Thu, 10 Sep 2026 11:06:22 +0300 Subject: [PATCH 12/18] test(migration): upgrade from a real 1.10.0 index The fixture was built by the 1.10.0 release code (abb67df) over tests/fixtures/sample_repo, not synthesised by 2.0. The index schema stays at version 3, so an upgraded project keeps its index: search works, update reparses nothing and built_at survives, a 1.x config.json without a memory section loads with defaults, memory.sqlite attaches on the first session, and clean removes the index while memory survives. --- .gitignore | 2 + tests/fixtures/index-1.10.0/index.sqlite | Bin 0 -> 102400 bytes tests/test_migration_1x.py | 104 +++++++++++++++++++++++ 3 files changed, 106 insertions(+) create mode 100644 tests/fixtures/index-1.10.0/index.sqlite create mode 100644 tests/test_migration_1x.py diff --git a/.gitignore b/.gitignore index 1c5d496..30036c7 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ *.sqlite *.sqlite-wal *.sqlite-shm +# Exception: a real 1.10.0 index, kept to test the 1.x -> 2.0 upgrade path. +!tests/fixtures/index-1.10.0/index.sqlite # The skill itself (.claude/skills/codebase-index/) MAY be committed so the team shares it. # The cache above must not be. diff --git a/tests/fixtures/index-1.10.0/index.sqlite b/tests/fixtures/index-1.10.0/index.sqlite new file mode 100644 index 0000000000000000000000000000000000000000..fa405406e84516db127cffe60fdc968bdfdba4fb GIT binary patch literal 102400 zcmeHwdvF{_df(1t-&hPEf>r=eV2g?kY(zX3z+wS1i~^A>k#`_K0-z~HR#vk!v%6z1 zcA%YEf}oB%L((Z%KD+Fs?vJb7IdzqCT>8Ujy{NLQu9%0NT&ab;?_5oleQIaKD*IOi}r|0Xhzwg)I*F8P`TD`nzdlo4>jdk54)1gBl zNecZ9LP8;VCKL({;(zeF4*@l}!By%A_XvW=_|M!Ero`G0Db2mw598_Z%j(nNE#>QK zUU^*pJ-H}-SNf7v2mk1{tr74QGnYnx@k{&xI*)5Ll)_TdQxkZ|XA3nT~XD1P5>fevj zM42^>v2^yaqKg7)_iw63oQT(zEDDN!d^dY^Gp(e%WGw4L(wQq3xR zZUCp@)XR2dUqrvper@5A#R3`G_k3jZ8o9J`;o`!|6Xa8cCr*;<8FKZ)+WF-xYh-2l z>V`{C@5luhz&_Rmp zp57sYiQ8mPA?VZI`j%>ragD27k44i11Jd@xOyhOSLuq?@yo)i7A)^U`-!*HCUi{x$ z;qkRjy-qTk>w2we1tf&gEG@5*r7MeziOfhml0H2kg_y;=&(u(d6!oUZJ)lT)DrV{5 z#Pjz?qUmGDq!$$~nPWEL66^dD?Us*6`XZu`JAMb1Ha-qeZASTvGe zIMx$Ejs-yEf!Is6a5SAxOFx%qI9g`Ka@n6q7lthA?hvF&ir&@+FY7J@eVrNxs~b6d zJqX&xwO47*#B{wqOxp4L(zYM5T8w49r$kz5_GU*`3Xc|63QK1TtAtg*s4gMRfe%w1`)=T|$e2Y`}p$ml-xVZdNT2fSFO# z?REEIGGg5TU&FJ^5psrDHxMLfwv8$>g2RuomV~!A#Ay(nG^`D0tXa38X<7kO)()H| zBX+%PH5#0gPO4V3VY{AfAc?Ox@*TC;H=Kq?sFRJk&2rhkkwH>@y<}IKPSYI;n4Upi zl<|b007}6HY9^^WNNv}N=YWC>LOi`uvAl7zYIy|j9HpiWLeEBXRoVg;ElDdir=-_N zrJ-+BA%g3kCUxt&)gYCo<-+(Wxn3uWixm7`wEj585g^JZFg zgOzrd3Aa4B1KdRBvp3afdT>zs{4#Sl{tChWhPxb&$K9RB`3*exr@-n+t4g_B11`Ei z^uXGQTQW8Bq7_tsdo7vM?5`eqJU1r1HZfs?v>g%#??K&8wp!o~1!n^EJ&V_$VAF*@ zC-9tWR=o)C>B=Plgwwzu&RKQ(np(qqg9-cE*dk7u8VrBK8*%-YGG>WruAPl$O^2^g zBin4yw_Ivu+#tYh)pKC}WR#kyPHOgbi~JPaFGF01F@1Dx6<%ytDptcKC94LfHDGRV zshZC0lsjPH0?hirk2jl6otRe5@~nyG1{!heG_r7;CHNtlm`&yy4NHe;fy@oC_o|Sz z!vWfrR)8lYa1GtG;PtTm8fCFXsyakB(byVQ3ok2%(`Ym|JiA_DYB0ID(tcr5M|Jm< zL)!~R5}6m%3Tv}J7iPxq7qNd)yUgE@{Lt*b-9Av$Il#1@X~JI0HVP&n;O4a7;9CyJ z37FTs51!@PmAdZH+P+(W4OISh?>%}JZCYTXDC%zJcD-m?l?ECg`U~7PaHv*c`!6j{ z)9WDbO`XWR@Ck{%g}jtzmciOr_DAlr3Kn&DHt|m1<7&FEEYqz0Qb0Qs#nP5XEqlMl zyhz=uK9!qg3GwFuV_ByaM6u3T^kP!6JzC*FL1Ox&C0fB%(Kp-Fx5C{ka1Ogus1h!+5ROo>RFpL!mJ&=oqHm%ZxzOgax;q&W; zBdUj9v}=190-BnfnjM>*AInazP3BMKvZr$Me{(y9 z9#nQv4n%Pq#soguTQzs0iSFR|#@5D`S9R*4Unot_XD4%$mXVz_a%OgV(lCwLypf-q z%$4TNd|98j<}5u6$tKOISxeVv%cWVPTry|oW{r8P%;b6fUwry)X`paQ z*}43ri6=(hoS)1i`|`}(Jj!Wuaz@XYIWs?#o6#-9(5JaPzY+fO+sHE@r$VO#%ZZ0* zak=Avm zw3#m%Q-)re%FpRJJ!?$PTMVZ!z4W=amg|NDgAOr+9)LlQ<0At;8{zw8crrUTJu?Rr zow2f0#wyP3H2`D9WsEPR|1&mjBP5ewBXC zI>>+%@ac&5LP*=y{)6_YQVv+hq!;Q?iWMR$GaQ3K-c4=zedc_7~Yz(B|Uqp69I z_PjRQ|C!YHQ@@pJ;#T|w0s;YnfIvVXAP^7;2m}NI0s(=*k2(Tp@8nV;>G<*cqa;8?2wKI%YUxT4fT%VL@4~e3GqjIYp3fhK&7nvV@gqr-t7cy)zB20|RMEdN34< z$HJ5w{g35iuB%4ah0uso+u=J?{UIrxzF(5AhPI=zxLR{6cAfnRUSpTFu3>7*3!Y1{P;=I=Yeu@h)8zMwhB#a=z#C4x)G~p$j*RDpnF;30@#MmZ)J75?u(7xVp7TxF$S$ zT)ui|5^~a|=#qgViz`^YVQns$>vo+L8oT8smSDKC2*0=j=?*||_?2hxOrR)^A4jpx zv!YO1Tw4KFbkQU)!B*@j%{g;wwOQg1uo{ZSoF;q~o@I`e8_qgkS;!aNG4(o2W1RN? zQ;D-7?T@uLwMhT5)HhOJ==<%yWb*6DX7YIAt9T@S0s(=5KtLcM5D*9m1Ox&CfldS} zyU}Dwic1g5+efA25yf9-UUJN>;Ww=tYzK+pZGC@Vh&+oG^w=Q)yG~f+=xmIhz*KH@ zZLp<*WuC~KPTV@&>OVQV5#)sKjlBJkG!RxJomsun(Z6Af^50p%iILJ3xwL0B&eoHk zVg0Ln=)3glZkXvd7}`!ssj$3>RlLsTi4!O2L&ApP2zHjbK^J(VFV=eVCYv@Ufp6`9 z-$v5Jq2*nb>2VbA`%+4cC$NtDJ$0kBg}!zW$CngA7gR_@1<(7|{}GSb&x8xUj!#+1YSpT3&@Cxi zW;aeVi!JYbro*O`XbdLso_dDbc4g-|AI=vJOUIRHmp#04aA!0C>^)^o6=IBdk)fmg z_8n^EzIHS)(w1Fv0ZF9U^E=p z<(u#D{g>!RgsxsTQ;2oO=@9)m|J1;H=`DNc{Bn|Dw0x9*yh*DXpO{uVi65Wb`7||< z_+aRQl=AJvuV`sa>XF9^ONA9|igym%1bL07bvib%^K2p%r%BTB6xZ&r((0D2|L^~` zkoJ$X&udq;6aC-Djra)!1Ofs9fq+0jARrJB2nYlO0s;YnfI#4*fxwU7Ug--B49cO< zQAvs&$4;*`{;_KKO>}A_a%+p)hi_kIgoAQUlH#3&ulosJJLnUPnrv;gJ9Ya~U+CDN zJSY!P4v5EIs;-~-761AM?U*akR-)|f<-XA1fHWXIh(!1E-aFlSPoaOp9X)$R4R+$4 zyuIY7dEkIFYWvH9q6A4EzJ0NWcCWsbc1l}2+W*I<|KcYQ5C{ka1Ofs9fq+0jARrJB z2nYlO0s;Ynz{eN?y8oZ(|9^}NSg0lt5C{ka1Ofs9fq+0jARrJB2nYlO0s;XbK*#^X zQSAT!9qqTYo6)Z!D1HI~fq+0jARrJB2>it(FdI=Ge5#70yo-jjzHWQd^D|~?x@6+; zrD>eMG>P+t^qJXQDL=j^n4m zsaf|I$`RobX zq!00=rsGI-gip||im}7+3B=Jem=Q&pdCo?W3gRcDr)^VWI{A@O(q+~xaqAo?8cWZB z?qJz*N)&y{546Vfsaq$>vnP77WNRyNu+%<=ou^=R@yos?Kf3zz0Cs!^XK={Z9)3?T zjtr|TB=5!wa!~TO2aC_(q_hEC2hCKa&%1!@sb@EC2XV(j+kNc1PV1siTjkJ=+kSp* zP4+&1J;lr3>VSmiK5(dvfbGhP{6NE}U`?|88ye@_!dj`j-6nqxG(gXipyRy94gw-(3d# zb+}uHyi4|X>l1s}Pu-#(;x-!_4coQr6`1goH0GHmwhhF7imWL)akZ*@#CC~SwMYY7 zh7fFe_cWZ*KThyzV%OO-ys&ob!791l}ZJ@0H1)4^)}rw=mxr^%+%xW))s4yA^r zUvr5K<_DAW?E~F*ioG%gCf`G%r*17P>Vw-Utk*Bwm10$Qt2m%CJ5#czt?6<}$AOil zd40xWR|97Wn%OxV8Cfdfgv8nDd1HRsoSrV}b6LC>==n098pmbj{#>bP*Sw; z5lP8t{J6v7b!;I4P^anme^~p~koG<8ziR(R`zHvBpFlt$AP^7;2m}NI0s(=5KtLcM z5D*9m1Ox&fDguXG;1Et%tPl zYj?E2-T%G*nbhy3&iDO(-&FE9lTReRooK}W6pzJEARrJB2nYlO0s?_|jlgG>fA-V( zNowe4@Znv)9G&F5{_fn84@>fWa*UoQLCrH zxfxSIB8AyT%sX4M90q-a@1%}Ik+9_Ld|Fmym2OXr#1Tose##(J!?Gr+bXvg9b8@*OJhUK^<_epYGR&eCd z&MmbM_xKl)qyBMNV5CU#3QkRG=-AmElBrUJZ;}iKu`m@RCCLd@p}F2twRltp*$p`y zkz*2cQ!rxz!iS|W+lv{pD9F76zEJ=r@jaD5uSA$O_ed%oHs9HbL?eo<;D{_FhD1&o zN=GCJ6ZJ=S!0C__IT8&k3I@RO^pK*`jmHtXS5oP=@H<;-BodYnD`7UygaiPZ?@*8I z-=qCcKYCZm1vRaM=73N%4C$a7RZWK_rEo?W+Ei6hm zmBv^b9z#aH(KN!4E7;Z_cZ#H7d=Y`D6!mxS$4#7zicnaMA4&D~$5WMG2{K+m#!s@0 zcWz?Gs}9SSq##fF#LtojTRGAjnjyUg`O&bCjHJf;`;w_xU*qKYalAdZDKdV4I&v&pjczouLL=*A*Y{=P5S2%?E3zRYcy~* zgkx$L9%h*c!k5^j1q2l-%#LS65E)a;LkQ-}%?sg(3N&F!iS57gWk0{?7m?rJ@ip7o zLV>XQ1qI?uuyX@0Kr0B8Vc_bLpFMrXU0Q1SE{c!)EFLk3MFjUh@bTbeAZ1cgj=p=~ z(1Soq#Hxu{Uo@GBCE~GoG?6sE7(mlm2s>LSR4%LH7XqUELfF}gwkr%;uv~yJF9gu} zg|M>)zlSgSg|IDuAwVStRNSCu-&IjbN_AY4sHy2TbqJz5rL`|YeUe($uhUQ>fC%AO zt8T&#M5Jj@$+-@$k)WVdRJN?{Kx9~@T2sG|C7=9-K<85sYK2@px)G|FeOv4P^04EnQ_bk)q1V zXw7_OWw2xR&KeP>wJaa;wU%kc%MAl*`cbpfLkM?eX5MWKV6xFvn8F~-DPA`tbd&)i z%YDo_WHr(`MJgIF^K2_)AGNKCxQL{dT~B+9g1ID`U-`X@f@hM1SK(g?}2no8vpO2|Wkkeu57#*?AcVd@Nc zC4lS_sx&BpT%xEoQM*b~LXsd_pCsYnN=XUbo7%oVii0Mw20;7ekoJEt1K>N_f75aq8EJVp%;u6sBv8A_+IP|W z|Bu@5YX4IEr`oS;uV}xl{i5~-JQ6>FfIvVXAP^7;2m}NI0s(=5KtLcM5D*9m1l~Uc zBD+x~bR;H)BD)a;k~9#eF(Cw08bbpL0&yCU5lB-SjO8C0P!I?qpy1KYXAr=^eq`r4 z1cpGfa|?lcY0Re)P-);<1TYhTj{i&A{|w=u_z46A0s;YnfIvVXAP^7;2m}NI0s(=5 zKtLeyF+w1z9FzS0|5V!lk7|D}r0r;br2Tu0{r_X_AL3U01Ofs9fq+0jARrJB2nYlO z0s;YnfIvVXAQ1RK5J)I#xqgEE>Z~u1Ofs9fq+0jARrJB2nYlO z0s;YnfIvVX@ZJzeDueR63D_f|F{wLm~1AP^7;2m}NI0s(=5KtLcM5D*9m1l}J6=wN+h9_LbGSbsjkFJXF7 z=jpkJxG4N0^NYkUA$n2f{+wU_AHVz=y~uO_H>||vxgXN`{}Jt1u>YU-$87!o*Ki|# z0s(=5KtLcM5D*9m1Ox&C0fB%(Kp-Fx5D0u|2=uFivg~fHmz>)0n?oy|x33Rf?!0|< z=wj#XD?^u(>VPDd?V6MQrZtf;qc3${&DSJ_1zOshhJ8o4sR)6SM$o_^6$w- z>ATXGq(W#n^p(&j_bPtra3uZEpmfcyo7N3`?MNg&l9smkoq3~ZnH9?|x($N`j-Fj9 zEUXpCg{5Yi^OM!h=WlG43FtocZl2 zO+h(cJ`_oh9h2;qoE%Y?VdV+(slpQ{$@L7mdSUJS@|88RvV8TzIkzvG&Ss_U!+SF{9mBP6c8cJ~ zqOH1Q3$2&>G3YcG=_{O2!7YBEqd{PYlX+xI`ulqXs+wErWKG7MzgfMMwYHD zE+#VC52R;K4@ecZVQNQuMYqcv*KL=v3mA=;o`#hx|taVvejsCPCBVt&4%rIwt*zR-pF^< zUf*yU9-&S)<~GY^`$h&y_4SfnX*x}JBw%_5c~Qm_egY^37pR$}?jW^YC!PZeE(r1T zM#b{R$*ScM-6N%@4MNXGb5+^`7A;9DHK(N4NTs1~R3U=vo+fn*`xcW*({f#u49N95 zSzNqGMv;uLx`+VEaMmPTZ2Ikr%C?`rB*CoMwOZ$m6*& z;kAhg8>H=!FnAB@cCyt1Zzwnupzm3{{sfyY^f>|B%&G|Q>B=Plgwwzu&RKP*v98zf z-eAJMHnxaUrUt{`@J3w!rHol3nrmmHS<~Sw)W|j)XrY*qaf1N2RnLL>lTm7-I;q*$ zE%H-vzYK95#`Mv(Rd}&osaOq{l&l(@)_}RerD{5}Q|^F)3oz>gKi+IQbz)jI%d;k$ z8)(F>)5yYYmf(kIVm6s;G%OvW1u{3l-m5~=4hLvgS^=Jrz%_K!g4e_LYm~(nsp=5j zL}P1IExfE4PNUJ>@a%epslnvpO8bRL9o5}a4s9+x4PpRT^k~=r3^Fz@b`&?Z31*O|OHzH+3TO!Y3s57V=V>Sq5ug*&n&f zDp=Ir*~B}2kE`jvvP`q~O9Aan6iZtkwe0;G^CESt`c!U~CB&ZtjAfm+Ld7~`(Thn1 z8|uL=RCW-&67IsI0`+^qmOpN3^iudQt9O?S4Vs~CMhyT&vikFG3V zUoUSV>TFuvuPswZJN%}f{cZwbKAxsD~T5d_mqVBmL*~OCE*YFf22!-YI*ZM zuH|EUw6xz+OWW7dH>$hP@*`EmPo<;j2OfyM5ak0(tr=-;r||vlCoZh4U0GNpd@5S| zF}bpOVd*jIPdPrfDX>Cj{(wVeWVIwl_NsrXd*;JiD zU%S1tA}Il_Ll{wO>f+J><3`6WG2yZ`g_2I*VTAYf;KpY6w79h<{A4cP!}ztQLNYEz zTHCNtx(7ct>e}K*r}og8%-YcdXMo~4<2KNP8=iN}0Hxfhf+qdEYGcua~?}^Wk9|{NiOmJ{~g-@|Bm)w zw12C8L%Z{l149TT5D*9m1Ox&C0fB%(Kp-Fx5C{ka1OftqA7upSL4bV69n9@-hnM=4 zV=}f1!k#bg&?VZ}$4)r4p_NwXaw~L^_y4~|*Z=Qof1>>X<^X)_M;TI~yFfr7AP^7; w2m}NI0s(=5KtLcM5D*9m1pe|MK#%x~46V?MI&_(x`!{rnUzYi0iC-@Me*)Mtu>b%7 literal 0 HcmV?d00001 diff --git a/tests/test_migration_1x.py b/tests/test_migration_1x.py new file mode 100644 index 0000000..2713d5f --- /dev/null +++ b/tests/test_migration_1x.py @@ -0,0 +1,104 @@ +"""Upgrading from 1.x: a real 1.10.0 index keeps working, and memory attaches beside it. + +`tests/fixtures/index-1.10.0/index.sqlite` was built by the released 1.10.0 code (commit +abb67df) over `tests/fixtures/sample_repo`, not synthesised by 2.0, so these tests check the +upgrade path users actually take. 2.0 leaves the index schema at version 3; evidence memory +lives in its own `memory.sqlite`, so no reindex is needed. +""" + +from __future__ import annotations + +import json +import shutil +import sqlite3 +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from codebase_index.cli import app +from codebase_index.config import load +from codebase_index.storage.db import SCHEMA_VERSION, peek_schema_version + +FIXTURES = Path(__file__).parent / "fixtures" +runner = CliRunner() + + +@pytest.fixture +def upgraded(tmp_path, monkeypatch): + for name in ("CBX_DB_PATH", "CBX_MEMORY_PATH", "CBX_MEMORY", "CBX_ROOT"): + monkeypatch.delenv(name, raising=False) + root = tmp_path / "repo" + # Other tests index sample_repo in place; never carry their runtime cache along. + shutil.copytree(FIXTURES / "sample_repo", root, ignore=shutil.ignore_patterns(".claude")) + cache = root / ".claude" / "cache" / "codebase-index" + cache.mkdir(parents=True, exist_ok=True) + shutil.copy(FIXTURES / "index-1.10.0" / "index.sqlite", cache / "index.sqlite") + # The config 1.10.0 `init` writes: every section 2.0 knows, except `memory`. + (cache / "config.json").write_text(json.dumps({ + "root": ".", "languages": "auto", "max_file_bytes": 1048576, + "ignore_files": [".gitignore", ".cursorignore", ".claudeignore", ".codeindexignore"], + "extra_ignore": [], "chunk": {"window_lines": 80, "overlap_lines": 10}, + "retrieval": {"default_mode": "hybrid", "rrf_k": 60, "token_budget": 1500, "limit": 10, + "compact_snippets": True, "compact_min_reduction": 0.25}, + "embeddings": {"backend": "noop", "enabled": False, "model": "all-MiniLM-L6-v2", + "allow_external": False, "endpoint": None}, + "graph": {"max_depth": 2, "node_cap": 40}, "redaction": {"enabled": True}, + }), encoding="utf-8") + return root, cache / "index.sqlite" + + +def _meta(db: Path, key: str): + conn = sqlite3.connect(db) + try: + row = conn.execute("SELECT value FROM meta WHERE key = ?", (key,)).fetchone() + return row[0] if row else None + finally: + conn.close() + + +def _cli_json(root: Path, *args: str) -> dict: + result = runner.invoke(app, ["--root", str(root), *args]) + assert result.exit_code == 0, result.output + return json.loads(result.output) + + +def test_fixture_is_a_real_1_10_index_on_the_current_schema(upgraded): + _root, db = upgraded + assert peek_schema_version(db) == 3 == SCHEMA_VERSION + conn = sqlite3.connect(db) + try: + assert conn.execute("SELECT COUNT(*) FROM files").fetchone()[0] == 6 + finally: + conn.close() + + +def test_1x_index_is_used_in_place_without_a_rebuild(upgraded): + root, db = upgraded + built_at = _meta(db, "built_at") + search = _cli_json(root, "search", "token", "--json") + assert search["results"] and "memory" not in search + update = _cli_json(root, "--json", "update") + assert update["indexed"] == 0 and update["deleted"] == 0 # same bytes: nothing reparsed + assert _meta(db, "built_at") == built_at # the 1.x database survived + assert peek_schema_version(db) == 3 + + +def test_memory_attaches_to_an_upgraded_project(upgraded): + root, db = upgraded + assert load(root).memory.enabled is True # 1.x config, 2.0 defaults + first = _cli_json(root, "search", "token", "--session", "upgrade", "--json") + second = _cli_json(root, "search", "token", "--session", "upgrade", "--json") + assert first["memory"]["reused"] == 0 and second["memory"]["reused"] >= 1 + assert (db.parent / "memory.sqlite").is_file() + assert _cli_json(root, "verify", "--session", "upgrade", "--json")["all_valid"] is True + stats = _cli_json(root, "stats", "--json") + assert stats["memory"]["sessions"] == 1 and stats["files"] == 6 + + +def test_clean_removes_the_index_but_keeps_memory(upgraded): + root, db = upgraded + _cli_json(root, "search", "token", "--session", "keep", "--json") + result = runner.invoke(app, ["--root", str(root), "clean", "--yes", "--json"]) + assert result.exit_code == 0, result.output + assert not db.exists() and (db.parent / "memory.sqlite").is_file() From 37120cc26704597e12d0ddfd8d2eeca3b10d31d4 Mon Sep 17 00:00:00 2001 From: denfry Date: Thu, 10 Sep 2026 11:06:23 +0300 Subject: [PATCH 13/18] bench(memory): sequential real-history evidence benchmark Replays each corpus's own history in a shared clone (sources are never modified): for every git-derived query the tree is checked out at the parent of the query's commit, the index is updated incrementally, and one retrieval call is made. Consecutive tasks form sessions of K tasks, so the repository really evolves inside a session. All arms share one packet per task: A (reread whole files), A-mem (skip unchanged files), B (1.10.0 packet), S (withhold by locator without checking the source) and C (2.0 evidence memory). The oracle is independent of memory's hashing: a withheld snippet is stale unless the session was handed that exact text or the whole current span, and notices are scored against a list-comparison re-implementation of span presence. Every task also checks that restoring withheld snippets reproduces the B packet exactly. --- tests/eval/memory_eval.py | 660 ++++++++++++++++++++++++++++++++ tests/test_memory_eval_smoke.py | 119 ++++++ 2 files changed, 779 insertions(+) create mode 100644 tests/eval/memory_eval.py create mode 100644 tests/test_memory_eval_smoke.py diff --git a/tests/eval/memory_eval.py b/tests/eval/memory_eval.py new file mode 100644 index 0000000..b6d15e7 --- /dev/null +++ b/tests/eval/memory_eval.py @@ -0,0 +1,660 @@ +#!/usr/bin/env python3 +"""Sequential real-history benchmark for evidence memory. + +Replays a repository's own history. For every git-derived query (a commit subject whose +answer is the files that commit changed) the working tree is checked out at the commit's +*parent* — what an agent sees before making the change — the index is updated +incrementally, and one retrieval call is made. Consecutive tasks form agent sessions of K +tasks, so the repository genuinely evolves between the calls of one session: unrelated +commits, edits to evidence already delivered, moves, deletions and refactors. + +Each task produces one retrieval packet. Every arm is computed from that same packet and +the same candidates, so arms differ only in what they do with evidence: + + A read the full file behind every delivered result ("just reread the file") + A-mem A, skipping a file this session already read whose bytes are unchanged + B the 1.10.0 packet: every snippet, every time + S unsafe dedup: withhold any result whose (path, lines) the session saw before, + without looking at the source ("trust what you already read") + C 2.0 evidence memory: withhold only byte-identical evidence; report changes + +The oracle is independent of memory's hashing and relocation code. It keeps the text each +session was actually handed; a withheld snippet is **stale** unless the text B would +deliver now is text that session already holds. Invalidation notices are scored against a +list-comparison re-implementation of span presence. + +Sources are never modified: each corpus is replayed in a `git clone --shared` inside a +temporary directory. + + python tests/eval/memory_eval.py + python tests/eval/memory_eval.py --corpus ../svc:/tmp/svc.yml --sessions 1,5,10,all +""" + +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import statistics +import subprocess +import sys +import tempfile +import time +from collections.abc import Callable, Sequence +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Optional + +REPO_ROOT = Path(__file__).resolve().parents[2] +if __package__ in (None, ""): + sys.path.insert(0, str(REPO_ROOT / "tests")) + sys.path.insert(0, str(REPO_ROOT / "src")) + from eval import harness, metrics # type: ignore[no-redef] +else: + from . import harness, metrics + +import yaml # noqa: E402 + +from codebase_index.config import Config # noqa: E402 +from codebase_index.indexer.pipeline import build_index, update_index # noqa: E402 +from codebase_index.memory import identity as ident # noqa: E402 +from codebase_index.memory.session import ( # noqa: E402 + EvidenceProcessor, + Session, + index_sha_lookup, +) +from codebase_index.memory.store import MemoryStore # noqa: E402 +from codebase_index.output.redact import redact_snippet # noqa: E402 +from codebase_index.parsers.line_chunker import estimate_tokens # noqa: E402 +from codebase_index.retrieval.pipeline import search # noqa: E402 +from codebase_index.storage.db import Database # noqa: E402 + +LIMIT = 10 +BUDGET = 1500 +SURVIVAL_HORIZONS = (1, 5, 10, 20) +_EPOCH = datetime(2026, 1, 1, tzinfo=timezone.utc) + + +# --- replay setup ------------------------------------------------------------------ + + +@dataclass(frozen=True) +class Task: + query: str + expected_files: tuple[str, ...] + commit: str + parent: str + + +def git(repo: Path, *args: str) -> str: + proc = subprocess.run(["git", "-C", str(repo), *args], capture_output=True, text=True, + encoding="utf-8", errors="replace") + if proc.returncode != 0: + raise RuntimeError(f"git {' '.join(args)}: {proc.stderr.strip()}") + return proc.stdout + + +def load_tasks(queries_path: Path, work: Path) -> tuple[list[Task], dict[str, int]]: + """Tasks oldest-first, plus each commit's `git log` position (0 = newest).""" + order: dict[str, int] = {} + parents: dict[str, Optional[str]] = {} + for i, line in enumerate(git(work, "log", "--format=%H %P", "HEAD").splitlines()): + parts = line.split() + order[parts[0]] = i + parents[parts[0]] = parts[1] if len(parts) > 1 else None + by_prefix = {sha[:12]: sha for sha in order} + tasks: list[Task] = [] + for entry in yaml.safe_load(queries_path.read_text(encoding="utf-8")) or []: + full = by_prefix.get(str(entry.get("commit", ""))[:12]) + if not full or not parents.get(full): + continue # unknown commit or a root commit: no "before" state to replay + tasks.append(Task( + query=entry["query"], + expected_files=tuple(f.replace("\\", "/") for f in entry.get("expected_files", ())), + commit=full, + parent=str(parents[full]), + )) + tasks.sort(key=lambda t: -order[t.commit]) + return tasks, order + + +def corpus_config(work: Path) -> Config: + cfg = Config() + cfg.root = str(work) + cfg.embeddings.enabled = False + cfg.extra_ignore = [*cfg.extra_ignore, *harness.CORPUS_EXCLUDES] + cfg.memory.retention_days = 36500 # the replay's synthetic clock must never expire a session + return cfg + + +# --- oracle ------------------------------------------------------------------------ + + +class FileCache: + """Working-tree reads for one task, shared by every arm's oracle.""" + + def __init__(self, work: Path) -> None: + self.work = work + self._bytes: dict[str, Optional[bytes]] = {} + + def raw(self, rel: str) -> Optional[bytes]: + if rel not in self._bytes: + path = self.work / rel + self._bytes[rel] = path.read_bytes() if path.is_file() else None + return self._bytes[rel] + + def lines(self, rel: str) -> Optional[list[str]]: + raw = self.raw(rel) + if raw is None: + return None + text = raw.decode("utf-8", "surrogateescape") + return text.replace("\r\n", "\n").replace("\r", "\n").splitlines() + + +def _sha16(lines: list[str]) -> str: + return hashlib.sha256("\n".join(lines).encode("utf-8", "surrogateescape")).hexdigest()[:16] + + +def span_present(lines: Optional[list[str]], span: list[str], line_start: int) -> bool: + """List comparison: the span sits at its recorded lines, or occurs exactly once.""" + if lines is None or not span: + return False + n = len(span) + if lines[line_start - 1:line_start - 1 + n] == span: + return True + return sum(1 for i in range(len(lines) - n + 1) if lines[i:i + n] == span) == 1 + + +def _occurrences(lines: Optional[list[str]], span: list[str]) -> Optional[int]: + if lines is None or not span: + return None + n = len(span) + return sum(1 for i in range(len(lines) - n + 1) if lines[i:i + n] == span) + + +def session_holds(truth: "SessionTruth", path: str, snippet: str, span_text: str) -> bool: + """Does the session hold what it would be sent now? + + Yes if it was handed this exact snippet before, or the whole current span (the + snippet — skeleton, signature or full text — is derived from those very bytes). + Both sides are plain strings read from packets and files, never memory's hashes. + """ + return (snippet in truth.held.get(path, set()) + or span_text in truth.full_spans.get(path, set())) + + +@dataclass +class Atom: + path: str + line_start: int + span: list[str] + delivered_position: int + last_valid: int = 0 + invalid_at: Optional[int] = None + + +@dataclass +class SessionTruth: + held: dict[str, set[str]] = field(default_factory=dict) + full_spans: dict[str, set[str]] = field(default_factory=dict) + seen_at: dict[tuple[str, int, int], str] = field(default_factory=dict) + files_read: dict[str, str] = field(default_factory=dict) + atoms: dict[tuple[str, str], Atom] = field(default_factory=dict) + + +@dataclass +class Totals: + tasks: int = 0 + sessions: int = 0 + deliveries: int = 0 + b_tokens: int = 0 + c_tokens: int = 0 + s_tokens: int = 0 + a_tokens: int = 0 + amem_tokens: int = 0 + b_packet_tokens: int = 0 + c_packet_tokens: int = 0 + c_reused: int = 0 + s_reused: int = 0 + c_stale_withheld: int = 0 + s_stale_withheld: int = 0 + s_stale_gold: int = 0 + c_stale_flags: int = 0 + notices: int = 0 + notice_tp: int = 0 + notice_fp: int = 0 + notice_fn: int = 0 + page_mismatches: int = 0 + distinct_atoms: int = 0 + store_bytes: int = 0 + survival: dict[int, list[int]] = field( + default_factory=lambda: {h: [0, 0] for h in SURVIVAL_HORIZONS}) + useful_b: list[float] = field(default_factory=list) + useful_s: list[float] = field(default_factory=list) + saved_per_task: list[float] = field(default_factory=list) + memory_ms: list[float] = field(default_factory=list) + examples: list[dict] = field(default_factory=list) + """Bounded diagnostics for any event that should not happen (never committed: they + can quote private source).""" + + def example(self, kind: str, **info: Any) -> None: + if sum(1 for e in self.examples if e["kind"] == kind) < 10: + self.examples.append({"kind": kind, **info}) + + def merge(self, other: "Totals") -> None: + for name, value in vars(other).items(): + mine = getattr(self, name) + if isinstance(value, int): + setattr(self, name, mine + value) + elif isinstance(value, list): + mine.extend(value) + elif isinstance(value, dict): + for h, (survived, observed) in value.items(): + mine[h][0] += survived + mine[h][1] += observed + + +def close_session(truth: SessionTruth, stat: Totals) -> None: + """Fold one session's evidence into survival counts (interval-censored).""" + stat.distinct_atoms += len(truth.atoms) + for atom in truth.atoms.values(): + for h in SURVIVAL_HORIZONS: + if atom.last_valid >= h: + stat.survival[h][0] += 1 + stat.survival[h][1] += 1 + elif atom.invalid_at is not None and atom.invalid_at <= h: + stat.survival[h][1] += 1 + + +# --- scoring one task -------------------------------------------------------------- + + +def score_task(*, files: FileCache, processor: EvidenceProcessor, task: Task, packet: dict, + candidates: list, truth: SessionTruth, stat: Totals, position: int, + token_budget: int) -> None: + stat.tasks += 1 + + # Ground truth for evidence this session holds, established before memory runs. + truth_invalid: set[tuple[str, str]] = set() + for key, atom in truth.atoms.items(): + if atom.invalid_at is not None: + continue + distance = atom.delivered_position - position + if span_present(files.lines(atom.path), atom.span, atom.line_start): + atom.last_valid = max(atom.last_valid, distance) + else: + atom.invalid_at = distance + truth_invalid.add(key) + + memory_packet = copy.deepcopy(packet) + started = time.perf_counter() + processor(memory_packet, candidates) + stat.memory_ms.append((time.perf_counter() - started) * 1000) + + notices = {(r.path, r.sha[:16]) for r in + (ident.parse_ref(n["ref"]) for n in memory_packet["memory"]["invalidated"])} + stat.notices += len(notices) + stat.notice_tp += len(notices & truth_invalid) + stat.notice_fp += len(notices - truth_invalid) + stat.notice_fn += len(truth_invalid - notices) + states = {(r.path, r.sha[:16]): n["state"] for n in memory_packet["memory"]["invalidated"] + for r in [ident.parse_ref(n["ref"])]} + for key in sorted(notices ^ truth_invalid): + atom = truth.atoms.get(key) + stat.example("notice_fp" if key in notices else "notice_fn", path=key[0], sha=key[1], + memory_state=states.get(key), oracle_known=atom is not None, + oracle_line_start=atom.line_start if atom else None, + oracle_occurrences=_occurrences(files.lines(key[0]), atom.span) + if atom else None) + + # Filling withheld snippets back in must reproduce the no-memory packet exactly. + restored = copy.deepcopy(memory_packet) + restored.pop("memory") + for mine, base in zip(restored["results"], packet["results"]): + if mine.pop("reused", False): + mine["snippet"] = base["snippet"] + mine.pop("stale", None) + if restored != packet: + stat.page_mismatches += 1 + + returned_s: list[tuple[str, int]] = [] + saved = 0 + delivered_files: set[str] = set() + padded = list(candidates) + [None] * max(0, len(packet["results"]) - len(candidates)) + for base, mine, candidate in zip(packet["results"], memory_packet["results"], padded): + path = base["path"].replace("\\", "/") + tokens = int(base.get("token_est") or 0) + snippet = base.get("snippet") + if not snippet: + returned_s.append((path, tokens)) + continue + stat.deliveries += 1 + stat.b_tokens += tokens + delivered_files.add(path) + span_now = (files.lines(path) or [])[base["line_start"] - 1:base["line_end"]] + + # C: evidence memory + if mine.get("stale"): + stat.c_stale_flags += 1 + stat.example("stale_flag", path=path, lines=[base["line_start"], base["line_end"]], + chunk_kind=getattr(candidate, "kind", None), + source=getattr(candidate, "source", None), + content=(getattr(candidate, "content", None) or "")[:240], + span="\n".join(span_now)[:240]) + span_text = "\n".join(span_now) + if mine.get("reused"): + stat.c_reused += 1 + saved += tokens + if not session_holds(truth, path, snippet, span_text): + stat.c_stale_withheld += 1 + stat.example("c_stale_withheld", path=path, + lines=[base["line_start"], base["line_end"]], + source=getattr(candidate, "source", None), + skeletonized=base.get("skeletonized"), snippet=snippet[:400], + held=[h[:400] for h in sorted(truth.held.get(path, set()))[:3]]) + else: + stat.c_tokens += tokens + truth.held.setdefault(path, set()).add(snippet) + if snippet == redact_snippet(span_text): + truth.full_spans.setdefault(path, set()).add(span_text) + if not mine.get("stale"): + span = (files.lines(path) or [])[base["line_start"] - 1:base["line_end"]] + key = (path, _sha16(span)) + known = truth.atoms.get(key) + if known is None or known.invalid_at is not None: + truth.atoms[key] = Atom(path, int(base["line_start"]), span, position) + + # S: unsafe dedup by locator + locator = (path, int(base["line_start"]), int(base["line_end"])) + if locator in truth.seen_at: + stat.s_reused += 1 + if truth.seen_at[locator] != snippet: + stat.s_stale_withheld += 1 + if path in task.expected_files: + stat.s_stale_gold += 1 + continue # the session holds a stale version: not useful evidence + else: + truth.seen_at[locator] = snippet + stat.s_tokens += tokens + returned_s.append((path, tokens)) + + # A / A-mem: whole-file reads of every delivered result's file + for path in sorted(delivered_files): + raw = files.raw(path) or b"" + cost = estimate_tokens(raw.decode("utf-8", "ignore")) if raw else 0 + stat.a_tokens += cost + digest = hashlib.sha256(raw).hexdigest() + if truth.files_read.get(path) != digest: + stat.amem_tokens += cost + truth.files_read[path] = digest + + stat.b_packet_tokens += estimate_tokens(json.dumps(packet, separators=(",", ":"))) + stat.c_packet_tokens += estimate_tokens(json.dumps(memory_packet, separators=(",", ":"))) + stat.saved_per_task.append(float(saved)) + returned_b = [(r["path"].replace("\\", "/"), int(r.get("token_est") or 0)) + for r in packet["results"]] + stat.useful_b.append( + metrics.useful_context_at_budget(returned_b, task.expected_files, token_budget)) + stat.useful_s.append( + metrics.useful_context_at_budget(returned_s, task.expected_files, token_budget)) + + +# --- one corpus -------------------------------------------------------------------- + + +@dataclass +class CorpusResult: + name: str + tasks: int + totals: dict[Optional[int], Totals] + search_ms: list[float] + update_ms: list[float] + + +def replay_corpus(repo: Path, queries: Path, sessions: Sequence[Optional[int]], *, + limit: int = LIMIT, token_budget: int = BUDGET, + max_tasks: Optional[int] = None, + log: Callable[[str], None] = lambda _m: None) -> CorpusResult: + with tempfile.TemporaryDirectory() as tmp_name: + tmp = Path(tmp_name) + work = tmp / "work" + subprocess.run(["git", "clone", "-q", "--shared", "--no-checkout", str(repo), str(work)], + check=True, capture_output=True) + tasks, order = load_tasks(queries, work) + if max_tasks is not None: + tasks = tasks[:max_tasks] + cfg = corpus_config(work) + repo_id = ident.repo_id_for(work) + totals = {k: Totals() for k in sessions} + truths: dict[Optional[int], SessionTruth] = {k: SessionTruth() for k in sessions} + active: dict[Optional[int], int] = {} + stores = {k: MemoryStore.open(tmp / f"memory-{k or 'all'}.sqlite") for k in sessions} + search_ms: list[float] = [] + update_ms: list[float] = [] + db = Database(tmp / "index.sqlite").open() + try: + for index, task in enumerate(tasks): + git(work, "checkout", "-q", "-f", "--detach", task.parent) + started = time.perf_counter() + if index == 0: + build_index(cfg, db, root=work) + else: + update_index(cfg, db, root=work) + update_ms.append((time.perf_counter() - started) * 1000) + + captured: dict[str, Any] = {} + started = time.perf_counter() + packet = search(db.conn, task.query, mode="hybrid", limit=limit, + token_budget=token_budget, no_fallback=True, + evidence=lambda _p, c: captured.update(candidates=list(c))) + search_ms.append((time.perf_counter() - started) * 1000) + + files = FileCache(work) + now = _EPOCH + timedelta(minutes=index) + for k in sessions: + session_index = 0 if k is None else index // k + if active.get(k) != session_index: + if k in active: + close_session(truths[k], totals[k]) + truths[k] = SessionTruth() + totals[k].sessions += 1 + active[k] = session_index + tag = f"k{k or 'all'}-s{session_index}" + store = stores[k] + session_id = store.touch_session( + repo_id, ident.session_key(repo_id, tag), now=now) + processor = EvidenceProcessor( + root=work, config=cfg, now=now, + session=Session(tag=tag, repo_id=repo_id, store=store, + session_id=session_id), + index_sha=index_sha_lookup(db.conn)) + score_task(files=files, processor=processor, task=task, packet=packet, + candidates=captured.get("candidates", []), truth=truths[k], + stat=totals[k], position=order[task.parent], + token_budget=token_budget) + if (index + 1) % 25 == 0: + log(f" {index + 1}/{len(tasks)} tasks") + for k in sessions: + if k in active: + close_session(truths[k], totals[k]) + finally: + db.close() + for k, store in stores.items(): + store.conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") + totals[k].store_bytes = store.stats(repo_id)["bytes"] + store.close() + return CorpusResult(Path(repo).resolve().name, len(tasks), totals, search_ms, update_ms) + + +# --- reporting --------------------------------------------------------------------- + + +def _pct(part: float, whole: float) -> str: + return f"{100.0 * part / whole:.1f}%" if whole else "—" + + +def _ratio(num: int, den: int) -> str: + return f"{num / den:.3f}" if den else "—" + + +def summarise(label: str, t: Totals) -> dict: + mean_saved, lo, hi = metrics.paired_bootstrap_ci(t.saved_per_task, resamples=2000) + return { + "sessions_k": label, + "tasks": t.tasks, + "sessions": t.sessions, + "deliveries": t.deliveries, + "distinct_atoms": t.distinct_atoms, + "B_tokens": t.b_tokens, + "C_tokens": t.c_tokens, + "S_tokens": t.s_tokens, + "A_tokens": t.a_tokens, + "A_mem_tokens": t.amem_tokens, + "C_token_reuse_rate": (t.b_tokens - t.c_tokens) / t.b_tokens if t.b_tokens else 0.0, + "C_evidence_reuse_rate": t.c_reused / t.deliveries if t.deliveries else 0.0, + "C_saved_per_task": mean_saved, + "C_saved_per_task_ci95": [lo, hi], + "C_stale_withheld": t.c_stale_withheld, + "C_validated_reuse_rate": ( + (t.c_reused - t.c_stale_withheld) / t.c_reused if t.c_reused else None), + "S_token_reuse_rate": (t.b_tokens - t.s_tokens) / t.b_tokens if t.b_tokens else 0.0, + "S_stale_withheld": t.s_stale_withheld, + "S_stale_reuse_rate": t.s_stale_withheld / t.s_reused if t.s_reused else None, + "S_stale_gold": t.s_stale_gold, + "notices": t.notices, + "invalidation_precision": t.notice_tp / (t.notice_tp + t.notice_fp) + if (t.notice_tp + t.notice_fp) else None, + "invalidation_recall": t.notice_tp / (t.notice_tp + t.notice_fn) + if (t.notice_tp + t.notice_fn) else None, + "page_mismatches": t.page_mismatches, + "stale_flags": t.c_stale_flags, + "B_packet_tokens": t.b_packet_tokens, + "C_packet_tokens": t.c_packet_tokens, + "useful_B_equals_C": statistics.fmean(t.useful_b) if t.useful_b else 0.0, + "useful_S": statistics.fmean(t.useful_s) if t.useful_s else 0.0, + "survival": {h: (s / o if o else None, o) for h, (s, o) in t.survival.items()}, + "memory_ms_p50": metrics.percentile(t.memory_ms, 50), + "memory_ms_p95": metrics.percentile(t.memory_ms, 95), + "store_bytes": t.store_bytes, + "examples": t.examples, + } + + +def format_report(rows: list[dict]) -> str: + out = [ + "| K | tasks | sessions | B tokens | C tokens (reuse) | 95% CI saved/task | " + "C stale withheld | S tokens (reuse) | S stale withheld | notices P/R | " + "A tokens | A-mem tokens | packet tok B→C |", + "|" + "---|" * 13, + ] + for r in rows: + lo, hi = r["C_saved_per_task_ci95"] + precision = r["invalidation_precision"] + recall = r["invalidation_recall"] + out.append( + f"| {r['sessions_k']} | {r['tasks']} | {r['sessions']} | {r['B_tokens']} | " + f"{r['C_tokens']} ({100 * r['C_token_reuse_rate']:.1f}%) | " + f"{r['C_saved_per_task']:.1f} [{lo:.1f}, {hi:.1f}] | {r['C_stale_withheld']} | " + f"{r['S_tokens']} ({100 * r['S_token_reuse_rate']:.1f}%) | " + f"{r['S_stale_withheld']} | " + f"{'—' if precision is None else f'{precision:.3f}'}/" + f"{'—' if recall is None else f'{recall:.3f}'} | " + f"{r['A_tokens']} | {r['A_mem_tokens']} | " + f"{r['B_packet_tokens']}→{r['C_packet_tokens']} |" + ) + out.append("") + out.append("| K | useful@budget B=C | useful@budget S | S stale gold | page mismatches | " + "stale flags | distinct atoms | survival h=1/5/10/20 | memory ms p50/p95 | " + "store bytes |") + out.append("|" + "---|" * 10) + for r in rows: + surv = "/".join("—" if v[0] is None else f"{v[0]:.2f}" for v in r["survival"].values()) + out.append( + f"| {r['sessions_k']} | {r['useful_B_equals_C']:.3f} | {r['useful_S']:.3f} | " + f"{r['S_stale_gold']} | {r['page_mismatches']} | {r['stale_flags']} | " + f"{r['distinct_atoms']} | {surv} | {r['memory_ms_p50']:.2f}/{r['memory_ms_p95']:.2f} | " + f"{r['store_bytes']} |" + ) + return "\n".join(out) + + +def _parse_corpus(spec: str) -> tuple[Path, Path]: + head, sep, tail = spec.rpartition(":") + if not sep or (len(head) == 1 and head.isalpha()): + raise argparse.ArgumentTypeError(f"--corpus expects ':', got {spec!r}") + return Path(head).resolve(), Path(tail) + + +def _parse_sessions(spec: str) -> list[Optional[int]]: + out: list[Optional[int]] = [] + for part in spec.split(","): + part = part.strip().lower() + if part == "all": + out.append(None) + elif part: + out.append(max(1, int(part))) + return out + + +def main(argv: Optional[list[str]] = None) -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--corpus", action="append", default=[], metavar="REPO:QUERIES") + ap.add_argument("--sessions", default="1,5,10,25,all", + help="session lengths in tasks; 'all' = one session per corpus") + ap.add_argument("--limit", type=int, default=LIMIT) + ap.add_argument("--token-budget", type=int, default=BUDGET) + ap.add_argument("--max-tasks", type=int, default=None) + ap.add_argument("--json-out", default=None) + args = ap.parse_args(argv) + if sys.platform == "win32": + sys.stdout.reconfigure(encoding="utf-8") # type: ignore[union-attr] + + corpora = [_parse_corpus(s) for s in args.corpus] or [ + (REPO_ROOT, harness.QUERY_DIR / "self_repo_git.yml")] + sessions = _parse_sessions(args.sessions) + results: list[CorpusResult] = [] + for repo, queries in corpora: + print(f"replaying {repo.name} ({queries})", file=sys.stderr, flush=True) + results.append(replay_corpus( + repo, queries, sessions, limit=args.limit, token_budget=args.token_budget, + max_tasks=args.max_tasks, log=lambda m: print(m, file=sys.stderr, flush=True))) + + def label(k: Optional[int]) -> str: + return "all" if k is None else str(k) + + report: dict[str, Any] = {"corpora": {}, "pooled": []} + for res in results: + rows = [summarise(label(k), res.totals[k]) for k in sessions] + report["corpora"][res.name] = { + "tasks": res.tasks, + "rows": rows, + "search_ms_p50": metrics.percentile(res.search_ms, 50), + "search_ms_p95": metrics.percentile(res.search_ms, 95), + "update_ms_p50": metrics.percentile(res.update_ms, 50), + } + print(f"\n## {res.name} ({res.tasks} tasks; search p50 " + f"{report['corpora'][res.name]['search_ms_p50']:.1f} ms, update p50 " + f"{report['corpora'][res.name]['update_ms_p50']:.1f} ms)\n") + print(format_report(rows)) + + pooled_rows = [] + for k in sessions: + merged = Totals() + for res in results: + merged.merge(res.totals[k]) + pooled_rows.append(summarise(label(k), merged)) + report["pooled"] = pooled_rows + if len(results) > 1: + print(f"\n## POOLED ({sum(r.tasks for r in results)} tasks, {len(results)} corpora)\n") + print(format_report(pooled_rows)) + if args.json_out: + Path(args.json_out).write_text(json.dumps(report, indent=2, default=str), + encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_memory_eval_smoke.py b/tests/test_memory_eval_smoke.py new file mode 100644 index 0000000..8a7bbea --- /dev/null +++ b/tests/test_memory_eval_smoke.py @@ -0,0 +1,119 @@ +"""A miniature of the sequential memory benchmark on a synthetic git history. + +The invariants asserted here must hold on any corpus: memory never withholds text the +session does not hold, withheld packets restore to the no-memory packet exactly, every +invalidation notice matches an independent oracle, and a one-task session reuses nothing. +The scenario is built so the unsafe arm demonstrably goes stale where memory does not. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import yaml + +from tests.eval import memory_eval + +INVOICE = '''def compute_invoice_total(lines, tax_rate): + """Sum invoice line amounts and apply the tax rate.""" + subtotal = sum(line.amount for line in lines) + return round(subtotal * (1 + tax_rate), 2) +''' + +REFUND = '''def issue_refund(invoice, amount): + """Refund part of an invoice total.""" + if amount > invoice.total: + raise ValueError("refund exceeds invoice total") + return invoice.total - amount +''' + +TAX = '''def tax_rate_for(region): + """Look up the sales tax rate for a region.""" + return {"eu": 0.2, "us": 0.07}.get(region, 0.0) +''' + + +def _git(repo: Path, *args: str) -> str: + return subprocess.run( + ["git", "-c", "user.name=bench", "-c", "user.email=bench@example.com", + "-c", "core.autocrlf=false", "-C", str(repo), *args], + check=True, capture_output=True, text=True, + ).stdout.strip() + + +def _commit(repo: Path, message: str, files: dict[str, str]) -> str: + for rel, text in files.items(): + path = repo / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8", newline="\n") + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "-m", message) + return _git(repo, "rev-parse", "HEAD") + + +def _history(tmp_path: Path) -> tuple[Path, Path]: + repo = tmp_path / "origin" + repo.mkdir() + _git(repo, "init", "-q") + _commit(repo, "initial billing", {"billing/invoice.py": INVOICE, + "billing/refund.py": REFUND, "billing/tax.py": TAX}) + queries = [] + # Task 1 sees the original refund; its own commit then changes the signature and the + # check without moving any line, so the same (path, lines) now holds different text. + c1 = _commit(repo, "tighten refund check", {"billing/refund.py": REFUND.replace( + "def issue_refund(invoice, amount):", "def issue_refund(invoice, amount, reason=None):" + ).replace(">", ">=")}) + queries.append({"query": "refund exceeds invoice total", "commit": c1, + "expected_files": ["billing/refund.py"]}) + # Unrelated change: evidence from task 1 about invoices stays valid. + c2 = _commit(repo, "add region", {"billing/tax.py": TAX.replace('"us": 0.07', '"us": 0.07, "ca": 0.05')}) + queries.append({"query": "refund exceeds invoice total amount", "commit": c2, + "expected_files": ["billing/tax.py"]}) + c3 = _commit(repo, "round tax", {"billing/tax.py": TAX.replace("0.0)", "0.0) or 0.0")}) + queries.append({"query": "invoice total refund tax rate", "commit": c3, + "expected_files": ["billing/tax.py"]}) + path = tmp_path / "queries.yml" + path.write_text(yaml.safe_dump(queries), encoding="utf-8") + return repo, path + + +def test_replay_is_sound_and_reuses_only_unchanged_evidence(tmp_path): + repo, queries = _history(tmp_path) + head_before = _git(repo, "rev-parse", "HEAD") + result = memory_eval.replay_corpus(repo, queries, [None, 1]) + assert result.tasks == 3 + assert _git(repo, "rev-parse", "HEAD") == head_before # source repository untouched + assert _git(repo, "status", "--porcelain") == "" + + one_session = result.totals[None] + assert one_session.c_stale_withheld == 0 + assert one_session.page_mismatches == 0 + assert one_session.notice_fp == 0 and one_session.notice_fn == 0 + assert one_session.c_reused > 0 + assert one_session.c_tokens < one_session.b_tokens + assert one_session.s_stale_withheld > 0 # trusting old reads goes stale here + assert one_session.notices >= 1 # ...and memory said so instead + + per_task = result.totals[1] + assert per_task.c_reused == 0 and per_task.c_tokens == per_task.b_tokens + assert per_task.s_reused == 0 + + summary = memory_eval.summarise("all", one_session) + assert summary["C_validated_reuse_rate"] == 1.0 + assert memory_eval.format_report([summary]).count("|") > 20 + + +def test_oracle_helpers(): + assert memory_eval.span_present(["a", "b", "c"], ["b", "c"], 2) + assert memory_eval.span_present(["x", "b", "c"], ["b", "c"], 1) # moved, unique + assert not memory_eval.span_present(["b", "c", "b", "c"], ["b", "c"], 9) # ambiguous + assert not memory_eval.span_present(None, ["b"], 1) + truth = memory_eval.SessionTruth() + span = "export default function App() {\n return 1\n}" + truth.full_spans["a.tsx"] = {span} + truth.held["b.py"] = {"def g(x):"} + assert memory_eval.session_holds(truth, "a.tsx", "function App()", span) # derived excerpt + assert not memory_eval.session_holds(truth, "a.tsx", "function App()", span + " ") + assert memory_eval.session_holds(truth, "b.py", "def g(x):", "def g(x):\n return 2") + assert not memory_eval.session_holds(truth, "b.py", "def g(y):", "def g(y):") From d78b2e5b84459506e44d4aebfce0818fd8219a63 Mon Sep 17 00:00:00 2001 From: denfry Date: Mon, 14 Sep 2026 10:53:25 +0300 Subject: [PATCH 14/18] fix(discovery): skip out-of-tree symlinks instead of crashing the walker os.walk lists symlinks lexically under the root, but _rel resolved them before relative_to(root), so any link pointing outside the repository (Bazel convenience symlinks being the common case) raised ValueError before a single file was indexed. Resolve, and skip entries that land outside the root, consistent with PathGate.read. Closes #27 --- CHANGELOG.md | 7 +++++++ src/codebase_index/discovery/walker.py | 25 ++++++++++++++++++++----- tests/test_discovery.py | 24 ++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc0b537..f32d30f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ All notable changes to this project are documented here. The format is based on ## [Unreleased] +### Fixed + +- `codebase-index index` no longer crashes with `ValueError: ... is not in the subpath of ...` + when the repository contains a symlink that points outside the tree (for example Bazel's + `bazel-out` / `bazel-bin` convenience symlinks). Such entries are now skipped by the walker, + matching the gate's existing "resolves outside the repository" exclusion (#27). + ## [1.10.0] - 2026-09-02 Ranking release. 1.9.0's own diagnostics showed that a perfect reranker over the diff --git a/src/codebase_index/discovery/walker.py b/src/codebase_index/discovery/walker.py index 9a3327e..e3199b8 100644 --- a/src/codebase_index/discovery/walker.py +++ b/src/codebase_index/discovery/walker.py @@ -27,13 +27,18 @@ def walk(root: Path, config: Config) -> Iterator[Candidate]: gate = PathGate(root, config) for dirpath, dirnames, filenames in os.walk(root): - dirnames[:] = [ - d for d in dirnames if gate.dir_allowed(d, _rel(root, Path(dirpath) / d)) - ] + kept: list[str] = [] + for d in dirnames: + rel_dir = _rel(root, Path(dirpath) / d) + if rel_dir is not None and gate.dir_allowed(d, rel_dir): + kept.append(d) + dirnames[:] = kept for fname in filenames: abs_path = Path(dirpath) / fname rel = _rel(root, abs_path) + if rel is None: + continue if gate.name_rejection(rel): continue @@ -62,5 +67,15 @@ def walk(root: Path, config: Config) -> Iterator[Candidate]: ) -def _rel(root: Path, path: Path) -> str: - return path.resolve().relative_to(root).as_posix() +def _rel(root: Path, path: Path) -> Optional[str]: + """Root-relative POSIX path of ``path``, or ``None`` if it resolves outside ``root``. + + ``os.walk`` lists symlinks lexically under ``root``, but a link may point elsewhere + (Bazel's ``bazel-out`` convenience symlinks are the common case). Such entries are + skipped rather than raising, matching the gate's "resolves outside the repository" + exclusion. + """ + try: + return path.resolve().relative_to(root).as_posix() + except (ValueError, OSError): + return None diff --git a/tests/test_discovery.py b/tests/test_discovery.py index b00817d..9108219 100644 --- a/tests/test_discovery.py +++ b/tests/test_discovery.py @@ -49,3 +49,27 @@ def read_bytes_should_not_be_used(self: Path) -> bytes: found = _walk_paths(root) assert "big.py" in found + + +def test_walk_skips_symlinks_pointing_outside_root(tmp_path): + import os + + import pytest + + outside = tmp_path / "outside" + (outside / "bazel-out").mkdir(parents=True) + (outside / "bazel-out" / "gen.py").write_text("x = 1\n", encoding="utf-8") + (outside / "leak.py").write_text("SECRET = 1\n", encoding="utf-8") + + root = tmp_path / "proj" + (root / "src").mkdir(parents=True) + (root / "src" / "keep.py").write_text("y = 2\n", encoding="utf-8") + try: + os.symlink(outside / "bazel-out", root / "bazel-out", target_is_directory=True) + os.symlink(outside / "leak.py", root / "src" / "leak.py") + except (OSError, NotImplementedError): + pytest.skip("symlinks unavailable") + + found = _walk_paths(root) + + assert set(found) == {"src/keep.py"} From 9e3a5b46bba25bc8a0fbee1d97226b3eaa40b900 Mon Sep 17 00:00:00 2001 From: denfry Date: Mon, 14 Sep 2026 11:27:30 +0300 Subject: [PATCH 15/18] docs(skill): session tags, verify step and evidence-memory notes in agent wrappers The skill and scaffolded AGENTS.md blocks now route every search and explain through one session tag per conversation, add a Verify step to the Find/Trace/Predict loop, and document reused and memory.invalidated payload fields in a new references/memory.md. --- .claude/skills/codebase-index/SKILL.md | 42 ++++++++++++---- .../codebase-index/references/commands.md | 15 ++++++ .../codebase-index/references/memory.md | 49 +++++++++++++++++++ .../references/response-contract.md | 8 +++ .codex/skills/codebase-index/SKILL.md | 42 ++++++++++++---- .../codebase-index/references/commands.md | 15 ++++++ .../codebase-index/references/memory.md | 49 +++++++++++++++++++ .../references/response-contract.md | 8 +++ .opencode/skills/codebase-index/SKILL.md | 42 ++++++++++++---- .../codebase-index/references/commands.md | 15 ++++++ .../codebase-index/references/memory.md | 49 +++++++++++++++++++ .../references/response-contract.md | 8 +++ AGENTS.md | 13 +++-- skill/SKILL.md | 42 ++++++++++++---- skill/references/commands.md | 15 ++++++ skill/references/memory.md | 49 +++++++++++++++++++ skill/references/response-contract.md | 8 +++ skills/codebase-index/SKILL.md | 42 ++++++++++++---- skills/codebase-index/references/commands.md | 15 ++++++ skills/codebase-index/references/memory.md | 49 +++++++++++++++++++ .../references/response-contract.md | 8 +++ src/codebase_index/scaffold.py | 16 ++++-- src/codebase_index/skill_template/SKILL.md | 42 ++++++++++++---- .../skill_template/references/commands.md | 15 ++++++ .../skill_template/references/memory.md | 49 +++++++++++++++++++ .../references/response-contract.md | 8 +++ 26 files changed, 645 insertions(+), 68 deletions(-) create mode 100644 .claude/skills/codebase-index/references/memory.md create mode 100644 .codex/skills/codebase-index/references/memory.md create mode 100644 .opencode/skills/codebase-index/references/memory.md create mode 100644 skill/references/memory.md create mode 100644 skills/codebase-index/references/memory.md create mode 100644 src/codebase_index/skill_template/references/memory.md diff --git a/.claude/skills/codebase-index/SKILL.md b/.claude/skills/codebase-index/SKILL.md index 9404686..432005b 100644 --- a/.claude/skills/codebase-index/SKILL.md +++ b/.claude/skills/codebase-index/SKILL.md @@ -1,25 +1,26 @@ --- name: codebase-index -description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository. -allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob +description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository, and verify evidence gathered earlier before relying on it. +allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index verify *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob --- # Codebase Index Use the local index before reading repository files. -The operating principle is **Find → Trace → Predict**: +The operating principle is **Find → Trace → Verify → Predict**: - **Find** the implementation with ranked retrieval. - **Trace** behavior through definitions, callers, dependencies, and paths. +- **Verify** that evidence you already hold is still true before relying on it. - **Predict** change impact while preserving an explicit evidence trail. ## Route the question | Intent | Command | |---|---| -| Where is X implemented? | `codebase-index search "X" --json` | -| How does X work? | `codebase-index explain "X" --json` | +| Where is X implemented? | `codebase-index search "X" --session --json` | +| How does X work? | `codebase-index explain "X" --session --json` | | What is this codebase? | `codebase-index architecture --json` | | Find a named symbol | `codebase-index symbol "X" --json` | | Who calls or references X? | `codebase-index refs "X" --json` | @@ -27,6 +28,7 @@ The operating principle is **Find → Trace → Predict**: | What does my current diff affect? | `codebase-index diff-impact --json` | | How are X and Y connected? | `codebase-index path "X" "Y" --json` | | Describe X and its neighborhood | `codebase-index describe "X" --json` | +| Is what I read earlier still true? | `codebase-index verify --session --json` | | Produce a human graph | `codebase-index graph "X" --output ` | Use `search --mode symbol` for exact symbol work, `--mode fts` for text and @@ -38,21 +40,41 @@ or routing remain unclear. ## Evidence protocol -1. Run the best-matching command with `--json`. -2. Check `index` before trusting the payload: +1. Pick one session tag for this conversation (for example `auth-fix-1`) and + pass `--session ` to every `search` and `explain`. +2. Run the best-matching command with `--json`. +3. Check `index` before trusting the payload: - missing → run `codebase-index index`, then repeat; - stale with fewer than 20 changed files → run `codebase-index update`; - stale with 20 or more changed files → run `codebase-index index`; - fresh → continue. -3. Start with ranks 1–3. Read only `recommended_reads` line ranges. -4. Trace one additional hop only when the question requires behavior, +4. Start with ranks 1–3. Read only `recommended_reads` line ranges. +5. Trace one additional hop only when the question requires behavior, ownership, or impact. -5. Answer with `file:line` evidence and state uncertainty explicitly. +6. Before answering or editing from evidence gathered earlier in the task, run + `codebase-index verify --session --json` and reread anything whose + state is not `valid` or `relocated`. +7. Answer with `file:line` evidence and state uncertainty explicitly. Do not open whole files when a line range is available. A snippet may already be sufficient. `skeletonized: true` means the response intentionally folded unrelated body lines; read the supplied range when the missing body matters. +## Evidence memory + +- `reused: true` with `snippet: null` — this session already received that + exact text and its source is unchanged. Use your earlier copy; if you can no + longer see it, Read the range. +- `memory.invalidated` — evidence this session received has changed since. + Treat your earlier copy as wrong and reread before relying on it. +- `stale: true` — the index is older than the file. Run `codebase-index update` + or Read the range. +- A tag belongs to one context. Never give it to a subagent or another + conversation. Start a new tag after the context is cleared or compacted, or + whenever earlier snippets are no longer visible to you. + +Verdict states and citing evidence in notes: [references/memory.md](references/memory.md). + ## Confidence contract - **high** — answer from the indexed evidence. diff --git a/.claude/skills/codebase-index/references/commands.md b/.claude/skills/codebase-index/references/commands.md index 0efb464..79c91a6 100644 --- a/.claude/skills/codebase-index/references/commands.md +++ b/.claude/skills/codebase-index/references/commands.md @@ -17,6 +17,9 @@ Useful search options: - `--offset ` - `--raw` to disable snippet skeletonization - `--no-fallback` to suppress fallback suggestions +- `--session ` to name this conversation's context: unchanged evidence it + already received comes back as `reused: true` without the snippet, and + evidence that changed is listed under `memory.invalidated` `explain` uses the HOW_IT_WORKS intent and a larger default token budget. Prefer it over repeatedly rewording a broad search. @@ -49,6 +52,18 @@ codebase-index graph "" --direction both --depth 2 --output graph.html For headless work, use `--output`; do not use `--open`. Exports also support `--format graphml|dot|neo4j`. +## Evidence + +```bash +codebase-index verify --session --json +codebase-index verify "" ... --json +``` + +- `verify` is read-only and needs no index: it checks evidence against the + working tree. `all_valid` is true only when every checked span still holds. +- `--strict` exits 1 when anything is invalid (useful in scripts). +- See [memory.md](memory.md) for verdict states and when to reread. + ## Index health ```bash diff --git a/.claude/skills/codebase-index/references/memory.md b/.claude/skills/codebase-index/references/memory.md new file mode 100644 index 0000000..fa533ad --- /dev/null +++ b/.claude/skills/codebase-index/references/memory.md @@ -0,0 +1,49 @@ +# Evidence Memory + +Load this when you use `--session`, when a packet contains `reused`, `stale` or +`memory`, or when you want to cite evidence for later. + +## What is guaranteed + +Evidence is identified by the exact bytes of a file span. `reused: true` is set +only when this session already received the same text — or the whole span — +and those bytes are unchanged now. Nothing is decided by query similarity, and +memory stores no source text. + +## Verdict states + +| state | still true? | do this | +|---|---|---| +| `valid` | yes | use it | +| `relocated` | yes — identical content moved inside the file | use it; cite the new lines | +| `changed` | no | reread the range before relying on it | +| `ambiguous` | no — identical content now occurs more than once | reread | +| `deleted` | no — file gone; a rename or move counts as deleted | search again | +| `excluded` | no — now ignored or secret-like | do not read it | +| `unreadable` | no | search again | + +## Reread even when evidence is valid + +- You are about to edit and need the exact current text. +- The earlier snippet was a skeleton or a signature and you need the body. +- Your context was cleared or compacted, or you cannot see the earlier snippet. +- A tool output was truncated. + +## Citing evidence across conversations + +`codebase-index verify --session --json` lists everything the session +received as `path:start-end@hash` references. Keep the references next to the +conclusions they support in notes or handoffs: + +```text +Refunds are capped at the invoice total [billing/refund.py:3-4@3f9a2c1b7d4e8a90] +``` + +Later, from any agent, check them before trusting the note: + +```bash +codebase-index verify "billing/refund.py:3-4@3f9a2c1b7d4e8a90" --json +``` + +`all_valid: true` means every cited span still holds exactly. Otherwise reread +the invalid spans and re-derive the conclusion; do not patch the old one. diff --git a/.claude/skills/codebase-index/references/response-contract.md b/.claude/skills/codebase-index/references/response-contract.md index 059d87a..44a88af 100644 --- a/.claude/skills/codebase-index/references/response-contract.md +++ b/.claude/skills/codebase-index/references/response-contract.md @@ -16,10 +16,18 @@ Each result can contain: - `snippet` - `skeletonized` - `elided_lines` +- `stale` — only when the index text no longer matches the file +- `reused` — only with `--session`, when `snippet` was withheld because this + session already received that exact text from unchanged source `recommended_reads` is the read plan. Start with its first one to three entries and use exact line ranges. +With `--session`, the packet also carries `memory`: `session`, `reused`, +`tokens_saved`, and `invalidated` — references to evidence the session received +that has since changed. `available: false` means nothing was withheld. See +[memory.md](memory.md). + `pagination.has_more` and `pagination.next_offset` indicate additional results. Prefer a more specific command or a larger token budget before paging. diff --git a/.codex/skills/codebase-index/SKILL.md b/.codex/skills/codebase-index/SKILL.md index 9404686..432005b 100644 --- a/.codex/skills/codebase-index/SKILL.md +++ b/.codex/skills/codebase-index/SKILL.md @@ -1,25 +1,26 @@ --- name: codebase-index -description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository. -allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob +description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository, and verify evidence gathered earlier before relying on it. +allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index verify *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob --- # Codebase Index Use the local index before reading repository files. -The operating principle is **Find → Trace → Predict**: +The operating principle is **Find → Trace → Verify → Predict**: - **Find** the implementation with ranked retrieval. - **Trace** behavior through definitions, callers, dependencies, and paths. +- **Verify** that evidence you already hold is still true before relying on it. - **Predict** change impact while preserving an explicit evidence trail. ## Route the question | Intent | Command | |---|---| -| Where is X implemented? | `codebase-index search "X" --json` | -| How does X work? | `codebase-index explain "X" --json` | +| Where is X implemented? | `codebase-index search "X" --session --json` | +| How does X work? | `codebase-index explain "X" --session --json` | | What is this codebase? | `codebase-index architecture --json` | | Find a named symbol | `codebase-index symbol "X" --json` | | Who calls or references X? | `codebase-index refs "X" --json` | @@ -27,6 +28,7 @@ The operating principle is **Find → Trace → Predict**: | What does my current diff affect? | `codebase-index diff-impact --json` | | How are X and Y connected? | `codebase-index path "X" "Y" --json` | | Describe X and its neighborhood | `codebase-index describe "X" --json` | +| Is what I read earlier still true? | `codebase-index verify --session --json` | | Produce a human graph | `codebase-index graph "X" --output ` | Use `search --mode symbol` for exact symbol work, `--mode fts` for text and @@ -38,21 +40,41 @@ or routing remain unclear. ## Evidence protocol -1. Run the best-matching command with `--json`. -2. Check `index` before trusting the payload: +1. Pick one session tag for this conversation (for example `auth-fix-1`) and + pass `--session ` to every `search` and `explain`. +2. Run the best-matching command with `--json`. +3. Check `index` before trusting the payload: - missing → run `codebase-index index`, then repeat; - stale with fewer than 20 changed files → run `codebase-index update`; - stale with 20 or more changed files → run `codebase-index index`; - fresh → continue. -3. Start with ranks 1–3. Read only `recommended_reads` line ranges. -4. Trace one additional hop only when the question requires behavior, +4. Start with ranks 1–3. Read only `recommended_reads` line ranges. +5. Trace one additional hop only when the question requires behavior, ownership, or impact. -5. Answer with `file:line` evidence and state uncertainty explicitly. +6. Before answering or editing from evidence gathered earlier in the task, run + `codebase-index verify --session --json` and reread anything whose + state is not `valid` or `relocated`. +7. Answer with `file:line` evidence and state uncertainty explicitly. Do not open whole files when a line range is available. A snippet may already be sufficient. `skeletonized: true` means the response intentionally folded unrelated body lines; read the supplied range when the missing body matters. +## Evidence memory + +- `reused: true` with `snippet: null` — this session already received that + exact text and its source is unchanged. Use your earlier copy; if you can no + longer see it, Read the range. +- `memory.invalidated` — evidence this session received has changed since. + Treat your earlier copy as wrong and reread before relying on it. +- `stale: true` — the index is older than the file. Run `codebase-index update` + or Read the range. +- A tag belongs to one context. Never give it to a subagent or another + conversation. Start a new tag after the context is cleared or compacted, or + whenever earlier snippets are no longer visible to you. + +Verdict states and citing evidence in notes: [references/memory.md](references/memory.md). + ## Confidence contract - **high** — answer from the indexed evidence. diff --git a/.codex/skills/codebase-index/references/commands.md b/.codex/skills/codebase-index/references/commands.md index 0efb464..79c91a6 100644 --- a/.codex/skills/codebase-index/references/commands.md +++ b/.codex/skills/codebase-index/references/commands.md @@ -17,6 +17,9 @@ Useful search options: - `--offset ` - `--raw` to disable snippet skeletonization - `--no-fallback` to suppress fallback suggestions +- `--session ` to name this conversation's context: unchanged evidence it + already received comes back as `reused: true` without the snippet, and + evidence that changed is listed under `memory.invalidated` `explain` uses the HOW_IT_WORKS intent and a larger default token budget. Prefer it over repeatedly rewording a broad search. @@ -49,6 +52,18 @@ codebase-index graph "" --direction both --depth 2 --output graph.html For headless work, use `--output`; do not use `--open`. Exports also support `--format graphml|dot|neo4j`. +## Evidence + +```bash +codebase-index verify --session --json +codebase-index verify "" ... --json +``` + +- `verify` is read-only and needs no index: it checks evidence against the + working tree. `all_valid` is true only when every checked span still holds. +- `--strict` exits 1 when anything is invalid (useful in scripts). +- See [memory.md](memory.md) for verdict states and when to reread. + ## Index health ```bash diff --git a/.codex/skills/codebase-index/references/memory.md b/.codex/skills/codebase-index/references/memory.md new file mode 100644 index 0000000..fa533ad --- /dev/null +++ b/.codex/skills/codebase-index/references/memory.md @@ -0,0 +1,49 @@ +# Evidence Memory + +Load this when you use `--session`, when a packet contains `reused`, `stale` or +`memory`, or when you want to cite evidence for later. + +## What is guaranteed + +Evidence is identified by the exact bytes of a file span. `reused: true` is set +only when this session already received the same text — or the whole span — +and those bytes are unchanged now. Nothing is decided by query similarity, and +memory stores no source text. + +## Verdict states + +| state | still true? | do this | +|---|---|---| +| `valid` | yes | use it | +| `relocated` | yes — identical content moved inside the file | use it; cite the new lines | +| `changed` | no | reread the range before relying on it | +| `ambiguous` | no — identical content now occurs more than once | reread | +| `deleted` | no — file gone; a rename or move counts as deleted | search again | +| `excluded` | no — now ignored or secret-like | do not read it | +| `unreadable` | no | search again | + +## Reread even when evidence is valid + +- You are about to edit and need the exact current text. +- The earlier snippet was a skeleton or a signature and you need the body. +- Your context was cleared or compacted, or you cannot see the earlier snippet. +- A tool output was truncated. + +## Citing evidence across conversations + +`codebase-index verify --session --json` lists everything the session +received as `path:start-end@hash` references. Keep the references next to the +conclusions they support in notes or handoffs: + +```text +Refunds are capped at the invoice total [billing/refund.py:3-4@3f9a2c1b7d4e8a90] +``` + +Later, from any agent, check them before trusting the note: + +```bash +codebase-index verify "billing/refund.py:3-4@3f9a2c1b7d4e8a90" --json +``` + +`all_valid: true` means every cited span still holds exactly. Otherwise reread +the invalid spans and re-derive the conclusion; do not patch the old one. diff --git a/.codex/skills/codebase-index/references/response-contract.md b/.codex/skills/codebase-index/references/response-contract.md index 059d87a..44a88af 100644 --- a/.codex/skills/codebase-index/references/response-contract.md +++ b/.codex/skills/codebase-index/references/response-contract.md @@ -16,10 +16,18 @@ Each result can contain: - `snippet` - `skeletonized` - `elided_lines` +- `stale` — only when the index text no longer matches the file +- `reused` — only with `--session`, when `snippet` was withheld because this + session already received that exact text from unchanged source `recommended_reads` is the read plan. Start with its first one to three entries and use exact line ranges. +With `--session`, the packet also carries `memory`: `session`, `reused`, +`tokens_saved`, and `invalidated` — references to evidence the session received +that has since changed. `available: false` means nothing was withheld. See +[memory.md](memory.md). + `pagination.has_more` and `pagination.next_offset` indicate additional results. Prefer a more specific command or a larger token budget before paging. diff --git a/.opencode/skills/codebase-index/SKILL.md b/.opencode/skills/codebase-index/SKILL.md index 9404686..432005b 100644 --- a/.opencode/skills/codebase-index/SKILL.md +++ b/.opencode/skills/codebase-index/SKILL.md @@ -1,25 +1,26 @@ --- name: codebase-index -description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository. -allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob +description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository, and verify evidence gathered earlier before relying on it. +allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index verify *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob --- # Codebase Index Use the local index before reading repository files. -The operating principle is **Find → Trace → Predict**: +The operating principle is **Find → Trace → Verify → Predict**: - **Find** the implementation with ranked retrieval. - **Trace** behavior through definitions, callers, dependencies, and paths. +- **Verify** that evidence you already hold is still true before relying on it. - **Predict** change impact while preserving an explicit evidence trail. ## Route the question | Intent | Command | |---|---| -| Where is X implemented? | `codebase-index search "X" --json` | -| How does X work? | `codebase-index explain "X" --json` | +| Where is X implemented? | `codebase-index search "X" --session --json` | +| How does X work? | `codebase-index explain "X" --session --json` | | What is this codebase? | `codebase-index architecture --json` | | Find a named symbol | `codebase-index symbol "X" --json` | | Who calls or references X? | `codebase-index refs "X" --json` | @@ -27,6 +28,7 @@ The operating principle is **Find → Trace → Predict**: | What does my current diff affect? | `codebase-index diff-impact --json` | | How are X and Y connected? | `codebase-index path "X" "Y" --json` | | Describe X and its neighborhood | `codebase-index describe "X" --json` | +| Is what I read earlier still true? | `codebase-index verify --session --json` | | Produce a human graph | `codebase-index graph "X" --output ` | Use `search --mode symbol` for exact symbol work, `--mode fts` for text and @@ -38,21 +40,41 @@ or routing remain unclear. ## Evidence protocol -1. Run the best-matching command with `--json`. -2. Check `index` before trusting the payload: +1. Pick one session tag for this conversation (for example `auth-fix-1`) and + pass `--session ` to every `search` and `explain`. +2. Run the best-matching command with `--json`. +3. Check `index` before trusting the payload: - missing → run `codebase-index index`, then repeat; - stale with fewer than 20 changed files → run `codebase-index update`; - stale with 20 or more changed files → run `codebase-index index`; - fresh → continue. -3. Start with ranks 1–3. Read only `recommended_reads` line ranges. -4. Trace one additional hop only when the question requires behavior, +4. Start with ranks 1–3. Read only `recommended_reads` line ranges. +5. Trace one additional hop only when the question requires behavior, ownership, or impact. -5. Answer with `file:line` evidence and state uncertainty explicitly. +6. Before answering or editing from evidence gathered earlier in the task, run + `codebase-index verify --session --json` and reread anything whose + state is not `valid` or `relocated`. +7. Answer with `file:line` evidence and state uncertainty explicitly. Do not open whole files when a line range is available. A snippet may already be sufficient. `skeletonized: true` means the response intentionally folded unrelated body lines; read the supplied range when the missing body matters. +## Evidence memory + +- `reused: true` with `snippet: null` — this session already received that + exact text and its source is unchanged. Use your earlier copy; if you can no + longer see it, Read the range. +- `memory.invalidated` — evidence this session received has changed since. + Treat your earlier copy as wrong and reread before relying on it. +- `stale: true` — the index is older than the file. Run `codebase-index update` + or Read the range. +- A tag belongs to one context. Never give it to a subagent or another + conversation. Start a new tag after the context is cleared or compacted, or + whenever earlier snippets are no longer visible to you. + +Verdict states and citing evidence in notes: [references/memory.md](references/memory.md). + ## Confidence contract - **high** — answer from the indexed evidence. diff --git a/.opencode/skills/codebase-index/references/commands.md b/.opencode/skills/codebase-index/references/commands.md index 0efb464..79c91a6 100644 --- a/.opencode/skills/codebase-index/references/commands.md +++ b/.opencode/skills/codebase-index/references/commands.md @@ -17,6 +17,9 @@ Useful search options: - `--offset ` - `--raw` to disable snippet skeletonization - `--no-fallback` to suppress fallback suggestions +- `--session ` to name this conversation's context: unchanged evidence it + already received comes back as `reused: true` without the snippet, and + evidence that changed is listed under `memory.invalidated` `explain` uses the HOW_IT_WORKS intent and a larger default token budget. Prefer it over repeatedly rewording a broad search. @@ -49,6 +52,18 @@ codebase-index graph "" --direction both --depth 2 --output graph.html For headless work, use `--output`; do not use `--open`. Exports also support `--format graphml|dot|neo4j`. +## Evidence + +```bash +codebase-index verify --session --json +codebase-index verify "" ... --json +``` + +- `verify` is read-only and needs no index: it checks evidence against the + working tree. `all_valid` is true only when every checked span still holds. +- `--strict` exits 1 when anything is invalid (useful in scripts). +- See [memory.md](memory.md) for verdict states and when to reread. + ## Index health ```bash diff --git a/.opencode/skills/codebase-index/references/memory.md b/.opencode/skills/codebase-index/references/memory.md new file mode 100644 index 0000000..fa533ad --- /dev/null +++ b/.opencode/skills/codebase-index/references/memory.md @@ -0,0 +1,49 @@ +# Evidence Memory + +Load this when you use `--session`, when a packet contains `reused`, `stale` or +`memory`, or when you want to cite evidence for later. + +## What is guaranteed + +Evidence is identified by the exact bytes of a file span. `reused: true` is set +only when this session already received the same text — or the whole span — +and those bytes are unchanged now. Nothing is decided by query similarity, and +memory stores no source text. + +## Verdict states + +| state | still true? | do this | +|---|---|---| +| `valid` | yes | use it | +| `relocated` | yes — identical content moved inside the file | use it; cite the new lines | +| `changed` | no | reread the range before relying on it | +| `ambiguous` | no — identical content now occurs more than once | reread | +| `deleted` | no — file gone; a rename or move counts as deleted | search again | +| `excluded` | no — now ignored or secret-like | do not read it | +| `unreadable` | no | search again | + +## Reread even when evidence is valid + +- You are about to edit and need the exact current text. +- The earlier snippet was a skeleton or a signature and you need the body. +- Your context was cleared or compacted, or you cannot see the earlier snippet. +- A tool output was truncated. + +## Citing evidence across conversations + +`codebase-index verify --session --json` lists everything the session +received as `path:start-end@hash` references. Keep the references next to the +conclusions they support in notes or handoffs: + +```text +Refunds are capped at the invoice total [billing/refund.py:3-4@3f9a2c1b7d4e8a90] +``` + +Later, from any agent, check them before trusting the note: + +```bash +codebase-index verify "billing/refund.py:3-4@3f9a2c1b7d4e8a90" --json +``` + +`all_valid: true` means every cited span still holds exactly. Otherwise reread +the invalid spans and re-derive the conclusion; do not patch the old one. diff --git a/.opencode/skills/codebase-index/references/response-contract.md b/.opencode/skills/codebase-index/references/response-contract.md index 059d87a..44a88af 100644 --- a/.opencode/skills/codebase-index/references/response-contract.md +++ b/.opencode/skills/codebase-index/references/response-contract.md @@ -16,10 +16,18 @@ Each result can contain: - `snippet` - `skeletonized` - `elided_lines` +- `stale` — only when the index text no longer matches the file +- `reused` — only with `--session`, when `snippet` was withheld because this + session already received that exact text from unchanged source `recommended_reads` is the read plan. Start with its first one to three entries and use exact line ranges. +With `--session`, the packet also carries `memory`: `session`, `reused`, +`tokens_saved`, and `invalidated` — references to evidence the session received +that has since changed. `available: false` means nothing was withheld. See +[memory.md](memory.md). + `pagination.has_more` and `pagination.next_offset` indicate additional results. Prefer a more specific command or a larger token budget before paging. diff --git a/AGENTS.md b/AGENTS.md index 65f18e5..0cd9881 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,10 +7,15 @@ Use the local codebase index before scanning repository files. Skill resources: `.codex/skills/codebase-index/SKILL.md` -Run `codebase-index search "" --json` for general questions, or use -`symbol`, `refs`, `impact`, and `graph` for symbol lookup, references, change -impact, and HTML graph export. Search/read commands auto-build the index when -it is missing; run `codebase-index update` when responses report stale data. +Run `codebase-index search "" --session --json` for general +questions, or use `symbol`, `refs`, `impact`, and `graph` for symbol lookup, +references, change impact, and HTML graph export. Use one session tag per +conversation: unchanged evidence you already received comes back as +`reused: true`, and evidence that changed is listed under `memory.invalidated`. +Before relying on something read earlier, run +`codebase-index verify --session --json`. Search/read commands auto-build +the index when it is missing; run `codebase-index update` when responses report +stale data. ## Versioning Policy diff --git a/skill/SKILL.md b/skill/SKILL.md index 9404686..432005b 100644 --- a/skill/SKILL.md +++ b/skill/SKILL.md @@ -1,25 +1,26 @@ --- name: codebase-index -description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository. -allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob +description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository, and verify evidence gathered earlier before relying on it. +allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index verify *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob --- # Codebase Index Use the local index before reading repository files. -The operating principle is **Find → Trace → Predict**: +The operating principle is **Find → Trace → Verify → Predict**: - **Find** the implementation with ranked retrieval. - **Trace** behavior through definitions, callers, dependencies, and paths. +- **Verify** that evidence you already hold is still true before relying on it. - **Predict** change impact while preserving an explicit evidence trail. ## Route the question | Intent | Command | |---|---| -| Where is X implemented? | `codebase-index search "X" --json` | -| How does X work? | `codebase-index explain "X" --json` | +| Where is X implemented? | `codebase-index search "X" --session --json` | +| How does X work? | `codebase-index explain "X" --session --json` | | What is this codebase? | `codebase-index architecture --json` | | Find a named symbol | `codebase-index symbol "X" --json` | | Who calls or references X? | `codebase-index refs "X" --json` | @@ -27,6 +28,7 @@ The operating principle is **Find → Trace → Predict**: | What does my current diff affect? | `codebase-index diff-impact --json` | | How are X and Y connected? | `codebase-index path "X" "Y" --json` | | Describe X and its neighborhood | `codebase-index describe "X" --json` | +| Is what I read earlier still true? | `codebase-index verify --session --json` | | Produce a human graph | `codebase-index graph "X" --output ` | Use `search --mode symbol` for exact symbol work, `--mode fts` for text and @@ -38,21 +40,41 @@ or routing remain unclear. ## Evidence protocol -1. Run the best-matching command with `--json`. -2. Check `index` before trusting the payload: +1. Pick one session tag for this conversation (for example `auth-fix-1`) and + pass `--session ` to every `search` and `explain`. +2. Run the best-matching command with `--json`. +3. Check `index` before trusting the payload: - missing → run `codebase-index index`, then repeat; - stale with fewer than 20 changed files → run `codebase-index update`; - stale with 20 or more changed files → run `codebase-index index`; - fresh → continue. -3. Start with ranks 1–3. Read only `recommended_reads` line ranges. -4. Trace one additional hop only when the question requires behavior, +4. Start with ranks 1–3. Read only `recommended_reads` line ranges. +5. Trace one additional hop only when the question requires behavior, ownership, or impact. -5. Answer with `file:line` evidence and state uncertainty explicitly. +6. Before answering or editing from evidence gathered earlier in the task, run + `codebase-index verify --session --json` and reread anything whose + state is not `valid` or `relocated`. +7. Answer with `file:line` evidence and state uncertainty explicitly. Do not open whole files when a line range is available. A snippet may already be sufficient. `skeletonized: true` means the response intentionally folded unrelated body lines; read the supplied range when the missing body matters. +## Evidence memory + +- `reused: true` with `snippet: null` — this session already received that + exact text and its source is unchanged. Use your earlier copy; if you can no + longer see it, Read the range. +- `memory.invalidated` — evidence this session received has changed since. + Treat your earlier copy as wrong and reread before relying on it. +- `stale: true` — the index is older than the file. Run `codebase-index update` + or Read the range. +- A tag belongs to one context. Never give it to a subagent or another + conversation. Start a new tag after the context is cleared or compacted, or + whenever earlier snippets are no longer visible to you. + +Verdict states and citing evidence in notes: [references/memory.md](references/memory.md). + ## Confidence contract - **high** — answer from the indexed evidence. diff --git a/skill/references/commands.md b/skill/references/commands.md index 0efb464..79c91a6 100644 --- a/skill/references/commands.md +++ b/skill/references/commands.md @@ -17,6 +17,9 @@ Useful search options: - `--offset ` - `--raw` to disable snippet skeletonization - `--no-fallback` to suppress fallback suggestions +- `--session ` to name this conversation's context: unchanged evidence it + already received comes back as `reused: true` without the snippet, and + evidence that changed is listed under `memory.invalidated` `explain` uses the HOW_IT_WORKS intent and a larger default token budget. Prefer it over repeatedly rewording a broad search. @@ -49,6 +52,18 @@ codebase-index graph "" --direction both --depth 2 --output graph.html For headless work, use `--output`; do not use `--open`. Exports also support `--format graphml|dot|neo4j`. +## Evidence + +```bash +codebase-index verify --session --json +codebase-index verify "" ... --json +``` + +- `verify` is read-only and needs no index: it checks evidence against the + working tree. `all_valid` is true only when every checked span still holds. +- `--strict` exits 1 when anything is invalid (useful in scripts). +- See [memory.md](memory.md) for verdict states and when to reread. + ## Index health ```bash diff --git a/skill/references/memory.md b/skill/references/memory.md new file mode 100644 index 0000000..fa533ad --- /dev/null +++ b/skill/references/memory.md @@ -0,0 +1,49 @@ +# Evidence Memory + +Load this when you use `--session`, when a packet contains `reused`, `stale` or +`memory`, or when you want to cite evidence for later. + +## What is guaranteed + +Evidence is identified by the exact bytes of a file span. `reused: true` is set +only when this session already received the same text — or the whole span — +and those bytes are unchanged now. Nothing is decided by query similarity, and +memory stores no source text. + +## Verdict states + +| state | still true? | do this | +|---|---|---| +| `valid` | yes | use it | +| `relocated` | yes — identical content moved inside the file | use it; cite the new lines | +| `changed` | no | reread the range before relying on it | +| `ambiguous` | no — identical content now occurs more than once | reread | +| `deleted` | no — file gone; a rename or move counts as deleted | search again | +| `excluded` | no — now ignored or secret-like | do not read it | +| `unreadable` | no | search again | + +## Reread even when evidence is valid + +- You are about to edit and need the exact current text. +- The earlier snippet was a skeleton or a signature and you need the body. +- Your context was cleared or compacted, or you cannot see the earlier snippet. +- A tool output was truncated. + +## Citing evidence across conversations + +`codebase-index verify --session --json` lists everything the session +received as `path:start-end@hash` references. Keep the references next to the +conclusions they support in notes or handoffs: + +```text +Refunds are capped at the invoice total [billing/refund.py:3-4@3f9a2c1b7d4e8a90] +``` + +Later, from any agent, check them before trusting the note: + +```bash +codebase-index verify "billing/refund.py:3-4@3f9a2c1b7d4e8a90" --json +``` + +`all_valid: true` means every cited span still holds exactly. Otherwise reread +the invalid spans and re-derive the conclusion; do not patch the old one. diff --git a/skill/references/response-contract.md b/skill/references/response-contract.md index 059d87a..44a88af 100644 --- a/skill/references/response-contract.md +++ b/skill/references/response-contract.md @@ -16,10 +16,18 @@ Each result can contain: - `snippet` - `skeletonized` - `elided_lines` +- `stale` — only when the index text no longer matches the file +- `reused` — only with `--session`, when `snippet` was withheld because this + session already received that exact text from unchanged source `recommended_reads` is the read plan. Start with its first one to three entries and use exact line ranges. +With `--session`, the packet also carries `memory`: `session`, `reused`, +`tokens_saved`, and `invalidated` — references to evidence the session received +that has since changed. `available: false` means nothing was withheld. See +[memory.md](memory.md). + `pagination.has_more` and `pagination.next_offset` indicate additional results. Prefer a more specific command or a larger token budget before paging. diff --git a/skills/codebase-index/SKILL.md b/skills/codebase-index/SKILL.md index 9404686..432005b 100644 --- a/skills/codebase-index/SKILL.md +++ b/skills/codebase-index/SKILL.md @@ -1,25 +1,26 @@ --- name: codebase-index -description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository. -allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob +description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository, and verify evidence gathered earlier before relying on it. +allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index verify *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob --- # Codebase Index Use the local index before reading repository files. -The operating principle is **Find → Trace → Predict**: +The operating principle is **Find → Trace → Verify → Predict**: - **Find** the implementation with ranked retrieval. - **Trace** behavior through definitions, callers, dependencies, and paths. +- **Verify** that evidence you already hold is still true before relying on it. - **Predict** change impact while preserving an explicit evidence trail. ## Route the question | Intent | Command | |---|---| -| Where is X implemented? | `codebase-index search "X" --json` | -| How does X work? | `codebase-index explain "X" --json` | +| Where is X implemented? | `codebase-index search "X" --session --json` | +| How does X work? | `codebase-index explain "X" --session --json` | | What is this codebase? | `codebase-index architecture --json` | | Find a named symbol | `codebase-index symbol "X" --json` | | Who calls or references X? | `codebase-index refs "X" --json` | @@ -27,6 +28,7 @@ The operating principle is **Find → Trace → Predict**: | What does my current diff affect? | `codebase-index diff-impact --json` | | How are X and Y connected? | `codebase-index path "X" "Y" --json` | | Describe X and its neighborhood | `codebase-index describe "X" --json` | +| Is what I read earlier still true? | `codebase-index verify --session --json` | | Produce a human graph | `codebase-index graph "X" --output ` | Use `search --mode symbol` for exact symbol work, `--mode fts` for text and @@ -38,21 +40,41 @@ or routing remain unclear. ## Evidence protocol -1. Run the best-matching command with `--json`. -2. Check `index` before trusting the payload: +1. Pick one session tag for this conversation (for example `auth-fix-1`) and + pass `--session ` to every `search` and `explain`. +2. Run the best-matching command with `--json`. +3. Check `index` before trusting the payload: - missing → run `codebase-index index`, then repeat; - stale with fewer than 20 changed files → run `codebase-index update`; - stale with 20 or more changed files → run `codebase-index index`; - fresh → continue. -3. Start with ranks 1–3. Read only `recommended_reads` line ranges. -4. Trace one additional hop only when the question requires behavior, +4. Start with ranks 1–3. Read only `recommended_reads` line ranges. +5. Trace one additional hop only when the question requires behavior, ownership, or impact. -5. Answer with `file:line` evidence and state uncertainty explicitly. +6. Before answering or editing from evidence gathered earlier in the task, run + `codebase-index verify --session --json` and reread anything whose + state is not `valid` or `relocated`. +7. Answer with `file:line` evidence and state uncertainty explicitly. Do not open whole files when a line range is available. A snippet may already be sufficient. `skeletonized: true` means the response intentionally folded unrelated body lines; read the supplied range when the missing body matters. +## Evidence memory + +- `reused: true` with `snippet: null` — this session already received that + exact text and its source is unchanged. Use your earlier copy; if you can no + longer see it, Read the range. +- `memory.invalidated` — evidence this session received has changed since. + Treat your earlier copy as wrong and reread before relying on it. +- `stale: true` — the index is older than the file. Run `codebase-index update` + or Read the range. +- A tag belongs to one context. Never give it to a subagent or another + conversation. Start a new tag after the context is cleared or compacted, or + whenever earlier snippets are no longer visible to you. + +Verdict states and citing evidence in notes: [references/memory.md](references/memory.md). + ## Confidence contract - **high** — answer from the indexed evidence. diff --git a/skills/codebase-index/references/commands.md b/skills/codebase-index/references/commands.md index 0efb464..79c91a6 100644 --- a/skills/codebase-index/references/commands.md +++ b/skills/codebase-index/references/commands.md @@ -17,6 +17,9 @@ Useful search options: - `--offset ` - `--raw` to disable snippet skeletonization - `--no-fallback` to suppress fallback suggestions +- `--session ` to name this conversation's context: unchanged evidence it + already received comes back as `reused: true` without the snippet, and + evidence that changed is listed under `memory.invalidated` `explain` uses the HOW_IT_WORKS intent and a larger default token budget. Prefer it over repeatedly rewording a broad search. @@ -49,6 +52,18 @@ codebase-index graph "" --direction both --depth 2 --output graph.html For headless work, use `--output`; do not use `--open`. Exports also support `--format graphml|dot|neo4j`. +## Evidence + +```bash +codebase-index verify --session --json +codebase-index verify "" ... --json +``` + +- `verify` is read-only and needs no index: it checks evidence against the + working tree. `all_valid` is true only when every checked span still holds. +- `--strict` exits 1 when anything is invalid (useful in scripts). +- See [memory.md](memory.md) for verdict states and when to reread. + ## Index health ```bash diff --git a/skills/codebase-index/references/memory.md b/skills/codebase-index/references/memory.md new file mode 100644 index 0000000..fa533ad --- /dev/null +++ b/skills/codebase-index/references/memory.md @@ -0,0 +1,49 @@ +# Evidence Memory + +Load this when you use `--session`, when a packet contains `reused`, `stale` or +`memory`, or when you want to cite evidence for later. + +## What is guaranteed + +Evidence is identified by the exact bytes of a file span. `reused: true` is set +only when this session already received the same text — or the whole span — +and those bytes are unchanged now. Nothing is decided by query similarity, and +memory stores no source text. + +## Verdict states + +| state | still true? | do this | +|---|---|---| +| `valid` | yes | use it | +| `relocated` | yes — identical content moved inside the file | use it; cite the new lines | +| `changed` | no | reread the range before relying on it | +| `ambiguous` | no — identical content now occurs more than once | reread | +| `deleted` | no — file gone; a rename or move counts as deleted | search again | +| `excluded` | no — now ignored or secret-like | do not read it | +| `unreadable` | no | search again | + +## Reread even when evidence is valid + +- You are about to edit and need the exact current text. +- The earlier snippet was a skeleton or a signature and you need the body. +- Your context was cleared or compacted, or you cannot see the earlier snippet. +- A tool output was truncated. + +## Citing evidence across conversations + +`codebase-index verify --session --json` lists everything the session +received as `path:start-end@hash` references. Keep the references next to the +conclusions they support in notes or handoffs: + +```text +Refunds are capped at the invoice total [billing/refund.py:3-4@3f9a2c1b7d4e8a90] +``` + +Later, from any agent, check them before trusting the note: + +```bash +codebase-index verify "billing/refund.py:3-4@3f9a2c1b7d4e8a90" --json +``` + +`all_valid: true` means every cited span still holds exactly. Otherwise reread +the invalid spans and re-derive the conclusion; do not patch the old one. diff --git a/skills/codebase-index/references/response-contract.md b/skills/codebase-index/references/response-contract.md index 059d87a..44a88af 100644 --- a/skills/codebase-index/references/response-contract.md +++ b/skills/codebase-index/references/response-contract.md @@ -16,10 +16,18 @@ Each result can contain: - `snippet` - `skeletonized` - `elided_lines` +- `stale` — only when the index text no longer matches the file +- `reused` — only with `--session`, when `snippet` was withheld because this + session already received that exact text from unchanged source `recommended_reads` is the read plan. Start with its first one to three entries and use exact line ranges. +With `--session`, the packet also carries `memory`: `session`, `reused`, +`tokens_saved`, and `invalidated` — references to evidence the session received +that has since changed. `available: false` means nothing was withheld. See +[memory.md](memory.md). + `pagination.has_more` and `pagination.next_offset` indicate additional results. Prefer a more specific command or a larger token budget before paging. diff --git a/src/codebase_index/scaffold.py b/src/codebase_index/scaffold.py index e49ea0b..72a65c4 100644 --- a/src/codebase_index/scaffold.py +++ b/src/codebase_index/scaffold.py @@ -166,10 +166,15 @@ def write_codex_agents(root: Path) -> Path: Skill resources: `{rel.as_posix()}` -Run `codebase-index search "" --json` for general questions, or use -`symbol`, `refs`, `impact`, and `graph` for symbol lookup, references, change -impact, and HTML graph export. Search/read commands auto-build the index when -it is missing; run `codebase-index update` when responses report stale data. +Run `codebase-index search "" --session --json` for general +questions, or use `symbol`, `refs`, `impact`, and `graph` for symbol lookup, +references, change impact, and HTML graph export. Use one session tag per +conversation: unchanged evidence you already received comes back as +`reused: true`, and evidence that changed is listed under `memory.invalidated`. +Before relying on something read earlier, run +`codebase-index verify --session --json`. Search/read commands auto-build +the index when it is missing; run `codebase-index update` when responses report +stale data. """ return _upsert_managed_block(root / "AGENTS.md", content) @@ -192,6 +197,9 @@ def write_opencode_files(root: Path) -> list[Path]: Use `symbol `, `refs `, or `impact ` when those match the request. If the index is missing, run `codebase-index index` first. + +In a multi-step task pass `--session ` (one tag per conversation) and check +evidence read earlier with `codebase-index verify --session --json`. """, encoding="utf-8", ) diff --git a/src/codebase_index/skill_template/SKILL.md b/src/codebase_index/skill_template/SKILL.md index 9404686..432005b 100644 --- a/src/codebase_index/skill_template/SKILL.md +++ b/src/codebase_index/skill_template/SKILL.md @@ -1,25 +1,26 @@ --- name: codebase-index -description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository. -allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob +description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository, and verify evidence gathered earlier before relying on it. +allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index verify *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob --- # Codebase Index Use the local index before reading repository files. -The operating principle is **Find → Trace → Predict**: +The operating principle is **Find → Trace → Verify → Predict**: - **Find** the implementation with ranked retrieval. - **Trace** behavior through definitions, callers, dependencies, and paths. +- **Verify** that evidence you already hold is still true before relying on it. - **Predict** change impact while preserving an explicit evidence trail. ## Route the question | Intent | Command | |---|---| -| Where is X implemented? | `codebase-index search "X" --json` | -| How does X work? | `codebase-index explain "X" --json` | +| Where is X implemented? | `codebase-index search "X" --session --json` | +| How does X work? | `codebase-index explain "X" --session --json` | | What is this codebase? | `codebase-index architecture --json` | | Find a named symbol | `codebase-index symbol "X" --json` | | Who calls or references X? | `codebase-index refs "X" --json` | @@ -27,6 +28,7 @@ The operating principle is **Find → Trace → Predict**: | What does my current diff affect? | `codebase-index diff-impact --json` | | How are X and Y connected? | `codebase-index path "X" "Y" --json` | | Describe X and its neighborhood | `codebase-index describe "X" --json` | +| Is what I read earlier still true? | `codebase-index verify --session --json` | | Produce a human graph | `codebase-index graph "X" --output ` | Use `search --mode symbol` for exact symbol work, `--mode fts` for text and @@ -38,21 +40,41 @@ or routing remain unclear. ## Evidence protocol -1. Run the best-matching command with `--json`. -2. Check `index` before trusting the payload: +1. Pick one session tag for this conversation (for example `auth-fix-1`) and + pass `--session ` to every `search` and `explain`. +2. Run the best-matching command with `--json`. +3. Check `index` before trusting the payload: - missing → run `codebase-index index`, then repeat; - stale with fewer than 20 changed files → run `codebase-index update`; - stale with 20 or more changed files → run `codebase-index index`; - fresh → continue. -3. Start with ranks 1–3. Read only `recommended_reads` line ranges. -4. Trace one additional hop only when the question requires behavior, +4. Start with ranks 1–3. Read only `recommended_reads` line ranges. +5. Trace one additional hop only when the question requires behavior, ownership, or impact. -5. Answer with `file:line` evidence and state uncertainty explicitly. +6. Before answering or editing from evidence gathered earlier in the task, run + `codebase-index verify --session --json` and reread anything whose + state is not `valid` or `relocated`. +7. Answer with `file:line` evidence and state uncertainty explicitly. Do not open whole files when a line range is available. A snippet may already be sufficient. `skeletonized: true` means the response intentionally folded unrelated body lines; read the supplied range when the missing body matters. +## Evidence memory + +- `reused: true` with `snippet: null` — this session already received that + exact text and its source is unchanged. Use your earlier copy; if you can no + longer see it, Read the range. +- `memory.invalidated` — evidence this session received has changed since. + Treat your earlier copy as wrong and reread before relying on it. +- `stale: true` — the index is older than the file. Run `codebase-index update` + or Read the range. +- A tag belongs to one context. Never give it to a subagent or another + conversation. Start a new tag after the context is cleared or compacted, or + whenever earlier snippets are no longer visible to you. + +Verdict states and citing evidence in notes: [references/memory.md](references/memory.md). + ## Confidence contract - **high** — answer from the indexed evidence. diff --git a/src/codebase_index/skill_template/references/commands.md b/src/codebase_index/skill_template/references/commands.md index 0efb464..79c91a6 100644 --- a/src/codebase_index/skill_template/references/commands.md +++ b/src/codebase_index/skill_template/references/commands.md @@ -17,6 +17,9 @@ Useful search options: - `--offset ` - `--raw` to disable snippet skeletonization - `--no-fallback` to suppress fallback suggestions +- `--session ` to name this conversation's context: unchanged evidence it + already received comes back as `reused: true` without the snippet, and + evidence that changed is listed under `memory.invalidated` `explain` uses the HOW_IT_WORKS intent and a larger default token budget. Prefer it over repeatedly rewording a broad search. @@ -49,6 +52,18 @@ codebase-index graph "" --direction both --depth 2 --output graph.html For headless work, use `--output`; do not use `--open`. Exports also support `--format graphml|dot|neo4j`. +## Evidence + +```bash +codebase-index verify --session --json +codebase-index verify "" ... --json +``` + +- `verify` is read-only and needs no index: it checks evidence against the + working tree. `all_valid` is true only when every checked span still holds. +- `--strict` exits 1 when anything is invalid (useful in scripts). +- See [memory.md](memory.md) for verdict states and when to reread. + ## Index health ```bash diff --git a/src/codebase_index/skill_template/references/memory.md b/src/codebase_index/skill_template/references/memory.md new file mode 100644 index 0000000..fa533ad --- /dev/null +++ b/src/codebase_index/skill_template/references/memory.md @@ -0,0 +1,49 @@ +# Evidence Memory + +Load this when you use `--session`, when a packet contains `reused`, `stale` or +`memory`, or when you want to cite evidence for later. + +## What is guaranteed + +Evidence is identified by the exact bytes of a file span. `reused: true` is set +only when this session already received the same text — or the whole span — +and those bytes are unchanged now. Nothing is decided by query similarity, and +memory stores no source text. + +## Verdict states + +| state | still true? | do this | +|---|---|---| +| `valid` | yes | use it | +| `relocated` | yes — identical content moved inside the file | use it; cite the new lines | +| `changed` | no | reread the range before relying on it | +| `ambiguous` | no — identical content now occurs more than once | reread | +| `deleted` | no — file gone; a rename or move counts as deleted | search again | +| `excluded` | no — now ignored or secret-like | do not read it | +| `unreadable` | no | search again | + +## Reread even when evidence is valid + +- You are about to edit and need the exact current text. +- The earlier snippet was a skeleton or a signature and you need the body. +- Your context was cleared or compacted, or you cannot see the earlier snippet. +- A tool output was truncated. + +## Citing evidence across conversations + +`codebase-index verify --session --json` lists everything the session +received as `path:start-end@hash` references. Keep the references next to the +conclusions they support in notes or handoffs: + +```text +Refunds are capped at the invoice total [billing/refund.py:3-4@3f9a2c1b7d4e8a90] +``` + +Later, from any agent, check them before trusting the note: + +```bash +codebase-index verify "billing/refund.py:3-4@3f9a2c1b7d4e8a90" --json +``` + +`all_valid: true` means every cited span still holds exactly. Otherwise reread +the invalid spans and re-derive the conclusion; do not patch the old one. diff --git a/src/codebase_index/skill_template/references/response-contract.md b/src/codebase_index/skill_template/references/response-contract.md index 059d87a..44a88af 100644 --- a/src/codebase_index/skill_template/references/response-contract.md +++ b/src/codebase_index/skill_template/references/response-contract.md @@ -16,10 +16,18 @@ Each result can contain: - `snippet` - `skeletonized` - `elided_lines` +- `stale` — only when the index text no longer matches the file +- `reused` — only with `--session`, when `snippet` was withheld because this + session already received that exact text from unchanged source `recommended_reads` is the read plan. Start with its first one to three entries and use exact line ranges. +With `--session`, the packet also carries `memory`: `session`, `reused`, +`tokens_saved`, and `invalidated` — references to evidence the session received +that has since changed. `available: false` means nothing was withheld. See +[memory.md](memory.md). + `pagination.has_more` and `pagination.next_offset` indicate additional results. Prefer a more specific command or a larger token budget before paging. From c2d85ee29368a175afec81e590dd7e99c9d58626 Mon Sep 17 00:00:00 2001 From: denfry Date: Mon, 14 Sep 2026 11:40:24 +0300 Subject: [PATCH 16/18] =?UTF-8?q?release:=20v2.0.0=20=E2=80=94=20evidence?= =?UTF-8?q?=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Version 2.0.0 across the package, plugin manifest, lock pin and skill stamps. Changelog section for 2.0.0 built from the evidence-memory commits, the unreleased 1.10.0 ranking work and the community-readiness items from #26. New docs/MEMORY.md; verify_evidence and session in docs/MCP.md; Verify added to the roadmap's shipped foundation; the logged evidence-memory benchmark run under tests/eval/results with its summary in docs/BENCHMARKS.md. Supported line in SECURITY/FAQ/README is 2.0.x. --- .claude-plugin/plugin.json | 2 +- .claude/skills/codebase-index/.skill_version | 2 +- .codex/skills/codebase-index/.skill_version | 2 +- .../skills/codebase-index/.skill_version | 2 +- CHANGELOG.md | 67 ++ README.md | 9 +- SECURITY.md | 4 +- docs/BENCHMARKS.md | 35 + docs/FAQ.md | 2 +- docs/MCP.md | 16 +- docs/MEMORY.md | 107 ++ docs/ROADMAP.md | 9 +- docs/installer.md | 2 +- requirements.lock | 2 +- src/codebase_index/__init__.py | 2 +- .../results/2026-09-14-evidence-memory.json | 960 ++++++++++++++++++ .../results/2026-09-14-evidence-memory.md | 19 + 17 files changed, 1224 insertions(+), 18 deletions(-) create mode 100644 docs/MEMORY.md create mode 100644 tests/eval/results/2026-09-14-evidence-memory.json create mode 100644 tests/eval/results/2026-09-14-evidence-memory.md diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 3ab5ec4..2a30b5b 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -3,7 +3,7 @@ "name": "codebase-index", "displayName": "Codebase Index", "description": "Give Claude a precise local map of your codebase: find implementations, trace behavior, and predict change impact with file-line evidence.", - "version": "1.10.0", + "version": "2.0.0", "author": { "name": "codebase-index contributors" }, diff --git a/.claude/skills/codebase-index/.skill_version b/.claude/skills/codebase-index/.skill_version index 81c871d..227cea2 100644 --- a/.claude/skills/codebase-index/.skill_version +++ b/.claude/skills/codebase-index/.skill_version @@ -1 +1 @@ -1.10.0 +2.0.0 diff --git a/.codex/skills/codebase-index/.skill_version b/.codex/skills/codebase-index/.skill_version index 81c871d..227cea2 100644 --- a/.codex/skills/codebase-index/.skill_version +++ b/.codex/skills/codebase-index/.skill_version @@ -1 +1 @@ -1.10.0 +2.0.0 diff --git a/.opencode/skills/codebase-index/.skill_version b/.opencode/skills/codebase-index/.skill_version index 81c871d..227cea2 100644 --- a/.opencode/skills/codebase-index/.skill_version +++ b/.opencode/skills/codebase-index/.skill_version @@ -1 +1 @@ -1.10.0 +2.0.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fda959..3942753 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,65 @@ All notable changes to this project are documented here. The format is based on ## [Unreleased] +## [2.0.0] - 2026-09-14 + +Evidence release. Everything an agent reads through `codebase-index` is now identified +by the exact bytes of the span it came from, can be re-checked against the working tree +at any later moment, is not resent to a session that already holds it, and is reported +when it changes. 2.0.0 also carries the unreleased 1.10.0 ranking work below and the +community-readiness work merged from #26. + +Measured by replaying this repository's own history in sessions of 5 to 87 tasks +(`tests/eval/results/2026-09-14-evidence-memory.md`): memory withheld **zero stale +snippets** at every session length while the locator-based alternative withheld +15–54, change notices had precision 1.000 and recall 0.87–0.97, restoring withheld +snippets reproduced the 1.10.0 packet on every task, and snippet tokens fell by +4–11%. The savings are modest on this corpus; correctness is the deliverable. Summary in +[docs/BENCHMARKS.md](docs/BENCHMARKS.md#evidence-memory), design in +[docs/MEMORY.md](docs/MEMORY.md). + ### Added +- **Evidence identity and verification.** Every delivered snippet carries a + `path:start-end@hash` reference whose hash covers the span's exact bytes under the + indexer's line model. `codebase-index verify [REF ...] [--session TAG] [--strict]` + and MCP `verify_evidence` re-check references against the working tree, read-only + and without an index, returning `valid`, `relocated`, `changed`, `ambiguous`, + `deleted`, `excluded` or `unreadable` per reference and `all_valid` overall. + References are untrusted input: absolute paths, drive letters, `..` and NUL are + rejected before any filesystem access. +- **Session-scoped evidence reuse.** `search`/`explain --session TAG` (MCP `session`) + name one agent context. A snippet the session already received from byte-identical + source comes back as `snippet: null, reused: true`; evidence the session received + that has since changed is listed once under `memory.invalidated`. The evidence hook + runs after ranking, budgeting and pagination are final, so memory can never change + which results are returned. Sessions are only ever named explicitly. +- **Stale marking without a session.** A result whose index text no longer matches the + working tree carries `stale: true`. Derived index text (config-key chunks, Markdown + section summaries) is never flagged: a mismatch counts only when the file's current + hash differs from the one the index was built from. +- **`memory.sqlite`**, a content-free ledger next to the index: hashes, paths, line + numbers, token counts and timestamps; session tags stored hashed. Separate from + `index.sqlite` so rebuilds and `clean` leave it alone and ledger writes never queue + behind an update. A newer schema is refused, a corrupt file is moved aside, lock + contention degrades to no-memory output. `memory gc` and `memory clear` are CLI-only + maintenance, deliberately absent from the skill wrappers and MCP, like `clean`. + `stats`, `doctor`, `index_stats` and `healthcheck` gain an additive `memory` block. + Config: `memory.enabled` (true), `memory.retention_days` (14), + `memory.max_deliveries` (50 000); `CBX_MEMORY=0` restores 1.10.0 output byte for byte. +- **Sequential real-history benchmark** (`tests/eval/memory_eval.py`) with an oracle + independent of memory's hashing, plus repository-lifecycle, security and + stale-context test suites driven by real git: branch switches, detached HEAD, + worktrees, dirty trees, renames, rebases, CRLF checkouts, and excluded content that + must never reach the ledger. +- **Upgrade test from a real 1.10.0 index.** The index schema stays at version 3; an + upgraded project keeps its index, a 1.x config loads with memory defaults, and + `memory.sqlite` attaches on the first `--session` call. +- **Skill: Find → Trace → Verify → Predict.** The agent wrappers pass one session tag + per conversation, run `verify` before relying on evidence gathered earlier, and + document `reused`, `stale` and `memory.invalidated` in `references/memory.md`. + `verify` joins the wrapper whitelists; `memory` does not. + - **Public baseline benchmark.** `tests/eval/run_baselines.py` compares the index with a disciplined `rg` + 80-line-window agent and with repo-map-style context on Flask, Gson and Fastify at pinned commits, using git-derived ground truth, one @@ -31,6 +88,15 @@ All notable changes to this project are documented here. The format is based on ### Changed +- **Indexing gates are shared with later working-tree reads.** Evidence validation + reads files long after they were indexed, so the walker and the validator now go + through one `PathGate`; a parity test asserts the gate admits exactly the files a + walk indexes. The resolved on-disk path is gated again, so a symlink or a + differently-cased path on a case-insensitive filesystem cannot reach a file the + walker would never have indexed. +- MCP `schema_version` stays **1**: every payload change is an added field. No CLI + command, flag or JSON field is removed or retyped. + - **Read plan is bounded.** `recommended_reads` entries are capped at `retrieval.max_read_lines` (default 120) and carry `truncated: true` plus `line_end_full` when capped. A symbol-aligned chunk can be a whole class; on the @@ -704,6 +770,7 @@ Pooled over 305 queries (Python, Java, TypeScript), v1.8.0 → 1.9.0: - `doctor`, `stats`, `clean` diagnostics/maintenance commands. [Unreleased]: https://github.com/denfry/codebase-index/compare/v1.10.0...HEAD +[2.0.0]: https://github.com/denfry/codebase-index/compare/v1.9.0...v2.0.0 [1.10.0]: https://github.com/denfry/codebase-index/compare/v1.9.0...v1.10.0 [1.9.0]: https://github.com/denfry/codebase-index/compare/v1.8.0...v1.9.0 [1.8.0]: https://github.com/denfry/codebase-index/compare/v1.7.0...v1.8.0 diff --git a/README.md b/README.md index 96340f7..5271de0 100644 --- a/README.md +++ b/README.md @@ -191,7 +191,7 @@ installed skills and the MCP server all call the same service layer, so behaviou cannot drift between surfaces. Deep dives: [Architecture](docs/ARCHITECTURE.md) · -[Retrieval](docs/RETRIEVAL.md) · [Schema](docs/SCHEMA.md) · +[Retrieval](docs/RETRIEVAL.md) · [Evidence memory](docs/MEMORY.md) · [Schema](docs/SCHEMA.md) · [Languages](docs/LANGUAGES.md) · [Skill design](docs/SKILL_DESIGN.md) ## Privacy and security @@ -225,14 +225,15 @@ silently presented as "no callers". Tiers and how to add a language: ## Project status -Current line: **1.9.x**, on PyPI, MIT. CI runs Linux, macOS and Windows on Python +Current line: **2.0.x**, on PyPI, MIT. CI runs Linux, macOS and Windows on Python 3.11–3.13 with an 80% coverage gate, golden snapshots for every CLI and MCP payload, a packaging smoke test on every PR, and a skill-copy drift check. What works today: hybrid retrieval with optional local vectors; Tree-sitter symbols and edges; `search`, `explain`, `symbol`, `refs`, `impact`, `diff-impact`, `path`, -`describe`, `architecture`, `graph`; token-budgeted, skeletonized packets; incremental -`update`, `watch`, and hooks; CLI, Claude Code plugin/skill, Codex, OpenCode and +`describe`, `architecture`, `graph`, `verify`; token-budgeted, skeletonized packets; +evidence memory that withholds byte-identical snippets within a session and reports +evidence that changed; incremental `update`, `watch`, and hooks; CLI, Claude Code plugin/skill, Codex, OpenCode and MCP delivery; a reproducible multi-repository benchmark. What does not exist yet: framework-aware typed edges (routes, DI, migrations), diff --git a/SECURITY.md b/SECURITY.md index 5d1e94f..21f1e86 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -11,8 +11,8 @@ Only the latest minor release line receives security fixes. | Version | Supported | |---|---| -| 1.9.x (latest) | Yes | -| < 1.9 | No — upgrade with `pip install -U codebase-index` | +| 2.0.x (latest) | Yes | +| < 2.0 | No — upgrade with `pip install -U codebase-index` | ## Reporting a vulnerability diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 7aadd2c..4b3c80f 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -178,6 +178,41 @@ Every ranking signal that ships has an ablation row. 1.9.0 removed two signals that could not demonstrate a benefit and rejected several plausible ones (IDF-weighted coverage, stemming, graph propagation, MMR, a file-length prior). +## Evidence memory + +`tests/eval/memory_eval.py` replays a repository's own history: for every git-derived +query the tree is checked out at the parent of the query's commit, the index is updated +incrementally, one retrieval call is made, and consecutive tasks form sessions of K +tasks. Every arm is computed from the same packet, so arms differ only in what they do +with evidence: **B** is the 1.10.0 packet, **C** is 2.0 evidence memory (withhold only +byte-identical evidence, report changes), **S** is the unsafe alternative (withhold by +locator without checking the source), **A** rereads whole files. The oracle keeps the +text each session was actually handed and is independent of memory's hashing. + +Logged run over this repository, 87 tasks (`tests/eval/results/2026-09-14-evidence-memory.md`): + +| K (tasks / session) | snippet tokens B → C | saved / task, 95% CI | C stale withheld | S saved | S stale withheld | change notices P / R | +|---|---|---|---|---|---|---| +| 5 | 98 222 → 94 358 (−3.9%) | 44 [13, 89] | 0 | −10.8% | 15 | 1.000 / 0.973 | +| 10 | 98 222 → 92 565 (−5.8%) | 65 [31, 112] | 0 | −14.7% | 20 | 1.000 / 0.925 | +| 25 | 98 222 → 92 268 (−6.1%) | 68 [35, 114] | 0 | −19.4% | 30 | 1.000 / 0.920 | +| all | 98 222 → 87 634 (−10.8%) | 122 [76, 179] | 0 | −34.5% | 54 | 1.000 / 0.871 | + +What this does and does not show: + +- **Correctness is the result.** Across every session length, memory withheld zero + stale snippets and every change notice was correct (precision 1.000). The unsafe + arm saves two to three times more tokens and pays for it with 15–54 stale + withholdings per run, six of which hide a gold answer at K = all. +- **Token savings are modest**, 4–11% of snippet tokens at 44–122 tokens per task on + this corpus. Restoring withheld snippets reproduced the B packet on every task (0 + page mismatches), and `useful@budget` is identical for B and C. +- **Cost:** memory adds 8–37 ms p50 per call (K = 5 … all) on a 31.6 ms p50 search; + the store stayed under 410 KB. +- One corpus, one language. The number to watch on other repositories is stale + withheld, which must stay at zero; the savings figure will vary with how often a + session revisits the same files. + ## Claims that must NOT be made Do not write, imply, or ship any of these until a run with published logs exists: diff --git a/docs/FAQ.md b/docs/FAQ.md index c381aa2..fde1586 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -160,7 +160,7 @@ Yes. Use any of these methods: ## Is it production-ready? -Yes, with the caveats below. The current line is **1.9.x** (see +Yes, with the caveats below. The current line is **2.0.x** (see [CHANGELOG.md](../CHANGELOG.md)). It ships: - Hybrid FTS5 / path / symbol retrieval with optional local embeddings; rank fusion that diff --git a/docs/MCP.md b/docs/MCP.md index 9cc6e84..ab7a9bf 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -41,7 +41,16 @@ The MCP server exposes the same retrieval contract as the CLI. | `architecture_overview` | Modules, god nodes, surprising connections, suggested questions | `architecture` | | `path_between` | Shortest dependency/call path between two symbols or files | `path` | | `describe_symbol` | Node card: definition, callers, callees, centrality, module | `describe` | -| `index_stats` | Return counts, language coverage, graph stats, freshness | `stats` | +| `index_stats` | Return counts, language coverage, graph stats, freshness, memory | `stats` | +| `verify_evidence` | Re-check `path:start-end@hash` references and/or a session's deliveries against the working tree (read-only) | `verify` | + +`search_code` and `explain_code` accept an optional `session` string. With a session, a +snippet the session already received from byte-identical source comes back as +`snippet: null, reused: true`, and evidence the session received that has since changed is +listed once under `memory.invalidated`. Without a session the payload is unchanged except +for `stale: true` on a result whose index text no longer matches the working tree. Memory +maintenance (`memory gc`, `memory clear`) is CLI-only, like `clean`. See +[MEMORY.md](MEMORY.md). ## Output contract @@ -69,7 +78,7 @@ branch on the contract without sniffing the shape: version. The current version is **1**. - `tool` (string) — the emitting tool name (`search_code`, `find_symbol`, `find_refs`, `impact_of`, `impact_of_diff`, `explain_code`, `architecture_overview`, - `path_between`, `describe_symbol`, `index_stats`, `healthcheck`). + `path_between`, `describe_symbol`, `index_stats`, `healthcheck`, `verify_evidence`). - The no-index / error path carries the same envelope plus an `"error"` field. Rules: @@ -165,7 +174,8 @@ same trust boundaries: - Done: `codebase-index mcp --root ` CLI entrypoint. - Done: `healthcheck`, `search_code`, `find_symbol`, `find_refs`, `impact_of`, `impact_of_diff`, `explain_code`, `architecture_overview`, `path_between`, - `describe_symbol`, and `index_stats` tools. + `describe_symbol`, `index_stats`, and `verify_evidence` tools; `session` on `search_code` + and `explain_code`; additive `memory` block in `index_stats` and `healthcheck`. - Done: focused tests for tool registration, missing-index behavior, config resolution, and run entrypoint. - Done: explicit `schema_version` + `tool` envelope on every structured tool payload (including the error path), asserted by `tests/test_mcp_server.py` and `tests/test_mcp_golden.py`. diff --git a/docs/MEMORY.md b/docs/MEMORY.md new file mode 100644 index 0000000..18bb09e --- /dev/null +++ b/docs/MEMORY.md @@ -0,0 +1,107 @@ +# Evidence memory + +Evidence memory is the 2.0 answer to one question: **is what the agent read earlier +still true?** It identifies every delivered snippet by the exact bytes of its span, +re-checks those bytes against the working tree on later calls, withholds snippets the +same session already holds, and reports evidence that has changed. It never decides +anything by query similarity and it stores no source text. + +## Evidence references + +A reference is `path:start-end@hash`. The hash covers the bytes of lines `start..end` +under the indexer's own line model (universal newlines); only line terminators are +normalised. References are printed by `verify` and can be kept next to the conclusions +they support: + +```text +Refunds are capped at the invoice total [billing/refund.py:3-4@3f9a2c1b7d4e8a90] +``` + +References are untrusted input. Absolute paths, drive letters, `..` segments and NUL +are rejected before any filesystem access, and every read goes through the same +`PathGate` the indexer uses, so a reference can never reach a file the walker would +refuse (ignored paths, dependency and build directories, secret filenames, oversized +or binary files, symlinks that resolve outside the repository). + +## Verdicts + +| state | still true? | meaning | +|---|---|---| +| `valid` | yes | bytes unchanged at the same lines | +| `relocated` | yes | identical bytes occur exactly once elsewhere in the same file | +| `changed` | no | bytes at the span differ and the text occurs nowhere else | +| `ambiguous` | no | identical bytes now occur more than once | +| `deleted` | no | file gone; a rename or move counts as deleted | +| `excluded` | no | the path is now ignored, secret-like or outside the repository | +| `unreadable` | no | the file could not be read | + +`all_valid` is true only when every verdict is `valid` or `relocated`; it is false for an +empty or unknown set. + +## Sessions + +A session is a tag the caller chooses, one per agent conversation. Sessions are never +inferred from a process, an environment variable or a time window, because none of +those identify what an agent still holds in context. + +```bash +codebase-index search "refund cap" --session auth-fix-1 --json +codebase-index explain "how are refunds capped" --session auth-fix-1 --json +codebase-index verify --session auth-fix-1 --json +codebase-index verify "billing/refund.py:3-4@3f9a2c1b7d4e8a90" --strict +``` + +With a session: + +- a snippet the session already received from byte-identical source is replaced by + `snippet: null, reused: true`. A skeleton is withheld only when the same skeleton or + the whole span was delivered before; +- evidence the session received earlier that has since changed is reported once under + `memory.invalidated`; +- ranking, budgeting and pagination are untouched. The evidence hook runs after they + are final, and restoring the withheld snippets reproduces the no-memory packet + exactly (asserted by tests and by the benchmark on every task). + +Without a session the packet is unchanged except for `stale: true` on a result whose +index text no longer matches the working tree. Index text that is derived rather than +copied (config-key chunks, Markdown section summaries) is never flagged: a mismatch +counts as stale only when the file's current hash differs from the one the index was +built from. + +## Storage + +`memory.sqlite` lives next to `index.sqlite` (override with `CBX_MEMORY_PATH`). It is a +separate file because rebuilds and `clean` delete the index, and so that ledger writes +never queue behind an update transaction. It holds hashes, paths, line numbers, token +counts and timestamps only; session tags are stored hashed. A newer schema is refused +without touching the file, a corrupt file is moved aside rather than deleted, lock +contention degrades to no-memory output instead of blocking, and GC only removes rows. + +Configuration (`.codeindex.json`, not part of `config_hash`): + +```json +{ "memory": { "enabled": true, "retention_days": 14, "max_deliveries": 50000 } } +``` + +`CBX_MEMORY=0` or `memory.enabled: false` disables everything and gives 1.10.0 output +byte for byte. + +Maintenance is CLI-only and is not exposed to the skill wrappers or MCP, like `clean`: + +```bash +codebase-index memory gc # drop expired sessions, compact +codebase-index memory clear --session TAG # forget one session +codebase-index memory clear --yes # forget everything +``` + +`stats`, `doctor`, MCP `index_stats` and `healthcheck` carry an additive `memory` block +(counts, schema, size; never paths or content). + +## What was measured + +The sequential real-history benchmark (`tests/eval/memory_eval.py`) replays a +repository's own commits: each git-derived query runs against the tree at the parent of +its commit, consecutive queries form sessions of K tasks, and the repository evolves +inside a session. The logged run over this repository is in +`tests/eval/results/2026-09-14-evidence-memory.md`; the summary is in +[BENCHMARKS.md](BENCHMARKS.md#evidence-memory). diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 57ae43b..823b786 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -1,7 +1,7 @@ # Roadmap `codebase-index` is becoming the evidence layer between a coding agent and a -repository: **Find implementations, Trace behavior, Predict change impact.** +repository: **Find implementations, Trace behavior, Verify evidence, Predict change impact.** This document separates shipped capability from forward work. Roadmap entries are not product claims until they appear under `[Unreleased]` or a tagged @@ -32,6 +32,13 @@ release in `CHANGELOG.md`. - Graph-coverage honesty for partially supported languages. - Diagnostics that surface stale indexes and incomplete graph extraction. +### Verify + +- Evidence references (`path:start-end@hash`) minted for every delivered snippet. +- `verify` and MCP `verify_evidence`: byte-identity verdicts against the working tree. +- Session-scoped reuse: byte-identical snippets withheld, changed evidence reported. +- Content-free `memory.sqlite`, gated by the same path rules as the indexer. + ### Delivery and trust - CLI, Claude Code plugin/skill, Codex CLI, OpenCode, and stdio MCP. diff --git a/docs/installer.md b/docs/installer.md index fd8bd05..abf2c1f 100644 --- a/docs/installer.md +++ b/docs/installer.md @@ -110,7 +110,7 @@ pwsh ./install.ps1 -Target claude -InstallDir "D:\skills\codebase-index" **Pin to a branch or tag** (reproducibility and safety): ```sh -sh install.sh --branch v1.9.0 +sh install.sh --branch v2.0.0 ``` --- diff --git a/requirements.lock b/requirements.lock index f83cde3..eb8619a 100644 --- a/requirements.lock +++ b/requirements.lock @@ -1,3 +1,3 @@ -codebase-index @ https://github.com/denfry/codebase-index/archive/refs/tags/v1.10.0.tar.gz +codebase-index @ https://github.com/denfry/codebase-index/archive/refs/tags/v2.0.0.tar.gz tree-sitter==0.25.2 tree-sitter-language-pack==1.8.1 diff --git a/src/codebase_index/__init__.py b/src/codebase_index/__init__.py index a8508b9..61ed905 100644 --- a/src/codebase_index/__init__.py +++ b/src/codebase_index/__init__.py @@ -4,4 +4,4 @@ See docs/ARCHITECTURE.md for the module map. """ -__version__ = "1.10.0" +__version__ = "2.0.0" diff --git a/tests/eval/results/2026-09-14-evidence-memory.json b/tests/eval/results/2026-09-14-evidence-memory.json new file mode 100644 index 0000000..7c6ebae --- /dev/null +++ b/tests/eval/results/2026-09-14-evidence-memory.json @@ -0,0 +1,960 @@ +{ + "corpora": { + "codebase-index": { + "tasks": 87, + "rows": [ + { + "sessions_k": "1", + "tasks": 87, + "sessions": 87, + "deliveries": 643, + "distinct_atoms": 643, + "B_tokens": 98222, + "C_tokens": 98222, + "S_tokens": 98222, + "A_tokens": 1928268, + "A_mem_tokens": 1928268, + "C_token_reuse_rate": 0.0, + "C_evidence_reuse_rate": 0.0, + "C_saved_per_task": 0.0, + "C_saved_per_task_ci95": [ + 0.0, + 0.0 + ], + "C_stale_withheld": 0, + "C_validated_reuse_rate": null, + "S_token_reuse_rate": 0.0, + "S_stale_withheld": 0, + "S_stale_reuse_rate": null, + "S_stale_gold": 0, + "notices": 0, + "invalidation_precision": null, + "invalidation_recall": null, + "page_mismatches": 0, + "stale_flags": 0, + "B_packet_tokens": 172851, + "C_packet_tokens": 174475, + "useful_B_equals_C": 0.27586206896551724, + "useful_S": 0.27586206896551724, + "survival": { + "1": [ + null, + 0 + ], + "5": [ + null, + 0 + ], + "10": [ + null, + 0 + ], + "20": [ + null, + 0 + ] + }, + "memory_ms_p50": 4.046000000016647, + "memory_ms_p95": 6.0016000006726244, + "store_bytes": 409600, + "examples": [] + }, + { + "sessions_k": "5", + "tasks": 87, + "sessions": 18, + "deliveries": 643, + "distinct_atoms": 591, + "B_tokens": 98222, + "C_tokens": 94358, + "S_tokens": 87633, + "A_tokens": 1928268, + "A_mem_tokens": 1291386, + "C_token_reuse_rate": 0.03933945551913014, + "C_evidence_reuse_rate": 0.05132192846034215, + "C_saved_per_task": 44.41379310344828, + "C_saved_per_task_ci95": [ + 12.89655172413793, + 88.70114942528735 + ], + "C_stale_withheld": 0, + "C_validated_reuse_rate": 1.0, + "S_token_reuse_rate": 0.10780680499277148, + "S_stale_withheld": 15, + "S_stale_reuse_rate": 0.28846153846153844, + "S_stale_gold": 2, + "notices": 36, + "invalidation_precision": 1.0, + "invalidation_recall": 0.972972972972973, + "page_mismatches": 0, + "stale_flags": 0, + "B_packet_tokens": 172851, + "C_packet_tokens": 171263, + "useful_B_equals_C": 0.27586206896551724, + "useful_S": 0.27011494252873564, + "survival": { + "1": [ + 0.9714912280701754, + 456 + ], + "5": [ + 0.8198757763975155, + 161 + ], + "10": [ + 0.65, + 100 + ], + "20": [ + 0.075, + 40 + ] + }, + "memory_ms_p50": 7.953800000905176, + "memory_ms_p95": 15.853599999900325, + "store_bytes": 372736, + "examples": [ + { + "kind": "notice_fn", + "path": "docs/FAQ.md", + "sha": "f481fe4537198f01", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 7, + "oracle_occurrences": 0 + } + ] + }, + { + "sessions_k": "10", + "tasks": 87, + "sessions": 9, + "deliveries": 643, + "distinct_atoms": 563, + "B_tokens": 98222, + "C_tokens": 92565, + "S_tokens": 83756, + "A_tokens": 1928268, + "A_mem_tokens": 1084176, + "C_token_reuse_rate": 0.05759402170593146, + "C_evidence_reuse_rate": 0.08242612752721618, + "C_saved_per_task": 65.02298850574712, + "C_saved_per_task_ci95": [ + 30.862068965517242, + 111.6896551724138 + ], + "C_stale_withheld": 0, + "C_validated_reuse_rate": 1.0, + "S_token_reuse_rate": 0.1472786137525198, + "S_stale_withheld": 20, + "S_stale_reuse_rate": 0.2597402597402597, + "S_stale_gold": 2, + "notices": 49, + "invalidation_precision": 1.0, + "invalidation_recall": 0.9245283018867925, + "page_mismatches": 0, + "stale_flags": 0, + "B_packet_tokens": 172851, + "C_packet_tokens": 169696, + "useful_B_equals_C": 0.27586206896551724, + "useful_S": 0.27586206896551724, + "survival": { + "1": [ + 0.9737373737373738, + 495 + ], + "5": [ + 0.9025069637883009, + 359 + ], + "10": [ + 0.7680412371134021, + 194 + ], + "20": [ + 0.3977272727272727, + 88 + ] + }, + "memory_ms_p50": 10.889099999985774, + "memory_ms_p95": 23.312499999519787, + "store_bytes": 368640, + "examples": [ + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "150e7a90d1bec372", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 49, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/FAQ.md", + "sha": "f481fe4537198f01", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 7, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/PRODUCT_UPGRADE_PLAN.md", + "sha": "2bbea03fa8d909c6", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 1, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/PRODUCT_UPGRADE_PLAN.md", + "sha": "802af198a186890e", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 141, + "oracle_occurrences": 0 + } + ] + }, + { + "sessions_k": "25", + "tasks": 87, + "sessions": 4, + "deliveries": 643, + "distinct_atoms": 542, + "B_tokens": 98222, + "C_tokens": 92268, + "S_tokens": 79119, + "A_tokens": 1928268, + "A_mem_tokens": 827557, + "C_token_reuse_rate": 0.06061778420313168, + "C_evidence_reuse_rate": 0.1104199066874028, + "C_saved_per_task": 68.4367816091954, + "C_saved_per_task_ci95": [ + 34.51724137931034, + 114.25287356321839 + ], + "C_stale_withheld": 0, + "C_validated_reuse_rate": 1.0, + "S_token_reuse_rate": 0.19448799657917779, + "S_stale_withheld": 30, + "S_stale_reuse_rate": 0.30303030303030304, + "S_stale_gold": 4, + "notices": 69, + "invalidation_precision": 1.0, + "invalidation_recall": 0.92, + "page_mismatches": 0, + "stale_flags": 0, + "B_packet_tokens": 172851, + "C_packet_tokens": 169861, + "useful_B_equals_C": 0.27586206896551724, + "useful_S": 0.27011494252873564, + "survival": { + "1": [ + 0.9778225806451613, + 496 + ], + "5": [ + 0.9225806451612903, + 465 + ], + "10": [ + 0.8777506112469438, + 409 + ], + "20": [ + 0.7800687285223368, + 291 + ] + }, + "memory_ms_p50": 19.56839999911608, + "memory_ms_p95": 38.00810000029742, + "store_bytes": 368640, + "examples": [ + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "039513f1b18c1364", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 38, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "e15784d0fcfe16d1", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 45, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "150e7a90d1bec372", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 49, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/FAQ.md", + "sha": "f481fe4537198f01", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 7, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/PRODUCT_UPGRADE_PLAN.md", + "sha": "2bbea03fa8d909c6", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 1, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/PRODUCT_UPGRADE_PLAN.md", + "sha": "802af198a186890e", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 141, + "oracle_occurrences": 0 + } + ] + }, + { + "sessions_k": "all", + "tasks": 87, + "sessions": 1, + "deliveries": 643, + "distinct_atoms": 473, + "B_tokens": 98222, + "C_tokens": 87634, + "S_tokens": 64318, + "A_tokens": 1928268, + "A_mem_tokens": 544122, + "C_token_reuse_rate": 0.10779662397426239, + "C_evidence_reuse_rate": 0.19440124416796267, + "C_saved_per_task": 121.70114942528735, + "C_saved_per_task_ci95": [ + 76.34482758620689, + 178.7816091954023 + ], + "C_stale_withheld": 0, + "C_validated_reuse_rate": 1.0, + "S_token_reuse_rate": 0.34517725153224327, + "S_stale_withheld": 54, + "S_stale_reuse_rate": 0.3157894736842105, + "S_stale_gold": 6, + "notices": 128, + "invalidation_precision": 1.0, + "invalidation_recall": 0.8707482993197279, + "page_mismatches": 0, + "stale_flags": 0, + "B_packet_tokens": 172851, + "C_packet_tokens": 166456, + "useful_B_equals_C": 0.27586206896551724, + "useful_S": 0.26436781609195403, + "survival": { + "1": [ + 0.9754464285714286, + 448 + ], + "5": [ + 0.9249448123620309, + 453 + ], + "10": [ + 0.8778280542986425, + 442 + ], + "20": [ + 0.8408071748878924, + 446 + ] + }, + "memory_ms_p50": 36.57369999928051, + "memory_ms_p95": 68.22990000000573, + "store_bytes": 352256, + "examples": [ + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "039513f1b18c1364", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 38, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "e15784d0fcfe16d1", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 45, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "150e7a90d1bec372", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 49, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "README.md", + "sha": "2353492da96d24f4", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 145, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "ROADMAP.md", + "sha": "1faec12ba8307640", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 49, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "ROADMAP.md", + "sha": "1fb676dca4d76452", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 37, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "ROADMAP.md", + "sha": "7b40838bf03af0d5", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 43, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "ROADMAP.md", + "sha": "ab9b49fee3918b21", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 31, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "ROADMAP.md", + "sha": "c64353992a16f581", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 25, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "afb56398eb3173a5", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 33, + "oracle_occurrences": 0 + } + ] + } + ], + "search_ms_p50": 31.602200000634184, + "search_ms_p95": 55.25899999884132, + "update_ms_p50": 154.52839999852586 + } + }, + "pooled": [ + { + "sessions_k": "1", + "tasks": 87, + "sessions": 87, + "deliveries": 643, + "distinct_atoms": 643, + "B_tokens": 98222, + "C_tokens": 98222, + "S_tokens": 98222, + "A_tokens": 1928268, + "A_mem_tokens": 1928268, + "C_token_reuse_rate": 0.0, + "C_evidence_reuse_rate": 0.0, + "C_saved_per_task": 0.0, + "C_saved_per_task_ci95": [ + 0.0, + 0.0 + ], + "C_stale_withheld": 0, + "C_validated_reuse_rate": null, + "S_token_reuse_rate": 0.0, + "S_stale_withheld": 0, + "S_stale_reuse_rate": null, + "S_stale_gold": 0, + "notices": 0, + "invalidation_precision": null, + "invalidation_recall": null, + "page_mismatches": 0, + "stale_flags": 0, + "B_packet_tokens": 172851, + "C_packet_tokens": 174475, + "useful_B_equals_C": 0.27586206896551724, + "useful_S": 0.27586206896551724, + "survival": { + "1": [ + null, + 0 + ], + "5": [ + null, + 0 + ], + "10": [ + null, + 0 + ], + "20": [ + null, + 0 + ] + }, + "memory_ms_p50": 4.046000000016647, + "memory_ms_p95": 6.0016000006726244, + "store_bytes": 409600, + "examples": [] + }, + { + "sessions_k": "5", + "tasks": 87, + "sessions": 18, + "deliveries": 643, + "distinct_atoms": 591, + "B_tokens": 98222, + "C_tokens": 94358, + "S_tokens": 87633, + "A_tokens": 1928268, + "A_mem_tokens": 1291386, + "C_token_reuse_rate": 0.03933945551913014, + "C_evidence_reuse_rate": 0.05132192846034215, + "C_saved_per_task": 44.41379310344828, + "C_saved_per_task_ci95": [ + 12.89655172413793, + 88.70114942528735 + ], + "C_stale_withheld": 0, + "C_validated_reuse_rate": 1.0, + "S_token_reuse_rate": 0.10780680499277148, + "S_stale_withheld": 15, + "S_stale_reuse_rate": 0.28846153846153844, + "S_stale_gold": 2, + "notices": 36, + "invalidation_precision": 1.0, + "invalidation_recall": 0.972972972972973, + "page_mismatches": 0, + "stale_flags": 0, + "B_packet_tokens": 172851, + "C_packet_tokens": 171263, + "useful_B_equals_C": 0.27586206896551724, + "useful_S": 0.27011494252873564, + "survival": { + "1": [ + 0.9714912280701754, + 456 + ], + "5": [ + 0.8198757763975155, + 161 + ], + "10": [ + 0.65, + 100 + ], + "20": [ + 0.075, + 40 + ] + }, + "memory_ms_p50": 7.953800000905176, + "memory_ms_p95": 15.853599999900325, + "store_bytes": 372736, + "examples": [ + { + "kind": "notice_fn", + "path": "docs/FAQ.md", + "sha": "f481fe4537198f01", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 7, + "oracle_occurrences": 0 + } + ] + }, + { + "sessions_k": "10", + "tasks": 87, + "sessions": 9, + "deliveries": 643, + "distinct_atoms": 563, + "B_tokens": 98222, + "C_tokens": 92565, + "S_tokens": 83756, + "A_tokens": 1928268, + "A_mem_tokens": 1084176, + "C_token_reuse_rate": 0.05759402170593146, + "C_evidence_reuse_rate": 0.08242612752721618, + "C_saved_per_task": 65.02298850574712, + "C_saved_per_task_ci95": [ + 30.862068965517242, + 111.6896551724138 + ], + "C_stale_withheld": 0, + "C_validated_reuse_rate": 1.0, + "S_token_reuse_rate": 0.1472786137525198, + "S_stale_withheld": 20, + "S_stale_reuse_rate": 0.2597402597402597, + "S_stale_gold": 2, + "notices": 49, + "invalidation_precision": 1.0, + "invalidation_recall": 0.9245283018867925, + "page_mismatches": 0, + "stale_flags": 0, + "B_packet_tokens": 172851, + "C_packet_tokens": 169696, + "useful_B_equals_C": 0.27586206896551724, + "useful_S": 0.27586206896551724, + "survival": { + "1": [ + 0.9737373737373738, + 495 + ], + "5": [ + 0.9025069637883009, + 359 + ], + "10": [ + 0.7680412371134021, + 194 + ], + "20": [ + 0.3977272727272727, + 88 + ] + }, + "memory_ms_p50": 10.889099999985774, + "memory_ms_p95": 23.312499999519787, + "store_bytes": 368640, + "examples": [ + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "150e7a90d1bec372", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 49, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/FAQ.md", + "sha": "f481fe4537198f01", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 7, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/PRODUCT_UPGRADE_PLAN.md", + "sha": "2bbea03fa8d909c6", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 1, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/PRODUCT_UPGRADE_PLAN.md", + "sha": "802af198a186890e", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 141, + "oracle_occurrences": 0 + } + ] + }, + { + "sessions_k": "25", + "tasks": 87, + "sessions": 4, + "deliveries": 643, + "distinct_atoms": 542, + "B_tokens": 98222, + "C_tokens": 92268, + "S_tokens": 79119, + "A_tokens": 1928268, + "A_mem_tokens": 827557, + "C_token_reuse_rate": 0.06061778420313168, + "C_evidence_reuse_rate": 0.1104199066874028, + "C_saved_per_task": 68.4367816091954, + "C_saved_per_task_ci95": [ + 34.51724137931034, + 114.25287356321839 + ], + "C_stale_withheld": 0, + "C_validated_reuse_rate": 1.0, + "S_token_reuse_rate": 0.19448799657917779, + "S_stale_withheld": 30, + "S_stale_reuse_rate": 0.30303030303030304, + "S_stale_gold": 4, + "notices": 69, + "invalidation_precision": 1.0, + "invalidation_recall": 0.92, + "page_mismatches": 0, + "stale_flags": 0, + "B_packet_tokens": 172851, + "C_packet_tokens": 169861, + "useful_B_equals_C": 0.27586206896551724, + "useful_S": 0.27011494252873564, + "survival": { + "1": [ + 0.9778225806451613, + 496 + ], + "5": [ + 0.9225806451612903, + 465 + ], + "10": [ + 0.8777506112469438, + 409 + ], + "20": [ + 0.7800687285223368, + 291 + ] + }, + "memory_ms_p50": 19.56839999911608, + "memory_ms_p95": 38.00810000029742, + "store_bytes": 368640, + "examples": [ + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "039513f1b18c1364", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 38, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "e15784d0fcfe16d1", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 45, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "150e7a90d1bec372", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 49, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/FAQ.md", + "sha": "f481fe4537198f01", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 7, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/PRODUCT_UPGRADE_PLAN.md", + "sha": "2bbea03fa8d909c6", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 1, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/PRODUCT_UPGRADE_PLAN.md", + "sha": "802af198a186890e", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 141, + "oracle_occurrences": 0 + } + ] + }, + { + "sessions_k": "all", + "tasks": 87, + "sessions": 1, + "deliveries": 643, + "distinct_atoms": 473, + "B_tokens": 98222, + "C_tokens": 87634, + "S_tokens": 64318, + "A_tokens": 1928268, + "A_mem_tokens": 544122, + "C_token_reuse_rate": 0.10779662397426239, + "C_evidence_reuse_rate": 0.19440124416796267, + "C_saved_per_task": 121.70114942528735, + "C_saved_per_task_ci95": [ + 76.34482758620689, + 178.7816091954023 + ], + "C_stale_withheld": 0, + "C_validated_reuse_rate": 1.0, + "S_token_reuse_rate": 0.34517725153224327, + "S_stale_withheld": 54, + "S_stale_reuse_rate": 0.3157894736842105, + "S_stale_gold": 6, + "notices": 128, + "invalidation_precision": 1.0, + "invalidation_recall": 0.8707482993197279, + "page_mismatches": 0, + "stale_flags": 0, + "B_packet_tokens": 172851, + "C_packet_tokens": 166456, + "useful_B_equals_C": 0.27586206896551724, + "useful_S": 0.26436781609195403, + "survival": { + "1": [ + 0.9754464285714286, + 448 + ], + "5": [ + 0.9249448123620309, + 453 + ], + "10": [ + 0.8778280542986425, + 442 + ], + "20": [ + 0.8408071748878924, + 446 + ] + }, + "memory_ms_p50": 36.57369999928051, + "memory_ms_p95": 68.22990000000573, + "store_bytes": 352256, + "examples": [ + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "039513f1b18c1364", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 38, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "e15784d0fcfe16d1", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 45, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "150e7a90d1bec372", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 49, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "README.md", + "sha": "2353492da96d24f4", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 145, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "ROADMAP.md", + "sha": "1faec12ba8307640", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 49, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "ROADMAP.md", + "sha": "1fb676dca4d76452", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 37, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "ROADMAP.md", + "sha": "7b40838bf03af0d5", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 43, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "ROADMAP.md", + "sha": "ab9b49fee3918b21", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 31, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "ROADMAP.md", + "sha": "c64353992a16f581", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 25, + "oracle_occurrences": 0 + }, + { + "kind": "notice_fn", + "path": "docs/ROADMAP.md", + "sha": "afb56398eb3173a5", + "memory_state": null, + "oracle_known": true, + "oracle_line_start": 33, + "oracle_occurrences": 0 + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/eval/results/2026-09-14-evidence-memory.md b/tests/eval/results/2026-09-14-evidence-memory.md new file mode 100644 index 0000000..bb7f4cd --- /dev/null +++ b/tests/eval/results/2026-09-14-evidence-memory.md @@ -0,0 +1,19 @@ +replaying codebase-index (C:\Projects\codebase-index\tests\eval\queries\self_repo_git.yml) + +## codebase-index (87 tasks; search p50 31.6 ms, update p50 154.5 ms) + +| K | tasks | sessions | B tokens | C tokens (reuse) | 95% CI saved/task | C stale withheld | S tokens (reuse) | S stale withheld | notices P/R | A tokens | A-mem tokens | packet tok B→C | +|---|---|---|---|---|---|---|---|---|---|---|---|---| +| 1 | 87 | 87 | 98222 | 98222 (0.0%) | 0.0 [0.0, 0.0] | 0 | 98222 (0.0%) | 0 | —/— | 1928268 | 1928268 | 172851→174475 | +| 5 | 87 | 18 | 98222 | 94358 (3.9%) | 44.4 [12.9, 88.7] | 0 | 87633 (10.8%) | 15 | 1.000/0.973 | 1928268 | 1291386 | 172851→171263 | +| 10 | 87 | 9 | 98222 | 92565 (5.8%) | 65.0 [30.9, 111.7] | 0 | 83756 (14.7%) | 20 | 1.000/0.925 | 1928268 | 1084176 | 172851→169696 | +| 25 | 87 | 4 | 98222 | 92268 (6.1%) | 68.4 [34.5, 114.3] | 0 | 79119 (19.4%) | 30 | 1.000/0.920 | 1928268 | 827557 | 172851→169861 | +| all | 87 | 1 | 98222 | 87634 (10.8%) | 121.7 [76.3, 178.8] | 0 | 64318 (34.5%) | 54 | 1.000/0.871 | 1928268 | 544122 | 172851→166456 | + +| K | useful@budget B=C | useful@budget S | S stale gold | page mismatches | stale flags | distinct atoms | survival h=1/5/10/20 | memory ms p50/p95 | store bytes | +|---|---|---|---|---|---|---|---|---|---| +| 1 | 0.276 | 0.276 | 0 | 0 | 0 | 643 | —/—/—/— | 4.05/6.00 | 409600 | +| 5 | 0.276 | 0.270 | 2 | 0 | 0 | 591 | 0.97/0.82/0.65/0.07 | 7.95/15.85 | 372736 | +| 10 | 0.276 | 0.276 | 2 | 0 | 0 | 563 | 0.97/0.90/0.77/0.40 | 10.89/23.31 | 368640 | +| 25 | 0.276 | 0.270 | 4 | 0 | 0 | 542 | 0.98/0.92/0.88/0.78 | 19.57/38.01 | 368640 | +| all | 0.276 | 0.264 | 6 | 0 | 0 | 473 | 0.98/0.92/0.88/0.84 | 36.57/68.23 | 352256 | From fe58fa991328bdc78aca6f829bda8b163c35acf7 Mon Sep 17 00:00:00 2001 From: denfry Date: Mon, 14 Sep 2026 11:51:10 +0300 Subject: [PATCH 17/18] test(migration): make the 1.10.0 fixture index checkout-independent tests/fixtures/sample_repo had no line-ending rule for web/app.ts, so a Windows clone with core.autocrlf=true checked it out with CRLF while Linux and macOS got LF. The fixture index had been built from the Windows bytes, so on CI update saw a different sha256 and reparsed the file (indexed == 1). -text on the fixture keeps checkout bytes equal to the repository bytes on every OS, and the index is rebuilt by the 1.10.0 release code (abb67df) over those bytes. Chunks, edges, symbols and FTS content are unchanged; only web/app.ts's recorded size and hash, internal row ids and the root-dependent config_hash (written, never read) differ. --- .gitattributes | 5 +++++ tests/fixtures/index-1.10.0/index.sqlite | Bin 102400 -> 102400 bytes 2 files changed, 5 insertions(+) diff --git a/.gitattributes b/.gitattributes index e1dc5cb..0885947 100644 --- a/.gitattributes +++ b/.gitattributes @@ -9,3 +9,8 @@ cbx text eol=lf # сравнивает байты, поэтому при core.autocrlf=true git подставлял CRLF в worktree и # test_real_repo_is_in_sync падал на любом свежем клоне под Windows. .skill_version text eol=lf +# sample_repo сравнивается побайтно: tests/fixtures/index-1.10.0/index.sqlite хранит sha256 +# её файлов. Файлы без правила (web/app.ts) git под Windows с core.autocrlf=true отдавал +# с CRLF, под Linux/macOS с LF, и update на CI переиндексировал файл. -text отключает +# любую конверсию: в рабочем дереве те же байты, что в репозитории, на любой ОС. +tests/fixtures/sample_repo/** -text diff --git a/tests/fixtures/index-1.10.0/index.sqlite b/tests/fixtures/index-1.10.0/index.sqlite index fa405406e84516db127cffe60fdc968bdfdba4fb..9b837b9fb9585d4e663de3b817360abbf2bb7adc 100644 GIT binary patch delta 5284 zcmds5d5~1a8Sm>I{f_CqhXs~hL{M;9W{+WaP(X#mD5nXeWD%A(GjDcBc4u&AmLtT> zE`&s*QQ^f5Wl)IGXeyRatW{E4MoH8X4)`ppb122=SjRhg>k zH?RBa?%((Oj{bUv_NYU9)aEZ;zA%|{n-b~FfRk%#M-_+=bYW8yLI}gR9=_!d-rn5S z0lh8Xs!^a1ixYCBR-7;onrqE5`Z(t>AJM|-Fs2k|Dd1HF!yieuiyip9ZHr53EM z#lv%RPPE@0aC46QRuy_y;}ztbs=~z8cnYiDQYQ*;uf}6>h4(e~F|?bNW?H#0P;ge@ z30M?{R^S$F7xv1xfP4!U-dus}u~qn|?6Qkp;ch$zn}x;QxB*6dsjc0(2^)olA?PxT zT`zUx#9ri!$U)p(l*2u()uUd3?GWE93xr@xxytdi% ztS_oI!VTxtW4r=;TV`KEm2(yE@a$<|jN`Mn34nAr!r^cO#)D`;_imcA(^^hoO7J|G z660QY?gFwId)wx2u9NsRaR(9~h=;{Kaff(991(AgRIa%}wAr)~N7LzCHkWbMX%=n3 zJw3@(GS}0i>a-Rw&$=0f(;)6o_a;+p3=L!Njh@=+vMb}J+zrH{jr^kcKr$soi#xr5 z#0l|{_=|W;{9e2wUVxnrin$8KSk+{!sD7)+mNCDjvZaJk%xQ6veMtOWJSkoizW~VZ ziics5TSZ<1Obn)Cxnw$}8MGclT=u2os>BoPcd}V!v;-UdLQjbkod&rtZUZ&8@nr;1 z9}qk+(LQ4DvYxks=C91HW{dF?7#jTz>Ig`C53%}MlPb&+BahRR9E{(ehZ+JA9hEo((PZyndN4a%IrCbqyw>(PHNDJqH$sfWa&8=mm^!YW z!*L>)g>E+$&lY865KyfwIfGdU1nV&f^3w&2;>zjr@mA_}_q2f-ZtMxwNWxWdK>FsZ zK=X&~|?W^^<@E=Eb?h5O2W_`w=9x zk07-@C4MXR!w$EL=%_`NQwu3{)NHCSI9Eq+PvuE#VdzFGKD8$!MNM`iaFdtBZgEKb zN!$(lTrVt*R?F0)`iTW-p01R$4ij_|9!SUC{%p&jWL4%_XE4{-l1s03QzS^k+=Ce< znjS;qn0Qt^2CBR#eg}*16HDb%(lh)_CVwa<9fwQyVlo{h2iB!CIX^QLe-e$>aw%Gb z{K{#W3|2iPCQ08_eXnKz=C!2^wtr2aiHM6wDu%M!$V1grkGW{#=ifJHu(`y>rt9{)YIuxKA7ue+HFz3Gc}6U~&|A-!!8zQJH4T zOOi@Dl|z-`4mdsQ-Aon|wJnYzT|0yVE#g_L=D#VxYR*)EL1nh|KUQX{IHqnozt+nu z^@TaBG5=JBc7lmxN~d>?GVTApNPD2>0{kv*mWNvvc^_HZt@F({&F`9@H;(bw_&$Ci z59*!TliGasId!$-u+8Lss6iel4^%D31C*qL-chJr>vz!l!p&FWb8(&LkPsdN@7|9I zoN0v_tDrZygRa0eUN@bF>*UbwbSfQQRbAL}1%9PmB#)^+lO;=~ba%02W|gNE(kICO zu7aLo>rWt|Hz9MD-XxwA+XHvmKd>vUgHRF7H-2V}=hx}S^yNU-aV^kpP}C0;BW&Bq z_Kxg5J8xrr)%`>8fh0J51q`ibA*c&&qGNut%CdEaF~J%{P&VD~+C;^wyHO@0n-Hv* z%Nv_eOd&VgM8jG(=VajIu?fd)eL4w06sA{Nrmb7N8?_~T7#6jV<>OenuMA=qj4V+@ zez9Z0^0H8{2*XSkc@|MHOTpd7QZC-|^Eaqmj20*|mgP2(HD+dEsX7VCs5IDa2ob~X-D5Eog z04?JtGH$ldAe3R4u3?f*8(_F1M1yd`h8u^pOfl!xwIBsbvbN9q0;7~aKLu3wcL?2MPr&_>$+ur zh{iAWchhyGpJGCqM8_1>*~1Na4Go{Wrvzb(#CA)W6rCA1u%eU75-U1vDBVRbv!j!$ zB0D;%D6^xJ09L?DlPTTzkCRmCI@4?g*wWz+#K7>day7mPc+&|uVlJaIwQORYh$4Tw zuIM&nmI_|RO<4_Gc=ShCKws@F(o7&R{@uliYefUG?N}sWKtp5IE66Ab=@>H+vI^8(b$Y3b8Ni~ zB%Ut26eDAZF7Jjid?k-CMe(?Nr z?aF3jY86~0q%j$(D4LFux?&Dvq*8hsV>Og$C50J~P*Ru?$VF0NMgm3uV`?=dXJ2;) zH&bR-fFt?@1i&F7#JDN0LBV{;=YdiLDwSv$UV1~BgR^1u!7>LFFi{Q+fZO!-{@vy$*c?Akc|Cu9g0xJPnA$k%2Y{9My1Rul9Gp{Nx`gC7OzZjh(Ko( zl*nLMwOBG!Qgf0aDxDk*us#`cDezv*SyU=7eU7q#T2z^`TE-x`8#adoZ*s%5EW;3g zL^acVec{hG$nv0X9(P__^6DIngy4LEvn&U9w(?0vU?1KKe-LCGNbBI$nB_8qE`j`L zn1PC~mu63a*%k{y{=gsuFJo)?26>A>m6Akxi6v#M36JW};XWL51n0Oxn0@#z|_YPVMO{tPIl5&e`3w=Y7uZ-cwI0r=C)-{m9|{*^=Eek}pgoOFc0uAOT1p=;@Ip zG9XD(2mUpGn;yBg|7M)s{>cq8&H;Z*tZd|O=|TA{yHB2>-)0GVC;4kK3_pj@!Yxu= zdP3?}e*^ zIZX|=x5NDIQZhAePuQiT_|qSG{cg}J9sA%)W}M!rI42WXr`}bV)7#Z%L~f5Hu8&8rj}JQU?;2Od(&As7ynN}ouKV4k zmw)$-%a`sAd%M0RKXLie&%z~l*JnTeiOZMHbbqv8$okX`3**_G-5=@Sab0Bay67c! z?lv&)3V%rAb^iDK&-v5*QNG*~fPz|$1m02G{InO2*{QzdNfb&i`2*i1` z@8+!*?bWxBW=q-8T)tpi$V#>PBz~H2^OvpHtnXS=R$sNZr^BK*t;C%hcim)tWM#mT zpzD1d0Cy4R;az*=aCO^C%z1Iw)nujr(f6NW^S`kYb$;*0FIq=eB1js%21(Pz*}40} za`^m-l|JXp?h~7ZD9gA>;{VPsa>d_iJ!?JY`<~A>ePAZ#D})+s*}dP)3-p7>E_yfiy3>QXFNZe&24io%=I|Gb#kWn<~lSt#39bs z3%Y`edCkWcFDARVM#aKL#xw>492>^|Rs)jNuIfvT&! zqDzOICqLg$mJT}K|9oW2?xh3op~gZ}4d;(P*S+`)e-BZ=z8nZZQQf&ip1_&S0hQh^G?$$&pyPKb}F-_&_4M{7?|Ssg@5>wq@Q* z=W`?3(c!UVaSS^`bRcEN?AS;ui5(&}m>jTOm`+C1ndr_)I+IGp2je?pgXzIoCKgL2 zcSezaGBJXy>25;1;;E_ZcxgCU`c=i*3iK*oOpn9epZ+1gKf0h3W|33c{{&cl> z=n1~Lgo=b=gchNuR_PK@O`CFPybm(gI`?(J~X++!V&BHm6Yl zLdfJ~?Ksg0ohpqXkp+4xpDz_L=s2%2Bmp@+o-7t?Gla-UujF&Z{J4#q6d6jjlY|mh z$m9AfQJ_q*B9M~FgaZ>jyEaD*AgT<4KqK0;T}Wq(wiwBjP38+F_QSV7IkBw zWG5yuf02VG3r#E_2BSjkEaSRLkZhXB3ef!j8+FQ$pxFu#3TmUOkW~PbR#m1nXCK4W0A*-o z4zZ2EiuO2qjVX#uTBuxjFq;;RKwp=V1!2f>qEjd!Z%t+!C>osdLNbS{qF#b*)Gi^f zvI*#9W*19ER8yA;j{QJP!bCNr$$kZtHjSYVr)>0(49rc2Y2@PO17qzB7(=!Q9?WNj zY#EeeSu}hGSt^XTXnL#Pry4#>^;?F&C1CM(!>^mBsrt&g*{U0U(`N>FTR`tL3JZ;N zoFBt19$^tYk?JaX`%;?KRWHp{EK!kJxQt5L(;_zzN5w`O0mcypij55s71W~V9USPM zu64CcWTv-TKEG~_o@+`zf|BoXCD)E)>spiF08OD{r6E(Gt0KifNEEOsP~Z*U0Mqj%l=FzH9y2eo4=oyO0B|~x0Fch&>JJVli+IgEK zm8GLf#LbPmf}RuAhzN!};U0KaooJ6J~0qY7TAR(m!PHA3V)DFOGDJottTiNc)f5~XMS}N@rkaAeq4`wP&gMsz5rHwe?R3i?x8MWCtaIz7H+KlYQ0jC;qsLf#gV@WpRFi#$9`b9%g#b%SQ zvoiKe$kiFvz3g5F5XH_a8Zlx%U}MP#1!gpzgt)faq;P;b77<{(E9Ma@%~&z)yKWD` zNtp?|Mg42$J)dr-#Pz7jP?AantO|=Q;e}@Pp8L{IaC3=DNuY4A)z#+3FH7Y+TA$MCfBJSO_M>oGthnx{$~ z{8ZLVOpS(`Wk5YCp57<`t!g+E0y5K71@o9`0UdqXW*B`Rb9JIp235N1QYlLq19i+E zpTuiPU2==n4TwYtv#f+cr4nwCh&BK0J(ATTN&>G4s61eAlPfpe2|=GEVGyK^)_j*L zef2*0GKZGRxjfu`#lsSRo&TKw8~+FX1O6TUr~GUDJpU4Zl0U{DuJ(Q)%gah#rBbT~ zl2TU?F$E%v8N314nV1m>MMM_}K_n<>P+n_0MMOe`;!>@Q2ws7d+8iPwq^Zp!vQf;O zM1%?C1R@|wNQN)&UrPL!%f=~qjc*haD*uecYy2ntZ&A~4@-Onw;MMJ8yll`QDc}|3 zSa_j>#u5GBx90E#LF6V^c+pS0$?B04zG%@dGCVBW@$hhXNvG{Z+?c~vLH0p$Idg~S zJq@0h_?!5``zild{zLu}e;(DUZhiTyd=DlcQNkAmki$zJs0zTsiypWj01YpApuz8~ z#Q&532YUZGyoxUIvuM$y{7#d05PYPMHTZ&1wR!YELF{s<>7e2PSpYUDN(ce<0Pz5LKoWrNeAff-c;IaT$j<+ksZMsjC0wNN za~L&zPKqFXou5O49_M90>ma1@p12q~vUa!_I=ptc5IV4Scs?{_vUVWiCQ3tx*5C&% z{)QyWXC?o3{+GTD|L0ek-rG>Kb-$q&MLdY_$0waVoxQ!)S)lEDv5rLhL2;4*9flB8-*!utf0cUx`jqnN;)rtXK5;^}f{v-YZf2P`V>M{Pvkbhl7 ztD&K_F}tunq8CC3R>LN*2_Xv|RIuhr=!keuwacxN|0n*h`S<#}tp)2^-?Hyk^HuXn z^ETrR6o>rbf((2~GKe6T7hhT!z`Fn$v>-WMB$^MO%|GEvr&>`Yi ziF_D>(C)O*Fl@q!2yAeYhzFfhv=7=a;9a;X2(Q==J@|p{*E9pj`@FAon~eAPkXJdn z5AK#P`(($o_22Xwp?q9CS$~YJ>wA20xn~ll-B`X?*9;wg&y@oKx`nv+$KrNzVGJG8 zXq#xTp(C93k|e%ex4ZX-(7|R$4Gnqev?6qPh5s{ukw42qS01lnnMgRZ+ From b45bd45bd60a08efe6c78869218635dca727e607 Mon Sep 17 00:00:00 2001 From: denfry Date: Mon, 14 Sep 2026 11:53:09 +0300 Subject: [PATCH 18/18] docs(changelog): 1.10.0 was never published on its own No v1.10.0 tag or PyPI release exists, so the 1.10.0 compare link and the Unreleased link pointed at a missing tag. Point 1.10.0 at its release commit and Unreleased at v2.0.0, and say in the 2.0.0 section, which becomes the GitHub release body, that 2.0.0 includes the 1.10.0 work. --- CHANGELOG.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3942753..1e6a1f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,8 +11,9 @@ All notable changes to this project are documented here. The format is based on Evidence release. Everything an agent reads through `codebase-index` is now identified by the exact bytes of the span it came from, can be re-checked against the working tree at any later moment, is not resent to a session that already holds it, and is reported -when it changes. 2.0.0 also carries the unreleased 1.10.0 ranking work below and the -community-readiness work merged from #26. +when it changes. 2.0.0 also includes the community-readiness work merged from #26 and the +1.10.0 ranking release, which was prepared but never published on its own; its notes are in +the [1.10.0 section of the changelog](https://github.com/denfry/codebase-index/blob/main/CHANGELOG.md#1100---2026-09-02). Measured by replaying this repository's own history in sessions of 5 to 87 tasks (`tests/eval/results/2026-09-14-evidence-memory.md`): memory withheld **zero stale @@ -141,6 +142,9 @@ snippets reproduced the 1.10.0 packet on every task, and snippet tokens fell by ## [1.10.0] - 2026-09-02 +Never published on its own: tagged releases go from 1.9.0 straight to 2.0.0, which +includes everything in this section. + Ranking release. 1.9.0's own diagnostics showed that a perfect reranker over the candidate pool it already generated would score MRR 0.902 against the 0.577 actually delivered — a ranking gap roughly three times larger than the remaining recall gap. @@ -769,9 +773,9 @@ Pooled over 305 queries (Python, Java, TypeScript), v1.8.0 → 1.9.0: - Hooks example + `watch` mode for keeping the index fresh without blocking the edit loop (M8). - `doctor`, `stats`, `clean` diagnostics/maintenance commands. -[Unreleased]: https://github.com/denfry/codebase-index/compare/v1.10.0...HEAD +[Unreleased]: https://github.com/denfry/codebase-index/compare/v2.0.0...HEAD [2.0.0]: https://github.com/denfry/codebase-index/compare/v1.9.0...v2.0.0 -[1.10.0]: https://github.com/denfry/codebase-index/compare/v1.9.0...v1.10.0 +[1.10.0]: https://github.com/denfry/codebase-index/compare/v1.9.0...abb67df [1.9.0]: https://github.com/denfry/codebase-index/compare/v1.8.0...v1.9.0 [1.8.0]: https://github.com/denfry/codebase-index/compare/v1.7.0...v1.8.0 [1.7.0]: https://github.com/denfry/codebase-index/compare/v1.6.0...v1.7.0