diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index ccfe407..29e7c00 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -3,7 +3,7 @@ "name": "codebase-index", "displayName": "Codebase Index", "description": "Give Claude a precise local map of your codebase: find implementations, trace behavior, and predict change impact with file-line evidence.", - "version": "1.8.0", + "version": "1.9.0", "author": { "name": "codebase-index contributors" }, diff --git a/.claude/skills/codebase-index/.skill_version b/.claude/skills/codebase-index/.skill_version index 27f9cd3..f8e233b 100644 --- a/.claude/skills/codebase-index/.skill_version +++ b/.claude/skills/codebase-index/.skill_version @@ -1 +1 @@ -1.8.0 +1.9.0 diff --git a/.codex/skills/codebase-index/.skill_version b/.codex/skills/codebase-index/.skill_version index 27f9cd3..f8e233b 100644 --- a/.codex/skills/codebase-index/.skill_version +++ b/.codex/skills/codebase-index/.skill_version @@ -1 +1 @@ -1.8.0 +1.9.0 diff --git a/.opencode/skills/codebase-index/.skill_version b/.opencode/skills/codebase-index/.skill_version index 27f9cd3..f8e233b 100644 --- a/.opencode/skills/codebase-index/.skill_version +++ b/.opencode/skills/codebase-index/.skill_version @@ -1 +1 @@ -1.8.0 +1.9.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index fe931d1..64f7fa8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,89 @@ All notable changes to this project are documented here. The format is based on ## [Unreleased] +## [1.9.0] - 2026-09-02 + +### Added + +- **Objective ground truth from git history.** `tests/eval/gen_queries.py` mints a + retrieval benchmark from any git repository by pairing a human-written commit + subject with the files that commit actually changed. Unlike docstring-derived + benchmarks the query text is not copied into the document being retrieved, so the + set is leak-free by construction; merges, reverts, releases, version bumps, + sweeping refactors, changelog-style answers and benchmark scaffolding are all + filtered out. +- **Multi-corpus, multi-language evaluation.** `run_eval.py --corpus REPO:QUERIES` + pools several repositories into one benchmark so ranking changes are validated + outside this repository and outside Python. The 1.9.0 defaults were measured on + 305 queries across Python, Java and TypeScript corpora. +- **Significance testing.** Every non-baseline row now reports a paired bootstrap + 95% confidence interval and a paired permutation p-value (both seeded, therefore + reproducible). Query sets of this size have a noise floor of several MRR points; + signals now ship on the strength of that test rather than the sign of a delta. +- **Context-noise metrics.** The report adds mean emitted snippet tokens, duplicate + rate of returned results, and p99 latency alongside the existing IR metrics. + +### Changed + +- **Fusion now scores cross-retriever agreement.** RRF fuses on + `(path, line-bucket)`, so a symbol definition at line 40 and a lexical hit at line + 120 in the same file fused as two unrelated candidates — two retrievers agreeing + on a file produced two weak results instead of one strong one, and cross-source + agreement never reached the score. Each candidate now also receives, at + `file_agreement_weight` (0.4), the RRF mass of every retriever that found its file + at another locator, excluding retrievers already counted at that locator. + Ablatable via `RetrievalTuning(file_agreement=False)`. +- **Documentation demotion deepened** from -0.05 to -0.20. Prose describing a + feature matches a natural-language question more literally than the code + implementing it, so design notes and plans were displacing the modules they + describe. Because this only reorders prose relative to code, documentation-seeking + queries improved as well (category MRR 0.579 → 0.612). Generated/vendor paths move + to -0.25 so they remain the least-preferred role, and `MAX_ABS_PRIOR` now pins the + invariant that priors stay tiebreakers. +- **Fuzzy identifier matching is now a recall fallback.** It runs only when the + precise symbol lookup named no symbol and returned fewer than + `fuzzy_fallback_min` (3) rows. It moved no ranking metric across 305 queries while + accounting for ~20% of query latency; typo and acronym recall is unchanged because + those are exactly the queries where the precise lookup comes up empty. +- **Candidate over-fetch is explicit.** `candidate_pool_multiplier` (default 2) + replaces the implicit widening that happened whenever dedup or MMR was enabled. + Making it explicit revealed that the quality previously credited to SimHash dedup + was really the wider pool; dedup is retained for what it does measurably do, which + is cutting the duplicate rate of returned snippets from ~1.6% to ~0%. + +### Fixed + +- **Synonym matches were reported as exact symbol matches.** `is_exact` came from + SQL and was relative to whichever needle retrieved the row, so a synonym + expansion ("config" for "configuration") marked an unrelated symbol as an exact + match — worth a +0.20 rerank bonus and an unconditional `high` confidence. + Exactness is now judged against the terms the user actually typed. +- **Duplicate suppression was order-dependent on ties.** Equal-scoring duplicates + handed the slot to whichever copy arrived last, contradicting the documented + "ties favor input order" and making the retained snippet depend on retriever + emission order. + +### Performance + +- Query latency roughly halves: p50 78.6 ms → 51.2 ms, p95 193.2 ms → 94.9 ms, + p99 277.1 ms → 152.2 ms on the pooled three-repository benchmark, from the fuzzy + fallback plus a SimHash fingerprint that folds repeated tokens by multiplicity and + caches token digests. Fingerprint output is bit-for-bit unchanged. +- Mean emitted snippet tokens fall 1183 → 1091 per query. + +### Retrieval quality + +Pooled over 305 queries (Python, Java, TypeScript), v1.8.0 → 1.9.0: + +| Metric | v1.8.0 | 1.9.0 | Δ | p | +|---|---|---|---|---| +| MRR | 0.564 | 0.591 | +0.027 | <0.001 | +| MAP | 0.433 | 0.461 | +0.028 | <0.001 | +| nDCG@10 | 0.503 | 0.527 | +0.024 | <0.001 | +| recall@5 | 0.529 | 0.560 | +0.031 | <0.001 | +| P@5 | 0.182 | 0.192 | +0.009 | 0.001 | +| hit@3 | 0.649 | 0.666 | +0.016 | 0.124 | + ## [1.8.0] - 2026-09-02 ### Added diff --git a/README.md b/README.md index 4b43c70..e6ad992 100644 --- a/README.md +++ b/README.md @@ -271,7 +271,7 @@ See [MCP.md](docs/MCP.md) for client configuration. ## Project status -The latest released line is **1.8.0**. It includes: +The latest released line is **1.9.0**. It includes: - hybrid and optional vector retrieval; - Tree-sitter symbol extraction across the documented language tiers; @@ -280,15 +280,21 @@ The latest released line is **1.8.0**. It includes: - shortest dependency paths and node descriptions; - token-budgeted and skeletonized retrieval packets; - benchmark-calibrated lexical expansion, fuzzy identifier matching, and source-aware ranking; +- rank fusion that scores cross-retriever agreement at file level, not just at a locator; - bounded, intent-directed graph discovery with optional diversity and duplicate suppression; - CLI, Skill, plugin, and MCP delivery; - incremental updates, watch hooks, diagnostics, skill rollback, and diff-aware - impact analysis. - -Planned work is deliberately separated from shipped capability. The next -product priorities are stronger real-repository evaluations, typed framework -edges, and an even more direct task-context workflow. See the -[roadmap](docs/ROADMAP.md). + impact analysis; +- a multi-repository retrieval evaluation with leak-free git-derived ground truth, + one-signal ablations, and paired significance tests + ([tests/eval](tests/eval/README.md)). + +Every shipped ranking signal has to survive that evaluation: 1.9.0 removed the +cost of two signals that could not demonstrate a benefit and rejected several +plausible ones outright (IDF-weighted coverage, stemming, graph propagation, MMR, +a file-length prior). Planned work is deliberately separated from shipped +capability. The next product priorities are typed framework edges and an even more +direct task-context workflow. See the [roadmap](docs/ROADMAP.md). ## Documentation diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 9c4bbf0..4bb88af 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -2,7 +2,7 @@ ## 1. Overview -`codebase-index` is a **local-first** code intelligence layer for AI coding agents. In `1.8.0` +`codebase-index` is a **local-first** code intelligence layer for AI coding agents. In `1.9.0` it has two shipped faces: 1. **A Claude Code Skill** (`.claude/skills/codebase-index/SKILL.md`) that Claude auto-invokes for @@ -97,8 +97,9 @@ codebase-index/ ├── graph/ # builder.py (edge resolution), expand.py (impact), │ # export.py (HTML graph) ├── storage/ # db.py (pragmas, schema, version guard), schema.sql, repo.py - ├── retrieval/ # intent.py, searchers.py, fusion.py, rerank.py, - │ # budget.py, pipeline.py, types.py + ├── retrieval/ # intent.py, searchers.py, fusion.py, rerank.py, priors.py, + │ # lexical.py, fuzzy.py, diversity.py, skeleton.py, + │ # tuning.py (ablation contract), budget.py, pipeline.py, types.py ├── embeddings/ # backend.py, noop.py (default), local.py, external.py — opt-in ├── output/ # markdown.py, json.py, redact.py ├── watch/ # watcher.py (optional, watchdog-based) diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 062da77..5b1d7db 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -1,10 +1,11 @@ # Benchmarks -`codebase-index` has three benchmark surfaces. Read them with their status in +`codebase-index` has four benchmark surfaces. Read them with their status in mind — the whole point of this page is to keep evidence and aspiration separate. | Surface | What it is | Status | Use it as | |---|---|---|---| +| Retrieval eval (`tests/eval/`) | Ranking quality vs a fixed baseline across multiple real repositories, with significance tests | **Proven (relative)** | The gate for ranking changes; measures *deltas*, not absolute superiority | | Public suite (`tests/benchmark_public.py`) | Deterministic synthetic multi-language fixture with the full metric framework | **Toy/synthetic** | CI regression gate + metric shape, **not** product-quality evidence | | Smoke/perf (`test_perf_smoke.py`, `test_benchmark_comparison.py`) | Latency + output-size guards on a tiny fixture | **Toy/smoke** | Regression checks only | | Honest real-repo (`tests/benchmark_honest.py`) | 55k LOC Java repo, recall@3 vs disciplined `rg` baseline, symmetric token accounting | **Proven (one repo)** | The only headline product-quality number we stand behind today | @@ -15,15 +16,33 @@ Do not write, imply, or ship any of these until a run with published logs exists - Any 10k / 100k / 1M LOC scale or speed claim (no real run at that size). - "Beats Cursor / Sourcegraph / Codebase-Memory MCP" — no head-to-head exists. -- Per-language quality claims beyond Java (the honest run is Java-only). +- Per-language *absolute* quality claims beyond Java. The retrieval eval covers + Python, Java and TypeScript, but it measures this system against its own earlier + versions — it says a change helped, not that the product beats an alternative. - Generic "Nx faster" / "Nx fewer tokens" without naming the baseline and repo. -- Latency claims — the honest run explicitly does not headline latency - (Python process start dominates; real `rg` is tens of ms). +- Latency claims against external tools — the honest run explicitly does not + headline latency (Python process start dominates; real `rg` is tens of ms). + Version-over-version latency from the retrieval eval is in-process and is only + comparable to other runs of that harness. The defensible headline today is exactly: **on one 55k LOC Java repo, recall@3 was 70% (index) vs 40% (`rg`+window), using ~13× fewer answer tokens.** Everything else is roadmap. +## Retrieval evaluation (ranking gate) + +See [tests/eval/README.md](../tests/eval/README.md) for the full protocol. In short: +ground truth comes from hand-written queries verified against the source tree and +from commit-subject → changed-files pairs mined from git history (leak-free: the +query text is not in the indexed corpus). Corpora are pooled across languages, one +index per corpus is shared by every variant, and every non-baseline row carries a +paired bootstrap CI and permutation p-value. + +`1.9.0` was measured over 305 queries across Python, Java and TypeScript corpora +against `1.8.0`: MRR +0.027, MAP +0.028, nDCG@10 +0.024, recall@5 +0.031 (all +p < 0.001), with p50 latency 78.6 ms → 51.2 ms. These are version-over-version +ranking deltas on those corpora, not a universal quality claim. + ## Public benchmark suite Run: diff --git a/docs/RETRIEVAL.md b/docs/RETRIEVAL.md index d4acf37..6102baa 100644 --- a/docs/RETRIEVAL.md +++ b/docs/RETRIEVAL.md @@ -65,6 +65,11 @@ source)` list so fusion is source-agnostic. first when the query clearly names a path. - **Symbol** — query against `symbols` (exact, identifier parts, bounded fuzzy matching). Carries `kind` (function/class/method/...) and signature. Primary for `locate_impl` / `find_refs`. + Fuzzy identifier matching (acronym / concatenation / edit distance) runs only as a **recall + fallback**, when the precise lookup named no symbol and returned fewer than + `fuzzy_fallback_min` rows. Measured over 305 queries on three repositories it moved no ranking + metric while costing ~20% of query latency, so it is kept for typos and abbreviations but no + longer runs when the query already spelled its identifier correctly. - **FTS** — FTS5 `bm25()` over the `fts_chunks` virtual table (chunk text + symbol names + summaries indexed). Query-time camelCase/snake_case splitting, small down-weighted synonym expansion, and soft coverage scoring make natural-language questions robust without weakening @@ -79,28 +84,66 @@ source)` list so fusion is source-agnostic. scores: ``` -RRF(d) = Σ_r w_r / (k + rank_r(d)) # k ≈ 60, w_r = per-intent retriever weight +RRF(d) = Σ_r w_r · k / (k + rank_r(d)) # k ≈ 60, w_r = per-intent retriever weight ``` - Robust to scale differences between BM25 and cosine. - Per-intent weights `w_r` let `locate_impl` favor the symbol list and `how_it_works` favor FTS. +- Scaled by `k` so fused scores and the reranker's bounded bonuses share an O(1) scale. This is a + monotonic rescale; fusion order is unchanged. - Ties broken by rerank features (next step). -## 4. Reranking (`retrieval/rerank.py`) +### Cross-locator file agreement + +Fusion keys on `(path, line-bucket)`, not `(path, start, end)`, because different retrievers report +different line ranges for the same place. Bucketing alone was not enough: a symbol defined at line +40 and a lexical hit at line 120 are genuinely different locators, so a file that **two retrievers +agreed on** still fused as two separate candidates, each carrying one retriever's evidence — and +cross-source agreement, the entire point of RRF, never fired. + +Each candidate therefore also receives, at weight `file_agreement_weight`, the RRF mass of every +retriever that found its *file* at some other locator: + +``` +score(d) = RRF(d) + α · Σ_{r ∉ sources(d)} w_r · k / (k + best_rank_r(path(d))) +``` + +Retrievers already counted at the candidate's own locator are excluded, so nothing double-counts, +and the term is bounded by the same weights as fusion itself. `α = 0.4`; the 0.3–0.6 plateau peaks +there. Set `RetrievalTuning(file_agreement=False)` to recover plain locator-only RRF. -A lightweight, explainable feature score (no external model required) layered on the fused order: +## 4. Reranking (`retrieval/rerank.py`) -| Feature | Intuition | -|---|---| -| symbol-kind match | a `def`/`class` outranks an incidental mention | -| path proximity | files near a query-named path score higher | -| graph centrality | high in/out-degree nodes matter more for `architecture` | -| recency | recently changed files (git mtime) slightly boosted | -| exact-name bonus | exact symbol-name match dominates fuzzy | -| test/generated penalty | test files and generated code demoted unless asked | +A lightweight, explainable feature score (no external model required) layered on the fused order. +Every term is bounded, so reranking reorders near-neighbours rather than overruling retrieval: + +| Feature | Effect | Intuition | +|---|---:|---| +| Exact symbol match | +0.20 | the user named a specific symbol | +| Symbol definition kind | +0.05 | a `def`/`class` outranks an incidental mention | +| Symbol name among query terms | +0.05 | the name was asked for, not just matched | +| Path term match | +0.05 | the user supplied a location clue | +| Graph centrality (`in_degree`) | ≤ +0.08 | `log1p`-damped, so a god class cannot dominate | +| Reference-count fallback | ≤ +0.04 | for names too common to resolve a precise `in_degree` | +| Source role prior | −0.25…+0.08 | see below | +| Generated, or test on a non-test query | −0.15 | supporting evidence, not the answer | + +### Source role priors (`retrieval/priors.py`) + +| Role | Prior | Rationale | +|---|---:|---| +| Implementation | +0.08 | the answer to a code question is usually code | +| Test | −0.06 | flips to +0.05 when the query or intent is test-oriented | +| Documentation | −0.20 | prose *about* a feature matches a natural-language question more literally than the code implementing it, so design notes and plans crowded out the modules they describe | +| Generated / vendor / build | −0.25 | never the answer; kept strictly below documentation | + +The documentation prior deepened from −0.05 in 1.8.0. Because it only reorders prose relative to +code — never below other prose — documentation-seeking queries improved too (MRR 0.579 → 0.612). +At −0.35 that reverses and the `docs` category collapses, so the optimum is interior, not a +"more is better" knob. `MAX_ABS_PRIOR` caps every prior so this stays a tiebreaker. The reranker also produces the human-readable **`reason`** string per result -(e.g. *"exact symbol match · called by 4 sites · in src/auth/"*). +(e.g. *"exact symbol match · 4 callers · in src/auth/"*). ## 5. Graph expansion (`graph/retrieval.py`; `graph/expand.py` for impact APIs) @@ -119,11 +162,17 @@ Expanded nodes retain edge confidence and receive distance-decayed scores so see ## 6. Diversity and duplicate control -`retrieval.diversity` provides bounded MMR selection and SimHash near-duplicate -suppression. MMR is disabled in the shipped default because the reproducible -benchmark favored relevance-only ranking; callers that need broader snippet -coverage can enable `RetrievalTuning(mmr=True)`. Duplicate suppression remains -available independently. +`retrieval.diversity` provides bounded MMR selection and SimHash near-duplicate suppression. + +MMR is disabled in the shipped default: it moved no ranking metric on the benchmark and roughly +doubled p50 latency. Callers that need broader snippet coverage can enable +`RetrievalTuning(mmr=True)`. + +SimHash duplicate suppression stays on, but on noise grounds rather than ranking grounds: it does +not move MRR, and it takes the duplicate rate of returned snippets from ~1.6% to ~0%. The +over-fetch that feeds selection is now an explicit `candidate_pool_multiplier` rather than an +implicit side effect of enabling dedup — an earlier ablation credited dedup with a quality win that +was really the wider pool doing the work. ## 7. Token budgeting (`retrieval/budget.py`) diff --git a/docs/RETRIEVAL_PIPELINE.md b/docs/RETRIEVAL_PIPELINE.md index 2a843e1..5bf0944 100644 --- a/docs/RETRIEVAL_PIPELINE.md +++ b/docs/RETRIEVAL_PIPELINE.md @@ -97,17 +97,21 @@ direct lexical or symbol evidence. ## 6. Diversity and duplicate control -SimHash suppresses near-duplicate snippets independently of MMR. Bounded -Maximal Marginal Relevance is available through `RetrievalTuning(mmr=True)`; -the shipped default keeps relevance-only ordering because the benchmark favored -direct hits. +SimHash suppresses near-duplicate snippets independently of MMR. It does not move +ranking metrics; it earns its place by taking the duplicate rate of returned +snippets from ~1.6% to ~0%. Bounded Maximal Marginal Relevance is available +through `RetrievalTuning(mmr=True)`; the shipped default keeps relevance-only +ordering because MMR moved no metric and roughly doubled p50 latency. + +The over-fetch that feeds these selection stages is `candidate_pool_multiplier` +(default 2), an explicit knob rather than a side effect of enabling dedup. ## 7. Reciprocal Rank Fusion (RRF) Combines ranked lists from the enabled retrievers: ``` -RRF_score(d) = Σ w_r · k / (k + rank_r(d)) +RRF_score(d) = Σ_r w_r · k / (k + rank_r(d)) ``` The implementation multiplies textbook RRF by `k` so fusion and bounded rerank @@ -118,8 +122,13 @@ Where: - `rank_r(d)` is the rank of document `d` in retriever `r` - `w_r` is the intent/tuning weight for retriever `r` -The implementation merges co-located chunks into one per-file bucket before -fusion, preventing a large file from dominating the result list. +Candidates are keyed by `(path, line-bucket)`, so co-located hits merge. Because a +symbol definition and a lexical hit in the same file are often *not* co-located, +each candidate additionally receives — at `file_agreement_weight` (0.4) — the RRF +mass of every retriever that found its file at a different locator. Retrievers +already counted at that locator are excluded, so nothing double-counts. Without +this, two retrievers agreeing on a file produced two weak candidates instead of one +strong one, and cross-source agreement never affected the score. ## 8. Reranking @@ -132,8 +141,8 @@ After fusion, apply bounded explainable boosts and penalties: | Path term match | +0.05 | User supplied a location clue | | Degree / reference evidence | up to +0.08 | Stable structural tiebreaker | | Implementation source prior | +0.08 | Prefer source over prose/tests | -| Documentation source prior | -0.05 | Avoid docs displacing implementation | -| Generated/vendor/build | -0.12 | Suppress low-value derived code | +| Documentation source prior | -0.20 | Prose describing a feature outmatches the code lexically | +| Generated/vendor/build | -0.25 | Suppress low-value derived code | | Test path on non-test query | -0.06 | Keep tests as supporting evidence | ## 9. Confidence diff --git a/requirements.lock b/requirements.lock index 4a600d6..6843b5b 100644 --- a/requirements.lock +++ b/requirements.lock @@ -1,3 +1,3 @@ -codebase-index @ https://github.com/denfry/codebase-index/archive/refs/tags/v1.8.0.tar.gz +codebase-index @ https://github.com/denfry/codebase-index/archive/refs/tags/v1.9.0.tar.gz tree-sitter==0.25.2 tree-sitter-language-pack==1.8.1 diff --git a/src/codebase_index/__init__.py b/src/codebase_index/__init__.py index f4bccfd..af8c17b 100644 --- a/src/codebase_index/__init__.py +++ b/src/codebase_index/__init__.py @@ -4,4 +4,4 @@ See docs/ARCHITECTURE.md for the module map. """ -__version__ = "1.8.0" +__version__ = "1.9.0" diff --git a/src/codebase_index/retrieval/diversity.py b/src/codebase_index/retrieval/diversity.py index dbbb2a5..d94be5d 100644 --- a/src/codebase_index/retrieval/diversity.py +++ b/src/codebase_index/retrieval/diversity.py @@ -10,7 +10,9 @@ import hashlib import math +from collections import Counter from collections.abc import Callable, Iterable, Sequence +from functools import lru_cache from .types import Candidate @@ -90,6 +92,18 @@ def normalize_code_tokens(content: str | None) -> tuple[str, ...]: return tuple(tokens) +@lru_cache(maxsize=1 << 16) +def _token_digest(token: str) -> int: + """64-bit blake2b digest of one token. Pure, so caching is safe and bounded. + + Source chunks repeat identifiers and keywords heavily both inside a chunk and + across the candidate pool, and hashing dominated fingerprint cost. + """ + return int.from_bytes( + hashlib.blake2b(token.encode("utf-8"), digest_size=8).digest(), "big", signed=False + ) + + def token_fingerprint(tokens: Iterable[str] | str | None) -> int: """Compute a stable unsigned 64-bit SimHash for tokens or source content.""" if tokens is None: @@ -100,12 +114,14 @@ def token_fingerprint(tokens: Iterable[str] | str | None) -> int: if not values: return 0 + # Weighting distinct tokens by multiplicity is arithmetically identical to + # summing each occurrence, and collapses the per-occurrence hash and 64-bit + # accumulation loop to per-distinct-token work. Output is bit-for-bit unchanged. weights = [0] * 64 - for token in values: - digest = hashlib.blake2b(str(token).encode("utf-8"), digest_size=8).digest() - hashed = int.from_bytes(digest, "big", signed=False) + for token, count in Counter(str(value) for value in values).items(): + hashed = _token_digest(token) for bit in range(64): - weights[bit] += 1 if (hashed >> bit) & 1 else -1 + weights[bit] += count if (hashed >> bit) & 1 else -count fingerprint = 0 for bit, weight in enumerate(weights): @@ -133,8 +149,14 @@ def _score_value(candidate: Candidate) -> float: def _same_or_better(left: Candidate, right: Candidate) -> bool: - """Whether left is the representative to retain; ties favor input order.""" - return _score_value(left) > _score_value(right) + """Whether `left` (the incumbent) should stay the representative. + + Ties favor the incumbent, i.e. input order. Candidates arrive sorted by score, + so a later duplicate is never strictly better; deciding ties by ">" instead + handed the slot to whichever equal-scoring copy happened to arrive last, making + the retained snippet depend on retriever emission order. + """ + return _score_value(left) >= _score_value(right) def deduplicate(candidates: Sequence[Candidate], hamming_distance: int = 3) -> list[Candidate]: @@ -146,14 +168,21 @@ def deduplicate(candidates: Sequence[Candidate], hamming_distance: int = 3) -> l fingerprints: list[int | None] = [] for candidate in candidates: - _, _, fingerprint = _features(candidate) + # Only the fingerprint is needed here; building the token set as well + # doubled the per-candidate cost of the pipeline's hottest stage. "No + # fingerprint" means "no tokens", never "fingerprint happened to be 0". + tokens = normalize_code_tokens(candidate.content) + fingerprint = token_fingerprint(tokens) if tokens else None if fingerprint is None: representatives.append(candidate) fingerprints.append(None) continue + # Carry each match's fingerprint alongside its index: it is known non-None + # by construction here, and re-reading it from the list later would lose + # that guarantee. matches = [ - index + (index, existing) for index, existing in enumerate(fingerprints) if existing is not None and simhash_distance(fingerprint, existing) <= threshold ] @@ -162,17 +191,23 @@ def deduplicate(candidates: Sequence[Candidate], hamming_distance: int = 3) -> l fingerprints.append(fingerprint) continue - first = matches[0] + indices = [index for index, _ in matches] + first = indices[0] winner = candidate - for index in matches: + # The winner is either this candidate or an incumbent, and both + # fingerprints are already known — recomputing one from content was pure + # duplicated work. + winner_fingerprint = fingerprint + for index, existing in matches: incumbent = representatives[index] if _same_or_better(incumbent, winner): winner = incumbent - for index in reversed(matches): + winner_fingerprint = existing + for index in reversed(indices): representatives.pop(index) fingerprints.pop(index) representatives.insert(first, winner) - fingerprints.insert(first, token_fingerprint(normalize_code_tokens(winner.content))) + fingerprints.insert(first, winner_fingerprint) return representatives diff --git a/src/codebase_index/retrieval/fusion.py b/src/codebase_index/retrieval/fusion.py index a410545..4a929c9 100644 --- a/src/codebase_index/retrieval/fusion.py +++ b/src/codebase_index/retrieval/fusion.py @@ -13,6 +13,14 @@ retrievers report different line ranges for the same place; an exact key almost never coincides across sources, so cross-source agreement — RRF's whole point — would never fire. `agreeing_sources` is therefore counted at file granularity. +* Cross-locator agreement is scored, not merely counted. Bucketing alone does not + rescue agreement: a symbol defined at line 40 and a lexical hit at line 120 sit + in different buckets of the same file, so two retrievers pointing at one file + still fused as two unrelated candidates, each carrying a single retriever's + evidence. `file_agreement` adds the missing evidence back at a discount — a + candidate also receives, weighted by `file_agreement_weight`, the RRF mass of + every retriever that found its *file* somewhere else. Retrievers already + counted at the candidate's own locator are excluded, so nothing double-counts. On merge, the candidate carrying the most signal (symbol > fts > path) is kept as the representative so downstream rerank/snippet logic has the richest fields. @@ -36,11 +44,21 @@ def fuse( *, weights: dict[str, float], k: int, + file_agreement: float = 0.0, ) -> list[Candidate]: + """Fuse per-source ranked lists into one ordered candidate list. + + `file_agreement` is the discount applied to same-file, different-locator + evidence; 0.0 reproduces plain locator-only RRF. + """ accum: dict[tuple, float] = {} rep: dict[tuple, Candidate] = {} seen: set[tuple] = set() file_sources: dict[str, set[str]] = {} + # Best (lowest) rank each source achieved for each file, and which sources + # already contributed at each fused locator. + best_file_rank: dict[tuple[str, str], int] = {} + key_sources: dict[tuple, set[str]] = {} for source, candidates in lists.items(): w = weights.get(source, 0.0) @@ -48,7 +66,12 @@ def fuse( continue for rank, cand in enumerate(candidates): file_sources.setdefault(cand.path, set()).add(source) + file_key = (source, cand.path) + previous = best_file_rank.get(file_key) + if previous is None or rank < previous: + best_file_rank[file_key] = rank key = cand.fuse_key() + key_sources.setdefault(key, set()).add(source) # One contribution per source per locator: a file matching three FTS # chunks in the same bucket is one lexical signal, not three. if (source, key) in seen: @@ -57,6 +80,21 @@ def fuse( accum[key] = accum.get(key, 0.0) + w * k / (k + rank) rep[key] = _richer(rep[key], cand) if key in rep else cand + if file_agreement > 0.0: + for key, base in accum.items(): + path = rep[key].path + own = key_sources[key] + extra = 0.0 + for source in file_sources.get(path, ()): + if source in own: + continue + w = weights.get(source, 0.0) + if w <= 0.0: + continue + extra += w * k / (k + best_file_rank[(source, path)]) + if extra: + accum[key] = base + file_agreement * extra + fused = [_replace(rep[key], score=score) for key, score in accum.items()] fused.sort(key=lambda c: c.score, reverse=True) return [_replace(c, agreeing_sources=len(file_sources[c.path])) for c in fused] diff --git a/src/codebase_index/retrieval/pipeline.py b/src/codebase_index/retrieval/pipeline.py index a6cf2e2..020c956 100644 --- a/src/codebase_index/retrieval/pipeline.py +++ b/src/codebase_index/retrieval/pipeline.py @@ -165,11 +165,13 @@ def search( if token_budget <= 0: token_budget = plan.token_budget fetch_limit = limit + offset - pool_limit = ( - max(fetch_limit * 2, 20) - if (tuning.mmr or tuning.dedup) - else fetch_limit - ) + # Selection (dedup / MMR / per-file diversification) removes candidates, so the + # pool is over-fetched. The multiplier is explicit rather than a side effect of + # which selection flags happen to be on. A widened pool also carries a floor so + # a tiny `limit` still leaves selection something to choose between; multiplier + # 1 means "no over-fetch at all" and takes the page size verbatim. + pool_mult = max(1, tuning.candidate_pool_multiplier) + pool_limit = fetch_limit if pool_mult == 1 else max(fetch_limit * pool_mult, 20) lists, weights = _run_retrievers( conn, query, @@ -182,7 +184,12 @@ def search( graph_node_cap=tuning.graph_node_cap, graph_strategy=plan.graph_strategy, ) - fused = fuse(lists, weights=weights, k=tuning.rrf_k) + fused = fuse( + lists, + weights=weights, + k=tuning.rrf_k, + file_agreement=tuning.file_agreement_weight if tuning.file_agreement else 0.0, + ) ranked = rerank(fused, query=query, intent=plan.intent, tuning=tuning) if tuning.dedup: ranked = deduplicate(ranked, hamming_distance=tuning.dedup_hamming) diff --git a/src/codebase_index/retrieval/priors.py b/src/codebase_index/retrieval/priors.py index 3318e82..c9e0bec 100644 --- a/src/codebase_index/retrieval/priors.py +++ b/src/codebase_index/retrieval/priors.py @@ -113,11 +113,27 @@ class SourceRole(str, Enum): re.IGNORECASE, ) +# No single prior may exceed this magnitude. Priors exist to break ties between +# comparably-matching files, never to overrule retrieval evidence: fused scores +# reach ~1.0-1.5 and the largest rerank bonus (exact symbol) is 0.20, so a prior +# capped here can reorder near-neighbours but cannot lift an unrelated file over a +# genuine match. Widening this constant is a deliberate ranking-policy change. +MAX_ABS_PRIOR = 0.25 + _ROLE_PRIORS = { SourceRole.IMPLEMENTATION: 0.08, SourceRole.TEST: -0.06, - SourceRole.DOCUMENTATION: -0.05, - SourceRole.GENERATED_VENDOR_BUILD: -0.12, + # Prose about a feature matches a natural-language question more literally than + # the code implementing it, so design notes and plans crowded out the modules + # they describe. Measured over 305 queries on three repositories, deepening the + # demotion from -0.05 to -0.20 raised MRR and — because it only reorders docs + # relative to code, never below other docs — also improved documentation-seeking + # queries. -0.35 overshoots and collapses them, so the optimum is interior. + SourceRole.DOCUMENTATION: -0.20, + # Kept below documentation: a vendored or generated copy is the least useful + # answer of all. Measured as quality-neutral on the benchmark corpora, so this + # value preserves the role ordering rather than chasing a score. + SourceRole.GENERATED_VENDOR_BUILD: -0.25, SourceRole.UNKNOWN: 0.0, } _TEST_QUERY_PRIOR = 0.05 diff --git a/src/codebase_index/retrieval/searchers.py b/src/codebase_index/retrieval/searchers.py index a53874a..fbae798 100644 --- a/src/codebase_index/retrieval/searchers.py +++ b/src/codebase_index/retrieval/searchers.py @@ -292,20 +292,37 @@ def symbol_candidates( symbol_terms = list(dict.fromkeys((*terms, *expanded_terms))) if not terms: return [] - fuzzy_enabled = tuning.fuzzy_symbols and ( - len(terms) <= 3 or any(char.isupper() for char in query) - ) - term_set = set(terms) joined = "".join(terms) + + # Precise lookup first (exact -> prefix -> substring). It answers most real + # queries on its own, and knowing its yield is what lets fuzzy matching stay + # off the hot path. rows_by_key: dict[tuple, sqlite3.Row] = {} - if fuzzy_enabled: - for row in _fuzzy_symbol_rows(conn, query, limit=limit, kind=kind): - rows_by_key.setdefault((row["path"], row["line_start"], row["name"]), row) for needle in symbol_terms: for row in repo.symbol_search(conn, needle, limit=limit, kind=kind): rows_by_key.setdefault((row["path"], row["line_start"], row["name"]), row) + fuzzy_enabled = tuning.fuzzy_symbols and ( + len(terms) <= 3 or any(char.isupper() for char in query) + ) + if fuzzy_enabled and tuning.fuzzy_fallback_min > 0: + # Bounded edit distance over a lexical neighborhood is the single most + # expensive step in the pipeline. Spend it only when the precise lookup + # came up short: naming a real symbol, or simply returning enough + # candidates, means the query spelled its identifier well enough that + # fuzzing adds cost and noise rather than recall. + original_names = {term.casefold() for term in terms} + named_a_symbol = any( + (row["name"] or "").casefold() in original_names for row in rows_by_key.values() + ) + fuzzy_enabled = not ( + named_a_symbol or len(rows_by_key) >= tuning.fuzzy_fallback_min + ) + if fuzzy_enabled: + for row in _fuzzy_symbol_rows(conn, query, limit=limit, kind=kind): + rows_by_key.setdefault((row["path"], row["line_start"], row["name"]), row) + scored: list[tuple] = [] for row in rows_by_key.values(): subs = _name_subtokens(row["name"]) @@ -313,7 +330,12 @@ def symbol_candidates( covered = sum(1 for t in terms if t in subs or t.casefold() in name_l) expanded_covered = sum(1 for t in expanded_terms if t in subs or t.casefold() in name_l) tightness = len(subs & term_set) / len(subs) if subs else 0.0 - exact = (len(terms) == 1 and bool(row["is_exact"])) or ( + # Exactness is judged against the user's own terms, never against an + # expansion. `row["is_exact"]` is relative to whichever needle happened to + # retrieve the row, so a synonym needle ("config" for "configuration") used + # to mark a merely-related symbol as an exact match — worth +0.20 at rerank + # and HIGH confidence. Comparing names locally is needle-independent. + exact = (len(terms) == 1 and name_l == terms[0].casefold()) or ( bool(joined) and name_l == joined.casefold() ) fuzzy = 0.0 diff --git a/src/codebase_index/retrieval/tuning.py b/src/codebase_index/retrieval/tuning.py index ba52676..9898d6e 100644 --- a/src/codebase_index/retrieval/tuning.py +++ b/src/codebase_index/retrieval/tuning.py @@ -27,10 +27,21 @@ class RetrievalTuning: # --- candidate generation ------------------------------------------------ fuzzy_symbols: bool = True """Acronym / concatenation / edit-distance identifier matching in the symbol - retriever, so `userid` and `get user` both reach `getUserById`.""" + retriever, so `userid` and `get user` both reach `getUserById`. + + Runs only as a *recall fallback* (see `fuzzy_fallback_min`). Measured on 305 + queries across three repositories it changed no ranking metric while costing + ~20% of query latency, because real queries name identifiers correctly often + enough that the precise lookup already answers them. Gating it behind an + empty-handed precise lookup keeps the typo/acronym capability at ~zero cost. + """ fuzzy_threshold: float = 0.55 """Minimum identifier similarity for fuzzy symbol candidates.""" + fuzzy_fallback_min: int = 3 + """Run fuzzy identifier matching only when the precise symbol lookup returned + fewer than this many rows and found no exact match. 0 restores always-on.""" + query_expansion: bool = True """Down-weighted code-synonym expansion (auth->authentication, ...). Original terms always keep a strictly higher weight so precision is preserved.""" @@ -52,6 +63,16 @@ class RetrievalTuning: min_term_coverage: float = 0.5 """Fraction of salient query terms a chunk must contain under soft matching.""" + + candidate_pool_multiplier: int = 2 + """Over-fetch factor for the pre-selection candidate pool. + + Selection stages (dedup, MMR, per-file diversification) drop or reorder + candidates, so the pool must be wider than the requested limit or the page + ends up short. Previously this widening was an implicit side effect of + `dedup or mmr` being enabled; making it explicit is what let the ablation + show that the measured "dedup win" was really a pool-size win. + """ # --- selection ----------------------------------------------------------- mmr: bool = False """Maximal Marginal Relevance re-selection of the ranked list. @@ -72,6 +93,19 @@ class RetrievalTuning: """Prefer implementation files over their tests and over prose docs when both match a code question. Measured: tests/test_fusion.py outranked fusion.py.""" + file_agreement: bool = True + """Credit a candidate for retrievers that found its *file* at another locator. + + RRF fuses on (path, line-bucket), but a symbol hit at line 40 and a lexical + hit at line 120 in the same file land in different buckets, so cross-retriever + agreement — the entire point of fusion — never fired. This adds the missing + evidence back at reduced weight without double-counting a retriever already + counted at the candidate's own locator.""" + + file_agreement_weight: float = 0.4 + """Discount applied to same-file, different-locator evidence. Tuned on 305 + queries over three repositories; the 0.3-0.6 plateau peaks here.""" + # --- fixed parameters ---------------------------------------------------- rrf_k: int = 60 max_per_file: int = 3 @@ -92,6 +126,9 @@ def baseline(cls) -> RetrievalTuning: mmr=False, dedup=False, source_priors=False, + file_agreement=False, + # 1.7.0 had no over-fetch: the pool was exactly the requested page. + candidate_pool_multiplier=1, ) def without(self, flag: str) -> RetrievalTuning: @@ -99,6 +136,9 @@ def without(self, flag: str) -> RetrievalTuning: field_names = {f.name for f in fields(self)} if flag not in field_names: raise KeyError(f"unknown tuning flag: {flag!r}") + value = getattr(self, flag) + if not isinstance(value, bool): + raise TypeError(f"tuning flag {flag!r} is not a boolean signal") return replace(self, **{flag: False}) diff --git a/tests/eval/README.md b/tests/eval/README.md new file mode 100644 index 0000000..58903a1 --- /dev/null +++ b/tests/eval/README.md @@ -0,0 +1,99 @@ +# Retrieval evaluation + +Measures whether a ranking change actually helps, on more than one repository, with +enough statistical care to tell an improvement from a reshuffle. + +```bash +# shipped default vs the 1.7.0 baseline, on this repository +python tests/eval/run_eval.py + +# + one-signal-off ablation, with significance for every row +python tests/eval/run_eval.py --ablate + +# pool several repositories into one benchmark +python tests/eval/run_eval.py --queries self_repo_git \ + --corpus ../some-java-service:/tmp/svc.yml \ + --corpus ../some-ts-app:/tmp/app.yml --ablate +``` + +## Why the results are trustworthy + +**The index cannot grade its own homework.** Two independent ground-truth sources, +neither produced by the retriever: + +| Query set | Source | Size | Leakage | +|---|---|---|---| +| `self_repo` | Hand-written from the source tree | 36 | Query wording is human; answers verified against the tree | +| `self_repo_git` | Commit subject → files that commit changed | 87 | None: subjects live in git metadata, not in the corpus | + +`harness.validate_queries()` fails the run if any expected file no longer exists, +so a stale expectation is a loud error rather than a silently deflated score. + +**Benchmark scaffolding is excluded from the corpus it grades** (`CORPUS_EXCLUDES`). +The ground-truth YAML quotes every query verbatim; leaving it indexed would make it +the top lexical hit for almost every query. + +**One index per corpus, shared by every variant.** Ablation deltas measure ranking, +never indexing variance. + +**Every non-baseline row gets a significance test.** Query sets of this size have a +noise floor of several MRR points, so each metric is reported with a paired +bootstrap 95% CI and a paired permutation p-value (`metrics.paired_bootstrap_ci`, +`metrics.paired_permutation_p`, both seeded and therefore reproducible). Signals +ship on the strength of that test, not the sign of a delta. A one-query move on a +three-query category is noise, and the table is designed to make that visible. + +## Adding a corpus + +Tuning a ranker against one repository in one language produces numbers that only +move on that repository. `gen_queries.py` mints an objective query set from any git +repository, so a new corpus costs two commands: + +```bash +python tests/eval/gen_queries.py --repo ../some-java-service --out /tmp/svc.yml +python tests/eval/run_eval.py --corpus ../some-java-service:/tmp/svc.yml +``` + +The generator pairs a human-written commit subject with the files that commit +actually changed. It keeps only localised, described changes: no merges, reverts, +releases, version bumps or formatting commits; at most `--max-files` files, all of +which must still exist at HEAD; at least `--min-words` content words after the +Conventional Commits prefix is stripped; duplicate subjects collapse. Changelog-like +files are never accepted as answers because they paraphrase commit subjects, and any +commit touching `tests/eval/` is dropped so the benchmark cannot grade itself. + +Corpora used to validate the 1.9.0 ranking changes, beyond this repository: + +| Corpus | Language | Files | Queries | +|---|---|---|---| +| Civitas | Java | 944 | 64 | +| PoliternalSite | TypeScript / TSX | 443 | 118 | + +Those repositories are not vendored here — shipping someone else's source to run a +benchmark is not reproducible either. The generator is the reproducible part: point +it at any git repository and the protocol is identical. + +## What is measured + +Ranking quality is scored at **file** granularity, because the agent's unit of +decision is "which file do I open"; several hits inside one file collapse to its +best rank. + +- `recall@5`, `recall@10`, `precision@5`, `hit@3` +- `MRR`, `MAP`, `nDCG@10` +- `useful@budget` — the fraction of the answer that fits in the token budget, i.e. + what the agent can actually afford to read, not what is merely ranked somewhere +- `tokens` — mean snippet tokens actually emitted per query (results past the + budget carry no snippet and are not billed) +- `dup%` — fraction of returned results that near-duplicate an earlier result +- `p50/p95/p99` latency, in-process, excluding interpreter start-up + +## Files + +| File | Role | +|---|---| +| `run_eval.py` | CLI: corpora, variants, ablation, significance | +| `harness.py` | Index build, query execution, aggregation, pooling, tables | +| `metrics.py` | IR metrics + paired bootstrap / permutation tests | +| `gen_queries.py` | Ground-truth generator from git history | +| `queries/` | Checked-in query sets | diff --git a/tests/eval/gen_queries.py b/tests/eval/gen_queries.py new file mode 100644 index 0000000..1e0d0c8 --- /dev/null +++ b/tests/eval/gen_queries.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python3 +"""Derive an objective retrieval ground-truth set from a repository's git history. + + python tests/eval/gen_queries.py --repo . --out tests/eval/queries/self_repo_git.yml + +Why git history +--------------- +Hand-written query sets are small and expensive; sets generated by the retriever +grade its own homework. A commit is neither: a human wrote the subject line +("tolerate missing FastMCP type export") *and*, independently, changed a specific +set of files. That pairing is exactly the agent question we care about — "given a +description of a change, which files implement it?" — and it is recorded before +this tool exists. + +Leakage properties +------------------ +The query text lives in git metadata, not in the indexed corpus, so unlike +docstring-derived benchmarks (CodeSearchNet-style) the answer is not copied into +the document being retrieved. The one exception is changelog-style files, which +routinely paraphrase commit subjects; they are excluded both as corpus documents +(`CHANGELOG_EXCLUDES`) and as expected answers. + +Filtering +--------- +Commits survive only if they look like a *localised, described* change: + +* not a merge, revert, release, version bump, or pure-formatting commit; +* subject carries at least ``--min-words`` content words after the + Conventional Commits ``type(scope):`` prefix is stripped; +* touches between 1 and ``--max-files`` files that still exist at HEAD and are + plausibly indexable source or prose; +* does not touch benchmark scaffolding (which would grade itself). + +Duplicate subjects collapse to their first occurrence, so a repeated +"fix lint" contributes one query, not thirty. +""" + +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +from collections import OrderedDict +from pathlib import Path + +# Conventional Commits prefix: `type`, optional `(scope)`, optional `!`, colon. +_PREFIX_RE = re.compile(r"^\s*(?P[a-z]+)\s*(?:\((?P[^)]*)\))?\s*!?\s*:\s*", re.I) + +# Subjects that describe bookkeeping rather than a locatable code change. +_SKIP_SUBJECT_RE = re.compile( + r"^\s*(?:" + r"merge\b|revert\b|release\b|bump\b|" + r"v?\d+\.\d+(?:\.\d+)?\s*$|" + r"(?:chore|style|ci|build|deps)\s*(?:\([^)]*\))?\s*:\s*(?:bump|format|lint|typo|whitespace)" + r")", + re.I, +) + +# Conventional type -> eval category. Unlisted types fall back to "change". +_TYPE_CATEGORY = { + "feat": "feature", + "fix": "bug", + "perf": "perf", + "refactor": "refactor", + "test": "test", + "docs": "docs", +} + +# Files that may be *changed* by a commit but are never a useful retrieval answer. +_ANSWER_DENY_RE = re.compile( + r"(?:^|/)(?:" + r"CHANGELOG[^/]*|HISTORY[^/]*|NEWS[^/]*|" + r"package-lock\.json|yarn\.lock|poetry\.lock|Cargo\.lock|requirements\.lock|" + r"go\.sum|Gemfile\.lock|pnpm-lock\.yaml" + r")$", + re.I, +) + +# Extensions worth retrieving. Binary assets and generated bundles are excluded so +# an image-only commit does not become an unanswerable query. +_ANSWER_EXTS = frozenset( + """ + py pyi js mjs cjs jsx ts tsx java kt kts go rs rb php cs c h cc cpp hpp + swift scala clj ex exs erl hs ml dart lua r sql sh bash ps1 psm1 + md rst txt adoc yml yaml toml json xml gradle cfg ini + """.split() +) + +# Benchmark scaffolding: a commit touching it must not become a benchmark query. +_SCAFFOLD_RE = re.compile(r"(?:^|/)tests/(?:eval|benchmark_)", re.I) + +# Corpus documents that paraphrase commit subjects. `harness` applies these on top +# of its own excludes whenever a git-derived query set is evaluated. +CHANGELOG_EXCLUDES = ("CHANGELOG*", "HISTORY*", "NEWS*", "**/CHANGELOG*") + +_WORD_RE = re.compile(r"[A-Za-z][A-Za-z0-9_-]*") + + +def _git(repo: Path, *args: str) -> str: + proc = subprocess.run( + ["git", "-C", str(repo), *args], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + ) + if proc.returncode != 0: + raise RuntimeError(f"git {' '.join(args)} failed: {proc.stderr.strip()}") + return proc.stdout + + +def strip_prefix(subject: str) -> tuple[str, str]: + """Split a Conventional Commits subject into (category, human text).""" + match = _PREFIX_RE.match(subject) + if not match: + return "change", subject.strip() + rest = subject[match.end():].strip() + return _TYPE_CATEGORY.get(match.group("type").lower(), "change"), rest + + +def _is_answerable(path: str) -> bool: + if _ANSWER_DENY_RE.search(path) or _SCAFFOLD_RE.search(path): + return False + ext = path.rsplit(".", 1)[-1].lower() if "." in path.rsplit("/", 1)[-1] else "" + return ext in _ANSWER_EXTS + + +def harvest( + repo: Path, + *, + max_commits: int, + max_files: int, + min_words: int, +) -> list[dict]: + """Return ground-truth records mined from `repo`'s history, newest first.""" + tracked = {p for p in _git(repo, "ls-files").splitlines() if p} + # `%x00` separates subject from the name-only file list without colliding with + # either; commit records are separated by a sentinel line. + raw = _git( + repo, + "log", + "--no-merges", + f"--max-count={max_commits}", + "--name-only", + "--pretty=format:%x01%H%x00%s", + "--diff-filter=ACMR", + ) + + seen_subjects: OrderedDict[str, dict] = OrderedDict() + for record in raw.split("\x01"): + record = record.strip("\n") + if not record: + continue + header, _, body = record.partition("\n") + sha, _, subject = header.partition("\x00") + subject = subject.strip() + if not subject or _SKIP_SUBJECT_RE.match(subject): + continue + + files = [line.strip() for line in body.splitlines() if line.strip()] + if any(_SCAFFOLD_RE.search(f) for f in files): + continue + answers = sorted({f for f in files if f in tracked and _is_answerable(f)}) + if not answers or len(answers) > max_files: + continue + + category, text = strip_prefix(subject) + # Backticks and quotes in subjects are prose decoration, not query intent. + # Collapse the resulting gaps so the query reads like something a user typed. + text = " ".join(text.replace("`", " ").replace('"', " ").split()).strip(" .;:") + words = [w for w in _WORD_RE.findall(text) if len(w) > 1] + if len(words) < min_words: + continue + key = text.casefold() + if key in seen_subjects: + continue + seen_subjects[key] = { + "query": text, + "category": category, + "expected_files": answers, + "commit": sha[:12], + } + return list(seen_subjects.values()) + + +def _yaml_scalar(text: str) -> str: + """Quote a scalar for YAML. Only double quotes and backslashes need escaping.""" + return '"' + text.replace("\\", "\\\\").replace('"', '\\"') + '"' + + +def render(records: list[dict], *, repo: Path, source: str) -> str: + lines = [ + "# Auto-generated by tests/eval/gen_queries.py -- DO NOT EDIT BY HAND.", + "#", + f"# corpus: {source}", + f"# queries: {len(records)}", + "#", + "# Ground truth is objective and leak-free: the query is a human-written commit", + "# subject (git metadata, absent from the indexed corpus) and the expected files", + "# are the files that commit actually changed. Neither side is produced by the", + "# retriever. Regenerate after history changes; `commit` records provenance.", + "", + ] + for rec in records: + lines.append(f"- query: {_yaml_scalar(rec['query'])}") + lines.append(f" category: {rec['category']}") + lines.append(f" commit: {rec['commit']}") + lines.append(" expected_files:") + lines.extend(f" - {path}" for path in rec["expected_files"]) + lines.append("") + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument("--repo", default=".", help="git repository to mine") + ap.add_argument("--out", required=True, help="destination YAML path") + ap.add_argument("--max-commits", type=int, default=4000) + ap.add_argument( + "--max-files", + type=int, + default=4, + help="drop sweeping commits: a 60-file refactor is not a retrieval target", + ) + ap.add_argument("--min-words", type=int, default=3) + ap.add_argument("--limit", type=int, default=0, help="keep at most N queries (0 = all)") + args = ap.parse_args(argv) + + repo = Path(args.repo).resolve() + records = harvest( + repo, + max_commits=args.max_commits, + max_files=args.max_files, + min_words=args.min_words, + ) + if args.limit > 0: + records = records[: args.limit] + if not records: + print("no usable commits found", file=sys.stderr) + return 1 + + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(render(records, repo=repo, source=repo.name), encoding="utf-8") + print(f"wrote {len(records)} queries to {out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/eval/harness.py b/tests/eval/harness.py index 279a086..58b9e2c 100644 --- a/tests/eval/harness.py +++ b/tests/eval/harness.py @@ -23,6 +23,11 @@ from codebase_index.config import Config from codebase_index.indexer.pipeline import build_index +from codebase_index.retrieval.diversity import ( + normalize_code_tokens, + simhash_distance, + token_fingerprint, +) from codebase_index.retrieval.pipeline import search from codebase_index.retrieval.tuning import RetrievalTuning from codebase_index.storage.db import Database @@ -48,6 +53,11 @@ class QueryOutcome: ranked_files: list[str] returned: list[tuple[str, int]] latency_ms: float + total_tokens: int = 0 + """Snippet tokens actually handed to the agent for this query.""" + duplicates: int = 0 + """Results whose snippet near-duplicates an earlier result in the same page.""" + n_results: int = 0 @dataclass @@ -68,7 +78,17 @@ class EvalReport: p95_ms: float p99_ms: float mean_ms: float + mean_tokens: float = 0.0 + """Mean tokens of snippet context returned per query — the agent's actual bill.""" + duplicate_rate: float = 0.0 + """Fraction of returned results that near-duplicate an earlier result.""" + mean_candidates: float = 0.0 + """Mean results returned per query, before the agent reads anything.""" per_category: dict[str, float] = field(default_factory=dict) + per_query: dict[str, list[float]] = field(default_factory=dict) + """Per-query metric vectors, in query order. Required for paired significance + testing: aggregate deltas alone cannot separate a real gain from resampling + noise on a set this size.""" def as_row(self) -> dict[str, float | str | int]: return { @@ -82,6 +102,8 @@ def as_row(self) -> dict[str, float | str | int]: "P@5": self.precision_at_5, "MAP": self.map_score, "useful@budget": self.useful_context, + "tokens": self.mean_tokens, + "dup%": self.duplicate_rate * 100.0, "p50_ms": self.p50_ms, "p95_ms": self.p95_ms, "p99_ms": self.p99_ms, @@ -164,15 +186,42 @@ def run_query( ranked_files: list[str] = [] returned: list[tuple[str, int]] = [] - for r in payload.get("results", []): + total_tokens = 0 + duplicates = 0 + fingerprints: list[int] = [] + results = payload.get("results", []) + for r in results: p = _normalise(r["path"]) # File-level ranking: the agent's unit of decision is "which file do I # open", so multiple hits inside one file collapse to its best rank. if p not in ranked_files: ranked_files.append(p) - returned.append((p, int(r.get("token_est") or 0))) - return QueryOutcome(query=q, ranked_files=ranked_files, returned=returned, - latency_ms=latency_ms) + tokens = int(r.get("token_est") or 0) + returned.append((p, tokens)) + snippet = r.get("snippet") + # Only snippets are actually placed in the agent's context. Results past + # the budget still carry a `token_est` for their (unread) chunk, so summing + # it would report a bill the agent never pays. + if snippet: + total_tokens += tokens + # Duplicate rate is measured on what the agent actually receives, using the + # same fingerprint the pipeline suppresses with — so the number reports the + # residual noise after selection, not the raw candidate overlap. + chunk_tokens = normalize_code_tokens(snippet or "") + if chunk_tokens: + fingerprint = token_fingerprint(chunk_tokens) + if any(simhash_distance(fingerprint, seen) <= 3 for seen in fingerprints): + duplicates += 1 + fingerprints.append(fingerprint) + return QueryOutcome( + query=q, + ranked_files=ranked_files, + returned=returned, + latency_ms=latency_ms, + total_tokens=total_tokens, + duplicates=duplicates, + n_results=len(results), + ) def evaluate( @@ -198,45 +247,74 @@ def evaluate( token_budget=token_budget).latency_ms ) - def mean(fn) -> float: - vals = [fn(o) for o in outcomes] + # Named metric extractors, so aggregate values and the per-query vectors used + # for significance testing can never be computed two different ways. + scorers = { + "recall@5": lambda o: metrics.recall_at_k(o.ranked_files, o.query.expected_files, 5), + "recall@10": lambda o: metrics.recall_at_k(o.ranked_files, o.query.expected_files, 10), + "MRR": lambda o: metrics.reciprocal_rank(o.ranked_files, o.query.expected_files), + "nDCG@10": lambda o: metrics.ndcg_at_k(o.ranked_files, o.query.expected_files, 10), + "hit@3": lambda o: metrics.hit_rate_at_k(o.ranked_files, o.query.expected_files, 3), + "P@5": lambda o: metrics.precision_at_k(o.ranked_files, o.query.expected_files, 5), + "MAP": lambda o: metrics.average_precision(o.ranked_files, o.query.expected_files), + "useful@budget": lambda o: metrics.useful_context_at_budget( + o.returned, o.query.expected_files, token_budget + ), + } + per_query = {name: [fn(o) for o in outcomes] for name, fn in scorers.items()} + + def mean(name: str) -> float: + vals = per_query[name] return statistics.fmean(vals) if vals else 0.0 per_category: dict[str, list[float]] = {} - for o in outcomes: - per_category.setdefault(o.query.category, []).append( - metrics.reciprocal_rank(o.ranked_files, o.query.expected_files) - ) + for o, rr in zip(outcomes, per_query["MRR"]): + per_category.setdefault(o.query.category, []).append(rr) + + returned_total = sum(o.n_results for o in outcomes) + duplicate_total = sum(o.duplicates for o in outcomes) return EvalReport( label=label, n_queries=len(outcomes), - recall_at_5=mean(lambda o: metrics.recall_at_k(o.ranked_files, o.query.expected_files, 5)), - recall_at_10=mean(lambda o: metrics.recall_at_k(o.ranked_files, o.query.expected_files, 10)), - mrr=mean(lambda o: metrics.reciprocal_rank(o.ranked_files, o.query.expected_files)), - ndcg_at_10=mean(lambda o: metrics.ndcg_at_k(o.ranked_files, o.query.expected_files, 10)), - hit_rate_at_3=mean(lambda o: metrics.hit_rate_at_k(o.ranked_files, o.query.expected_files, 3)), - precision_at_5=mean(lambda o: metrics.precision_at_k(o.ranked_files, o.query.expected_files, 5)), - map_score=mean(lambda o: metrics.average_precision(o.ranked_files, o.query.expected_files)), - useful_context=mean( - lambda o: metrics.useful_context_at_budget( - o.returned, o.query.expected_files, token_budget - ) - ), + recall_at_5=mean("recall@5"), + recall_at_10=mean("recall@10"), + mrr=mean("MRR"), + ndcg_at_10=mean("nDCG@10"), + hit_rate_at_3=mean("hit@3"), + precision_at_5=mean("P@5"), + map_score=mean("MAP"), + useful_context=mean("useful@budget"), p50_ms=metrics.percentile(latencies, 50), p95_ms=metrics.percentile(latencies, 95), p99_ms=metrics.percentile(latencies, 99), mean_ms=statistics.fmean(latencies) if latencies else 0.0, + mean_tokens=( + statistics.fmean([o.total_tokens for o in outcomes]) if outcomes else 0.0 + ), + duplicate_rate=(duplicate_total / returned_total) if returned_total else 0.0, + mean_candidates=(returned_total / len(outcomes)) if outcomes else 0.0, per_category={ cat: statistics.fmean(vals) for cat, vals in sorted(per_category.items()) }, + per_query=per_query, ) -def format_table(reports: Sequence[EvalReport], *, baseline: EvalReport | None = None) -> str: +_WIDE_COLS = ("tokens", "dup%") + + +def format_table( + reports: Sequence[EvalReport], + *, + baseline: EvalReport | None = None, + columns: Sequence[str] | None = None, +) -> str: """Render reports as a Markdown table, with deltas against `baseline`.""" - cols = ["label", "recall@5", "recall@10", "MRR", "nDCG@10", "hit@3", "P@5", - "MAP", "useful@budget", "p50_ms", "p95_ms"] + cols = list(columns) if columns else [ + "label", "recall@5", "recall@10", "MRR", "nDCG@10", "hit@3", "P@5", + "MAP", "useful@budget", "tokens", "dup%", "p50_ms", "p95_ms", + ] lines = ["| " + " | ".join(cols) + " |", "|" + "|".join("---" for _ in cols) + "|"] for rep in reports: @@ -245,13 +323,97 @@ def format_table(reports: Sequence[EvalReport], *, baseline: EvalReport | None = for c in cols: v = row[c] if isinstance(v, float): - cell = f"{v:.3f}" if c.endswith("_ms") is False else f"{v:.1f}" + coarse = c.endswith("_ms") or c in _WIDE_COLS + cell = f"{v:.1f}" if coarse else f"{v:.3f}" if baseline is not None and rep is not baseline: delta = v - float(baseline.as_row()[c]) - if abs(delta) >= 0.0005: - cell += f" ({delta:+.3f})" if not c.endswith("_ms") else f" ({delta:+.1f})" + if abs(delta) >= (0.05 if coarse else 0.0005): + cell += f" ({delta:+.1f})" if coarse else f" ({delta:+.3f})" cells.append(cell) else: cells.append(str(v)) lines.append("| " + " | ".join(cells) + " |") return "\n".join(lines) + + +def pool(reports: Sequence[EvalReport], *, label: str) -> EvalReport: + """Concatenate per-corpus reports into one, weighting every query equally. + + Averaging the per-corpus averages would let a 36-query corpus outvote a + 118-query one; pooling the raw per-query vectors keeps one query = one vote and + is what makes the significance tests valid across a mixed-language benchmark. + """ + reports = list(reports) + if not reports: + raise ValueError("cannot pool an empty report list") + + per_query: dict[str, list[float]] = {} + for rep in reports: + for name, values in rep.per_query.items(): + per_query.setdefault(name, []).extend(values) + + def mean(name: str) -> float: + vals = per_query.get(name, []) + return statistics.fmean(vals) if vals else 0.0 + + total = sum(rep.n_queries for rep in reports) or 1 + + def weighted(attr: str) -> float: + return sum(getattr(rep, attr) * rep.n_queries for rep in reports) / total + + per_category: dict[str, list[float]] = {} + for rep in reports: + for cat, value in rep.per_category.items(): + per_category.setdefault(cat, []).append(value) + + return EvalReport( + label=label, + n_queries=total, + recall_at_5=mean("recall@5"), + recall_at_10=mean("recall@10"), + mrr=mean("MRR"), + ndcg_at_10=mean("nDCG@10"), + hit_rate_at_3=mean("hit@3"), + precision_at_5=mean("P@5"), + map_score=mean("MAP"), + useful_context=mean("useful@budget"), + # Latency percentiles cannot be pooled from percentiles; report the + # query-weighted mean of each, which is honest about being an approximation + # only when corpora differ wildly in size. + p50_ms=weighted("p50_ms"), + p95_ms=weighted("p95_ms"), + p99_ms=weighted("p99_ms"), + mean_ms=weighted("mean_ms"), + mean_tokens=weighted("mean_tokens"), + duplicate_rate=weighted("duplicate_rate"), + mean_candidates=weighted("mean_candidates"), + per_category={c: statistics.fmean(v) for c, v in sorted(per_category.items())}, + per_query=per_query, + ) + + +def format_significance( + baseline: EvalReport, + candidate: EvalReport, + *, + resamples: int = 5000, +) -> str: + """Paired bootstrap CI + permutation p-value per metric, candidate vs baseline.""" + lines = [ + f"paired comparison: {candidate.label} vs {baseline.label} " + f"(n={min(baseline.n_queries, candidate.n_queries)})", + f"| {'metric':13} | {'delta':>8} | {'95% CI':>19} | {'p':>6} | sig |", + "|" + "|".join("---" for _ in range(5)) + "|", + ] + for name, base_values in baseline.per_query.items(): + cand_values = candidate.per_query.get(name, []) + if len(cand_values) != len(base_values): + continue + deltas = [c - b for b, c in zip(base_values, cand_values)] + delta, lo, hi = metrics.paired_bootstrap_ci(deltas, resamples=resamples) + p = metrics.paired_permutation_p(deltas, resamples=resamples) + lines.append( + f"| {name:13} | {delta:+8.4f} | [{lo:+.4f},{hi:+.4f}] | {p:6.3f} | " + f"{'yes' if p < 0.05 else 'no':3} |" + ) + return "\n".join(lines) diff --git a/tests/eval/metrics.py b/tests/eval/metrics.py index 5ab1cef..fac5df0 100644 --- a/tests/eval/metrics.py +++ b/tests/eval/metrics.py @@ -14,6 +14,7 @@ from __future__ import annotations import math +import random from collections.abc import Iterable, Sequence @@ -127,3 +128,80 @@ def percentile(values: Sequence[float], pct: float) -> float: ordered = sorted(values) idx = math.ceil(pct / 100.0 * len(ordered)) - 1 return ordered[min(max(idx, 0), len(ordered) - 1)] + + +# --- significance ----------------------------------------------------------- +# +# A 36-query set cannot distinguish a +0.03 MRR improvement from noise, and eyeballing +# a delta column is how ranking systems accumulate changes that never actually helped. +# Both routines below are paired (same queries, two systems) and seeded, so a reported +# interval is reproducible rather than a different number on every run. + +_BOOTSTRAP_SEED = 20260902 + + +def paired_bootstrap_ci( + deltas: Sequence[float], + *, + resamples: int = 5000, + confidence: float = 0.95, + seed: int = _BOOTSTRAP_SEED, +) -> tuple[float, float, float]: + """Return (mean delta, lower, upper) for a per-query difference vector. + + Resampling queries — not scores — is what makes the interval answer the + question we care about: would this improvement survive a different, equally + plausible sample of user questions? + """ + values = list(deltas) + n = len(values) + if n == 0: + return 0.0, 0.0, 0.0 + observed = math.fsum(values) / n + if n == 1: + return observed, observed, observed + + rng = random.Random(seed) + means = [] + for _ in range(max(1, resamples)): + total = 0.0 + for _ in range(n): + total += values[rng.randrange(n)] + means.append(total / n) + means.sort() + tail = (1.0 - confidence) / 2.0 + lo = means[min(len(means) - 1, int(tail * len(means)))] + hi = means[min(len(means) - 1, int((1.0 - tail) * len(means)))] + return observed, lo, hi + + +def paired_permutation_p( + deltas: Sequence[float], + *, + resamples: int = 5000, + seed: int = _BOOTSTRAP_SEED, +) -> float: + """Two-sided paired permutation test on a per-query difference vector. + + Under the null the two systems are interchangeable on each query, so flipping + the sign of any subset of deltas is equally likely. Reports the fraction of + sign-flipped resamples at least as extreme as what we measured. + """ + values = list(deltas) + n = len(values) + if n == 0: + return 1.0 + observed = abs(math.fsum(values) / n) + if observed == 0.0: + return 1.0 + + rng = random.Random(seed + 1) + extreme = 0 + trials = max(1, resamples) + for _ in range(trials): + total = 0.0 + for value in values: + total += value if rng.getrandbits(1) else -value + if abs(total / n) >= observed - 1e-12: + extreme += 1 + return extreme / trials diff --git a/tests/eval/queries/self_repo_git.yml b/tests/eval/queries/self_repo_git.yml new file mode 100644 index 0000000..e1a6d18 --- /dev/null +++ b/tests/eval/queries/self_repo_git.yml @@ -0,0 +1,627 @@ +# Auto-generated by tests/eval/gen_queries.py -- DO NOT EDIT BY HAND. +# +# corpus: codebase-index +# queries: 87 +# +# Ground truth is objective and leak-free: the query is a human-written commit +# subject (git metadata, absent from the indexed corpus) and the expected files +# are the files that commit actually changed. Neither side is produced by the +# retriever. Regenerate after history changes; `commit` records provenance. + +- query: "tolerate missing FastMCP type export" + category: bug + commit: 96e034b997bc + expected_files: + - src/codebase_index/mcp/server.py + +- query: "stabilize lint policy across Ruff upgrades" + category: bug + commit: 0e42fe36b83f + expected_files: + - .github/workflows/ci.yml + - pyproject.toml + +- query: "mark PyPI distribution as shipped (1.6.0) in roadmap + upgrade plan" + category: docs + commit: 731b7af9f2f3 + expected_files: + - docs/PRODUCT_UPGRADE_PLAN.md + - docs/ROADMAP.md + +- query: "PyPI-first install across QUICKSTART/INSTALLATION/MCP/FAQ" + category: docs + commit: 601743ebb48e + expected_files: + - docs/FAQ.md + - docs/INSTALLATION.md + - docs/MCP.md + - docs/QUICKSTART.md + +- query: "feature pip install codebase-index from PyPI" + category: docs + commit: c782c5acc6f8 + expected_files: + - README.md + +- query: "publish to PyPI via Trusted Publishing (OIDC)" + category: change + commit: ec7a614547d4 + expected_files: + - .github/workflows/release.yml + - docs/RELEASE_CHECKLIST.md + +- query: "inject snippet compactor; emit skeletonized/elided_lines" + category: feature + commit: f096e534e76d + expected_files: + - src/codebase_index/retrieval/budget.py + - tests/test_budget.py + +- query: "make_compactor factory with intent->context policy" + category: feature + commit: 433bb5f6a088 + expected_files: + - src/codebase_index/retrieval/skeleton.py + - tests/test_skeleton.py + +- query: "markdown heading + structured key classifiers" + category: feature + commit: 2840ece8a6b2 + expected_files: + - src/codebase_index/retrieval/skeleton.py + - tests/test_skeleton.py + +- query: "code classifier + compact() with focus, guard, raw fallback" + category: feature + commit: be5b7dc608cb + expected_files: + - src/codebase_index/retrieval/skeleton.py + - tests/test_skeleton.py + +- query: "render_skeleton collapses keep/elide mask into markers" + category: feature + commit: f2e2f74e0f26 + expected_files: + - src/codebase_index/retrieval/skeleton.py + - tests/test_skeleton.py + +- query: "implementation plan for snippet skeletonization" + category: docs + commit: e2a60dca91ef + expected_files: + - docs/superpowers/plans/2026-06-24-snippet-skeletonization.md + +- query: "design spec for snippet skeletonization + content-aware rendering" + category: docs + commit: f6529c6d20d9 + expected_files: + - docs/superpowers/specs/2026-06-24-snippet-skeletonization-design.md + +- query: "key analytics by stable content keys, not volatile symbol ids" + category: bug + commit: 16d2980eb3e5 + expected_files: + - src/codebase_index/graph/analysis.py + - src/codebase_index/storage/repo.py + +- query: "discount test paths when labelling communities" + category: feature + commit: 61e89b49fc68 + expected_files: + - src/codebase_index/graph/analysis.py + - tests/test_analysis.py + +- query: "keep enable_vectors on Database (mypy/CI fix)" + category: bug + commit: e996fc7685cd + expected_files: + - src/codebase_index/storage/db.py + +- query: "sync roadmap with shipped MCP, add trust-model callout, changelog" + category: docs + commit: e92dd61c49bd + expected_files: + - README.md + - docs/PRODUCT_UPGRADE_PLAN.md + - docs/ROADMAP.md + - docs/SECURITY.md + +- query: "make bootstrap and golden tests pass on bare CI runners" + category: bug + commit: 07dd01b6196f + expected_files: + - tests/test_bootstrap.py + - tests/test_cli_golden.py + +- query: "single accurate repo layout, drop stale module claims" + category: docs + commit: ca285ebd565f + expected_files: + - docs/ARCHITECTURE.md + +- query: "shared CLI/MCP service layer; close surface drift" + category: refactor + commit: f767e1978afc + expected_files: + - src/codebase_index/cli.py + - src/codebase_index/mcp/server.py + - src/codebase_index/service.py + - tests/test_service.py + +- query: "assert _PARSE_CONFIG is set before use in worker parse" + category: bug + commit: 8e9f8b701bea + expected_files: + - src/codebase_index/indexer/pipeline.py + +- query: "sync version to 1.2.2 across plugin manifest and lock" + category: bug + commit: 980b31b7049e + expected_files: + - .claude-plugin/plugin.json + +- query: "define versioning and fork workflow" + category: docs + commit: 4948d7a5e111 + expected_files: + - AGENTS.md + - CONTRIBUTING.md + +- query: "add auto-update, skill-update, and skill-rollback" + category: feature + commit: 104fef57d890 + expected_files: + - src/codebase_index/cli.py + - src/codebase_index/scaffold.py + - src/codebase_index/skill_update.py + +- query: "add --json option to symbol, refs, and impact commands" + category: bug + commit: 799eb0f4b5c2 + expected_files: + - src/codebase_index/cli.py + - tests/test_impact_cli.py + +- query: "add Python 3.11+ pipx troubleshooting" + category: docs + commit: 732a3831bb4f + expected_files: + - README.md + - pyproject.toml + +- query: "add mcp to dev deps + tests to pass coverage gate" + category: bug + commit: 4281d6436431 + expected_files: + - .claude-plugin/plugin.json + - pyproject.toml + - tests/test_mcp_server.py + +- query: "add generalized skill research plan" + category: docs + commit: 99bbe0e0bc79 + expected_files: + - docs/superpowers/plans/2026-05-29-generalize-skill-research-discipline.md + +- query: "fix Windows/Git Bash path and PATH assumptions" + category: test + commit: b6b9d8e675ac + expected_files: + - tests/test_bootstrap.py + +- query: "enforce coverage floor + register slow marker" + category: test + commit: 6855f959b05e + expected_files: + - pyproject.toml + - tests/test_search_cli.py + +- query: "finalize for 0.1.0 (status, roadmap M9, clean-install recipe, examples)" + category: docs + commit: 1b0e4bc35c73 + expected_files: + - README.md + - docs/INSTALLATION.md + - docs/ROADMAP.md + +- query: "release pipeline (build, smoke, GitHub release, PyPI trusted publishing)" + category: change + commit: 03333d012f79 + expected_files: + - .github/workflows/release.yml + +- query: "finalize package metadata + add 0.1.0 changelog" + category: change + commit: 31dc30f32f7b + expected_files: + - pyproject.toml + +- query: "medium-repo index+search budget smoke test (--runslow)" + category: test + commit: ac4a47dfbb2a + expected_files: + - tests/conftest.py + - tests/test_perf_smoke.py + +- query: "add --json normalizer + golden compare helper" + category: test + commit: 1b0f2b6ab337 + expected_files: + - tests/__init__.py + - tests/golden_utils.py + - tests/test_cli_golden.py + +- query: "mark M8 complete (update/watch/hooks/doctor) + installation updates" + category: docs + commit: 0fa021b808bf + expected_files: + - docs/INSTALLATION.md + - docs/ROADMAP.md + - tests/test_watcher.py + +- query: "report hooks, cache-gitignore, and freshness (--strict)" + category: feature + commit: 08dd286458f0 + expected_files: + - src/codebase_index/cli.py + - src/codebase_index/doctor.py + - tests/test_doctor.py + +- query: "implement watch command (graceful without the watch extra)" + category: feature + commit: 1811ba3b6302 + expected_files: + - src/codebase_index/cli.py + - tests/test_watcher.py + +- query: "debounced live indexer + lazy watchdog observer" + category: feature + commit: aaea59ef8552 + expected_files: + - src/codebase_index/watch/watcher.py + - tests/test_watcher.py + +- query: "init --with-hooks auto-merges the update hook into settings.json" + category: feature + commit: 56548ae938da + expected_files: + - src/codebase_index/cli.py + - tests/test_init_cli.py + +- query: "add honest benchmark results to README" + category: docs + commit: 6b7a4aabe0e7 + expected_files: + - README.md + +- query: "idempotent PostToolUse hook merge + enabled_hooks reader" + category: feature + commit: 92122ddc1f09 + expected_files: + - src/codebase_index/scaffold.py + - tests/test_hooks_merge.py + +- query: "label doc chunks in retrieval results" + category: feature + commit: 94a15982d5f4 + expected_files: + - src/codebase_index/retrieval/searchers.py + - src/codebase_index/storage/repo.py + +- query: "integrate doc_chunks into indexer pipeline for FTS5" + category: feature + commit: d542629d3d5e + expected_files: + - src/codebase_index/indexer/pipeline.py + - src/codebase_index/storage/repo.py + +- query: "implement incremental update (--since/--all, --json, --quiet)" + category: feature + commit: ee441669740c + expected_files: + - src/codebase_index/cli.py + - tests/test_update_cli.py + +- query: "add doc_chunks extractor for markdown, tests, docstrings, exceptions, config" + category: feature + commit: 6830de973a99 + expected_files: + - src/codebase_index/indexer/doc_chunks.py + - tests/test_doc_chunks.py + +- query: "add cold/warm benchmark, build time, DB size metrics" + category: test + commit: 59ce7809a0c3 + expected_files: + - tests/test_benchmark_comparison.py + +- query: "incremental update_index (mtime fast-path + sha verify + prune)" + category: feature + commit: 009dddb7aa71 + expected_files: + - src/codebase_index/indexer/pipeline.py + - tests/test_update.py + +- query: "add quality benchmark test runner with pyyaml dependency" + category: test + commit: 8e6c15ea1aed + expected_files: + - pyproject.toml + - tests/test_quality_benchmark.py + +- query: "add expected_answers.yml quality benchmark fixture" + category: test + commit: e61a42b9ae0b + expected_files: + - tests/fixtures/expected_answers.yml + +- query: "rename token efficiency to output compression ratio" + category: bug + commit: 2b92e3c67a4d + expected_files: + - tests/test_benchmark_comparison.py + +- query: "add fingerprints accessor for incremental update" + category: feature + commit: 44f418339ac8 + expected_files: + - src/codebase_index/storage/repo.py + - tests/test_storage.py + +- query: "honor freshness contract in search responses" + category: feature + commit: a9da97735792 + expected_files: + - src/codebase_index/cli.py + - src/codebase_index/retrieval/pipeline.py + - src/codebase_index/retrieval/searchers.py + - tests/test_search_cli.py + +- query: "compute index freshness (git fast-path + mtime diff)" + category: feature + commit: 09abfee14e57 + expected_files: + - src/codebase_index/indexer/freshness.py + - src/codebase_index/storage/repo.py + - tests/test_freshness.py + - tests/test_storage.py + +- query: "implement init (skill + config + gitignore + hooks example)" + category: feature + commit: 8a9f1e689e0a + expected_files: + - src/codebase_index/cli.py + - tests/test_init_cli.py + +- query: "skill/config/gitignore materialization helpers" + category: feature + commit: c218f2cb9e9b + expected_files: + - src/codebase_index/scaffold.py + - tests/test_scaffold.py + +- query: "ship skill template (scripts + hooks) in the wheel" + category: feature + commit: 997bd8d42dcd + expected_files: + - pyproject.toml + - src/codebase_index/skill_template/SKILL.md + - src/codebase_index/skill_template/examples/hooks/settings.json + - tests/test_packaging.py + +- query: "real vector mode + backend resolution + external warning" + category: feature + commit: 8b3a5a4ad7ce + expected_files: + - src/codebase_index/cli.py + - tests/test_search_cli.py + +- query: "wire optional vector retriever + vector mode" + category: feature + commit: 8bffadf4a65d + expected_files: + - src/codebase_index/retrieval/pipeline.py + +- query: "vector retriever over sqlite-vec KNN" + category: feature + commit: 2b7e25c9bf06 + expected_files: + - src/codebase_index/retrieval/searchers.py + - tests/test_vector_search.py + +- query: "per-intent vector weights + fusion richness" + category: feature + commit: ac8d4eee19f4 + expected_files: + - src/codebase_index/retrieval/fusion.py + - src/codebase_index/retrieval/intent.py + - tests/test_fusion.py + - tests/test_intent.py + +- query: "embed + store chunk vectors when embeddings enabled" + category: feature + commit: 8f2f7652fa19 + expected_files: + - src/codebase_index/indexer/pipeline.py + - tests/conftest.py + - tests/test_pipeline_vectors.py + +- query: "sqlite-vec vector store (load, upsert, KNN, meta)" + category: feature + commit: 4df3653b97df + expected_files: + - src/codebase_index/storage/db.py + - src/codebase_index/storage/repo.py + - tests/test_vectors_storage.py + +- query: "gated external backend with injectable transport" + category: feature + commit: 84e91fdc56e1 + expected_files: + - src/codebase_index/embeddings/external.py + - tests/test_embeddings_external.py + +- query: "lazy on-device sentence-transformers backend" + category: feature + commit: 45b3c55089d1 + expected_files: + - src/codebase_index/embeddings/local.py + - tests/test_embeddings_local.py + +- query: "backend protocol, noop default, gating factory" + category: feature + commit: 2c7cb9879142 + expected_files: + - src/codebase_index/embeddings/__init__.py + - src/codebase_index/embeddings/backend.py + - src/codebase_index/embeddings/noop.py + - tests/test_embeddings_backend.py + +- query: "embeddings toggle/model participate in config_hash" + category: feature + commit: d0c9563d9a83 + expected_files: + - src/codebase_index/config.py + - tests/test_config.py + +- query: "mark M5 complete + graph-edge language recipe" + category: docs + commit: 34a0256b19ed + expected_files: + - docs/LANGUAGES.md + - docs/ROADMAP.md + +- query: "wire impact command" + category: feature + commit: 6e00c0d925d5 + expected_files: + - src/codebase_index/cli.py + - tests/test_impact_cli.py + +- query: "bounded impact BFS + impact_lookup" + category: feature + commit: 9934925e7575 + expected_files: + - src/codebase_index/graph/expand.py + - tests/test_graph.py + +- query: "run graph resolution pass after build" + category: feature + commit: db340ac41fa0 + expected_files: + - src/codebase_index/indexer/pipeline.py + - tests/fixtures/sample_repo/src/api/service.py + - tests/test_pipeline.py + +- query: "cross-file edge resolution + degree denormalization" + category: feature + commit: 44ed7e1fdf1f + expected_files: + - src/codebase_index/graph/builder.py + - tests/test_graph.py + +- query: "graph resolution + traversal accessors" + category: feature + commit: 2a72244bf64f + expected_files: + - src/codebase_index/storage/repo.py + - tests/test_storage.py + +- query: "extract import + inheritance edges" + category: feature + commit: 76f7c35ff267 + expected_files: + - src/codebase_index/parsers/treesitter.py + - tests/test_treesitter.py + +- query: "import + inheritance edge queries (py/js/ts)" + category: feature + commit: 0d3c2b44791d + expected_files: + - src/codebase_index/parsers/languages.py + - tests/test_languages.py + +- query: "acceptance — hybrid outranks single retrievers + budget enforced" + category: test + commit: 17501214265f + expected_files: + - tests/conftest.py + - tests/test_hybrid_ranking.py + +- query: "wire hybrid search and explain commands" + category: feature + commit: 988dc112ee96 + expected_files: + - src/codebase_index/cli.py + - tests/test_search_cli.py + +- query: "JSON and compact Markdown renderers for dict payloads" + category: feature + commit: 7f14840957bb + expected_files: + - src/codebase_index/output/json.py + - src/codebase_index/output/markdown.py + - tests/test_output.py + +- query: "pipeline orchestrator with confidence and fallback" + category: feature + commit: dca27fafa5fa + expected_files: + - src/codebase_index/retrieval/pipeline.py + - tests/test_pipeline_search.py + +- query: "greedy token budgeting with redaction" + category: feature + commit: 21f9ba031014 + expected_files: + - src/codebase_index/retrieval/budget.py + - tests/test_budget.py + +- query: "explainable feature reranker with reason strings" + category: feature + commit: b4199dd1e532 + expected_files: + - src/codebase_index/retrieval/rerank.py + - tests/test_rerank.py + +- query: "reciprocal rank fusion with per-intent weights" + category: feature + commit: 878788c0e234 + expected_files: + - src/codebase_index/retrieval/fusion.py + - tests/test_fusion.py + +- query: "path/symbol/fts retrievers with uniform Candidate" + category: feature + commit: 799e3a75351d + expected_files: + - src/codebase_index/retrieval/searchers.py + - tests/test_searchers.py + +- query: "path_search and symbol_search accessors" + category: feature + commit: b4f48e65e979 + expected_files: + - src/codebase_index/storage/repo.py + - tests/test_repo_search.py + +- query: "deterministic seeded_index fixture" + category: test + commit: 41c1f678c26a + expected_files: + - tests/conftest.py + - tests/test_seeded_index.py + +- query: "rule-first intent detection with per-intent weights" + category: feature + commit: 4a253036b51c + expected_files: + - src/codebase_index/retrieval/intent.py + - tests/test_intent.py + +- query: "shared Candidate and IntentPlan types" + category: feature + commit: 8955935159dd + expected_files: + - src/codebase_index/retrieval/types.py + - tests/test_retrieval_types.py diff --git a/tests/eval/run_eval.py b/tests/eval/run_eval.py index d2a5da8..3cad5fa 100644 --- a/tests/eval/run_eval.py +++ b/tests/eval/run_eval.py @@ -1,12 +1,31 @@ #!/usr/bin/env python3 -"""Run the retrieval benchmark and ablation sweep. +"""Run the retrieval benchmark, ablation sweep, and significance tests. - python tests/eval/run_eval.py # baseline vs shipped default - python tests/eval/run_eval.py --ablate # + one-signal-off sweep + python tests/eval/run_eval.py # baseline vs shipped default + python tests/eval/run_eval.py --ablate # + one-signal-off sweep + python tests/eval/run_eval.py --corpus : # add an external corpus python tests/eval/run_eval.py --repo --queries -The index is built once and shared by every variant, so reported deltas isolate -ranking changes from indexing variance. +Each corpus is indexed once and shared by every variant, so reported deltas +isolate ranking changes from indexing variance. + +Why more than one corpus +------------------------ +The shipped query sets are Python and self-hosted. Tuning a ranker against a +single repository in a single language is how you get numbers that only move on +that repository. Pass `--corpus` to pool additional repositories into one +benchmark; `tests/eval/gen_queries.py` mints an objective query set for any git +repository, so adding one is a two-command operation: + + python tests/eval/gen_queries.py --repo ../some-java-service --out /tmp/svc.yml + python tests/eval/run_eval.py --corpus ../some-java-service:/tmp/svc.yml --ablate + +Why significance +---------------- +Query sets of this size have a noise floor of several MRR points. Every non-baseline +row is accompanied by a paired bootstrap CI and permutation p-value against the +shipped default, because a delta column alone cannot tell an improvement from a +reshuffle. Signals are kept on the strength of that test, not the sign of the delta. """ from __future__ import annotations @@ -15,7 +34,7 @@ import json import sys import tempfile -from dataclasses import fields +from dataclasses import fields, replace from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[2] @@ -27,75 +46,138 @@ # Signals swept by --ablate. Numeric parameters are excluded: turning off a bool # answers "does this signal earn its complexity", which is the decision we make. -ABLATABLE = ("soft_lexical", "query_expansion", "fuzzy_symbols", "graph_source", - "mmr", "dedup", "source_priors") +ABLATABLE = ( + "soft_lexical", + "query_expansion", + "fuzzy_symbols", + "graph_source", + "mmr", + "dedup", + "source_priors", + "file_agreement", +) + + +def _parse_corpus(spec: str) -> tuple[Path, str]: + """Split `:`, tolerating a Windows drive letter.""" + head, sep, tail = spec.rpartition(":") + if not sep or (len(head) == 1 and head.isalpha()): + raise argparse.ArgumentTypeError( + f"--corpus expects ':', got {spec!r}" + ) + return Path(head).resolve(), tail def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--repo", default=str(REPO_ROOT), - help="corpus to index and query (default: this repository)") + help="primary corpus to index and query (default: this repository)") ap.add_argument("--queries", default="self_repo", help="query set name under tests/eval/queries or a path") + ap.add_argument("--corpus", action="append", default=[], metavar="REPO:QUERIES", + help="additional corpus to pool into the benchmark (repeatable)") ap.add_argument("--ablate", action="store_true", help="also run a one-signal-off sweep") ap.add_argument("--limit", type=int, default=harness.DEFAULT_LIMIT) ap.add_argument("--token-budget", type=int, default=harness.DEFAULT_BUDGET) ap.add_argument("--repeats", type=int, default=3, help="latency repeats per query (quality is unaffected)") + ap.add_argument("--resamples", type=int, default=5000, + help="bootstrap/permutation resamples for significance testing") ap.add_argument("--json", dest="as_json", action="store_true") args = ap.parse_args(argv) - root = Path(args.repo).resolve() - queries = harness.load_queries(args.queries) - problems = harness.validate_queries(queries, root) - if problems: - print("Ground-truth validation FAILED:", file=sys.stderr) - for p in problems: - print(f" - {p}", file=sys.stderr) - return 2 - - print(f"corpus: {root}") - print(f"queries: {len(queries)} from {args.queries}") - print("building index (once, shared by all variants)...", flush=True) + corpora: list[tuple[str, Path, str]] = [ + (Path(args.repo).resolve().name, Path(args.repo).resolve(), args.queries) + ] + for spec in args.corpus: + root, queries = _parse_corpus(spec) + corpora.append((root.name, root, queries)) + + loaded: list[tuple[str, Path, list]] = [] + for name, root, query_spec in corpora: + queries = harness.load_queries(query_spec) + problems = harness.validate_queries(queries, root) + if problems: + print(f"Ground-truth validation FAILED for {name}:", file=sys.stderr) + for p in problems: + print(f" - {p}", file=sys.stderr) + return 2 + loaded.append((name, root, queries)) + print(f"corpus: {name} ({root}) — {len(queries)} queries from {query_spec}") + + total = sum(len(q) for _, _, q in loaded) + print(f"pooled: {total} queries across {len(loaded)} corpora") + + variants: list[tuple[str, RetrievalTuning]] = [ + ("baseline (1.7.0)", RetrievalTuning.baseline()), + ("default (all signals)", RetrievalTuning()), + ] + if args.ablate: + default = RetrievalTuning() + known = {f.name for f in fields(default)} + for flag in ABLATABLE: + if flag in known: + variants.append((f" -{flag}", default.without(flag))) + + # variant label -> per-corpus reports, pooled after every corpus is measured. + collected: dict[str, list[harness.EvalReport]] = {label: [] for label, _ in variants} + per_corpus_default: dict[str, harness.EvalReport] = {} with tempfile.TemporaryDirectory() as tmp: - db = harness.build_corpus_index(root, Path(tmp) / "eval.sqlite") - try: - variants: list[tuple[str, RetrievalTuning]] = [ - ("baseline (1.7.0)", RetrievalTuning.baseline()), - ("default (all signals)", RetrievalTuning()), - ] - if args.ablate: - default = RetrievalTuning() - known = {f.name for f in fields(default)} - for flag in ABLATABLE: - if flag in known: - variants.append((f" -{flag}", default.without(flag))) - - reports = [] - for label, tuning in variants: - print(f"running: {label}", flush=True) - reports.append( - harness.evaluate( + for name, root, queries in loaded: + print(f"building index for {name} (once, shared by all variants)...", flush=True) + db = harness.build_corpus_index(root, Path(tmp) / f"{name}.sqlite") + try: + for label, tuning in variants: + print(f" running: {label}", flush=True) + report = harness.evaluate( db.conn, queries, tuning=tuning, label=label, limit=args.limit, token_budget=args.token_budget, repeats=args.repeats, ) - ) - finally: - db.close() + collected[label].append(report) + if label == "default (all signals)": + # Relabel so the per-corpus table identifies the repository + # rather than repeating the variant name on every row. + per_corpus_default[name] = replace(report, label=name) + finally: + db.close() + + reports = [harness.pool(collected[label], label=label) for label, _ in variants] if args.as_json: - print(json.dumps([r.as_row() for r in reports], indent=2)) + payload = { + "pooled": [r.as_row() for r in reports], + "per_corpus_default": {k: v.as_row() for k, v in per_corpus_default.items()}, + "per_category_default": reports[1].per_category, + } + print(json.dumps(payload, indent=2)) return 0 print() print(harness.format_table(reports, baseline=reports[0])) + + default = reports[1] + print() + print(harness.format_significance(reports[0], default, resamples=args.resamples)) + + if len(reports) > 2: + print() + print("Ablation significance (each row vs the shipped default):") + for rep in reports[2:]: + print() + print(harness.format_significance(default, rep, resamples=args.resamples)) + + if len(loaded) > 1: + print() + print("Per-corpus (default config):") + print(harness.format_table(list(per_corpus_default.values()))) + print() print("Per-category MRR (default config):") - for cat, val in reports[1].per_category.items(): + for cat, val in default.per_category.items(): print(f" {cat:14} {val:.3f}") return 0 diff --git a/tests/test_diversity.py b/tests/test_diversity.py index 6956d87..f277a3d 100644 --- a/tests/test_diversity.py +++ b/tests/test_diversity.py @@ -1,4 +1,10 @@ -from codebase_index.retrieval.diversity import deduplicate, mmr_select +from codebase_index.retrieval.diversity import ( + deduplicate, + mmr_select, + normalize_code_tokens, + simhash_distance, + token_fingerprint, +) from codebase_index.retrieval.types import Candidate @@ -47,3 +53,68 @@ def test_selection_handles_empty_and_singleton_inputs_without_mutation(): assert deduplicate([]) == [] assert mmr_select([], 3, lambda_=0.5) == [] assert mmr_select([candidate], 3, lambda_=0.5) == [candidate] + + +def _reference_fingerprint(tokens: tuple[str, ...]) -> int: + """Naive per-occurrence SimHash: the definition the fast path must match.""" + import hashlib + + if not tokens: + return 0 + weights = [0] * 64 + for token in tokens: + hashed = int.from_bytes( + hashlib.blake2b(str(token).encode("utf-8"), digest_size=8).digest(), "big" + ) + for bit in range(64): + weights[bit] += 1 if (hashed >> bit) & 1 else -1 + fingerprint = 0 + for bit, weight in enumerate(weights): + if weight >= 0: + fingerprint |= 1 << bit + return fingerprint & ((1 << 64) - 1) + + +def test_fingerprint_matches_per_occurrence_definition(): + """Folding duplicate tokens by count is an optimisation, not a behaviour change.""" + cases = [ + (), + ("a",), + ("a", "a", "a"), + ("def", "add", "a", "b", "return", "a", "b"), + ("ключ", "значение", "ключ"), # non-ASCII identifiers + ("x" * 4096,), # pathological single token + tuple(f"tok{i % 7}" for i in range(500)), # heavy repetition + ] + for tokens in cases: + assert token_fingerprint(tokens) == _reference_fingerprint(tokens), tokens + + +def test_fingerprint_is_stable_across_calls_and_order_sensitive_only_by_content(): + tokens = normalize_code_tokens("def add(a, b):\n return a + b") + assert token_fingerprint(tokens) == token_fingerprint(tokens) + assert simhash_distance(token_fingerprint(tokens), token_fingerprint(tokens)) == 0 + + +def test_fingerprint_handles_empty_and_none_content(): + assert token_fingerprint(None) == 0 + assert token_fingerprint("") == 0 + assert token_fingerprint(()) == 0 + + +def test_deduplicate_keeps_untokenizable_candidates_distinct(): + """Comment-only snippets normalise to no tokens; they must not collapse together.""" + left = _candidate("a.py", 0.9, "# just a comment") + right = _candidate("b.py", 0.8, "# another comment") + assert deduplicate([left, right]) == [left, right] + + +def test_deduplicate_is_deterministic_for_identical_scores(): + first = _candidate("a.py", 0.5, "def f(): return 1") + second = _candidate("b.py", 0.5, "def f(): return 1") + assert deduplicate([first, second]) == deduplicate([first, second]) == [first] + + +def test_deduplicate_collapses_a_long_duplicate_run_to_one(): + dupes = [_candidate(f"f{i}.py", 1.0 - i / 100, "def f(a):\n return a") for i in range(12)] + assert deduplicate(dupes) == [dupes[0]] diff --git a/tests/test_fusion.py b/tests/test_fusion.py index 693068f..1c6d30f 100644 --- a/tests/test_fusion.py +++ b/tests/test_fusion.py @@ -64,3 +64,82 @@ def test_fuse_dedupes_repeated_source_hits_in_one_bucket(): fused = fuse({"fts": fts}, weights={"fts": 1.0}, k=60) assert len(fused) == 1 assert fused[0].score <= 1.0 # single best-rank contribution, not summed 3x + + +def test_file_agreement_credits_cross_locator_evidence(): + """Two retrievers pointing at one file from different regions is agreement. + + Line bucketing alone cannot merge them — a symbol defined at line 5 and a + lexical hit at line 400 are genuinely different locators — so without this the + file's evidence stayed split across two weakly-scored candidates. + """ + lists = { + "fts": [_c("other.py", "fts", 0.9), Candidate( + path="target.py", line_start=400, line_end=420, source="fts", score=0.5)], + "symbol": [Candidate( + path="target.py", line_start=5, line_end=9, source="symbol", score=0.8)], + } + weights = {"fts": 1.0, "symbol": 1.0} + split = fuse(lists, weights=weights, k=60, file_agreement=0.0) + joined = fuse(lists, weights=weights, k=60, file_agreement=0.4) + + assert split[0].path == "other.py" # rank-0 lexical hit wins outright + assert joined[0].path == "target.py" # ...until both retrievers are heard + + +def test_file_agreement_zero_reproduces_plain_rrf(): + """The default must be opt-in-able away for a clean single-signal ablation.""" + lists = { + "fts": [_c("a.py", "fts", 0.9), _c("b.py", "fts", 0.4)], + "symbol": [Candidate(path="a.py", line_start=90, line_end=95, + source="symbol", score=0.8)], + } + weights = {"fts": 1.0, "symbol": 1.0} + plain = fuse(lists, weights=weights, k=60, file_agreement=0.0) + assert [(c.path, round(c.score, 9)) for c in plain] == [ + (c.path, round(c.score, 9)) + for c in fuse(lists, weights=weights, k=60) + ] + + +def test_file_agreement_does_not_double_count_own_source(): + """A retriever already counted at this locator must not be counted again.""" + only_fts = {"fts": [_c("a.py", "fts", 0.9)]} + weights = {"fts": 1.0} + without = fuse(only_fts, weights=weights, k=60, file_agreement=0.0) + with_agreement = fuse(only_fts, weights=weights, k=60, file_agreement=1.0) + assert without[0].score == with_agreement[0].score + + +def test_file_agreement_ignores_zero_weight_sources(): + lists = { + "fts": [_c("a.py", "fts", 0.9)], + "path": [Candidate(path="a.py", line_start=800, line_end=800, + source="path", score=1.0)], + } + weights = {"fts": 1.0, "path": 0.0} + scored = fuse(lists, weights=weights, k=60, file_agreement=1.0) + assert len(scored) == 1 + assert scored[0].score == fuse({"fts": lists["fts"]}, weights=weights, k=60)[0].score + + +def test_file_agreement_uses_best_rank_per_source_and_is_deterministic(): + """Repeated same-file hits from one source contribute once, at their best rank.""" + lists = { + "fts": [_c("x.py", "fts", 0.9)], + "symbol": [ + Candidate(path="x.py", line_start=100, line_end=110, source="symbol", score=0.7), + Candidate(path="x.py", line_start=200, line_end=210, source="symbol", score=0.6), + ], + } + weights = {"fts": 1.0, "symbol": 1.0} + runs = [ + [(c.path, c.line_start, round(c.score, 12)) for c in + fuse(lists, weights=weights, k=60, file_agreement=0.4)] + for _ in range(3) + ] + assert runs[0] == runs[1] == runs[2] + top = fuse(lists, weights=weights, k=60, file_agreement=0.4)[0] + # fts rank 0 (1.0) + 0.4 * symbol best rank 0 (1.0) == 1.4, not 1.4 + a second + # symbol contribution from the line-200 hit. + assert top.score == 1.0 + 0.4 * 1.0 diff --git a/tests/test_gen_queries.py b/tests/test_gen_queries.py new file mode 100644 index 0000000..e6e0ebe --- /dev/null +++ b/tests/test_gen_queries.py @@ -0,0 +1,152 @@ +"""Tests for the git-history ground-truth generator. + +The generator is benchmark infrastructure, so its failure mode is silent: a bad +filter produces a plausible-looking query set that measures the wrong thing. +These tests pin the filtering rules that keep the set objective and leak-free. +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from eval import gen_queries # noqa: E402 + + +def _git(repo: Path, *args: str, **kw) -> None: + subprocess.run( + ["git", "-C", str(repo), *args], + check=True, + capture_output=True, + text=True, + **kw, + ) + + +@pytest.fixture +def repo(tmp_path: Path) -> Path: + root = tmp_path / "corpus" + root.mkdir() + _git(root, "init", "-q") + _git(root, "config", "user.email", "dev@example.test") + _git(root, "config", "user.name", "Dev") + return root + + +def _commit(repo: Path, subject: str, files: dict[str, str]) -> None: + for rel, body in files.items(): + target = repo / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(body, encoding="utf-8") + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "-m", subject) + + +def _harvest(repo: Path, **kw): + params = {"max_commits": 100, "max_files": 4, "min_words": 3} + params.update(kw) + return gen_queries.harvest(repo, **params) + + +def test_conventional_prefix_becomes_category_and_is_stripped(repo): + _commit(repo, "feat(auth): refresh the access token", {"src/auth.py": "def refresh(): ..."}) + (record,) = _harvest(repo) + assert record["query"] == "refresh the access token" + assert record["category"] == "feature" + assert record["expected_files"] == ["src/auth.py"] + assert record["commit"] + + +def test_bookkeeping_commits_are_excluded(repo): + _commit(repo, "feat: add the parser entry point", {"src/parse.py": "x = 1"}) + _commit(repo, "release: v2.1.0", {"src/parse.py": "x = 2"}) + _commit(repo, "chore(deps): bump pytest", {"src/parse.py": "x = 3"}) + _commit(repo, "Revert \"add the parser entry point\"", {"src/parse.py": "x = 4"}) + assert [r["query"] for r in _harvest(repo)] == ["add the parser entry point"] + + +def test_sweeping_commits_are_dropped(repo): + _commit( + repo, + "refactor: rename everything everywhere at once", + {f"src/mod{i}.py": "y = 1" for i in range(9)}, + ) + assert _harvest(repo, max_files=4) == [] + assert len(_harvest(repo, max_files=20)) == 1 + + +def test_changelog_is_never_an_expected_answer(repo): + """Changelogs paraphrase commit subjects; grading against them measures leakage.""" + _commit( + repo, + "fix: guard against an empty token budget", + {"CHANGELOG.md": "- guard against an empty token budget", "src/budget.py": "b = 0"}, + ) + (record,) = _harvest(repo) + assert record["expected_files"] == ["src/budget.py"] + + +def test_commit_touching_only_unanswerable_files_is_dropped(repo): + _commit(repo, "docs: add the architecture diagram", {"assets/diagram.png": "binary-ish"}) + assert _harvest(repo) == [] + + +def test_benchmark_scaffolding_never_grades_itself(repo): + _commit( + repo, + "test: extend the retrieval evaluation set", + {"tests/eval/queries/extra.yml": "- query: x", "src/thing.py": "z = 1"}, + ) + assert _harvest(repo) == [] + + +def test_short_subjects_and_duplicates_are_filtered(repo): + _commit(repo, "fix: typo", {"src/a.py": "a = 1"}) + _commit(repo, "fix: correct the retry backoff", {"src/b.py": "b = 1"}) + _commit(repo, "fix: correct the retry backoff", {"src/c.py": "c = 1"}) + records = _harvest(repo) + assert [r["query"] for r in records] == ["correct the retry backoff"] + # First occurrence wins, so provenance stays stable across regeneration. + assert records[0]["expected_files"] == ["src/c.py"] + + +def test_deleted_files_are_not_expected_answers(repo): + _commit(repo, "feat: add a temporary shim layer", {"src/shim.py": "s = 1"}) + (repo / "src/shim.py").unlink() + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "-m", "refactor: drop the shim layer entirely") + # The add-commit's only answer no longer exists at HEAD, so it must not ship. + assert all("src/shim.py" not in r["expected_files"] for r in _harvest(repo)) + + +def test_rendered_yaml_round_trips_through_the_harness(repo, tmp_path): + _commit( + repo, + 'fix: escape a "quoted" path\\name safely', + {"src/escape.py": "e = 1"}, + ) + out = tmp_path / "generated.yml" + assert gen_queries.main(["--repo", str(repo), "--out", str(out)]) == 0 + + from eval import harness + + queries = harness.load_queries(out) + assert len(queries) == 1 + # Quotes are stripped as prose decoration; a backslash is content and must + # survive YAML escaping intact. + assert queries[0].query == "escape a quoted path\\name safely" + assert harness.validate_queries(queries, repo) == [] + + +def test_strip_prefix_handles_unprefixed_and_breaking_subjects(): + assert gen_queries.strip_prefix("just do the thing") == ("change", "just do the thing") + assert gen_queries.strip_prefix("feat!: drop python 3.9") == ("feature", "drop python 3.9") + assert gen_queries.strip_prefix("perf(index): speed up walking") == ( + "perf", + "speed up walking", + ) diff --git a/tests/test_priors.py b/tests/test_priors.py index 574f0ec..9a1dd88 100644 --- a/tests/test_priors.py +++ b/tests/test_priors.py @@ -1,6 +1,7 @@ from __future__ import annotations from codebase_index.retrieval.priors import ( + MAX_ABS_PRIOR, SourceRole, classify_source_role, is_test_intent, @@ -38,8 +39,15 @@ def test_prior_preserves_order_and_is_bounded(): } assert values[SourceRole.IMPLEMENTATION] > values[SourceRole.TEST] assert values[SourceRole.IMPLEMENTATION] > values[SourceRole.DOCUMENTATION] + # Prose is demoted below tests but stays above vendored/generated output: a + # design note can still answer a question, a minified bundle never can. + assert values[SourceRole.TEST] > values[SourceRole.DOCUMENTATION] assert values[SourceRole.DOCUMENTATION] > values[SourceRole.GENERATED_VENDOR_BUILD] - assert all(-0.15 <= value <= 0.15 for value in values.values()) + # Priors must stay tiebreakers. The bound is read from the module so the + # contract cannot drift by editing one number in isolation, and is asserted + # small here so widening the constant is itself a visible change. + assert MAX_ABS_PRIOR <= 0.25 + assert all(abs(value) <= MAX_ABS_PRIOR for value in values.values()) assert ( source_role_prior("src/search.py", query="where is search implemented") == values[SourceRole.IMPLEMENTATION] diff --git a/tests/test_symbol_ranking.py b/tests/test_symbol_ranking.py index 93ec481..ed3bf45 100644 --- a/tests/test_symbol_ranking.py +++ b/tests/test_symbol_ranking.py @@ -186,3 +186,79 @@ def test_symbol_synonym_expansion_retrieves_alias(tmp_path): assert cands[0].symbol == "Config" assert cands[0].exact_symbol is False + + +def test_synonym_match_is_never_reported_as_an_exact_symbol(tmp_path): + """`is_exact` from SQL is relative to whichever needle found the row. + + A synonym needle ("config" expanded from "configuration") matches `Config` + exactly, which used to mark the candidate `exact_symbol=True` — worth +0.20 at + rerank and an unconditional HIGH confidence. Exactness must be judged against + the terms the user actually typed. + """ + db = _db(tmp_path) + conn = db.conn + f = _insert_file(conn, path="src/config.py", lang="python", mtime_ns=1) + _insert_symbol( + conn, f, name="Config", kind="class", line_start=1, line_end=12, + signature="class Config", + ) + conn.commit() + + cands = symbol_candidates(conn, "configuration", limit=10) + exact = symbol_candidates(conn, "Config", limit=10) + db.close() + + assert cands[0].symbol == "Config" + assert cands[0].exact_symbol is False + # ...while naming the symbol outright still registers as exact. + assert exact[0].exact_symbol is True + + +def test_fuzzy_runs_only_when_the_precise_lookup_comes_up_short(tmp_path): + """Fuzzy matching is a recall fallback, not a default cost. + + A typo that no precise lookup can resolve must still reach the symbol, but a + query that names an indexed symbol must not pay for edit-distance scoring. + """ + db = _db(tmp_path) + conn = db.conn + f = _insert_file(conn, path="src/user.py", lang="python", mtime_ns=1) + _insert_symbol( + conn, f, name="getUserById", kind="function", line_start=1, line_end=9, + signature="def getUserById(user_id)", + ) + conn.commit() + + typo = symbol_candidates(conn, "getUsrById", limit=10) + named = symbol_candidates(conn, "getUserById", limit=10) + db.close() + + assert [c.symbol for c in typo] == ["getUserById"] # fallback still fires + assert named[0].exact_symbol is True # precise path satisfied + + +def test_fuzzy_fallback_can_be_restored_to_always_on(tmp_path): + from dataclasses import replace + + from codebase_index.retrieval.tuning import DEFAULT_TUNING + + db = _db(tmp_path) + conn = db.conn + f = _insert_file(conn, path="src/user.py", lang="python", mtime_ns=1) + for name in ("getUserById", "getUserByName", "getUserByEmail"): + _insert_symbol( + conn, f, name=name, kind="function", line_start=1, line_end=9, + signature=f"def {name}(x)", + ) + conn.commit() + + always_on = replace(DEFAULT_TUNING, fuzzy_fallback_min=0) + gated = symbol_candidates(conn, "GUBI", limit=10) + ungated = symbol_candidates(conn, "GUBI", limit=10, tuning=always_on) + db.close() + + # The acronym reaches the symbol either way; the gate only decides whether the + # edit-distance pass runs when precise results already exist. + assert "getUserById" in {c.symbol for c in ungated} + assert all(c.exact_symbol is False for c in gated + ungated) diff --git a/tests/test_tuning_contract.py b/tests/test_tuning_contract.py new file mode 100644 index 0000000..b335997 --- /dev/null +++ b/tests/test_tuning_contract.py @@ -0,0 +1,141 @@ +"""The ablation contract: every ranking signal must be independently switchable. + +These tests protect the property that makes `tests/eval/run_eval.py --ablate` +meaningful. If a signal stops being individually disablable, or if `baseline()` +drifts away from the pre-1.8.0 pipeline, every published benchmark delta silently +becomes unverifiable. +""" + +from __future__ import annotations + +from dataclasses import fields + +import pytest + +from codebase_index.retrieval.pipeline import search +from codebase_index.retrieval.tuning import DEFAULT_TUNING, RetrievalTuning + + +def test_every_boolean_signal_is_independently_disablable(): + tuning = RetrievalTuning() + booleans = [f.name for f in fields(tuning) if isinstance(getattr(tuning, f.name), bool)] + assert booleans, "a tuning with no boolean signals cannot be ablated" + for flag in booleans: + assert getattr(tuning.without(flag), flag) is False + + +def test_without_rejects_unknown_and_non_boolean_fields(): + with pytest.raises(KeyError): + RetrievalTuning().without("no_such_flag") + # Silently "disabling" a numeric knob by setting it to False would produce a + # nonsense ablation row rather than an error. + with pytest.raises(TypeError): + RetrievalTuning().without("rrf_k") + + +def test_baseline_disables_every_post_170_signal(): + """`baseline()` is the honest 'before' column; it must stay signal-free.""" + baseline = RetrievalTuning.baseline() + for flag in ( + "fuzzy_symbols", + "query_expansion", + "graph_source", + "soft_lexical", + "mmr", + "dedup", + "source_priors", + "file_agreement", + ): + assert getattr(baseline, flag) is False, flag + # 1.7.0 took the page size verbatim; over-fetching is a later addition. + assert baseline.candidate_pool_multiplier == 1 + + +def test_shipped_defaults_are_the_measured_configuration(): + """Guards against a default drifting without a benchmark run behind it.""" + assert DEFAULT_TUNING.file_agreement is True + assert DEFAULT_TUNING.file_agreement_weight == pytest.approx(0.4) + assert DEFAULT_TUNING.candidate_pool_multiplier == 2 + assert DEFAULT_TUNING.fuzzy_fallback_min == 3 + # Signals measured as neutral-or-harmful stay off by default. + assert DEFAULT_TUNING.graph_source is False + assert DEFAULT_TUNING.mmr is False + + +def test_tuning_is_immutable_and_hashable(): + tuning = RetrievalTuning() + with pytest.raises(Exception): + tuning.file_agreement_weight = 0.9 # type: ignore[misc] + assert hash(tuning) == hash(RetrievalTuning()) + + +@pytest.mark.parametrize("multiplier", [1, 2, 5]) +def test_pool_multiplier_does_not_change_the_page_size(seeded_index, multiplier): + """Over-fetching feeds selection; it must never inflate the returned page.""" + from dataclasses import replace + + payload = search( + seeded_index.conn, + "token", + mode="hybrid", + limit=3, + token_budget=1500, + no_fallback=True, + tuning=replace(RetrievalTuning(), candidate_pool_multiplier=multiplier), + ) + assert len(payload["results"]) <= 3 + + +def test_baseline_and_default_both_answer_a_direct_symbol_query(seeded_index): + for tuning in (RetrievalTuning.baseline(), RetrievalTuning()): + payload = search( + seeded_index.conn, + "refresh_access_token", + mode="hybrid", + limit=10, + token_budget=1500, + no_fallback=True, + tuning=tuning, + ) + assert payload["results"][0]["path"] == "src/auth/token.py" + + +@pytest.mark.parametrize( + "query", + [ + "", + " ", + "the and of for", # stopwords only + "!!! ??? ***", # punctuation only + "\"; DROP TABLE files;--", # would be an FTS/SQL injection if unquoted + "a" * 512, # pathological identifier + "поиск конфигурации", # non-ASCII + "getUserById" * 40, # pathological camelCase run + ], +) +def test_degenerate_queries_are_answered_without_raising(seeded_index, query): + payload = search( + seeded_index.conn, + query, + mode="hybrid", + limit=5, + token_budget=800, + no_fallback=True, + ) + assert isinstance(payload["results"], list) + assert payload["confidence"] in {"high", "medium", "low"} + + +def test_ranking_is_deterministic_across_repeated_identical_queries(seeded_index): + def run() -> list[tuple[str, int, int]]: + payload = search( + seeded_index.conn, + "how does token refresh work", + mode="hybrid", + limit=10, + token_budget=1500, + no_fallback=True, + ) + return [(r["path"], r["line_start"], r["line_end"]) for r in payload["results"]] + + assert run() == run() == run()