From edc4b9aa88100d09abdec00d43b5327f475c94d3 Mon Sep 17 00:00:00 2001 From: denfry Date: Wed, 2 Sep 2026 10:15:48 +0300 Subject: [PATCH 1/3] feat(retrieval): score cross-retriever agreement; cut query latency in half MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RRF fused on (path, line-bucket), so a symbol definition at line 40 and a lexical hit at line 120 in the same file produced two weakly-scored candidates instead of one strong one — two retrievers agreeing on a file never reached the score at all. Each candidate now also receives, at file_agreement_weight (0.4), the RRF mass of every retriever that found its file at another locator, excluding retrievers already counted at that locator so nothing double-counts. Deepen the documentation prior from -0.05 to -0.20: prose describing a feature matches a natural-language question more literally than the code implementing it, so design notes were displacing the modules they describe. Because this only reorders prose relative to code, documentation-seeking queries improved too (category MRR 0.579 -> 0.612); -0.35 overshoots and collapses them. Generated and vendored paths move to -0.25 to stay the least-preferred role, and MAX_ABS_PRIOR pins the invariant that priors remain tiebreakers. Fuzzy identifier matching becomes a recall fallback, running only when the precise lookup named no symbol and returned fewer than fuzzy_fallback_min rows. It moved no ranking metric across 305 queries while accounting for ~20% of query latency; typo and acronym recall is unchanged because those are exactly the queries where the precise lookup comes up empty. Make candidate over-fetch explicit as candidate_pool_multiplier instead of an implicit side effect of enabling dedup or MMR. That alone showed the quality previously credited to SimHash dedup was really the wider pool; dedup is kept for what it measurably does, which is cutting the duplicate rate of returned snippets from ~1.6% to ~0%. SimHash itself now folds repeated tokens by multiplicity and caches token digests, bit-for-bit identical but ~2x faster. Also fixes two latent bugs: exactness was read from SQL relative to whichever needle retrieved the row, so a synonym expansion ("config" for "configuration") marked an unrelated symbol as an exact match, worth +0.20 at rerank and an unconditional high confidence; and duplicate suppression broke ties with ">", handing the slot to whichever equal-scoring copy arrived last and contradicting its own documented "ties favor input order". Measured against 1.8.0 over 305 queries on Python, Java and TypeScript corpora: MRR +0.027, MAP +0.028, nDCG@10 +0.024, recall@5 +0.031 (all p < 0.001), with p50 latency 78.6ms -> 51.2ms and p95 193.2ms -> 94.9ms. --- src/codebase_index/retrieval/diversity.py | 59 +++++++-- src/codebase_index/retrieval/fusion.py | 38 ++++++ src/codebase_index/retrieval/pipeline.py | 19 ++- src/codebase_index/retrieval/priors.py | 20 ++- src/codebase_index/retrieval/searchers.py | 38 ++++-- src/codebase_index/retrieval/tuning.py | 42 ++++++- tests/test_diversity.py | 73 ++++++++++- tests/test_fusion.py | 79 ++++++++++++ tests/test_priors.py | 10 +- tests/test_symbol_ranking.py | 76 ++++++++++++ tests/test_tuning_contract.py | 141 ++++++++++++++++++++++ 11 files changed, 564 insertions(+), 31 deletions(-) create mode 100644 tests/test_tuning_contract.py diff --git a/src/codebase_index/retrieval/diversity.py b/src/codebase_index/retrieval/diversity.py index dbbb2a5..d94be5d 100644 --- a/src/codebase_index/retrieval/diversity.py +++ b/src/codebase_index/retrieval/diversity.py @@ -10,7 +10,9 @@ import hashlib import math +from collections import Counter from collections.abc import Callable, Iterable, Sequence +from functools import lru_cache from .types import Candidate @@ -90,6 +92,18 @@ def normalize_code_tokens(content: str | None) -> tuple[str, ...]: return tuple(tokens) +@lru_cache(maxsize=1 << 16) +def _token_digest(token: str) -> int: + """64-bit blake2b digest of one token. Pure, so caching is safe and bounded. + + Source chunks repeat identifiers and keywords heavily both inside a chunk and + across the candidate pool, and hashing dominated fingerprint cost. + """ + return int.from_bytes( + hashlib.blake2b(token.encode("utf-8"), digest_size=8).digest(), "big", signed=False + ) + + def token_fingerprint(tokens: Iterable[str] | str | None) -> int: """Compute a stable unsigned 64-bit SimHash for tokens or source content.""" if tokens is None: @@ -100,12 +114,14 @@ def token_fingerprint(tokens: Iterable[str] | str | None) -> int: if not values: return 0 + # Weighting distinct tokens by multiplicity is arithmetically identical to + # summing each occurrence, and collapses the per-occurrence hash and 64-bit + # accumulation loop to per-distinct-token work. Output is bit-for-bit unchanged. weights = [0] * 64 - for token in values: - digest = hashlib.blake2b(str(token).encode("utf-8"), digest_size=8).digest() - hashed = int.from_bytes(digest, "big", signed=False) + for token, count in Counter(str(value) for value in values).items(): + hashed = _token_digest(token) for bit in range(64): - weights[bit] += 1 if (hashed >> bit) & 1 else -1 + weights[bit] += count if (hashed >> bit) & 1 else -count fingerprint = 0 for bit, weight in enumerate(weights): @@ -133,8 +149,14 @@ def _score_value(candidate: Candidate) -> float: def _same_or_better(left: Candidate, right: Candidate) -> bool: - """Whether left is the representative to retain; ties favor input order.""" - return _score_value(left) > _score_value(right) + """Whether `left` (the incumbent) should stay the representative. + + Ties favor the incumbent, i.e. input order. Candidates arrive sorted by score, + so a later duplicate is never strictly better; deciding ties by ">" instead + handed the slot to whichever equal-scoring copy happened to arrive last, making + the retained snippet depend on retriever emission order. + """ + return _score_value(left) >= _score_value(right) def deduplicate(candidates: Sequence[Candidate], hamming_distance: int = 3) -> list[Candidate]: @@ -146,14 +168,21 @@ def deduplicate(candidates: Sequence[Candidate], hamming_distance: int = 3) -> l fingerprints: list[int | None] = [] for candidate in candidates: - _, _, fingerprint = _features(candidate) + # Only the fingerprint is needed here; building the token set as well + # doubled the per-candidate cost of the pipeline's hottest stage. "No + # fingerprint" means "no tokens", never "fingerprint happened to be 0". + tokens = normalize_code_tokens(candidate.content) + fingerprint = token_fingerprint(tokens) if tokens else None if fingerprint is None: representatives.append(candidate) fingerprints.append(None) continue + # Carry each match's fingerprint alongside its index: it is known non-None + # by construction here, and re-reading it from the list later would lose + # that guarantee. matches = [ - index + (index, existing) for index, existing in enumerate(fingerprints) if existing is not None and simhash_distance(fingerprint, existing) <= threshold ] @@ -162,17 +191,23 @@ def deduplicate(candidates: Sequence[Candidate], hamming_distance: int = 3) -> l fingerprints.append(fingerprint) continue - first = matches[0] + indices = [index for index, _ in matches] + first = indices[0] winner = candidate - for index in matches: + # The winner is either this candidate or an incumbent, and both + # fingerprints are already known — recomputing one from content was pure + # duplicated work. + winner_fingerprint = fingerprint + for index, existing in matches: incumbent = representatives[index] if _same_or_better(incumbent, winner): winner = incumbent - for index in reversed(matches): + winner_fingerprint = existing + for index in reversed(indices): representatives.pop(index) fingerprints.pop(index) representatives.insert(first, winner) - fingerprints.insert(first, token_fingerprint(normalize_code_tokens(winner.content))) + fingerprints.insert(first, winner_fingerprint) return representatives diff --git a/src/codebase_index/retrieval/fusion.py b/src/codebase_index/retrieval/fusion.py index a410545..4a929c9 100644 --- a/src/codebase_index/retrieval/fusion.py +++ b/src/codebase_index/retrieval/fusion.py @@ -13,6 +13,14 @@ retrievers report different line ranges for the same place; an exact key almost never coincides across sources, so cross-source agreement — RRF's whole point — would never fire. `agreeing_sources` is therefore counted at file granularity. +* Cross-locator agreement is scored, not merely counted. Bucketing alone does not + rescue agreement: a symbol defined at line 40 and a lexical hit at line 120 sit + in different buckets of the same file, so two retrievers pointing at one file + still fused as two unrelated candidates, each carrying a single retriever's + evidence. `file_agreement` adds the missing evidence back at a discount — a + candidate also receives, weighted by `file_agreement_weight`, the RRF mass of + every retriever that found its *file* somewhere else. Retrievers already + counted at the candidate's own locator are excluded, so nothing double-counts. On merge, the candidate carrying the most signal (symbol > fts > path) is kept as the representative so downstream rerank/snippet logic has the richest fields. @@ -36,11 +44,21 @@ def fuse( *, weights: dict[str, float], k: int, + file_agreement: float = 0.0, ) -> list[Candidate]: + """Fuse per-source ranked lists into one ordered candidate list. + + `file_agreement` is the discount applied to same-file, different-locator + evidence; 0.0 reproduces plain locator-only RRF. + """ accum: dict[tuple, float] = {} rep: dict[tuple, Candidate] = {} seen: set[tuple] = set() file_sources: dict[str, set[str]] = {} + # Best (lowest) rank each source achieved for each file, and which sources + # already contributed at each fused locator. + best_file_rank: dict[tuple[str, str], int] = {} + key_sources: dict[tuple, set[str]] = {} for source, candidates in lists.items(): w = weights.get(source, 0.0) @@ -48,7 +66,12 @@ def fuse( continue for rank, cand in enumerate(candidates): file_sources.setdefault(cand.path, set()).add(source) + file_key = (source, cand.path) + previous = best_file_rank.get(file_key) + if previous is None or rank < previous: + best_file_rank[file_key] = rank key = cand.fuse_key() + key_sources.setdefault(key, set()).add(source) # One contribution per source per locator: a file matching three FTS # chunks in the same bucket is one lexical signal, not three. if (source, key) in seen: @@ -57,6 +80,21 @@ def fuse( accum[key] = accum.get(key, 0.0) + w * k / (k + rank) rep[key] = _richer(rep[key], cand) if key in rep else cand + if file_agreement > 0.0: + for key, base in accum.items(): + path = rep[key].path + own = key_sources[key] + extra = 0.0 + for source in file_sources.get(path, ()): + if source in own: + continue + w = weights.get(source, 0.0) + if w <= 0.0: + continue + extra += w * k / (k + best_file_rank[(source, path)]) + if extra: + accum[key] = base + file_agreement * extra + fused = [_replace(rep[key], score=score) for key, score in accum.items()] fused.sort(key=lambda c: c.score, reverse=True) return [_replace(c, agreeing_sources=len(file_sources[c.path])) for c in fused] diff --git a/src/codebase_index/retrieval/pipeline.py b/src/codebase_index/retrieval/pipeline.py index a6cf2e2..020c956 100644 --- a/src/codebase_index/retrieval/pipeline.py +++ b/src/codebase_index/retrieval/pipeline.py @@ -165,11 +165,13 @@ def search( if token_budget <= 0: token_budget = plan.token_budget fetch_limit = limit + offset - pool_limit = ( - max(fetch_limit * 2, 20) - if (tuning.mmr or tuning.dedup) - else fetch_limit - ) + # Selection (dedup / MMR / per-file diversification) removes candidates, so the + # pool is over-fetched. The multiplier is explicit rather than a side effect of + # which selection flags happen to be on. A widened pool also carries a floor so + # a tiny `limit` still leaves selection something to choose between; multiplier + # 1 means "no over-fetch at all" and takes the page size verbatim. + pool_mult = max(1, tuning.candidate_pool_multiplier) + pool_limit = fetch_limit if pool_mult == 1 else max(fetch_limit * pool_mult, 20) lists, weights = _run_retrievers( conn, query, @@ -182,7 +184,12 @@ def search( graph_node_cap=tuning.graph_node_cap, graph_strategy=plan.graph_strategy, ) - fused = fuse(lists, weights=weights, k=tuning.rrf_k) + fused = fuse( + lists, + weights=weights, + k=tuning.rrf_k, + file_agreement=tuning.file_agreement_weight if tuning.file_agreement else 0.0, + ) ranked = rerank(fused, query=query, intent=plan.intent, tuning=tuning) if tuning.dedup: ranked = deduplicate(ranked, hamming_distance=tuning.dedup_hamming) diff --git a/src/codebase_index/retrieval/priors.py b/src/codebase_index/retrieval/priors.py index 3318e82..c9e0bec 100644 --- a/src/codebase_index/retrieval/priors.py +++ b/src/codebase_index/retrieval/priors.py @@ -113,11 +113,27 @@ class SourceRole(str, Enum): re.IGNORECASE, ) +# No single prior may exceed this magnitude. Priors exist to break ties between +# comparably-matching files, never to overrule retrieval evidence: fused scores +# reach ~1.0-1.5 and the largest rerank bonus (exact symbol) is 0.20, so a prior +# capped here can reorder near-neighbours but cannot lift an unrelated file over a +# genuine match. Widening this constant is a deliberate ranking-policy change. +MAX_ABS_PRIOR = 0.25 + _ROLE_PRIORS = { SourceRole.IMPLEMENTATION: 0.08, SourceRole.TEST: -0.06, - SourceRole.DOCUMENTATION: -0.05, - SourceRole.GENERATED_VENDOR_BUILD: -0.12, + # Prose about a feature matches a natural-language question more literally than + # the code implementing it, so design notes and plans crowded out the modules + # they describe. Measured over 305 queries on three repositories, deepening the + # demotion from -0.05 to -0.20 raised MRR and — because it only reorders docs + # relative to code, never below other docs — also improved documentation-seeking + # queries. -0.35 overshoots and collapses them, so the optimum is interior. + SourceRole.DOCUMENTATION: -0.20, + # Kept below documentation: a vendored or generated copy is the least useful + # answer of all. Measured as quality-neutral on the benchmark corpora, so this + # value preserves the role ordering rather than chasing a score. + SourceRole.GENERATED_VENDOR_BUILD: -0.25, SourceRole.UNKNOWN: 0.0, } _TEST_QUERY_PRIOR = 0.05 diff --git a/src/codebase_index/retrieval/searchers.py b/src/codebase_index/retrieval/searchers.py index a53874a..fbae798 100644 --- a/src/codebase_index/retrieval/searchers.py +++ b/src/codebase_index/retrieval/searchers.py @@ -292,20 +292,37 @@ def symbol_candidates( symbol_terms = list(dict.fromkeys((*terms, *expanded_terms))) if not terms: return [] - fuzzy_enabled = tuning.fuzzy_symbols and ( - len(terms) <= 3 or any(char.isupper() for char in query) - ) - term_set = set(terms) joined = "".join(terms) + + # Precise lookup first (exact -> prefix -> substring). It answers most real + # queries on its own, and knowing its yield is what lets fuzzy matching stay + # off the hot path. rows_by_key: dict[tuple, sqlite3.Row] = {} - if fuzzy_enabled: - for row in _fuzzy_symbol_rows(conn, query, limit=limit, kind=kind): - rows_by_key.setdefault((row["path"], row["line_start"], row["name"]), row) for needle in symbol_terms: for row in repo.symbol_search(conn, needle, limit=limit, kind=kind): rows_by_key.setdefault((row["path"], row["line_start"], row["name"]), row) + fuzzy_enabled = tuning.fuzzy_symbols and ( + len(terms) <= 3 or any(char.isupper() for char in query) + ) + if fuzzy_enabled and tuning.fuzzy_fallback_min > 0: + # Bounded edit distance over a lexical neighborhood is the single most + # expensive step in the pipeline. Spend it only when the precise lookup + # came up short: naming a real symbol, or simply returning enough + # candidates, means the query spelled its identifier well enough that + # fuzzing adds cost and noise rather than recall. + original_names = {term.casefold() for term in terms} + named_a_symbol = any( + (row["name"] or "").casefold() in original_names for row in rows_by_key.values() + ) + fuzzy_enabled = not ( + named_a_symbol or len(rows_by_key) >= tuning.fuzzy_fallback_min + ) + if fuzzy_enabled: + for row in _fuzzy_symbol_rows(conn, query, limit=limit, kind=kind): + rows_by_key.setdefault((row["path"], row["line_start"], row["name"]), row) + scored: list[tuple] = [] for row in rows_by_key.values(): subs = _name_subtokens(row["name"]) @@ -313,7 +330,12 @@ def symbol_candidates( covered = sum(1 for t in terms if t in subs or t.casefold() in name_l) expanded_covered = sum(1 for t in expanded_terms if t in subs or t.casefold() in name_l) tightness = len(subs & term_set) / len(subs) if subs else 0.0 - exact = (len(terms) == 1 and bool(row["is_exact"])) or ( + # Exactness is judged against the user's own terms, never against an + # expansion. `row["is_exact"]` is relative to whichever needle happened to + # retrieve the row, so a synonym needle ("config" for "configuration") used + # to mark a merely-related symbol as an exact match — worth +0.20 at rerank + # and HIGH confidence. Comparing names locally is needle-independent. + exact = (len(terms) == 1 and name_l == terms[0].casefold()) or ( bool(joined) and name_l == joined.casefold() ) fuzzy = 0.0 diff --git a/src/codebase_index/retrieval/tuning.py b/src/codebase_index/retrieval/tuning.py index ba52676..9898d6e 100644 --- a/src/codebase_index/retrieval/tuning.py +++ b/src/codebase_index/retrieval/tuning.py @@ -27,10 +27,21 @@ class RetrievalTuning: # --- candidate generation ------------------------------------------------ fuzzy_symbols: bool = True """Acronym / concatenation / edit-distance identifier matching in the symbol - retriever, so `userid` and `get user` both reach `getUserById`.""" + retriever, so `userid` and `get user` both reach `getUserById`. + + Runs only as a *recall fallback* (see `fuzzy_fallback_min`). Measured on 305 + queries across three repositories it changed no ranking metric while costing + ~20% of query latency, because real queries name identifiers correctly often + enough that the precise lookup already answers them. Gating it behind an + empty-handed precise lookup keeps the typo/acronym capability at ~zero cost. + """ fuzzy_threshold: float = 0.55 """Minimum identifier similarity for fuzzy symbol candidates.""" + fuzzy_fallback_min: int = 3 + """Run fuzzy identifier matching only when the precise symbol lookup returned + fewer than this many rows and found no exact match. 0 restores always-on.""" + query_expansion: bool = True """Down-weighted code-synonym expansion (auth->authentication, ...). Original terms always keep a strictly higher weight so precision is preserved.""" @@ -52,6 +63,16 @@ class RetrievalTuning: min_term_coverage: float = 0.5 """Fraction of salient query terms a chunk must contain under soft matching.""" + + candidate_pool_multiplier: int = 2 + """Over-fetch factor for the pre-selection candidate pool. + + Selection stages (dedup, MMR, per-file diversification) drop or reorder + candidates, so the pool must be wider than the requested limit or the page + ends up short. Previously this widening was an implicit side effect of + `dedup or mmr` being enabled; making it explicit is what let the ablation + show that the measured "dedup win" was really a pool-size win. + """ # --- selection ----------------------------------------------------------- mmr: bool = False """Maximal Marginal Relevance re-selection of the ranked list. @@ -72,6 +93,19 @@ class RetrievalTuning: """Prefer implementation files over their tests and over prose docs when both match a code question. Measured: tests/test_fusion.py outranked fusion.py.""" + file_agreement: bool = True + """Credit a candidate for retrievers that found its *file* at another locator. + + RRF fuses on (path, line-bucket), but a symbol hit at line 40 and a lexical + hit at line 120 in the same file land in different buckets, so cross-retriever + agreement — the entire point of fusion — never fired. This adds the missing + evidence back at reduced weight without double-counting a retriever already + counted at the candidate's own locator.""" + + file_agreement_weight: float = 0.4 + """Discount applied to same-file, different-locator evidence. Tuned on 305 + queries over three repositories; the 0.3-0.6 plateau peaks here.""" + # --- fixed parameters ---------------------------------------------------- rrf_k: int = 60 max_per_file: int = 3 @@ -92,6 +126,9 @@ def baseline(cls) -> RetrievalTuning: mmr=False, dedup=False, source_priors=False, + file_agreement=False, + # 1.7.0 had no over-fetch: the pool was exactly the requested page. + candidate_pool_multiplier=1, ) def without(self, flag: str) -> RetrievalTuning: @@ -99,6 +136,9 @@ def without(self, flag: str) -> RetrievalTuning: field_names = {f.name for f in fields(self)} if flag not in field_names: raise KeyError(f"unknown tuning flag: {flag!r}") + value = getattr(self, flag) + if not isinstance(value, bool): + raise TypeError(f"tuning flag {flag!r} is not a boolean signal") return replace(self, **{flag: False}) diff --git a/tests/test_diversity.py b/tests/test_diversity.py index 6956d87..f277a3d 100644 --- a/tests/test_diversity.py +++ b/tests/test_diversity.py @@ -1,4 +1,10 @@ -from codebase_index.retrieval.diversity import deduplicate, mmr_select +from codebase_index.retrieval.diversity import ( + deduplicate, + mmr_select, + normalize_code_tokens, + simhash_distance, + token_fingerprint, +) from codebase_index.retrieval.types import Candidate @@ -47,3 +53,68 @@ def test_selection_handles_empty_and_singleton_inputs_without_mutation(): assert deduplicate([]) == [] assert mmr_select([], 3, lambda_=0.5) == [] assert mmr_select([candidate], 3, lambda_=0.5) == [candidate] + + +def _reference_fingerprint(tokens: tuple[str, ...]) -> int: + """Naive per-occurrence SimHash: the definition the fast path must match.""" + import hashlib + + if not tokens: + return 0 + weights = [0] * 64 + for token in tokens: + hashed = int.from_bytes( + hashlib.blake2b(str(token).encode("utf-8"), digest_size=8).digest(), "big" + ) + for bit in range(64): + weights[bit] += 1 if (hashed >> bit) & 1 else -1 + fingerprint = 0 + for bit, weight in enumerate(weights): + if weight >= 0: + fingerprint |= 1 << bit + return fingerprint & ((1 << 64) - 1) + + +def test_fingerprint_matches_per_occurrence_definition(): + """Folding duplicate tokens by count is an optimisation, not a behaviour change.""" + cases = [ + (), + ("a",), + ("a", "a", "a"), + ("def", "add", "a", "b", "return", "a", "b"), + ("ключ", "значение", "ключ"), # non-ASCII identifiers + ("x" * 4096,), # pathological single token + tuple(f"tok{i % 7}" for i in range(500)), # heavy repetition + ] + for tokens in cases: + assert token_fingerprint(tokens) == _reference_fingerprint(tokens), tokens + + +def test_fingerprint_is_stable_across_calls_and_order_sensitive_only_by_content(): + tokens = normalize_code_tokens("def add(a, b):\n return a + b") + assert token_fingerprint(tokens) == token_fingerprint(tokens) + assert simhash_distance(token_fingerprint(tokens), token_fingerprint(tokens)) == 0 + + +def test_fingerprint_handles_empty_and_none_content(): + assert token_fingerprint(None) == 0 + assert token_fingerprint("") == 0 + assert token_fingerprint(()) == 0 + + +def test_deduplicate_keeps_untokenizable_candidates_distinct(): + """Comment-only snippets normalise to no tokens; they must not collapse together.""" + left = _candidate("a.py", 0.9, "# just a comment") + right = _candidate("b.py", 0.8, "# another comment") + assert deduplicate([left, right]) == [left, right] + + +def test_deduplicate_is_deterministic_for_identical_scores(): + first = _candidate("a.py", 0.5, "def f(): return 1") + second = _candidate("b.py", 0.5, "def f(): return 1") + assert deduplicate([first, second]) == deduplicate([first, second]) == [first] + + +def test_deduplicate_collapses_a_long_duplicate_run_to_one(): + dupes = [_candidate(f"f{i}.py", 1.0 - i / 100, "def f(a):\n return a") for i in range(12)] + assert deduplicate(dupes) == [dupes[0]] diff --git a/tests/test_fusion.py b/tests/test_fusion.py index 693068f..1c6d30f 100644 --- a/tests/test_fusion.py +++ b/tests/test_fusion.py @@ -64,3 +64,82 @@ def test_fuse_dedupes_repeated_source_hits_in_one_bucket(): fused = fuse({"fts": fts}, weights={"fts": 1.0}, k=60) assert len(fused) == 1 assert fused[0].score <= 1.0 # single best-rank contribution, not summed 3x + + +def test_file_agreement_credits_cross_locator_evidence(): + """Two retrievers pointing at one file from different regions is agreement. + + Line bucketing alone cannot merge them — a symbol defined at line 5 and a + lexical hit at line 400 are genuinely different locators — so without this the + file's evidence stayed split across two weakly-scored candidates. + """ + lists = { + "fts": [_c("other.py", "fts", 0.9), Candidate( + path="target.py", line_start=400, line_end=420, source="fts", score=0.5)], + "symbol": [Candidate( + path="target.py", line_start=5, line_end=9, source="symbol", score=0.8)], + } + weights = {"fts": 1.0, "symbol": 1.0} + split = fuse(lists, weights=weights, k=60, file_agreement=0.0) + joined = fuse(lists, weights=weights, k=60, file_agreement=0.4) + + assert split[0].path == "other.py" # rank-0 lexical hit wins outright + assert joined[0].path == "target.py" # ...until both retrievers are heard + + +def test_file_agreement_zero_reproduces_plain_rrf(): + """The default must be opt-in-able away for a clean single-signal ablation.""" + lists = { + "fts": [_c("a.py", "fts", 0.9), _c("b.py", "fts", 0.4)], + "symbol": [Candidate(path="a.py", line_start=90, line_end=95, + source="symbol", score=0.8)], + } + weights = {"fts": 1.0, "symbol": 1.0} + plain = fuse(lists, weights=weights, k=60, file_agreement=0.0) + assert [(c.path, round(c.score, 9)) for c in plain] == [ + (c.path, round(c.score, 9)) + for c in fuse(lists, weights=weights, k=60) + ] + + +def test_file_agreement_does_not_double_count_own_source(): + """A retriever already counted at this locator must not be counted again.""" + only_fts = {"fts": [_c("a.py", "fts", 0.9)]} + weights = {"fts": 1.0} + without = fuse(only_fts, weights=weights, k=60, file_agreement=0.0) + with_agreement = fuse(only_fts, weights=weights, k=60, file_agreement=1.0) + assert without[0].score == with_agreement[0].score + + +def test_file_agreement_ignores_zero_weight_sources(): + lists = { + "fts": [_c("a.py", "fts", 0.9)], + "path": [Candidate(path="a.py", line_start=800, line_end=800, + source="path", score=1.0)], + } + weights = {"fts": 1.0, "path": 0.0} + scored = fuse(lists, weights=weights, k=60, file_agreement=1.0) + assert len(scored) == 1 + assert scored[0].score == fuse({"fts": lists["fts"]}, weights=weights, k=60)[0].score + + +def test_file_agreement_uses_best_rank_per_source_and_is_deterministic(): + """Repeated same-file hits from one source contribute once, at their best rank.""" + lists = { + "fts": [_c("x.py", "fts", 0.9)], + "symbol": [ + Candidate(path="x.py", line_start=100, line_end=110, source="symbol", score=0.7), + Candidate(path="x.py", line_start=200, line_end=210, source="symbol", score=0.6), + ], + } + weights = {"fts": 1.0, "symbol": 1.0} + runs = [ + [(c.path, c.line_start, round(c.score, 12)) for c in + fuse(lists, weights=weights, k=60, file_agreement=0.4)] + for _ in range(3) + ] + assert runs[0] == runs[1] == runs[2] + top = fuse(lists, weights=weights, k=60, file_agreement=0.4)[0] + # fts rank 0 (1.0) + 0.4 * symbol best rank 0 (1.0) == 1.4, not 1.4 + a second + # symbol contribution from the line-200 hit. + assert top.score == 1.0 + 0.4 * 1.0 diff --git a/tests/test_priors.py b/tests/test_priors.py index 574f0ec..9a1dd88 100644 --- a/tests/test_priors.py +++ b/tests/test_priors.py @@ -1,6 +1,7 @@ from __future__ import annotations from codebase_index.retrieval.priors import ( + MAX_ABS_PRIOR, SourceRole, classify_source_role, is_test_intent, @@ -38,8 +39,15 @@ def test_prior_preserves_order_and_is_bounded(): } assert values[SourceRole.IMPLEMENTATION] > values[SourceRole.TEST] assert values[SourceRole.IMPLEMENTATION] > values[SourceRole.DOCUMENTATION] + # Prose is demoted below tests but stays above vendored/generated output: a + # design note can still answer a question, a minified bundle never can. + assert values[SourceRole.TEST] > values[SourceRole.DOCUMENTATION] assert values[SourceRole.DOCUMENTATION] > values[SourceRole.GENERATED_VENDOR_BUILD] - assert all(-0.15 <= value <= 0.15 for value in values.values()) + # Priors must stay tiebreakers. The bound is read from the module so the + # contract cannot drift by editing one number in isolation, and is asserted + # small here so widening the constant is itself a visible change. + assert MAX_ABS_PRIOR <= 0.25 + assert all(abs(value) <= MAX_ABS_PRIOR for value in values.values()) assert ( source_role_prior("src/search.py", query="where is search implemented") == values[SourceRole.IMPLEMENTATION] diff --git a/tests/test_symbol_ranking.py b/tests/test_symbol_ranking.py index 93ec481..ed3bf45 100644 --- a/tests/test_symbol_ranking.py +++ b/tests/test_symbol_ranking.py @@ -186,3 +186,79 @@ def test_symbol_synonym_expansion_retrieves_alias(tmp_path): assert cands[0].symbol == "Config" assert cands[0].exact_symbol is False + + +def test_synonym_match_is_never_reported_as_an_exact_symbol(tmp_path): + """`is_exact` from SQL is relative to whichever needle found the row. + + A synonym needle ("config" expanded from "configuration") matches `Config` + exactly, which used to mark the candidate `exact_symbol=True` — worth +0.20 at + rerank and an unconditional HIGH confidence. Exactness must be judged against + the terms the user actually typed. + """ + db = _db(tmp_path) + conn = db.conn + f = _insert_file(conn, path="src/config.py", lang="python", mtime_ns=1) + _insert_symbol( + conn, f, name="Config", kind="class", line_start=1, line_end=12, + signature="class Config", + ) + conn.commit() + + cands = symbol_candidates(conn, "configuration", limit=10) + exact = symbol_candidates(conn, "Config", limit=10) + db.close() + + assert cands[0].symbol == "Config" + assert cands[0].exact_symbol is False + # ...while naming the symbol outright still registers as exact. + assert exact[0].exact_symbol is True + + +def test_fuzzy_runs_only_when_the_precise_lookup_comes_up_short(tmp_path): + """Fuzzy matching is a recall fallback, not a default cost. + + A typo that no precise lookup can resolve must still reach the symbol, but a + query that names an indexed symbol must not pay for edit-distance scoring. + """ + db = _db(tmp_path) + conn = db.conn + f = _insert_file(conn, path="src/user.py", lang="python", mtime_ns=1) + _insert_symbol( + conn, f, name="getUserById", kind="function", line_start=1, line_end=9, + signature="def getUserById(user_id)", + ) + conn.commit() + + typo = symbol_candidates(conn, "getUsrById", limit=10) + named = symbol_candidates(conn, "getUserById", limit=10) + db.close() + + assert [c.symbol for c in typo] == ["getUserById"] # fallback still fires + assert named[0].exact_symbol is True # precise path satisfied + + +def test_fuzzy_fallback_can_be_restored_to_always_on(tmp_path): + from dataclasses import replace + + from codebase_index.retrieval.tuning import DEFAULT_TUNING + + db = _db(tmp_path) + conn = db.conn + f = _insert_file(conn, path="src/user.py", lang="python", mtime_ns=1) + for name in ("getUserById", "getUserByName", "getUserByEmail"): + _insert_symbol( + conn, f, name=name, kind="function", line_start=1, line_end=9, + signature=f"def {name}(x)", + ) + conn.commit() + + always_on = replace(DEFAULT_TUNING, fuzzy_fallback_min=0) + gated = symbol_candidates(conn, "GUBI", limit=10) + ungated = symbol_candidates(conn, "GUBI", limit=10, tuning=always_on) + db.close() + + # The acronym reaches the symbol either way; the gate only decides whether the + # edit-distance pass runs when precise results already exist. + assert "getUserById" in {c.symbol for c in ungated} + assert all(c.exact_symbol is False for c in gated + ungated) diff --git a/tests/test_tuning_contract.py b/tests/test_tuning_contract.py new file mode 100644 index 0000000..b335997 --- /dev/null +++ b/tests/test_tuning_contract.py @@ -0,0 +1,141 @@ +"""The ablation contract: every ranking signal must be independently switchable. + +These tests protect the property that makes `tests/eval/run_eval.py --ablate` +meaningful. If a signal stops being individually disablable, or if `baseline()` +drifts away from the pre-1.8.0 pipeline, every published benchmark delta silently +becomes unverifiable. +""" + +from __future__ import annotations + +from dataclasses import fields + +import pytest + +from codebase_index.retrieval.pipeline import search +from codebase_index.retrieval.tuning import DEFAULT_TUNING, RetrievalTuning + + +def test_every_boolean_signal_is_independently_disablable(): + tuning = RetrievalTuning() + booleans = [f.name for f in fields(tuning) if isinstance(getattr(tuning, f.name), bool)] + assert booleans, "a tuning with no boolean signals cannot be ablated" + for flag in booleans: + assert getattr(tuning.without(flag), flag) is False + + +def test_without_rejects_unknown_and_non_boolean_fields(): + with pytest.raises(KeyError): + RetrievalTuning().without("no_such_flag") + # Silently "disabling" a numeric knob by setting it to False would produce a + # nonsense ablation row rather than an error. + with pytest.raises(TypeError): + RetrievalTuning().without("rrf_k") + + +def test_baseline_disables_every_post_170_signal(): + """`baseline()` is the honest 'before' column; it must stay signal-free.""" + baseline = RetrievalTuning.baseline() + for flag in ( + "fuzzy_symbols", + "query_expansion", + "graph_source", + "soft_lexical", + "mmr", + "dedup", + "source_priors", + "file_agreement", + ): + assert getattr(baseline, flag) is False, flag + # 1.7.0 took the page size verbatim; over-fetching is a later addition. + assert baseline.candidate_pool_multiplier == 1 + + +def test_shipped_defaults_are_the_measured_configuration(): + """Guards against a default drifting without a benchmark run behind it.""" + assert DEFAULT_TUNING.file_agreement is True + assert DEFAULT_TUNING.file_agreement_weight == pytest.approx(0.4) + assert DEFAULT_TUNING.candidate_pool_multiplier == 2 + assert DEFAULT_TUNING.fuzzy_fallback_min == 3 + # Signals measured as neutral-or-harmful stay off by default. + assert DEFAULT_TUNING.graph_source is False + assert DEFAULT_TUNING.mmr is False + + +def test_tuning_is_immutable_and_hashable(): + tuning = RetrievalTuning() + with pytest.raises(Exception): + tuning.file_agreement_weight = 0.9 # type: ignore[misc] + assert hash(tuning) == hash(RetrievalTuning()) + + +@pytest.mark.parametrize("multiplier", [1, 2, 5]) +def test_pool_multiplier_does_not_change_the_page_size(seeded_index, multiplier): + """Over-fetching feeds selection; it must never inflate the returned page.""" + from dataclasses import replace + + payload = search( + seeded_index.conn, + "token", + mode="hybrid", + limit=3, + token_budget=1500, + no_fallback=True, + tuning=replace(RetrievalTuning(), candidate_pool_multiplier=multiplier), + ) + assert len(payload["results"]) <= 3 + + +def test_baseline_and_default_both_answer_a_direct_symbol_query(seeded_index): + for tuning in (RetrievalTuning.baseline(), RetrievalTuning()): + payload = search( + seeded_index.conn, + "refresh_access_token", + mode="hybrid", + limit=10, + token_budget=1500, + no_fallback=True, + tuning=tuning, + ) + assert payload["results"][0]["path"] == "src/auth/token.py" + + +@pytest.mark.parametrize( + "query", + [ + "", + " ", + "the and of for", # stopwords only + "!!! ??? ***", # punctuation only + "\"; DROP TABLE files;--", # would be an FTS/SQL injection if unquoted + "a" * 512, # pathological identifier + "поиск конфигурации", # non-ASCII + "getUserById" * 40, # pathological camelCase run + ], +) +def test_degenerate_queries_are_answered_without_raising(seeded_index, query): + payload = search( + seeded_index.conn, + query, + mode="hybrid", + limit=5, + token_budget=800, + no_fallback=True, + ) + assert isinstance(payload["results"], list) + assert payload["confidence"] in {"high", "medium", "low"} + + +def test_ranking_is_deterministic_across_repeated_identical_queries(seeded_index): + def run() -> list[tuple[str, int, int]]: + payload = search( + seeded_index.conn, + "how does token refresh work", + mode="hybrid", + limit=10, + token_budget=1500, + no_fallback=True, + ) + return [(r["path"], r["line_start"], r["line_end"]) for r in payload["results"]] + + assert run() == run() == run() From 341855cfeb8e3cefd154b24347682b69eff40276 Mon Sep 17 00:00:00 2001 From: denfry Date: Wed, 2 Sep 2026 10:16:00 +0300 Subject: [PATCH 2/3] test(eval): multi-corpus git-derived benchmark with significance testing The benchmark was 36 hand-written queries against this repository in Python. At that size a +-0.03 MRR move is a single query, so a delta column could not tell an improvement from a reshuffle, and nothing guarded against tuning the ranker to one repository in one language. gen_queries.py mints objective ground truth from any git repository by pairing a human-written commit subject with the files that commit actually changed. Unlike docstring-derived benchmarks the query text lives in git metadata rather than in the indexed corpus, so the answer is not copied into the document being retrieved. Only localised, described changes survive: no merges, reverts, releases, version bumps or formatting commits; at most --max-files files, all of which must still exist at HEAD; duplicate subjects collapse. Changelog-style files are never accepted as answers because they paraphrase subjects, and any commit touching tests/eval is dropped so the set cannot grade itself. run_eval.py gains --corpus REPO:QUERIES to pool several repositories into one benchmark, weighting every query equally rather than averaging per-corpus averages. Every non-baseline row now reports a seeded paired bootstrap 95% confidence interval and a paired permutation p-value, so signals ship on the strength of that test rather than the sign of a delta. The report also tracks agent-facing noise: mean emitted snippet tokens (results past the budget carry no snippet and are not billed), duplicate rate of returned results, and p99 latency. --- tests/eval/README.md | 99 +++++ tests/eval/gen_queries.py | 253 +++++++++++ tests/eval/harness.py | 218 ++++++++-- tests/eval/metrics.py | 78 ++++ tests/eval/queries/self_repo_git.yml | 627 +++++++++++++++++++++++++++ tests/eval/run_eval.py | 170 ++++++-- tests/test_gen_queries.py | 152 +++++++ 7 files changed, 1525 insertions(+), 72 deletions(-) create mode 100644 tests/eval/README.md create mode 100644 tests/eval/gen_queries.py create mode 100644 tests/eval/queries/self_repo_git.yml create mode 100644 tests/test_gen_queries.py diff --git a/tests/eval/README.md b/tests/eval/README.md new file mode 100644 index 0000000..58903a1 --- /dev/null +++ b/tests/eval/README.md @@ -0,0 +1,99 @@ +# Retrieval evaluation + +Measures whether a ranking change actually helps, on more than one repository, with +enough statistical care to tell an improvement from a reshuffle. + +```bash +# shipped default vs the 1.7.0 baseline, on this repository +python tests/eval/run_eval.py + +# + one-signal-off ablation, with significance for every row +python tests/eval/run_eval.py --ablate + +# pool several repositories into one benchmark +python tests/eval/run_eval.py --queries self_repo_git \ + --corpus ../some-java-service:/tmp/svc.yml \ + --corpus ../some-ts-app:/tmp/app.yml --ablate +``` + +## Why the results are trustworthy + +**The index cannot grade its own homework.** Two independent ground-truth sources, +neither produced by the retriever: + +| Query set | Source | Size | Leakage | +|---|---|---|---| +| `self_repo` | Hand-written from the source tree | 36 | Query wording is human; answers verified against the tree | +| `self_repo_git` | Commit subject → files that commit changed | 87 | None: subjects live in git metadata, not in the corpus | + +`harness.validate_queries()` fails the run if any expected file no longer exists, +so a stale expectation is a loud error rather than a silently deflated score. + +**Benchmark scaffolding is excluded from the corpus it grades** (`CORPUS_EXCLUDES`). +The ground-truth YAML quotes every query verbatim; leaving it indexed would make it +the top lexical hit for almost every query. + +**One index per corpus, shared by every variant.** Ablation deltas measure ranking, +never indexing variance. + +**Every non-baseline row gets a significance test.** Query sets of this size have a +noise floor of several MRR points, so each metric is reported with a paired +bootstrap 95% CI and a paired permutation p-value (`metrics.paired_bootstrap_ci`, +`metrics.paired_permutation_p`, both seeded and therefore reproducible). Signals +ship on the strength of that test, not the sign of a delta. A one-query move on a +three-query category is noise, and the table is designed to make that visible. + +## Adding a corpus + +Tuning a ranker against one repository in one language produces numbers that only +move on that repository. `gen_queries.py` mints an objective query set from any git +repository, so a new corpus costs two commands: + +```bash +python tests/eval/gen_queries.py --repo ../some-java-service --out /tmp/svc.yml +python tests/eval/run_eval.py --corpus ../some-java-service:/tmp/svc.yml +``` + +The generator pairs a human-written commit subject with the files that commit +actually changed. It keeps only localised, described changes: no merges, reverts, +releases, version bumps or formatting commits; at most `--max-files` files, all of +which must still exist at HEAD; at least `--min-words` content words after the +Conventional Commits prefix is stripped; duplicate subjects collapse. Changelog-like +files are never accepted as answers because they paraphrase commit subjects, and any +commit touching `tests/eval/` is dropped so the benchmark cannot grade itself. + +Corpora used to validate the 1.9.0 ranking changes, beyond this repository: + +| Corpus | Language | Files | Queries | +|---|---|---|---| +| Civitas | Java | 944 | 64 | +| PoliternalSite | TypeScript / TSX | 443 | 118 | + +Those repositories are not vendored here — shipping someone else's source to run a +benchmark is not reproducible either. The generator is the reproducible part: point +it at any git repository and the protocol is identical. + +## What is measured + +Ranking quality is scored at **file** granularity, because the agent's unit of +decision is "which file do I open"; several hits inside one file collapse to its +best rank. + +- `recall@5`, `recall@10`, `precision@5`, `hit@3` +- `MRR`, `MAP`, `nDCG@10` +- `useful@budget` — the fraction of the answer that fits in the token budget, i.e. + what the agent can actually afford to read, not what is merely ranked somewhere +- `tokens` — mean snippet tokens actually emitted per query (results past the + budget carry no snippet and are not billed) +- `dup%` — fraction of returned results that near-duplicate an earlier result +- `p50/p95/p99` latency, in-process, excluding interpreter start-up + +## Files + +| File | Role | +|---|---| +| `run_eval.py` | CLI: corpora, variants, ablation, significance | +| `harness.py` | Index build, query execution, aggregation, pooling, tables | +| `metrics.py` | IR metrics + paired bootstrap / permutation tests | +| `gen_queries.py` | Ground-truth generator from git history | +| `queries/` | Checked-in query sets | diff --git a/tests/eval/gen_queries.py b/tests/eval/gen_queries.py new file mode 100644 index 0000000..1e0d0c8 --- /dev/null +++ b/tests/eval/gen_queries.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python3 +"""Derive an objective retrieval ground-truth set from a repository's git history. + + python tests/eval/gen_queries.py --repo . --out tests/eval/queries/self_repo_git.yml + +Why git history +--------------- +Hand-written query sets are small and expensive; sets generated by the retriever +grade its own homework. A commit is neither: a human wrote the subject line +("tolerate missing FastMCP type export") *and*, independently, changed a specific +set of files. That pairing is exactly the agent question we care about — "given a +description of a change, which files implement it?" — and it is recorded before +this tool exists. + +Leakage properties +------------------ +The query text lives in git metadata, not in the indexed corpus, so unlike +docstring-derived benchmarks (CodeSearchNet-style) the answer is not copied into +the document being retrieved. The one exception is changelog-style files, which +routinely paraphrase commit subjects; they are excluded both as corpus documents +(`CHANGELOG_EXCLUDES`) and as expected answers. + +Filtering +--------- +Commits survive only if they look like a *localised, described* change: + +* not a merge, revert, release, version bump, or pure-formatting commit; +* subject carries at least ``--min-words`` content words after the + Conventional Commits ``type(scope):`` prefix is stripped; +* touches between 1 and ``--max-files`` files that still exist at HEAD and are + plausibly indexable source or prose; +* does not touch benchmark scaffolding (which would grade itself). + +Duplicate subjects collapse to their first occurrence, so a repeated +"fix lint" contributes one query, not thirty. +""" + +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +from collections import OrderedDict +from pathlib import Path + +# Conventional Commits prefix: `type`, optional `(scope)`, optional `!`, colon. +_PREFIX_RE = re.compile(r"^\s*(?P[a-z]+)\s*(?:\((?P[^)]*)\))?\s*!?\s*:\s*", re.I) + +# Subjects that describe bookkeeping rather than a locatable code change. +_SKIP_SUBJECT_RE = re.compile( + r"^\s*(?:" + r"merge\b|revert\b|release\b|bump\b|" + r"v?\d+\.\d+(?:\.\d+)?\s*$|" + r"(?:chore|style|ci|build|deps)\s*(?:\([^)]*\))?\s*:\s*(?:bump|format|lint|typo|whitespace)" + r")", + re.I, +) + +# Conventional type -> eval category. Unlisted types fall back to "change". +_TYPE_CATEGORY = { + "feat": "feature", + "fix": "bug", + "perf": "perf", + "refactor": "refactor", + "test": "test", + "docs": "docs", +} + +# Files that may be *changed* by a commit but are never a useful retrieval answer. +_ANSWER_DENY_RE = re.compile( + r"(?:^|/)(?:" + r"CHANGELOG[^/]*|HISTORY[^/]*|NEWS[^/]*|" + r"package-lock\.json|yarn\.lock|poetry\.lock|Cargo\.lock|requirements\.lock|" + r"go\.sum|Gemfile\.lock|pnpm-lock\.yaml" + r")$", + re.I, +) + +# Extensions worth retrieving. Binary assets and generated bundles are excluded so +# an image-only commit does not become an unanswerable query. +_ANSWER_EXTS = frozenset( + """ + py pyi js mjs cjs jsx ts tsx java kt kts go rs rb php cs c h cc cpp hpp + swift scala clj ex exs erl hs ml dart lua r sql sh bash ps1 psm1 + md rst txt adoc yml yaml toml json xml gradle cfg ini + """.split() +) + +# Benchmark scaffolding: a commit touching it must not become a benchmark query. +_SCAFFOLD_RE = re.compile(r"(?:^|/)tests/(?:eval|benchmark_)", re.I) + +# Corpus documents that paraphrase commit subjects. `harness` applies these on top +# of its own excludes whenever a git-derived query set is evaluated. +CHANGELOG_EXCLUDES = ("CHANGELOG*", "HISTORY*", "NEWS*", "**/CHANGELOG*") + +_WORD_RE = re.compile(r"[A-Za-z][A-Za-z0-9_-]*") + + +def _git(repo: Path, *args: str) -> str: + proc = subprocess.run( + ["git", "-C", str(repo), *args], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + ) + if proc.returncode != 0: + raise RuntimeError(f"git {' '.join(args)} failed: {proc.stderr.strip()}") + return proc.stdout + + +def strip_prefix(subject: str) -> tuple[str, str]: + """Split a Conventional Commits subject into (category, human text).""" + match = _PREFIX_RE.match(subject) + if not match: + return "change", subject.strip() + rest = subject[match.end():].strip() + return _TYPE_CATEGORY.get(match.group("type").lower(), "change"), rest + + +def _is_answerable(path: str) -> bool: + if _ANSWER_DENY_RE.search(path) or _SCAFFOLD_RE.search(path): + return False + ext = path.rsplit(".", 1)[-1].lower() if "." in path.rsplit("/", 1)[-1] else "" + return ext in _ANSWER_EXTS + + +def harvest( + repo: Path, + *, + max_commits: int, + max_files: int, + min_words: int, +) -> list[dict]: + """Return ground-truth records mined from `repo`'s history, newest first.""" + tracked = {p for p in _git(repo, "ls-files").splitlines() if p} + # `%x00` separates subject from the name-only file list without colliding with + # either; commit records are separated by a sentinel line. + raw = _git( + repo, + "log", + "--no-merges", + f"--max-count={max_commits}", + "--name-only", + "--pretty=format:%x01%H%x00%s", + "--diff-filter=ACMR", + ) + + seen_subjects: OrderedDict[str, dict] = OrderedDict() + for record in raw.split("\x01"): + record = record.strip("\n") + if not record: + continue + header, _, body = record.partition("\n") + sha, _, subject = header.partition("\x00") + subject = subject.strip() + if not subject or _SKIP_SUBJECT_RE.match(subject): + continue + + files = [line.strip() for line in body.splitlines() if line.strip()] + if any(_SCAFFOLD_RE.search(f) for f in files): + continue + answers = sorted({f for f in files if f in tracked and _is_answerable(f)}) + if not answers or len(answers) > max_files: + continue + + category, text = strip_prefix(subject) + # Backticks and quotes in subjects are prose decoration, not query intent. + # Collapse the resulting gaps so the query reads like something a user typed. + text = " ".join(text.replace("`", " ").replace('"', " ").split()).strip(" .;:") + words = [w for w in _WORD_RE.findall(text) if len(w) > 1] + if len(words) < min_words: + continue + key = text.casefold() + if key in seen_subjects: + continue + seen_subjects[key] = { + "query": text, + "category": category, + "expected_files": answers, + "commit": sha[:12], + } + return list(seen_subjects.values()) + + +def _yaml_scalar(text: str) -> str: + """Quote a scalar for YAML. Only double quotes and backslashes need escaping.""" + return '"' + text.replace("\\", "\\\\").replace('"', '\\"') + '"' + + +def render(records: list[dict], *, repo: Path, source: str) -> str: + lines = [ + "# Auto-generated by tests/eval/gen_queries.py -- DO NOT EDIT BY HAND.", + "#", + f"# corpus: {source}", + f"# queries: {len(records)}", + "#", + "# Ground truth is objective and leak-free: the query is a human-written commit", + "# subject (git metadata, absent from the indexed corpus) and the expected files", + "# are the files that commit actually changed. Neither side is produced by the", + "# retriever. Regenerate after history changes; `commit` records provenance.", + "", + ] + for rec in records: + lines.append(f"- query: {_yaml_scalar(rec['query'])}") + lines.append(f" category: {rec['category']}") + lines.append(f" commit: {rec['commit']}") + lines.append(" expected_files:") + lines.extend(f" - {path}" for path in rec["expected_files"]) + lines.append("") + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument("--repo", default=".", help="git repository to mine") + ap.add_argument("--out", required=True, help="destination YAML path") + ap.add_argument("--max-commits", type=int, default=4000) + ap.add_argument( + "--max-files", + type=int, + default=4, + help="drop sweeping commits: a 60-file refactor is not a retrieval target", + ) + ap.add_argument("--min-words", type=int, default=3) + ap.add_argument("--limit", type=int, default=0, help="keep at most N queries (0 = all)") + args = ap.parse_args(argv) + + repo = Path(args.repo).resolve() + records = harvest( + repo, + max_commits=args.max_commits, + max_files=args.max_files, + min_words=args.min_words, + ) + if args.limit > 0: + records = records[: args.limit] + if not records: + print("no usable commits found", file=sys.stderr) + return 1 + + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(render(records, repo=repo, source=repo.name), encoding="utf-8") + print(f"wrote {len(records)} queries to {out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/eval/harness.py b/tests/eval/harness.py index 279a086..58b9e2c 100644 --- a/tests/eval/harness.py +++ b/tests/eval/harness.py @@ -23,6 +23,11 @@ from codebase_index.config import Config from codebase_index.indexer.pipeline import build_index +from codebase_index.retrieval.diversity import ( + normalize_code_tokens, + simhash_distance, + token_fingerprint, +) from codebase_index.retrieval.pipeline import search from codebase_index.retrieval.tuning import RetrievalTuning from codebase_index.storage.db import Database @@ -48,6 +53,11 @@ class QueryOutcome: ranked_files: list[str] returned: list[tuple[str, int]] latency_ms: float + total_tokens: int = 0 + """Snippet tokens actually handed to the agent for this query.""" + duplicates: int = 0 + """Results whose snippet near-duplicates an earlier result in the same page.""" + n_results: int = 0 @dataclass @@ -68,7 +78,17 @@ class EvalReport: p95_ms: float p99_ms: float mean_ms: float + mean_tokens: float = 0.0 + """Mean tokens of snippet context returned per query — the agent's actual bill.""" + duplicate_rate: float = 0.0 + """Fraction of returned results that near-duplicate an earlier result.""" + mean_candidates: float = 0.0 + """Mean results returned per query, before the agent reads anything.""" per_category: dict[str, float] = field(default_factory=dict) + per_query: dict[str, list[float]] = field(default_factory=dict) + """Per-query metric vectors, in query order. Required for paired significance + testing: aggregate deltas alone cannot separate a real gain from resampling + noise on a set this size.""" def as_row(self) -> dict[str, float | str | int]: return { @@ -82,6 +102,8 @@ def as_row(self) -> dict[str, float | str | int]: "P@5": self.precision_at_5, "MAP": self.map_score, "useful@budget": self.useful_context, + "tokens": self.mean_tokens, + "dup%": self.duplicate_rate * 100.0, "p50_ms": self.p50_ms, "p95_ms": self.p95_ms, "p99_ms": self.p99_ms, @@ -164,15 +186,42 @@ def run_query( ranked_files: list[str] = [] returned: list[tuple[str, int]] = [] - for r in payload.get("results", []): + total_tokens = 0 + duplicates = 0 + fingerprints: list[int] = [] + results = payload.get("results", []) + for r in results: p = _normalise(r["path"]) # File-level ranking: the agent's unit of decision is "which file do I # open", so multiple hits inside one file collapse to its best rank. if p not in ranked_files: ranked_files.append(p) - returned.append((p, int(r.get("token_est") or 0))) - return QueryOutcome(query=q, ranked_files=ranked_files, returned=returned, - latency_ms=latency_ms) + tokens = int(r.get("token_est") or 0) + returned.append((p, tokens)) + snippet = r.get("snippet") + # Only snippets are actually placed in the agent's context. Results past + # the budget still carry a `token_est` for their (unread) chunk, so summing + # it would report a bill the agent never pays. + if snippet: + total_tokens += tokens + # Duplicate rate is measured on what the agent actually receives, using the + # same fingerprint the pipeline suppresses with — so the number reports the + # residual noise after selection, not the raw candidate overlap. + chunk_tokens = normalize_code_tokens(snippet or "") + if chunk_tokens: + fingerprint = token_fingerprint(chunk_tokens) + if any(simhash_distance(fingerprint, seen) <= 3 for seen in fingerprints): + duplicates += 1 + fingerprints.append(fingerprint) + return QueryOutcome( + query=q, + ranked_files=ranked_files, + returned=returned, + latency_ms=latency_ms, + total_tokens=total_tokens, + duplicates=duplicates, + n_results=len(results), + ) def evaluate( @@ -198,45 +247,74 @@ def evaluate( token_budget=token_budget).latency_ms ) - def mean(fn) -> float: - vals = [fn(o) for o in outcomes] + # Named metric extractors, so aggregate values and the per-query vectors used + # for significance testing can never be computed two different ways. + scorers = { + "recall@5": lambda o: metrics.recall_at_k(o.ranked_files, o.query.expected_files, 5), + "recall@10": lambda o: metrics.recall_at_k(o.ranked_files, o.query.expected_files, 10), + "MRR": lambda o: metrics.reciprocal_rank(o.ranked_files, o.query.expected_files), + "nDCG@10": lambda o: metrics.ndcg_at_k(o.ranked_files, o.query.expected_files, 10), + "hit@3": lambda o: metrics.hit_rate_at_k(o.ranked_files, o.query.expected_files, 3), + "P@5": lambda o: metrics.precision_at_k(o.ranked_files, o.query.expected_files, 5), + "MAP": lambda o: metrics.average_precision(o.ranked_files, o.query.expected_files), + "useful@budget": lambda o: metrics.useful_context_at_budget( + o.returned, o.query.expected_files, token_budget + ), + } + per_query = {name: [fn(o) for o in outcomes] for name, fn in scorers.items()} + + def mean(name: str) -> float: + vals = per_query[name] return statistics.fmean(vals) if vals else 0.0 per_category: dict[str, list[float]] = {} - for o in outcomes: - per_category.setdefault(o.query.category, []).append( - metrics.reciprocal_rank(o.ranked_files, o.query.expected_files) - ) + for o, rr in zip(outcomes, per_query["MRR"]): + per_category.setdefault(o.query.category, []).append(rr) + + returned_total = sum(o.n_results for o in outcomes) + duplicate_total = sum(o.duplicates for o in outcomes) return EvalReport( label=label, n_queries=len(outcomes), - recall_at_5=mean(lambda o: metrics.recall_at_k(o.ranked_files, o.query.expected_files, 5)), - recall_at_10=mean(lambda o: metrics.recall_at_k(o.ranked_files, o.query.expected_files, 10)), - mrr=mean(lambda o: metrics.reciprocal_rank(o.ranked_files, o.query.expected_files)), - ndcg_at_10=mean(lambda o: metrics.ndcg_at_k(o.ranked_files, o.query.expected_files, 10)), - hit_rate_at_3=mean(lambda o: metrics.hit_rate_at_k(o.ranked_files, o.query.expected_files, 3)), - precision_at_5=mean(lambda o: metrics.precision_at_k(o.ranked_files, o.query.expected_files, 5)), - map_score=mean(lambda o: metrics.average_precision(o.ranked_files, o.query.expected_files)), - useful_context=mean( - lambda o: metrics.useful_context_at_budget( - o.returned, o.query.expected_files, token_budget - ) - ), + recall_at_5=mean("recall@5"), + recall_at_10=mean("recall@10"), + mrr=mean("MRR"), + ndcg_at_10=mean("nDCG@10"), + hit_rate_at_3=mean("hit@3"), + precision_at_5=mean("P@5"), + map_score=mean("MAP"), + useful_context=mean("useful@budget"), p50_ms=metrics.percentile(latencies, 50), p95_ms=metrics.percentile(latencies, 95), p99_ms=metrics.percentile(latencies, 99), mean_ms=statistics.fmean(latencies) if latencies else 0.0, + mean_tokens=( + statistics.fmean([o.total_tokens for o in outcomes]) if outcomes else 0.0 + ), + duplicate_rate=(duplicate_total / returned_total) if returned_total else 0.0, + mean_candidates=(returned_total / len(outcomes)) if outcomes else 0.0, per_category={ cat: statistics.fmean(vals) for cat, vals in sorted(per_category.items()) }, + per_query=per_query, ) -def format_table(reports: Sequence[EvalReport], *, baseline: EvalReport | None = None) -> str: +_WIDE_COLS = ("tokens", "dup%") + + +def format_table( + reports: Sequence[EvalReport], + *, + baseline: EvalReport | None = None, + columns: Sequence[str] | None = None, +) -> str: """Render reports as a Markdown table, with deltas against `baseline`.""" - cols = ["label", "recall@5", "recall@10", "MRR", "nDCG@10", "hit@3", "P@5", - "MAP", "useful@budget", "p50_ms", "p95_ms"] + cols = list(columns) if columns else [ + "label", "recall@5", "recall@10", "MRR", "nDCG@10", "hit@3", "P@5", + "MAP", "useful@budget", "tokens", "dup%", "p50_ms", "p95_ms", + ] lines = ["| " + " | ".join(cols) + " |", "|" + "|".join("---" for _ in cols) + "|"] for rep in reports: @@ -245,13 +323,97 @@ def format_table(reports: Sequence[EvalReport], *, baseline: EvalReport | None = for c in cols: v = row[c] if isinstance(v, float): - cell = f"{v:.3f}" if c.endswith("_ms") is False else f"{v:.1f}" + coarse = c.endswith("_ms") or c in _WIDE_COLS + cell = f"{v:.1f}" if coarse else f"{v:.3f}" if baseline is not None and rep is not baseline: delta = v - float(baseline.as_row()[c]) - if abs(delta) >= 0.0005: - cell += f" ({delta:+.3f})" if not c.endswith("_ms") else f" ({delta:+.1f})" + if abs(delta) >= (0.05 if coarse else 0.0005): + cell += f" ({delta:+.1f})" if coarse else f" ({delta:+.3f})" cells.append(cell) else: cells.append(str(v)) lines.append("| " + " | ".join(cells) + " |") return "\n".join(lines) + + +def pool(reports: Sequence[EvalReport], *, label: str) -> EvalReport: + """Concatenate per-corpus reports into one, weighting every query equally. + + Averaging the per-corpus averages would let a 36-query corpus outvote a + 118-query one; pooling the raw per-query vectors keeps one query = one vote and + is what makes the significance tests valid across a mixed-language benchmark. + """ + reports = list(reports) + if not reports: + raise ValueError("cannot pool an empty report list") + + per_query: dict[str, list[float]] = {} + for rep in reports: + for name, values in rep.per_query.items(): + per_query.setdefault(name, []).extend(values) + + def mean(name: str) -> float: + vals = per_query.get(name, []) + return statistics.fmean(vals) if vals else 0.0 + + total = sum(rep.n_queries for rep in reports) or 1 + + def weighted(attr: str) -> float: + return sum(getattr(rep, attr) * rep.n_queries for rep in reports) / total + + per_category: dict[str, list[float]] = {} + for rep in reports: + for cat, value in rep.per_category.items(): + per_category.setdefault(cat, []).append(value) + + return EvalReport( + label=label, + n_queries=total, + recall_at_5=mean("recall@5"), + recall_at_10=mean("recall@10"), + mrr=mean("MRR"), + ndcg_at_10=mean("nDCG@10"), + hit_rate_at_3=mean("hit@3"), + precision_at_5=mean("P@5"), + map_score=mean("MAP"), + useful_context=mean("useful@budget"), + # Latency percentiles cannot be pooled from percentiles; report the + # query-weighted mean of each, which is honest about being an approximation + # only when corpora differ wildly in size. + p50_ms=weighted("p50_ms"), + p95_ms=weighted("p95_ms"), + p99_ms=weighted("p99_ms"), + mean_ms=weighted("mean_ms"), + mean_tokens=weighted("mean_tokens"), + duplicate_rate=weighted("duplicate_rate"), + mean_candidates=weighted("mean_candidates"), + per_category={c: statistics.fmean(v) for c, v in sorted(per_category.items())}, + per_query=per_query, + ) + + +def format_significance( + baseline: EvalReport, + candidate: EvalReport, + *, + resamples: int = 5000, +) -> str: + """Paired bootstrap CI + permutation p-value per metric, candidate vs baseline.""" + lines = [ + f"paired comparison: {candidate.label} vs {baseline.label} " + f"(n={min(baseline.n_queries, candidate.n_queries)})", + f"| {'metric':13} | {'delta':>8} | {'95% CI':>19} | {'p':>6} | sig |", + "|" + "|".join("---" for _ in range(5)) + "|", + ] + for name, base_values in baseline.per_query.items(): + cand_values = candidate.per_query.get(name, []) + if len(cand_values) != len(base_values): + continue + deltas = [c - b for b, c in zip(base_values, cand_values)] + delta, lo, hi = metrics.paired_bootstrap_ci(deltas, resamples=resamples) + p = metrics.paired_permutation_p(deltas, resamples=resamples) + lines.append( + f"| {name:13} | {delta:+8.4f} | [{lo:+.4f},{hi:+.4f}] | {p:6.3f} | " + f"{'yes' if p < 0.05 else 'no':3} |" + ) + return "\n".join(lines) diff --git a/tests/eval/metrics.py b/tests/eval/metrics.py index 5ab1cef..fac5df0 100644 --- a/tests/eval/metrics.py +++ b/tests/eval/metrics.py @@ -14,6 +14,7 @@ from __future__ import annotations import math +import random from collections.abc import Iterable, Sequence @@ -127,3 +128,80 @@ def percentile(values: Sequence[float], pct: float) -> float: ordered = sorted(values) idx = math.ceil(pct / 100.0 * len(ordered)) - 1 return ordered[min(max(idx, 0), len(ordered) - 1)] + + +# --- significance ----------------------------------------------------------- +# +# A 36-query set cannot distinguish a +0.03 MRR improvement from noise, and eyeballing +# a delta column is how ranking systems accumulate changes that never actually helped. +# Both routines below are paired (same queries, two systems) and seeded, so a reported +# interval is reproducible rather than a different number on every run. + +_BOOTSTRAP_SEED = 20260902 + + +def paired_bootstrap_ci( + deltas: Sequence[float], + *, + resamples: int = 5000, + confidence: float = 0.95, + seed: int = _BOOTSTRAP_SEED, +) -> tuple[float, float, float]: + """Return (mean delta, lower, upper) for a per-query difference vector. + + Resampling queries — not scores — is what makes the interval answer the + question we care about: would this improvement survive a different, equally + plausible sample of user questions? + """ + values = list(deltas) + n = len(values) + if n == 0: + return 0.0, 0.0, 0.0 + observed = math.fsum(values) / n + if n == 1: + return observed, observed, observed + + rng = random.Random(seed) + means = [] + for _ in range(max(1, resamples)): + total = 0.0 + for _ in range(n): + total += values[rng.randrange(n)] + means.append(total / n) + means.sort() + tail = (1.0 - confidence) / 2.0 + lo = means[min(len(means) - 1, int(tail * len(means)))] + hi = means[min(len(means) - 1, int((1.0 - tail) * len(means)))] + return observed, lo, hi + + +def paired_permutation_p( + deltas: Sequence[float], + *, + resamples: int = 5000, + seed: int = _BOOTSTRAP_SEED, +) -> float: + """Two-sided paired permutation test on a per-query difference vector. + + Under the null the two systems are interchangeable on each query, so flipping + the sign of any subset of deltas is equally likely. Reports the fraction of + sign-flipped resamples at least as extreme as what we measured. + """ + values = list(deltas) + n = len(values) + if n == 0: + return 1.0 + observed = abs(math.fsum(values) / n) + if observed == 0.0: + return 1.0 + + rng = random.Random(seed + 1) + extreme = 0 + trials = max(1, resamples) + for _ in range(trials): + total = 0.0 + for value in values: + total += value if rng.getrandbits(1) else -value + if abs(total / n) >= observed - 1e-12: + extreme += 1 + return extreme / trials diff --git a/tests/eval/queries/self_repo_git.yml b/tests/eval/queries/self_repo_git.yml new file mode 100644 index 0000000..e1a6d18 --- /dev/null +++ b/tests/eval/queries/self_repo_git.yml @@ -0,0 +1,627 @@ +# Auto-generated by tests/eval/gen_queries.py -- DO NOT EDIT BY HAND. +# +# corpus: codebase-index +# queries: 87 +# +# Ground truth is objective and leak-free: the query is a human-written commit +# subject (git metadata, absent from the indexed corpus) and the expected files +# are the files that commit actually changed. Neither side is produced by the +# retriever. Regenerate after history changes; `commit` records provenance. + +- query: "tolerate missing FastMCP type export" + category: bug + commit: 96e034b997bc + expected_files: + - src/codebase_index/mcp/server.py + +- query: "stabilize lint policy across Ruff upgrades" + category: bug + commit: 0e42fe36b83f + expected_files: + - .github/workflows/ci.yml + - pyproject.toml + +- query: "mark PyPI distribution as shipped (1.6.0) in roadmap + upgrade plan" + category: docs + commit: 731b7af9f2f3 + expected_files: + - docs/PRODUCT_UPGRADE_PLAN.md + - docs/ROADMAP.md + +- query: "PyPI-first install across QUICKSTART/INSTALLATION/MCP/FAQ" + category: docs + commit: 601743ebb48e + expected_files: + - docs/FAQ.md + - docs/INSTALLATION.md + - docs/MCP.md + - docs/QUICKSTART.md + +- query: "feature pip install codebase-index from PyPI" + category: docs + commit: c782c5acc6f8 + expected_files: + - README.md + +- query: "publish to PyPI via Trusted Publishing (OIDC)" + category: change + commit: ec7a614547d4 + expected_files: + - .github/workflows/release.yml + - docs/RELEASE_CHECKLIST.md + +- query: "inject snippet compactor; emit skeletonized/elided_lines" + category: feature + commit: f096e534e76d + expected_files: + - src/codebase_index/retrieval/budget.py + - tests/test_budget.py + +- query: "make_compactor factory with intent->context policy" + category: feature + commit: 433bb5f6a088 + expected_files: + - src/codebase_index/retrieval/skeleton.py + - tests/test_skeleton.py + +- query: "markdown heading + structured key classifiers" + category: feature + commit: 2840ece8a6b2 + expected_files: + - src/codebase_index/retrieval/skeleton.py + - tests/test_skeleton.py + +- query: "code classifier + compact() with focus, guard, raw fallback" + category: feature + commit: be5b7dc608cb + expected_files: + - src/codebase_index/retrieval/skeleton.py + - tests/test_skeleton.py + +- query: "render_skeleton collapses keep/elide mask into markers" + category: feature + commit: f2e2f74e0f26 + expected_files: + - src/codebase_index/retrieval/skeleton.py + - tests/test_skeleton.py + +- query: "implementation plan for snippet skeletonization" + category: docs + commit: e2a60dca91ef + expected_files: + - docs/superpowers/plans/2026-06-24-snippet-skeletonization.md + +- query: "design spec for snippet skeletonization + content-aware rendering" + category: docs + commit: f6529c6d20d9 + expected_files: + - docs/superpowers/specs/2026-06-24-snippet-skeletonization-design.md + +- query: "key analytics by stable content keys, not volatile symbol ids" + category: bug + commit: 16d2980eb3e5 + expected_files: + - src/codebase_index/graph/analysis.py + - src/codebase_index/storage/repo.py + +- query: "discount test paths when labelling communities" + category: feature + commit: 61e89b49fc68 + expected_files: + - src/codebase_index/graph/analysis.py + - tests/test_analysis.py + +- query: "keep enable_vectors on Database (mypy/CI fix)" + category: bug + commit: e996fc7685cd + expected_files: + - src/codebase_index/storage/db.py + +- query: "sync roadmap with shipped MCP, add trust-model callout, changelog" + category: docs + commit: e92dd61c49bd + expected_files: + - README.md + - docs/PRODUCT_UPGRADE_PLAN.md + - docs/ROADMAP.md + - docs/SECURITY.md + +- query: "make bootstrap and golden tests pass on bare CI runners" + category: bug + commit: 07dd01b6196f + expected_files: + - tests/test_bootstrap.py + - tests/test_cli_golden.py + +- query: "single accurate repo layout, drop stale module claims" + category: docs + commit: ca285ebd565f + expected_files: + - docs/ARCHITECTURE.md + +- query: "shared CLI/MCP service layer; close surface drift" + category: refactor + commit: f767e1978afc + expected_files: + - src/codebase_index/cli.py + - src/codebase_index/mcp/server.py + - src/codebase_index/service.py + - tests/test_service.py + +- query: "assert _PARSE_CONFIG is set before use in worker parse" + category: bug + commit: 8e9f8b701bea + expected_files: + - src/codebase_index/indexer/pipeline.py + +- query: "sync version to 1.2.2 across plugin manifest and lock" + category: bug + commit: 980b31b7049e + expected_files: + - .claude-plugin/plugin.json + +- query: "define versioning and fork workflow" + category: docs + commit: 4948d7a5e111 + expected_files: + - AGENTS.md + - CONTRIBUTING.md + +- query: "add auto-update, skill-update, and skill-rollback" + category: feature + commit: 104fef57d890 + expected_files: + - src/codebase_index/cli.py + - src/codebase_index/scaffold.py + - src/codebase_index/skill_update.py + +- query: "add --json option to symbol, refs, and impact commands" + category: bug + commit: 799eb0f4b5c2 + expected_files: + - src/codebase_index/cli.py + - tests/test_impact_cli.py + +- query: "add Python 3.11+ pipx troubleshooting" + category: docs + commit: 732a3831bb4f + expected_files: + - README.md + - pyproject.toml + +- query: "add mcp to dev deps + tests to pass coverage gate" + category: bug + commit: 4281d6436431 + expected_files: + - .claude-plugin/plugin.json + - pyproject.toml + - tests/test_mcp_server.py + +- query: "add generalized skill research plan" + category: docs + commit: 99bbe0e0bc79 + expected_files: + - docs/superpowers/plans/2026-05-29-generalize-skill-research-discipline.md + +- query: "fix Windows/Git Bash path and PATH assumptions" + category: test + commit: b6b9d8e675ac + expected_files: + - tests/test_bootstrap.py + +- query: "enforce coverage floor + register slow marker" + category: test + commit: 6855f959b05e + expected_files: + - pyproject.toml + - tests/test_search_cli.py + +- query: "finalize for 0.1.0 (status, roadmap M9, clean-install recipe, examples)" + category: docs + commit: 1b0e4bc35c73 + expected_files: + - README.md + - docs/INSTALLATION.md + - docs/ROADMAP.md + +- query: "release pipeline (build, smoke, GitHub release, PyPI trusted publishing)" + category: change + commit: 03333d012f79 + expected_files: + - .github/workflows/release.yml + +- query: "finalize package metadata + add 0.1.0 changelog" + category: change + commit: 31dc30f32f7b + expected_files: + - pyproject.toml + +- query: "medium-repo index+search budget smoke test (--runslow)" + category: test + commit: ac4a47dfbb2a + expected_files: + - tests/conftest.py + - tests/test_perf_smoke.py + +- query: "add --json normalizer + golden compare helper" + category: test + commit: 1b0f2b6ab337 + expected_files: + - tests/__init__.py + - tests/golden_utils.py + - tests/test_cli_golden.py + +- query: "mark M8 complete (update/watch/hooks/doctor) + installation updates" + category: docs + commit: 0fa021b808bf + expected_files: + - docs/INSTALLATION.md + - docs/ROADMAP.md + - tests/test_watcher.py + +- query: "report hooks, cache-gitignore, and freshness (--strict)" + category: feature + commit: 08dd286458f0 + expected_files: + - src/codebase_index/cli.py + - src/codebase_index/doctor.py + - tests/test_doctor.py + +- query: "implement watch command (graceful without the watch extra)" + category: feature + commit: 1811ba3b6302 + expected_files: + - src/codebase_index/cli.py + - tests/test_watcher.py + +- query: "debounced live indexer + lazy watchdog observer" + category: feature + commit: aaea59ef8552 + expected_files: + - src/codebase_index/watch/watcher.py + - tests/test_watcher.py + +- query: "init --with-hooks auto-merges the update hook into settings.json" + category: feature + commit: 56548ae938da + expected_files: + - src/codebase_index/cli.py + - tests/test_init_cli.py + +- query: "add honest benchmark results to README" + category: docs + commit: 6b7a4aabe0e7 + expected_files: + - README.md + +- query: "idempotent PostToolUse hook merge + enabled_hooks reader" + category: feature + commit: 92122ddc1f09 + expected_files: + - src/codebase_index/scaffold.py + - tests/test_hooks_merge.py + +- query: "label doc chunks in retrieval results" + category: feature + commit: 94a15982d5f4 + expected_files: + - src/codebase_index/retrieval/searchers.py + - src/codebase_index/storage/repo.py + +- query: "integrate doc_chunks into indexer pipeline for FTS5" + category: feature + commit: d542629d3d5e + expected_files: + - src/codebase_index/indexer/pipeline.py + - src/codebase_index/storage/repo.py + +- query: "implement incremental update (--since/--all, --json, --quiet)" + category: feature + commit: ee441669740c + expected_files: + - src/codebase_index/cli.py + - tests/test_update_cli.py + +- query: "add doc_chunks extractor for markdown, tests, docstrings, exceptions, config" + category: feature + commit: 6830de973a99 + expected_files: + - src/codebase_index/indexer/doc_chunks.py + - tests/test_doc_chunks.py + +- query: "add cold/warm benchmark, build time, DB size metrics" + category: test + commit: 59ce7809a0c3 + expected_files: + - tests/test_benchmark_comparison.py + +- query: "incremental update_index (mtime fast-path + sha verify + prune)" + category: feature + commit: 009dddb7aa71 + expected_files: + - src/codebase_index/indexer/pipeline.py + - tests/test_update.py + +- query: "add quality benchmark test runner with pyyaml dependency" + category: test + commit: 8e6c15ea1aed + expected_files: + - pyproject.toml + - tests/test_quality_benchmark.py + +- query: "add expected_answers.yml quality benchmark fixture" + category: test + commit: e61a42b9ae0b + expected_files: + - tests/fixtures/expected_answers.yml + +- query: "rename token efficiency to output compression ratio" + category: bug + commit: 2b92e3c67a4d + expected_files: + - tests/test_benchmark_comparison.py + +- query: "add fingerprints accessor for incremental update" + category: feature + commit: 44f418339ac8 + expected_files: + - src/codebase_index/storage/repo.py + - tests/test_storage.py + +- query: "honor freshness contract in search responses" + category: feature + commit: a9da97735792 + expected_files: + - src/codebase_index/cli.py + - src/codebase_index/retrieval/pipeline.py + - src/codebase_index/retrieval/searchers.py + - tests/test_search_cli.py + +- query: "compute index freshness (git fast-path + mtime diff)" + category: feature + commit: 09abfee14e57 + expected_files: + - src/codebase_index/indexer/freshness.py + - src/codebase_index/storage/repo.py + - tests/test_freshness.py + - tests/test_storage.py + +- query: "implement init (skill + config + gitignore + hooks example)" + category: feature + commit: 8a9f1e689e0a + expected_files: + - src/codebase_index/cli.py + - tests/test_init_cli.py + +- query: "skill/config/gitignore materialization helpers" + category: feature + commit: c218f2cb9e9b + expected_files: + - src/codebase_index/scaffold.py + - tests/test_scaffold.py + +- query: "ship skill template (scripts + hooks) in the wheel" + category: feature + commit: 997bd8d42dcd + expected_files: + - pyproject.toml + - src/codebase_index/skill_template/SKILL.md + - src/codebase_index/skill_template/examples/hooks/settings.json + - tests/test_packaging.py + +- query: "real vector mode + backend resolution + external warning" + category: feature + commit: 8b3a5a4ad7ce + expected_files: + - src/codebase_index/cli.py + - tests/test_search_cli.py + +- query: "wire optional vector retriever + vector mode" + category: feature + commit: 8bffadf4a65d + expected_files: + - src/codebase_index/retrieval/pipeline.py + +- query: "vector retriever over sqlite-vec KNN" + category: feature + commit: 2b7e25c9bf06 + expected_files: + - src/codebase_index/retrieval/searchers.py + - tests/test_vector_search.py + +- query: "per-intent vector weights + fusion richness" + category: feature + commit: ac8d4eee19f4 + expected_files: + - src/codebase_index/retrieval/fusion.py + - src/codebase_index/retrieval/intent.py + - tests/test_fusion.py + - tests/test_intent.py + +- query: "embed + store chunk vectors when embeddings enabled" + category: feature + commit: 8f2f7652fa19 + expected_files: + - src/codebase_index/indexer/pipeline.py + - tests/conftest.py + - tests/test_pipeline_vectors.py + +- query: "sqlite-vec vector store (load, upsert, KNN, meta)" + category: feature + commit: 4df3653b97df + expected_files: + - src/codebase_index/storage/db.py + - src/codebase_index/storage/repo.py + - tests/test_vectors_storage.py + +- query: "gated external backend with injectable transport" + category: feature + commit: 84e91fdc56e1 + expected_files: + - src/codebase_index/embeddings/external.py + - tests/test_embeddings_external.py + +- query: "lazy on-device sentence-transformers backend" + category: feature + commit: 45b3c55089d1 + expected_files: + - src/codebase_index/embeddings/local.py + - tests/test_embeddings_local.py + +- query: "backend protocol, noop default, gating factory" + category: feature + commit: 2c7cb9879142 + expected_files: + - src/codebase_index/embeddings/__init__.py + - src/codebase_index/embeddings/backend.py + - src/codebase_index/embeddings/noop.py + - tests/test_embeddings_backend.py + +- query: "embeddings toggle/model participate in config_hash" + category: feature + commit: d0c9563d9a83 + expected_files: + - src/codebase_index/config.py + - tests/test_config.py + +- query: "mark M5 complete + graph-edge language recipe" + category: docs + commit: 34a0256b19ed + expected_files: + - docs/LANGUAGES.md + - docs/ROADMAP.md + +- query: "wire impact command" + category: feature + commit: 6e00c0d925d5 + expected_files: + - src/codebase_index/cli.py + - tests/test_impact_cli.py + +- query: "bounded impact BFS + impact_lookup" + category: feature + commit: 9934925e7575 + expected_files: + - src/codebase_index/graph/expand.py + - tests/test_graph.py + +- query: "run graph resolution pass after build" + category: feature + commit: db340ac41fa0 + expected_files: + - src/codebase_index/indexer/pipeline.py + - tests/fixtures/sample_repo/src/api/service.py + - tests/test_pipeline.py + +- query: "cross-file edge resolution + degree denormalization" + category: feature + commit: 44ed7e1fdf1f + expected_files: + - src/codebase_index/graph/builder.py + - tests/test_graph.py + +- query: "graph resolution + traversal accessors" + category: feature + commit: 2a72244bf64f + expected_files: + - src/codebase_index/storage/repo.py + - tests/test_storage.py + +- query: "extract import + inheritance edges" + category: feature + commit: 76f7c35ff267 + expected_files: + - src/codebase_index/parsers/treesitter.py + - tests/test_treesitter.py + +- query: "import + inheritance edge queries (py/js/ts)" + category: feature + commit: 0d3c2b44791d + expected_files: + - src/codebase_index/parsers/languages.py + - tests/test_languages.py + +- query: "acceptance — hybrid outranks single retrievers + budget enforced" + category: test + commit: 17501214265f + expected_files: + - tests/conftest.py + - tests/test_hybrid_ranking.py + +- query: "wire hybrid search and explain commands" + category: feature + commit: 988dc112ee96 + expected_files: + - src/codebase_index/cli.py + - tests/test_search_cli.py + +- query: "JSON and compact Markdown renderers for dict payloads" + category: feature + commit: 7f14840957bb + expected_files: + - src/codebase_index/output/json.py + - src/codebase_index/output/markdown.py + - tests/test_output.py + +- query: "pipeline orchestrator with confidence and fallback" + category: feature + commit: dca27fafa5fa + expected_files: + - src/codebase_index/retrieval/pipeline.py + - tests/test_pipeline_search.py + +- query: "greedy token budgeting with redaction" + category: feature + commit: 21f9ba031014 + expected_files: + - src/codebase_index/retrieval/budget.py + - tests/test_budget.py + +- query: "explainable feature reranker with reason strings" + category: feature + commit: b4199dd1e532 + expected_files: + - src/codebase_index/retrieval/rerank.py + - tests/test_rerank.py + +- query: "reciprocal rank fusion with per-intent weights" + category: feature + commit: 878788c0e234 + expected_files: + - src/codebase_index/retrieval/fusion.py + - tests/test_fusion.py + +- query: "path/symbol/fts retrievers with uniform Candidate" + category: feature + commit: 799e3a75351d + expected_files: + - src/codebase_index/retrieval/searchers.py + - tests/test_searchers.py + +- query: "path_search and symbol_search accessors" + category: feature + commit: b4f48e65e979 + expected_files: + - src/codebase_index/storage/repo.py + - tests/test_repo_search.py + +- query: "deterministic seeded_index fixture" + category: test + commit: 41c1f678c26a + expected_files: + - tests/conftest.py + - tests/test_seeded_index.py + +- query: "rule-first intent detection with per-intent weights" + category: feature + commit: 4a253036b51c + expected_files: + - src/codebase_index/retrieval/intent.py + - tests/test_intent.py + +- query: "shared Candidate and IntentPlan types" + category: feature + commit: 8955935159dd + expected_files: + - src/codebase_index/retrieval/types.py + - tests/test_retrieval_types.py diff --git a/tests/eval/run_eval.py b/tests/eval/run_eval.py index d2a5da8..3cad5fa 100644 --- a/tests/eval/run_eval.py +++ b/tests/eval/run_eval.py @@ -1,12 +1,31 @@ #!/usr/bin/env python3 -"""Run the retrieval benchmark and ablation sweep. +"""Run the retrieval benchmark, ablation sweep, and significance tests. - python tests/eval/run_eval.py # baseline vs shipped default - python tests/eval/run_eval.py --ablate # + one-signal-off sweep + python tests/eval/run_eval.py # baseline vs shipped default + python tests/eval/run_eval.py --ablate # + one-signal-off sweep + python tests/eval/run_eval.py --corpus : # add an external corpus python tests/eval/run_eval.py --repo --queries -The index is built once and shared by every variant, so reported deltas isolate -ranking changes from indexing variance. +Each corpus is indexed once and shared by every variant, so reported deltas +isolate ranking changes from indexing variance. + +Why more than one corpus +------------------------ +The shipped query sets are Python and self-hosted. Tuning a ranker against a +single repository in a single language is how you get numbers that only move on +that repository. Pass `--corpus` to pool additional repositories into one +benchmark; `tests/eval/gen_queries.py` mints an objective query set for any git +repository, so adding one is a two-command operation: + + python tests/eval/gen_queries.py --repo ../some-java-service --out /tmp/svc.yml + python tests/eval/run_eval.py --corpus ../some-java-service:/tmp/svc.yml --ablate + +Why significance +---------------- +Query sets of this size have a noise floor of several MRR points. Every non-baseline +row is accompanied by a paired bootstrap CI and permutation p-value against the +shipped default, because a delta column alone cannot tell an improvement from a +reshuffle. Signals are kept on the strength of that test, not the sign of the delta. """ from __future__ import annotations @@ -15,7 +34,7 @@ import json import sys import tempfile -from dataclasses import fields +from dataclasses import fields, replace from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[2] @@ -27,75 +46,138 @@ # Signals swept by --ablate. Numeric parameters are excluded: turning off a bool # answers "does this signal earn its complexity", which is the decision we make. -ABLATABLE = ("soft_lexical", "query_expansion", "fuzzy_symbols", "graph_source", - "mmr", "dedup", "source_priors") +ABLATABLE = ( + "soft_lexical", + "query_expansion", + "fuzzy_symbols", + "graph_source", + "mmr", + "dedup", + "source_priors", + "file_agreement", +) + + +def _parse_corpus(spec: str) -> tuple[Path, str]: + """Split `:`, tolerating a Windows drive letter.""" + head, sep, tail = spec.rpartition(":") + if not sep or (len(head) == 1 and head.isalpha()): + raise argparse.ArgumentTypeError( + f"--corpus expects ':', got {spec!r}" + ) + return Path(head).resolve(), tail def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--repo", default=str(REPO_ROOT), - help="corpus to index and query (default: this repository)") + help="primary corpus to index and query (default: this repository)") ap.add_argument("--queries", default="self_repo", help="query set name under tests/eval/queries or a path") + ap.add_argument("--corpus", action="append", default=[], metavar="REPO:QUERIES", + help="additional corpus to pool into the benchmark (repeatable)") ap.add_argument("--ablate", action="store_true", help="also run a one-signal-off sweep") ap.add_argument("--limit", type=int, default=harness.DEFAULT_LIMIT) ap.add_argument("--token-budget", type=int, default=harness.DEFAULT_BUDGET) ap.add_argument("--repeats", type=int, default=3, help="latency repeats per query (quality is unaffected)") + ap.add_argument("--resamples", type=int, default=5000, + help="bootstrap/permutation resamples for significance testing") ap.add_argument("--json", dest="as_json", action="store_true") args = ap.parse_args(argv) - root = Path(args.repo).resolve() - queries = harness.load_queries(args.queries) - problems = harness.validate_queries(queries, root) - if problems: - print("Ground-truth validation FAILED:", file=sys.stderr) - for p in problems: - print(f" - {p}", file=sys.stderr) - return 2 - - print(f"corpus: {root}") - print(f"queries: {len(queries)} from {args.queries}") - print("building index (once, shared by all variants)...", flush=True) + corpora: list[tuple[str, Path, str]] = [ + (Path(args.repo).resolve().name, Path(args.repo).resolve(), args.queries) + ] + for spec in args.corpus: + root, queries = _parse_corpus(spec) + corpora.append((root.name, root, queries)) + + loaded: list[tuple[str, Path, list]] = [] + for name, root, query_spec in corpora: + queries = harness.load_queries(query_spec) + problems = harness.validate_queries(queries, root) + if problems: + print(f"Ground-truth validation FAILED for {name}:", file=sys.stderr) + for p in problems: + print(f" - {p}", file=sys.stderr) + return 2 + loaded.append((name, root, queries)) + print(f"corpus: {name} ({root}) — {len(queries)} queries from {query_spec}") + + total = sum(len(q) for _, _, q in loaded) + print(f"pooled: {total} queries across {len(loaded)} corpora") + + variants: list[tuple[str, RetrievalTuning]] = [ + ("baseline (1.7.0)", RetrievalTuning.baseline()), + ("default (all signals)", RetrievalTuning()), + ] + if args.ablate: + default = RetrievalTuning() + known = {f.name for f in fields(default)} + for flag in ABLATABLE: + if flag in known: + variants.append((f" -{flag}", default.without(flag))) + + # variant label -> per-corpus reports, pooled after every corpus is measured. + collected: dict[str, list[harness.EvalReport]] = {label: [] for label, _ in variants} + per_corpus_default: dict[str, harness.EvalReport] = {} with tempfile.TemporaryDirectory() as tmp: - db = harness.build_corpus_index(root, Path(tmp) / "eval.sqlite") - try: - variants: list[tuple[str, RetrievalTuning]] = [ - ("baseline (1.7.0)", RetrievalTuning.baseline()), - ("default (all signals)", RetrievalTuning()), - ] - if args.ablate: - default = RetrievalTuning() - known = {f.name for f in fields(default)} - for flag in ABLATABLE: - if flag in known: - variants.append((f" -{flag}", default.without(flag))) - - reports = [] - for label, tuning in variants: - print(f"running: {label}", flush=True) - reports.append( - harness.evaluate( + for name, root, queries in loaded: + print(f"building index for {name} (once, shared by all variants)...", flush=True) + db = harness.build_corpus_index(root, Path(tmp) / f"{name}.sqlite") + try: + for label, tuning in variants: + print(f" running: {label}", flush=True) + report = harness.evaluate( db.conn, queries, tuning=tuning, label=label, limit=args.limit, token_budget=args.token_budget, repeats=args.repeats, ) - ) - finally: - db.close() + collected[label].append(report) + if label == "default (all signals)": + # Relabel so the per-corpus table identifies the repository + # rather than repeating the variant name on every row. + per_corpus_default[name] = replace(report, label=name) + finally: + db.close() + + reports = [harness.pool(collected[label], label=label) for label, _ in variants] if args.as_json: - print(json.dumps([r.as_row() for r in reports], indent=2)) + payload = { + "pooled": [r.as_row() for r in reports], + "per_corpus_default": {k: v.as_row() for k, v in per_corpus_default.items()}, + "per_category_default": reports[1].per_category, + } + print(json.dumps(payload, indent=2)) return 0 print() print(harness.format_table(reports, baseline=reports[0])) + + default = reports[1] + print() + print(harness.format_significance(reports[0], default, resamples=args.resamples)) + + if len(reports) > 2: + print() + print("Ablation significance (each row vs the shipped default):") + for rep in reports[2:]: + print() + print(harness.format_significance(default, rep, resamples=args.resamples)) + + if len(loaded) > 1: + print() + print("Per-corpus (default config):") + print(harness.format_table(list(per_corpus_default.values()))) + print() print("Per-category MRR (default config):") - for cat, val in reports[1].per_category.items(): + for cat, val in default.per_category.items(): print(f" {cat:14} {val:.3f}") return 0 diff --git a/tests/test_gen_queries.py b/tests/test_gen_queries.py new file mode 100644 index 0000000..e6e0ebe --- /dev/null +++ b/tests/test_gen_queries.py @@ -0,0 +1,152 @@ +"""Tests for the git-history ground-truth generator. + +The generator is benchmark infrastructure, so its failure mode is silent: a bad +filter produces a plausible-looking query set that measures the wrong thing. +These tests pin the filtering rules that keep the set objective and leak-free. +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from eval import gen_queries # noqa: E402 + + +def _git(repo: Path, *args: str, **kw) -> None: + subprocess.run( + ["git", "-C", str(repo), *args], + check=True, + capture_output=True, + text=True, + **kw, + ) + + +@pytest.fixture +def repo(tmp_path: Path) -> Path: + root = tmp_path / "corpus" + root.mkdir() + _git(root, "init", "-q") + _git(root, "config", "user.email", "dev@example.test") + _git(root, "config", "user.name", "Dev") + return root + + +def _commit(repo: Path, subject: str, files: dict[str, str]) -> None: + for rel, body in files.items(): + target = repo / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(body, encoding="utf-8") + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "-m", subject) + + +def _harvest(repo: Path, **kw): + params = {"max_commits": 100, "max_files": 4, "min_words": 3} + params.update(kw) + return gen_queries.harvest(repo, **params) + + +def test_conventional_prefix_becomes_category_and_is_stripped(repo): + _commit(repo, "feat(auth): refresh the access token", {"src/auth.py": "def refresh(): ..."}) + (record,) = _harvest(repo) + assert record["query"] == "refresh the access token" + assert record["category"] == "feature" + assert record["expected_files"] == ["src/auth.py"] + assert record["commit"] + + +def test_bookkeeping_commits_are_excluded(repo): + _commit(repo, "feat: add the parser entry point", {"src/parse.py": "x = 1"}) + _commit(repo, "release: v2.1.0", {"src/parse.py": "x = 2"}) + _commit(repo, "chore(deps): bump pytest", {"src/parse.py": "x = 3"}) + _commit(repo, "Revert \"add the parser entry point\"", {"src/parse.py": "x = 4"}) + assert [r["query"] for r in _harvest(repo)] == ["add the parser entry point"] + + +def test_sweeping_commits_are_dropped(repo): + _commit( + repo, + "refactor: rename everything everywhere at once", + {f"src/mod{i}.py": "y = 1" for i in range(9)}, + ) + assert _harvest(repo, max_files=4) == [] + assert len(_harvest(repo, max_files=20)) == 1 + + +def test_changelog_is_never_an_expected_answer(repo): + """Changelogs paraphrase commit subjects; grading against them measures leakage.""" + _commit( + repo, + "fix: guard against an empty token budget", + {"CHANGELOG.md": "- guard against an empty token budget", "src/budget.py": "b = 0"}, + ) + (record,) = _harvest(repo) + assert record["expected_files"] == ["src/budget.py"] + + +def test_commit_touching_only_unanswerable_files_is_dropped(repo): + _commit(repo, "docs: add the architecture diagram", {"assets/diagram.png": "binary-ish"}) + assert _harvest(repo) == [] + + +def test_benchmark_scaffolding_never_grades_itself(repo): + _commit( + repo, + "test: extend the retrieval evaluation set", + {"tests/eval/queries/extra.yml": "- query: x", "src/thing.py": "z = 1"}, + ) + assert _harvest(repo) == [] + + +def test_short_subjects_and_duplicates_are_filtered(repo): + _commit(repo, "fix: typo", {"src/a.py": "a = 1"}) + _commit(repo, "fix: correct the retry backoff", {"src/b.py": "b = 1"}) + _commit(repo, "fix: correct the retry backoff", {"src/c.py": "c = 1"}) + records = _harvest(repo) + assert [r["query"] for r in records] == ["correct the retry backoff"] + # First occurrence wins, so provenance stays stable across regeneration. + assert records[0]["expected_files"] == ["src/c.py"] + + +def test_deleted_files_are_not_expected_answers(repo): + _commit(repo, "feat: add a temporary shim layer", {"src/shim.py": "s = 1"}) + (repo / "src/shim.py").unlink() + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "-m", "refactor: drop the shim layer entirely") + # The add-commit's only answer no longer exists at HEAD, so it must not ship. + assert all("src/shim.py" not in r["expected_files"] for r in _harvest(repo)) + + +def test_rendered_yaml_round_trips_through_the_harness(repo, tmp_path): + _commit( + repo, + 'fix: escape a "quoted" path\\name safely', + {"src/escape.py": "e = 1"}, + ) + out = tmp_path / "generated.yml" + assert gen_queries.main(["--repo", str(repo), "--out", str(out)]) == 0 + + from eval import harness + + queries = harness.load_queries(out) + assert len(queries) == 1 + # Quotes are stripped as prose decoration; a backslash is content and must + # survive YAML escaping intact. + assert queries[0].query == "escape a quoted path\\name safely" + assert harness.validate_queries(queries, repo) == [] + + +def test_strip_prefix_handles_unprefixed_and_breaking_subjects(): + assert gen_queries.strip_prefix("just do the thing") == ("change", "just do the thing") + assert gen_queries.strip_prefix("feat!: drop python 3.9") == ("feature", "drop python 3.9") + assert gen_queries.strip_prefix("perf(index): speed up walking") == ( + "perf", + "speed up walking", + ) From bb73c49ed5b2f96538453ba322a373bb5286d2d9 Mon Sep 17 00:00:00 2001 From: denfry Date: Wed, 2 Sep 2026 10:16:09 +0300 Subject: [PATCH 3/3] release: v1.9.0 Ranking quality and latency both improve against 1.8.0, measured over 305 queries on Python, Java and TypeScript corpora: MRR 0.564 -> 0.591, MAP 0.433 -> 0.461, nDCG@10 0.503 -> 0.527, recall@5 0.529 -> 0.560 (all p < 0.001), emitted tokens 1183 -> 1091, p50 78.6ms -> 51.2ms, p99 277.1ms -> 152.2ms. Documentation now describes the shipped implementation rather than the plan. RETRIEVAL.md gains the cross-locator agreement formula and a source-prior table, and drops a "recency" rerank feature that was listed but never implemented. BENCHMARKS.md adds the retrieval eval as a fourth surface and is explicit that it measures version-over-version deltas, not absolute superiority over any other tool. --- .claude-plugin/plugin.json | 2 +- .claude/skills/codebase-index/.skill_version | 2 +- .codex/skills/codebase-index/.skill_version | 2 +- .../skills/codebase-index/.skill_version | 2 +- CHANGELOG.md | 83 +++++++++++++++++++ README.md | 20 +++-- docs/ARCHITECTURE.md | 7 +- docs/BENCHMARKS.md | 27 +++++- docs/RETRIEVAL.md | 83 +++++++++++++++---- docs/RETRIEVAL_PIPELINE.md | 27 ++++-- requirements.lock | 2 +- src/codebase_index/__init__.py | 2 +- 12 files changed, 213 insertions(+), 46 deletions(-) diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index ccfe407..29e7c00 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -3,7 +3,7 @@ "name": "codebase-index", "displayName": "Codebase Index", "description": "Give Claude a precise local map of your codebase: find implementations, trace behavior, and predict change impact with file-line evidence.", - "version": "1.8.0", + "version": "1.9.0", "author": { "name": "codebase-index contributors" }, diff --git a/.claude/skills/codebase-index/.skill_version b/.claude/skills/codebase-index/.skill_version index 27f9cd3..f8e233b 100644 --- a/.claude/skills/codebase-index/.skill_version +++ b/.claude/skills/codebase-index/.skill_version @@ -1 +1 @@ -1.8.0 +1.9.0 diff --git a/.codex/skills/codebase-index/.skill_version b/.codex/skills/codebase-index/.skill_version index 27f9cd3..f8e233b 100644 --- a/.codex/skills/codebase-index/.skill_version +++ b/.codex/skills/codebase-index/.skill_version @@ -1 +1 @@ -1.8.0 +1.9.0 diff --git a/.opencode/skills/codebase-index/.skill_version b/.opencode/skills/codebase-index/.skill_version index 27f9cd3..f8e233b 100644 --- a/.opencode/skills/codebase-index/.skill_version +++ b/.opencode/skills/codebase-index/.skill_version @@ -1 +1 @@ -1.8.0 +1.9.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index fe931d1..64f7fa8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,89 @@ All notable changes to this project are documented here. The format is based on ## [Unreleased] +## [1.9.0] - 2026-09-02 + +### Added + +- **Objective ground truth from git history.** `tests/eval/gen_queries.py` mints a + retrieval benchmark from any git repository by pairing a human-written commit + subject with the files that commit actually changed. Unlike docstring-derived + benchmarks the query text is not copied into the document being retrieved, so the + set is leak-free by construction; merges, reverts, releases, version bumps, + sweeping refactors, changelog-style answers and benchmark scaffolding are all + filtered out. +- **Multi-corpus, multi-language evaluation.** `run_eval.py --corpus REPO:QUERIES` + pools several repositories into one benchmark so ranking changes are validated + outside this repository and outside Python. The 1.9.0 defaults were measured on + 305 queries across Python, Java and TypeScript corpora. +- **Significance testing.** Every non-baseline row now reports a paired bootstrap + 95% confidence interval and a paired permutation p-value (both seeded, therefore + reproducible). Query sets of this size have a noise floor of several MRR points; + signals now ship on the strength of that test rather than the sign of a delta. +- **Context-noise metrics.** The report adds mean emitted snippet tokens, duplicate + rate of returned results, and p99 latency alongside the existing IR metrics. + +### Changed + +- **Fusion now scores cross-retriever agreement.** RRF fuses on + `(path, line-bucket)`, so a symbol definition at line 40 and a lexical hit at line + 120 in the same file fused as two unrelated candidates — two retrievers agreeing + on a file produced two weak results instead of one strong one, and cross-source + agreement never reached the score. Each candidate now also receives, at + `file_agreement_weight` (0.4), the RRF mass of every retriever that found its file + at another locator, excluding retrievers already counted at that locator. + Ablatable via `RetrievalTuning(file_agreement=False)`. +- **Documentation demotion deepened** from -0.05 to -0.20. Prose describing a + feature matches a natural-language question more literally than the code + implementing it, so design notes and plans were displacing the modules they + describe. Because this only reorders prose relative to code, documentation-seeking + queries improved as well (category MRR 0.579 → 0.612). Generated/vendor paths move + to -0.25 so they remain the least-preferred role, and `MAX_ABS_PRIOR` now pins the + invariant that priors stay tiebreakers. +- **Fuzzy identifier matching is now a recall fallback.** It runs only when the + precise symbol lookup named no symbol and returned fewer than + `fuzzy_fallback_min` (3) rows. It moved no ranking metric across 305 queries while + accounting for ~20% of query latency; typo and acronym recall is unchanged because + those are exactly the queries where the precise lookup comes up empty. +- **Candidate over-fetch is explicit.** `candidate_pool_multiplier` (default 2) + replaces the implicit widening that happened whenever dedup or MMR was enabled. + Making it explicit revealed that the quality previously credited to SimHash dedup + was really the wider pool; dedup is retained for what it does measurably do, which + is cutting the duplicate rate of returned snippets from ~1.6% to ~0%. + +### Fixed + +- **Synonym matches were reported as exact symbol matches.** `is_exact` came from + SQL and was relative to whichever needle retrieved the row, so a synonym + expansion ("config" for "configuration") marked an unrelated symbol as an exact + match — worth a +0.20 rerank bonus and an unconditional `high` confidence. + Exactness is now judged against the terms the user actually typed. +- **Duplicate suppression was order-dependent on ties.** Equal-scoring duplicates + handed the slot to whichever copy arrived last, contradicting the documented + "ties favor input order" and making the retained snippet depend on retriever + emission order. + +### Performance + +- Query latency roughly halves: p50 78.6 ms → 51.2 ms, p95 193.2 ms → 94.9 ms, + p99 277.1 ms → 152.2 ms on the pooled three-repository benchmark, from the fuzzy + fallback plus a SimHash fingerprint that folds repeated tokens by multiplicity and + caches token digests. Fingerprint output is bit-for-bit unchanged. +- Mean emitted snippet tokens fall 1183 → 1091 per query. + +### Retrieval quality + +Pooled over 305 queries (Python, Java, TypeScript), v1.8.0 → 1.9.0: + +| Metric | v1.8.0 | 1.9.0 | Δ | p | +|---|---|---|---|---| +| MRR | 0.564 | 0.591 | +0.027 | <0.001 | +| MAP | 0.433 | 0.461 | +0.028 | <0.001 | +| nDCG@10 | 0.503 | 0.527 | +0.024 | <0.001 | +| recall@5 | 0.529 | 0.560 | +0.031 | <0.001 | +| P@5 | 0.182 | 0.192 | +0.009 | 0.001 | +| hit@3 | 0.649 | 0.666 | +0.016 | 0.124 | + ## [1.8.0] - 2026-09-02 ### Added diff --git a/README.md b/README.md index 4b43c70..e6ad992 100644 --- a/README.md +++ b/README.md @@ -271,7 +271,7 @@ See [MCP.md](docs/MCP.md) for client configuration. ## Project status -The latest released line is **1.8.0**. It includes: +The latest released line is **1.9.0**. It includes: - hybrid and optional vector retrieval; - Tree-sitter symbol extraction across the documented language tiers; @@ -280,15 +280,21 @@ The latest released line is **1.8.0**. It includes: - shortest dependency paths and node descriptions; - token-budgeted and skeletonized retrieval packets; - benchmark-calibrated lexical expansion, fuzzy identifier matching, and source-aware ranking; +- rank fusion that scores cross-retriever agreement at file level, not just at a locator; - bounded, intent-directed graph discovery with optional diversity and duplicate suppression; - CLI, Skill, plugin, and MCP delivery; - incremental updates, watch hooks, diagnostics, skill rollback, and diff-aware - impact analysis. - -Planned work is deliberately separated from shipped capability. The next -product priorities are stronger real-repository evaluations, typed framework -edges, and an even more direct task-context workflow. See the -[roadmap](docs/ROADMAP.md). + impact analysis; +- a multi-repository retrieval evaluation with leak-free git-derived ground truth, + one-signal ablations, and paired significance tests + ([tests/eval](tests/eval/README.md)). + +Every shipped ranking signal has to survive that evaluation: 1.9.0 removed the +cost of two signals that could not demonstrate a benefit and rejected several +plausible ones outright (IDF-weighted coverage, stemming, graph propagation, MMR, +a file-length prior). Planned work is deliberately separated from shipped +capability. The next product priorities are typed framework edges and an even more +direct task-context workflow. See the [roadmap](docs/ROADMAP.md). ## Documentation diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 9c4bbf0..4bb88af 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -2,7 +2,7 @@ ## 1. Overview -`codebase-index` is a **local-first** code intelligence layer for AI coding agents. In `1.8.0` +`codebase-index` is a **local-first** code intelligence layer for AI coding agents. In `1.9.0` it has two shipped faces: 1. **A Claude Code Skill** (`.claude/skills/codebase-index/SKILL.md`) that Claude auto-invokes for @@ -97,8 +97,9 @@ codebase-index/ ├── graph/ # builder.py (edge resolution), expand.py (impact), │ # export.py (HTML graph) ├── storage/ # db.py (pragmas, schema, version guard), schema.sql, repo.py - ├── retrieval/ # intent.py, searchers.py, fusion.py, rerank.py, - │ # budget.py, pipeline.py, types.py + ├── retrieval/ # intent.py, searchers.py, fusion.py, rerank.py, priors.py, + │ # lexical.py, fuzzy.py, diversity.py, skeleton.py, + │ # tuning.py (ablation contract), budget.py, pipeline.py, types.py ├── embeddings/ # backend.py, noop.py (default), local.py, external.py — opt-in ├── output/ # markdown.py, json.py, redact.py ├── watch/ # watcher.py (optional, watchdog-based) diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 062da77..5b1d7db 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -1,10 +1,11 @@ # Benchmarks -`codebase-index` has three benchmark surfaces. Read them with their status in +`codebase-index` has four benchmark surfaces. Read them with their status in mind — the whole point of this page is to keep evidence and aspiration separate. | Surface | What it is | Status | Use it as | |---|---|---|---| +| Retrieval eval (`tests/eval/`) | Ranking quality vs a fixed baseline across multiple real repositories, with significance tests | **Proven (relative)** | The gate for ranking changes; measures *deltas*, not absolute superiority | | Public suite (`tests/benchmark_public.py`) | Deterministic synthetic multi-language fixture with the full metric framework | **Toy/synthetic** | CI regression gate + metric shape, **not** product-quality evidence | | Smoke/perf (`test_perf_smoke.py`, `test_benchmark_comparison.py`) | Latency + output-size guards on a tiny fixture | **Toy/smoke** | Regression checks only | | Honest real-repo (`tests/benchmark_honest.py`) | 55k LOC Java repo, recall@3 vs disciplined `rg` baseline, symmetric token accounting | **Proven (one repo)** | The only headline product-quality number we stand behind today | @@ -15,15 +16,33 @@ Do not write, imply, or ship any of these until a run with published logs exists - Any 10k / 100k / 1M LOC scale or speed claim (no real run at that size). - "Beats Cursor / Sourcegraph / Codebase-Memory MCP" — no head-to-head exists. -- Per-language quality claims beyond Java (the honest run is Java-only). +- Per-language *absolute* quality claims beyond Java. The retrieval eval covers + Python, Java and TypeScript, but it measures this system against its own earlier + versions — it says a change helped, not that the product beats an alternative. - Generic "Nx faster" / "Nx fewer tokens" without naming the baseline and repo. -- Latency claims — the honest run explicitly does not headline latency - (Python process start dominates; real `rg` is tens of ms). +- Latency claims against external tools — the honest run explicitly does not + headline latency (Python process start dominates; real `rg` is tens of ms). + Version-over-version latency from the retrieval eval is in-process and is only + comparable to other runs of that harness. The defensible headline today is exactly: **on one 55k LOC Java repo, recall@3 was 70% (index) vs 40% (`rg`+window), using ~13× fewer answer tokens.** Everything else is roadmap. +## Retrieval evaluation (ranking gate) + +See [tests/eval/README.md](../tests/eval/README.md) for the full protocol. In short: +ground truth comes from hand-written queries verified against the source tree and +from commit-subject → changed-files pairs mined from git history (leak-free: the +query text is not in the indexed corpus). Corpora are pooled across languages, one +index per corpus is shared by every variant, and every non-baseline row carries a +paired bootstrap CI and permutation p-value. + +`1.9.0` was measured over 305 queries across Python, Java and TypeScript corpora +against `1.8.0`: MRR +0.027, MAP +0.028, nDCG@10 +0.024, recall@5 +0.031 (all +p < 0.001), with p50 latency 78.6 ms → 51.2 ms. These are version-over-version +ranking deltas on those corpora, not a universal quality claim. + ## Public benchmark suite Run: diff --git a/docs/RETRIEVAL.md b/docs/RETRIEVAL.md index d4acf37..6102baa 100644 --- a/docs/RETRIEVAL.md +++ b/docs/RETRIEVAL.md @@ -65,6 +65,11 @@ source)` list so fusion is source-agnostic. first when the query clearly names a path. - **Symbol** — query against `symbols` (exact, identifier parts, bounded fuzzy matching). Carries `kind` (function/class/method/...) and signature. Primary for `locate_impl` / `find_refs`. + Fuzzy identifier matching (acronym / concatenation / edit distance) runs only as a **recall + fallback**, when the precise lookup named no symbol and returned fewer than + `fuzzy_fallback_min` rows. Measured over 305 queries on three repositories it moved no ranking + metric while costing ~20% of query latency, so it is kept for typos and abbreviations but no + longer runs when the query already spelled its identifier correctly. - **FTS** — FTS5 `bm25()` over the `fts_chunks` virtual table (chunk text + symbol names + summaries indexed). Query-time camelCase/snake_case splitting, small down-weighted synonym expansion, and soft coverage scoring make natural-language questions robust without weakening @@ -79,28 +84,66 @@ source)` list so fusion is source-agnostic. scores: ``` -RRF(d) = Σ_r w_r / (k + rank_r(d)) # k ≈ 60, w_r = per-intent retriever weight +RRF(d) = Σ_r w_r · k / (k + rank_r(d)) # k ≈ 60, w_r = per-intent retriever weight ``` - Robust to scale differences between BM25 and cosine. - Per-intent weights `w_r` let `locate_impl` favor the symbol list and `how_it_works` favor FTS. +- Scaled by `k` so fused scores and the reranker's bounded bonuses share an O(1) scale. This is a + monotonic rescale; fusion order is unchanged. - Ties broken by rerank features (next step). -## 4. Reranking (`retrieval/rerank.py`) +### Cross-locator file agreement + +Fusion keys on `(path, line-bucket)`, not `(path, start, end)`, because different retrievers report +different line ranges for the same place. Bucketing alone was not enough: a symbol defined at line +40 and a lexical hit at line 120 are genuinely different locators, so a file that **two retrievers +agreed on** still fused as two separate candidates, each carrying one retriever's evidence — and +cross-source agreement, the entire point of RRF, never fired. + +Each candidate therefore also receives, at weight `file_agreement_weight`, the RRF mass of every +retriever that found its *file* at some other locator: + +``` +score(d) = RRF(d) + α · Σ_{r ∉ sources(d)} w_r · k / (k + best_rank_r(path(d))) +``` + +Retrievers already counted at the candidate's own locator are excluded, so nothing double-counts, +and the term is bounded by the same weights as fusion itself. `α = 0.4`; the 0.3–0.6 plateau peaks +there. Set `RetrievalTuning(file_agreement=False)` to recover plain locator-only RRF. -A lightweight, explainable feature score (no external model required) layered on the fused order: +## 4. Reranking (`retrieval/rerank.py`) -| Feature | Intuition | -|---|---| -| symbol-kind match | a `def`/`class` outranks an incidental mention | -| path proximity | files near a query-named path score higher | -| graph centrality | high in/out-degree nodes matter more for `architecture` | -| recency | recently changed files (git mtime) slightly boosted | -| exact-name bonus | exact symbol-name match dominates fuzzy | -| test/generated penalty | test files and generated code demoted unless asked | +A lightweight, explainable feature score (no external model required) layered on the fused order. +Every term is bounded, so reranking reorders near-neighbours rather than overruling retrieval: + +| Feature | Effect | Intuition | +|---|---:|---| +| Exact symbol match | +0.20 | the user named a specific symbol | +| Symbol definition kind | +0.05 | a `def`/`class` outranks an incidental mention | +| Symbol name among query terms | +0.05 | the name was asked for, not just matched | +| Path term match | +0.05 | the user supplied a location clue | +| Graph centrality (`in_degree`) | ≤ +0.08 | `log1p`-damped, so a god class cannot dominate | +| Reference-count fallback | ≤ +0.04 | for names too common to resolve a precise `in_degree` | +| Source role prior | −0.25…+0.08 | see below | +| Generated, or test on a non-test query | −0.15 | supporting evidence, not the answer | + +### Source role priors (`retrieval/priors.py`) + +| Role | Prior | Rationale | +|---|---:|---| +| Implementation | +0.08 | the answer to a code question is usually code | +| Test | −0.06 | flips to +0.05 when the query or intent is test-oriented | +| Documentation | −0.20 | prose *about* a feature matches a natural-language question more literally than the code implementing it, so design notes and plans crowded out the modules they describe | +| Generated / vendor / build | −0.25 | never the answer; kept strictly below documentation | + +The documentation prior deepened from −0.05 in 1.8.0. Because it only reorders prose relative to +code — never below other prose — documentation-seeking queries improved too (MRR 0.579 → 0.612). +At −0.35 that reverses and the `docs` category collapses, so the optimum is interior, not a +"more is better" knob. `MAX_ABS_PRIOR` caps every prior so this stays a tiebreaker. The reranker also produces the human-readable **`reason`** string per result -(e.g. *"exact symbol match · called by 4 sites · in src/auth/"*). +(e.g. *"exact symbol match · 4 callers · in src/auth/"*). ## 5. Graph expansion (`graph/retrieval.py`; `graph/expand.py` for impact APIs) @@ -119,11 +162,17 @@ Expanded nodes retain edge confidence and receive distance-decayed scores so see ## 6. Diversity and duplicate control -`retrieval.diversity` provides bounded MMR selection and SimHash near-duplicate -suppression. MMR is disabled in the shipped default because the reproducible -benchmark favored relevance-only ranking; callers that need broader snippet -coverage can enable `RetrievalTuning(mmr=True)`. Duplicate suppression remains -available independently. +`retrieval.diversity` provides bounded MMR selection and SimHash near-duplicate suppression. + +MMR is disabled in the shipped default: it moved no ranking metric on the benchmark and roughly +doubled p50 latency. Callers that need broader snippet coverage can enable +`RetrievalTuning(mmr=True)`. + +SimHash duplicate suppression stays on, but on noise grounds rather than ranking grounds: it does +not move MRR, and it takes the duplicate rate of returned snippets from ~1.6% to ~0%. The +over-fetch that feeds selection is now an explicit `candidate_pool_multiplier` rather than an +implicit side effect of enabling dedup — an earlier ablation credited dedup with a quality win that +was really the wider pool doing the work. ## 7. Token budgeting (`retrieval/budget.py`) diff --git a/docs/RETRIEVAL_PIPELINE.md b/docs/RETRIEVAL_PIPELINE.md index 2a843e1..5bf0944 100644 --- a/docs/RETRIEVAL_PIPELINE.md +++ b/docs/RETRIEVAL_PIPELINE.md @@ -97,17 +97,21 @@ direct lexical or symbol evidence. ## 6. Diversity and duplicate control -SimHash suppresses near-duplicate snippets independently of MMR. Bounded -Maximal Marginal Relevance is available through `RetrievalTuning(mmr=True)`; -the shipped default keeps relevance-only ordering because the benchmark favored -direct hits. +SimHash suppresses near-duplicate snippets independently of MMR. It does not move +ranking metrics; it earns its place by taking the duplicate rate of returned +snippets from ~1.6% to ~0%. Bounded Maximal Marginal Relevance is available +through `RetrievalTuning(mmr=True)`; the shipped default keeps relevance-only +ordering because MMR moved no metric and roughly doubled p50 latency. + +The over-fetch that feeds these selection stages is `candidate_pool_multiplier` +(default 2), an explicit knob rather than a side effect of enabling dedup. ## 7. Reciprocal Rank Fusion (RRF) Combines ranked lists from the enabled retrievers: ``` -RRF_score(d) = Σ w_r · k / (k + rank_r(d)) +RRF_score(d) = Σ_r w_r · k / (k + rank_r(d)) ``` The implementation multiplies textbook RRF by `k` so fusion and bounded rerank @@ -118,8 +122,13 @@ Where: - `rank_r(d)` is the rank of document `d` in retriever `r` - `w_r` is the intent/tuning weight for retriever `r` -The implementation merges co-located chunks into one per-file bucket before -fusion, preventing a large file from dominating the result list. +Candidates are keyed by `(path, line-bucket)`, so co-located hits merge. Because a +symbol definition and a lexical hit in the same file are often *not* co-located, +each candidate additionally receives — at `file_agreement_weight` (0.4) — the RRF +mass of every retriever that found its file at a different locator. Retrievers +already counted at that locator are excluded, so nothing double-counts. Without +this, two retrievers agreeing on a file produced two weak candidates instead of one +strong one, and cross-source agreement never affected the score. ## 8. Reranking @@ -132,8 +141,8 @@ After fusion, apply bounded explainable boosts and penalties: | Path term match | +0.05 | User supplied a location clue | | Degree / reference evidence | up to +0.08 | Stable structural tiebreaker | | Implementation source prior | +0.08 | Prefer source over prose/tests | -| Documentation source prior | -0.05 | Avoid docs displacing implementation | -| Generated/vendor/build | -0.12 | Suppress low-value derived code | +| Documentation source prior | -0.20 | Prose describing a feature outmatches the code lexically | +| Generated/vendor/build | -0.25 | Suppress low-value derived code | | Test path on non-test query | -0.06 | Keep tests as supporting evidence | ## 9. Confidence diff --git a/requirements.lock b/requirements.lock index 4a600d6..6843b5b 100644 --- a/requirements.lock +++ b/requirements.lock @@ -1,3 +1,3 @@ -codebase-index @ https://github.com/denfry/codebase-index/archive/refs/tags/v1.8.0.tar.gz +codebase-index @ https://github.com/denfry/codebase-index/archive/refs/tags/v1.9.0.tar.gz tree-sitter==0.25.2 tree-sitter-language-pack==1.8.1 diff --git a/src/codebase_index/__init__.py b/src/codebase_index/__init__.py index f4bccfd..af8c17b 100644 --- a/src/codebase_index/__init__.py +++ b/src/codebase_index/__init__.py @@ -4,4 +4,4 @@ See docs/ARCHITECTURE.md for the module map. """ -__version__ = "1.8.0" +__version__ = "1.9.0"