Skip to content

fix: validate the .git marker, and stop serving stale traversal caches #17

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
maxgfr merged 2 commits into from
Sep 3, 2026
Merged

fix: validate the .git marker, and stop serving stale traversal caches #17

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8ea9ccb
fix: only a valid .git marks a repo boundary, and stop serving stale …
maxgfr Sep 2, 2026
65febc1
fix(walk): parse a gitfile the way git does, and bound the read
maxgfr Sep 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
152 changes: 76 additions & 76 deletions scripts/engine.browser.mjs
View file
Open in desktop

Large diffs are not rendered by default.

91 changes: 62 additions & 29 deletions scripts/engine.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -358,18 +358,29 @@ import { join, resolve, sep, extname } from "path";
function isIgnoredDirectory(name2, ignoreDirs) {
return name2 === GIT_ENTRY || ignoreDirs.has(name2) || name2.startsWith(".codeindex-edit-");
}
function readInfoExclude(root, entries) {
function gitDirOf(dir, entries) {
const marker = entries.find((e) => e.name === GIT_ENTRY);
if (!marker) return "";
let gitDir = join(root, GIT_ENTRY);
if (!marker) return void 0;
const path = join(dir, GIT_ENTRY);
try {
if (marker.isDirectory()) return path;
const st = statSync(path);
if (st.isDirectory()) return path;
if (!st.isFile() || st.size > MAX_GITFILE_BYTES) return void 0;
const content = readFileSync(path, "utf8");
if (!content.startsWith(GITFILE_PREFIX)) return void 0;
const target = content.slice(GITFILE_PREFIX.length).replace(/\s+$/, "");
if (!target) return void 0;
const gitDir = resolve(dir, target);
const common = join(gitDir, "commondir");
return existsSync(common) ? resolve(gitDir, readFileSync(common, "utf8").trim()) : gitDir;
} catch {
return void 0;
}
}
function readInfoExclude(gitDir) {
if (!gitDir) return "";
try {
if (!marker.isDirectory()) {
const m = /^gitdir:[ \t]*(.+?)[ \t]*$/m.exec(readFileSync(gitDir, "utf8"));
if (!m) return "";
gitDir = resolve(root, m[1]);
const common = join(gitDir, "commondir");
if (existsSync(common)) gitDir = resolve(gitDir, readFileSync(common, "utf8").trim());
}
const exclude = join(gitDir, "info", "exclude");
return existsSync(exclude) ? readText(exclude) : "";
} catch {
Expand Down Expand Up @@ -414,13 +425,14 @@ function walk(root, opts = {}) {
} catch {
continue;
}
if (frame.rel && entries.some((e) => e.name === GIT_ENTRY)) {
const gitDir = entries.some((e) => e.name === GIT_ENTRY) ? gitDirOf(frame.dir, entries) : void 0;
if (frame.rel && gitDir) {
excluded++;
continue;
}
let rules = frame.rules;
if (useGitignore && !frame.rel) {
const parsed = parseGitignore(readInfoExclude(frame.dir, entries), "");
const parsed = parseGitignore(readInfoExclude(gitDir), "");
if (parsed.length) rules = [...rules, ...parsed];
}
if (useGitignore && entries.some((e) => e.name === ".gitignore")) {
Expand Down Expand Up @@ -504,7 +516,7 @@ function readText(abs) {
return "";
}
}
var IGNORE_DIRS, GIT_ENTRY, LOCKFILES, BINARY_EXT, DEFAULT_MAX_FILES;
var IGNORE_DIRS, GIT_ENTRY, GITFILE_PREFIX, MAX_GITFILE_BYTES, LOCKFILES, BINARY_EXT, DEFAULT_MAX_FILES;
var init_walk = __esm({
"src/walk.ts"() {
"use strict";
Expand Down Expand Up @@ -544,6 +556,8 @@ var init_walk = __esm({
".dart_tool"
]);
GIT_ENTRY = ".git";
GITFILE_PREFIX = "gitdir: ";
MAX_GITFILE_BYTES = 4096;
LOCKFILES = /* @__PURE__ */ new Set([
"package-lock.json",
"npm-shrinkwrap.json",
Expand Down Expand Up @@ -16208,26 +16222,39 @@ init_viz();
// src/traverse.ts
init_sort();
var DEPENDS_KINDS = /* @__PURE__ */ new Set(["import", "use", "call"]);
var dependentsMemo = /* @__PURE__ */ new WeakMap();
function dependentsOf(edges) {
const hit = dependentsMemo.get(edges);
if (hit && hit.length === edges.length) return hit.map;
const map = /* @__PURE__ */ new Map();
for (const e of edges) {
if (e.dangling || !DEPENDS_KINDS.has(e.kind)) continue;
let arr = map.get(e.to);
if (!arr) map.set(e.to, arr = []);
arr.push(e);
var FIELDS2 = 6;
function snapshot(edges) {
const snap = new Array(edges.length * FIELDS2);
for (let i2 = 0; i2 < edges.length; i2++) {
const e = edges[i2];
const o = i2 * FIELDS2;
snap[o] = e.from;
snap[o + 1] = e.to;
snap[o + 2] = e.kind;
snap[o + 3] = e.weight;
snap[o + 4] = e.dangling;
snap[o + 5] = e.confidence;
}
return snap;
}
function unchanged(edges, snap) {
if (snap.length !== edges.length * FIELDS2) return false;
for (let i2 = 0; i2 < edges.length; i2++) {
const e = edges[i2];
const o = i2 * FIELDS2;
if (snap[o] !== e.from || snap[o + 1] !== e.to || snap[o + 2] !== e.kind || snap[o + 3] !== e.weight || snap[o + 4] !== e.dangling || snap[o + 5] !== e.confidence) {
return false;
}
}
for (const arr of map.values()) arr.sort((a, b) => byStr(a.from, b.from));
dependentsMemo.set(edges, { length: edges.length, map });
return map;
return true;
}
var adjacencyMemo = /* @__PURE__ */ new WeakMap();
function adjacencyOf(edges, kinds) {
const viewKey = kinds ? [...kinds].sort(byStr).join(",") : "*";
let entry = adjacencyMemo.get(edges);
if (!entry || entry.length !== edges.length) adjacencyMemo.set(edges, entry = { length: edges.length, views: /* @__PURE__ */ new Map() });
if (!entry || !unchanged(edges, entry.snap)) {
adjacencyMemo.set(edges, entry = { snap: snapshot(edges), views: /* @__PURE__ */ new Map() });
}
const cached = entry.views.get(viewKey);
if (cached) return cached;
const out2 = /* @__PURE__ */ new Map();
Expand All @@ -16254,14 +16281,20 @@ function hubThreshold(degrees) {
return Math.max(50, p99);
}
function reverseClosure(edges, seeds, depth = Infinity) {
const dependents = dependentsOf(edges);
const dependents = /* @__PURE__ */ new Map();
for (const e of edges) {
if (e.dangling || !DEPENDS_KINDS.has(e.kind)) continue;
let arr = dependents.get(e.to);
if (!arr) dependents.set(e.to, arr = []);
arr.push(e);
}
const depthOf = /* @__PURE__ */ new Map();
const seen = new Set(seeds);
let frontier = [...seeds];
for (let d = 1; d <= depth && frontier.length; d++) {
const next = [];
for (const node of frontier) {
for (const e of dependents.get(node) ?? []) {
for (const e of (dependents.get(node) ?? []).slice().sort((a, b) => byStr(a.from, b.from))) {
if (seen.has(e.from)) continue;
seen.add(e.from);
depthOf.set(e.from, d);
Expand Down
97 changes: 72 additions & 25 deletions src/traverse.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,41 +13,75 @@ import { byStr } from "./sort.js";
// mention says something references the name, not that it would break.
const DEPENDS_KINDS = new Set(["import", "use", "call"]);

// Adjacency, built ONCE per edge list and kept as long as the list itself.
// Both traversals rebuilt their maps — and re-sorted every neighbour list —
// on every call, while a consumer such as the change-risk scorer (delta.ts)
// asks impactOf() once per changed file of every module. Keyed on the edge
// ARRAY's identity (a Graph's `edges` is never rebuilt in place); the length
// check guards the one cheap-to-detect mutation. Lists are pre-sorted with the
// comparator the per-call sort used, on the same insertion order, so the
// traversal visits exactly the same sequence.
const dependentsMemo = new WeakMap<Edge[], { length: number; map: Map<string, Edge[]> }>();
function dependentsOf(edges: Edge[]): Map<string, Edge[]> {
const hit = dependentsMemo.get(edges);
if (hit && hit.length === edges.length) return hit.map;
const map = new Map<string, Edge[]>(); // target file → incoming depends-on edges
for (const e of edges) {
if (e.dangling || !DEPENDS_KINDS.has(e.kind)) continue;
let arr = map.get(e.to);
if (!arr) map.set(e.to, (arr = []));
arr.push(e);
// The traversal-relevant fields of an edge list, captured when a derived view
// is built so a later call can prove the list still describes the same graph.
//
// WHY A SNAPSHOT AND NOT AN IDENTITY CHECK. `Graph.fileEdges` / `moduleEdges`
// are public, MUTABLE arrays. A consumer can retarget an edge IN PLACE, which
// leaves both the array's identity and its length unchanged — so a cache keyed
// on those alone answers with the pre-edit graph, which is the one behaviour a
// cache must never have. Values are copied BY REFERENCE into one flat array (no
// new strings), so checking is a handful of pointer comparisons per edge.
//
// It is not free, though: on a 9 153-edge graph a check costs roughly what
// rebuilding the dependents map costs, which is exactly why reverseClosure
// above does not cache at all and this one does.
const FIELDS = 6;
function snapshot(edges: Edge[]): unknown[] {
const snap = new Array<unknown>(edges.length * FIELDS);
for (let i = 0; i < edges.length; i++) {
const e = edges[i]!;
const o = i * FIELDS;
snap[o] = e.from;
snap[o + 1] = e.to;
snap[o + 2] = e.kind;
snap[o + 3] = e.weight;
snap[o + 4] = e.dangling;
snap[o + 5] = e.confidence;
}
return snap;
}
function unchanged(edges: Edge[], snap: unknown[]): boolean {
if (snap.length !== edges.length * FIELDS) return false;
for (let i = 0; i < edges.length; i++) {
const e = edges[i]!;
const o = i * FIELDS;
if (
snap[o] !== e.from ||
snap[o + 1] !== e.to ||
snap[o + 2] !== e.kind ||
snap[o + 3] !== e.weight ||
snap[o + 4] !== e.dangling ||
snap[o + 5] !== e.confidence
) {
return false;
}
}
for (const arr of map.values()) arr.sort((a, b) => byStr(a.from, b.from));
dependentsMemo.set(edges, { length: edges.length, map });
return map;
return true;
}


interface Adjacency {
out: Map<string, Edge[]>; // from → edges, sorted by `to`
inn: Map<string, Edge[]>; // to → edges, sorted by `from`
degree: Map<string, number>;
threshold: number; // hubThreshold over this view's degree distribution
}
const adjacencyMemo = new WeakMap<Edge[], { length: number; views: Map<string, Adjacency> }>();
// Adjacency, built ONCE per edge list and kept as long as the list itself, then
// re-validated against the snapshot above on every hit. bfs() otherwise rebuilt
// two maps, re-sorted every neighbour list and recomputed the degree
// distribution on each call, while an MCP session answers many neighbors calls
// over one graph: 200 calls over a 9 153-edge graph, 189 ms → 38 ms. Lists are
// pre-sorted with the comparator the per-call sort used, on the same insertion
// order, so the traversal visits exactly the same sequence.
const adjacencyMemo = new WeakMap<Edge[], { snap: unknown[]; views: Map<string, Adjacency> }>();
function adjacencyOf(edges: Edge[], kinds?: Set<string>): Adjacency {
const viewKey = kinds ? [...kinds].sort(byStr).join(",") : "*";
let entry = adjacencyMemo.get(edges);
if (!entry || entry.length !== edges.length) adjacencyMemo.set(edges, (entry = { length: edges.length, views: new Map() }));
// One snapshot per edge list covers every kind-filtered view built from it.
if (!entry || !unchanged(edges, entry.snap)) {
adjacencyMemo.set(edges, (entry = { snap: snapshot(edges), views: new Map() }));
}
const cached = entry.views.get(viewKey);
if (cached) return cached;
const out = new Map<string, Edge[]>();
Expand Down Expand Up @@ -101,15 +135,28 @@ export interface ImpactResult {

// Reverse dependency closure: every file that transitively IMPORTS, USES, or
// CALLS one of `seeds`, out to `depth` hops (default: the full closure).
// Deliberately UNCACHED, and sorting only the buckets the walk actually visits.
// Both matter: an impact closure usually reaches a handful of nodes, so sorting
// every bucket up front costs more than the walk itself (200 impactOf calls
// over a 9 153-edge graph: 78 ms this way, 132 ms sorting eagerly), and proving
// a cached map still matches its edge list costs about as much as rebuilding
// it. adjacencyOf below is the opposite trade — two maps, two sorts and a
// degree distribution per call — and does earn its cache.
export function reverseClosure(edges: Edge[], seeds: string[], depth = Infinity): Map<string, number> {
const dependents = dependentsOf(edges);
const dependents = new Map<string, Edge[]>(); // target file → incoming depends-on edges
for (const e of edges) {
if (e.dangling || !DEPENDS_KINDS.has(e.kind)) continue;
let arr = dependents.get(e.to);
if (!arr) dependents.set(e.to, (arr = []));
arr.push(e);
}
const depthOf = new Map<string, number>();
const seen = new Set<string>(seeds);
let frontier = [...seeds];
for (let d = 1; d <= depth && frontier.length; d++) {
const next: string[] = [];
for (const node of frontier) {
for (const e of dependents.get(node) ?? []) {
for (const e of (dependents.get(node) ?? []).slice().sort((a, b) => byStr(a.from, b.from))) {
if (seen.has(e.from)) continue;
seen.add(e.from);
depthOf.set(e.from, d);
Expand Down
95 changes: 70 additions & 25 deletions src/walk.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,24 +31,67 @@ function isIgnoredDirectory(name: string, ignoreDirs: Set<string>): boolean {
return name === GIT_ENTRY || ignoreDirs.has(name) || name.startsWith(".codeindex-edit-");
}

// Locate this checkout's `info/exclude` — git's per-clone, never-committed
// ignore file (the walker honored only .gitignore, so local junk a developer
// excluded there was still indexed). A directory `.git` holds it directly; a
// gitfile points at the real git dir, and a linked worktree's git dir further
// points at the shared common dir (`commondir` file), which is where git keeps
// `info/` for every worktree. Returns "" when absent or unreadable.
function readInfoExclude(root: string, entries: readonly Dirent[]): string {
// A gitfile's mandatory opening bytes. Git's parser (read_gitfile_gently)
// compares the first 8 bytes against exactly this — verified against real git:
// `gitdir:` without the space, leading whitespace, or the line appearing
// anywhere but the start are all rejected as "invalid gitfile format".
const GITFILE_PREFIX = "gitdir: ";
// A real gitfile is one short line. The cap keeps a file that merely CARRIES the
// name `.git` — a stray archive, a truncated dump — from being read whole just
// to discover it is not a gitfile.
const MAX_GITFILE_BYTES = 4096;

// The git directory a `.git` entry in `dir` points at, or undefined when there
// is none, or when the entry is not a repository marker.
//
// Why validity is checked instead of assumed: the boundary used to trigger on
// the NAME alone, so a file named `.git` holding anything else — a truncated
// write, an unrelated file carrying the name, a dangling symlink — silently
// dropped its whole subtree from the index. Silent truncation is the one
// failure this walk does not allow.
//
// A DIRECTORY named `.git` is the git dir. A FILE is a marker only when it
// opens with `gitdir: ` (above); the rest, trailing whitespace trimmed, is the
// path. Symlinks are followed — git supports a symlinked `.git`, and a link's
// dirent is neither file nor directory, so its target decides.
//
// DELIBERATE DEVIATION: git additionally requires the TARGET to look like a
// repository (HEAD, objects/, refs/) and reports "not a git repository" when it
// does not. This walk stops at a well-formed marker whatever its target, and
// the tests pin that: a stale gitfile left by a pruned or moved worktree still
// sits on a full checkout, and indexing it would duplicate the parent's sources
// — exactly what the boundary exists to prevent.
//
// The returned dir is the COMMON one where relevant: a linked worktree's git
// dir points at the shared common dir via its `commondir` file, and that is
// where git keeps `info/` for every worktree.
function gitDirOf(dir: string, entries: readonly Dirent[]): string | undefined {
const marker = entries.find((e) => e.name === GIT_ENTRY);
if (!marker) return "";
let gitDir = join(root, GIT_ENTRY);
if (!marker) return undefined;
const path = join(dir, GIT_ENTRY);
try {
if (marker.isDirectory()) return path; // a plain clone — decided on the dirent, no syscall
const st = statSync(path); // a file, or a symlink resolved through its target
if (st.isDirectory()) return path;
if (!st.isFile() || st.size > MAX_GITFILE_BYTES) return undefined;
const content = readFileSync(path, "utf8");
if (!content.startsWith(GITFILE_PREFIX)) return undefined; // not a gitfile — not a marker
const target = content.slice(GITFILE_PREFIX.length).replace(/\s+$/, "");
if (!target) return undefined;
const gitDir = resolve(dir, target);
const common = join(gitDir, "commondir");
return existsSync(common) ? resolve(gitDir, readFileSync(common, "utf8").trim()) : gitDir;
} catch {
return undefined;
}
}

// This checkout's `info/exclude` — git's per-clone, never-committed ignore file
// (the walker honored only .gitignore, so local junk a developer excluded there
// was still indexed). Returns "" when absent or unreadable.
function readInfoExclude(gitDir: string | undefined): string {
if (!gitDir) return "";
try {
if (!marker.isDirectory()) {
const m = /^gitdir:[ \t]*(.+?)[ \t]*$/m.exec(readFileSync(gitDir, "utf8"));
if (!m) return "";
gitDir = resolve(root, m[1]!);
const common = join(gitDir, "commondir");
if (existsSync(common)) gitDir = resolve(gitDir, readFileSync(common, "utf8").trim());
}
const exclude = join(gitDir, "info", "exclude");
return existsSync(exclude) ? readText(exclude) : "";
} catch {
Expand Down Expand Up @@ -176,22 +219,24 @@ export function walk(root: string, opts: WalkOptions = {}): WalkResult {
} catch {
continue;
}
// Nested-repository boundary: a subdirectory with its own `.git` (directory
// or gitfile) is another repo — a linked worktree under .claude/worktrees/,
// a vendored clone, a submodule. Its files belong to THAT repo's index, and
// walking them here produced thousands of phantom duplicates of the same
// sources (a repo with four worktrees indexed five copies of itself); git
// itself never lists them. Decided on the dirents already listed — zero
// extra syscalls — and structural: independent of the gitignore layer.
if (frame.rel && entries.some((e) => e.name === GIT_ENTRY)) {
// Resolved at most once per directory, and only when a `.git` entry is
// actually listed — so an ordinary directory costs one name comparison.
const gitDir = entries.some((e) => e.name === GIT_ENTRY) ? gitDirOf(frame.dir, entries) : undefined;
// Nested-repository boundary: a subdirectory that IS another repo — a
// linked worktree under .claude/worktrees/, a vendored clone, a submodule.
// Its files belong to THAT repo's index, and walking them here produced
// thousands of phantom duplicates of the same sources (a repo with four
// worktrees indexed five copies of itself); git itself never lists them.
// Structural: independent of the gitignore layer.
if (frame.rel && gitDir) {
excluded++;
continue;
}
let rules = frame.rules;
if (useGitignore && !frame.rel) {
// `.git/info/exclude` sits BEFORE every .gitignore in git's own
// precedence (a .gitignore rule can still negate it — later rules win).
const parsed = parseGitignore(readInfoExclude(frame.dir, entries), "");
const parsed = parseGitignore(readInfoExclude(gitDir), "");
if (parsed.length) rules = [...rules, ...parsed];
}
if (useGitignore && entries.some((e) => e.name === ".gitignore")) {
Expand Down
Loading
Loading