fix(extraction): keep the git-aware scan working on git older than 2.36 (#1549)

CHANGELOG.md

@@ -25,6 +25,8 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ### Fixes
 
+- Files opted back in through `includeIgnored` are no longer dropped on older git. On git before 2.36 — which includes Ubuntu 22.04 LTS and Debian 11 — the command CodeGraph used to list tracked files refused to run at all, and the failure quietly took the whole git-aware scan with it: `includeIgnored`, submodule and embedded-repo recursion, and the `include` allowlist in `codegraph.json` all stopped applying, so files went missing from the index with no error and no warning. CodeGraph now falls back to a form those versions accept, and every one of those settings works there as it does on newer git. Thanks @newshowardz777. (#1549)
+
 - Indexing no longer hangs on a Swift Vapor project containing a call with a long argument list. A single `.get(...)`-style call with many labeled arguments and no `use:` handler — the shape generated request builders produce — could stall `codegraph index`, `codegraph sync`, and the MCP server indefinitely. Route detection now handles such files in milliseconds, and every previously-recognized route shape still parses exactly as before. Thanks @maxmilian. (#1544) (Swift)
 
 - `codegraph status` now sees new files inside brand-new directories. Git reports an entirely-untracked directory as a single collapsed entry, so source files created there — a freshly scaffolded `frontend/`, for example — were missing from the pending-changes report, which could claim everything was up to date while those files had not yet been indexed. Thanks @maxmilian. (#1213)

__tests__/extraction.test.ts

@@ -8,6 +8,7 @@ import { describe, it, expect, beforeAll, beforeEach, afterEach } from 'vitest';
 import * as fs from 'fs';
 import * as path from 'path';
 import * as os from 'os';
+import { execFileSync } from 'child_process';
 import { CodeGraph } from '../src';
 import { extractFromSource, scanDirectory, buildDefaultIgnore, discoverEmbeddedRepoRoots, buildScopeIgnore } from '../src/extraction';
 import { detectLanguage, isLanguageSupported, getSupportedLanguages, initGrammars, loadAllGrammars, isSourceFile } from '../src/extraction/grammars';
@@ -11637,3 +11638,92 @@ describe('C/C++ kernel-port preParse blanks (R7a)', () => {
     expect(result.nodes.some((n) => n.kind === 'method' && n.name === 'size')).toBe(true);
   });
 });
+
+// git < 2.36 rejects `ls-files -s --recurse-submodules` outright: the guard in
+// builtin/ls-files.c listed `show_stage` among the modes that die, and it was
+// only dropped in 2.36. The die is unconditional — it does not check whether the
+// repo has submodules — so on Ubuntu 22.04 (git 2.34.1), Debian 11 (2.30.2) and
+// older, every call threw, `getGitVisibleFiles` swallowed it, and the whole
+// git-visible path went with it: `includeIgnored`, gitlink recursion and the
+// `codegraph.json` `include` allowlist all silently stopped applying (#1549).
+//
+// A PATH shim reproduces that on any git version, which is what makes this
+// testable in CI at all.
+describe('Old git without `ls-files -s --recurse-submodules` support (#1549)', () => {
+  let tempDir: string;
+  let originalPath: string | undefined;
+
+  const runGit = (cwd: string, ...args: string[]) =>
+    execFileSync('git', args, { cwd, stdio: 'pipe' });
+
+  const makeRepo = (dir: string, base: string) => {
+    fs.mkdirSync(dir, { recursive: true });
+    runGit(dir, 'init', '-q');
+    runGit(dir, 'config', 'user.email', 'test@test.com');
+    runGit(dir, 'config', 'user.name', 'Test');
+    fs.writeFileSync(path.join(dir, `${base}.ts`), `export const ${base} = 1;`);
+    runGit(dir, 'add', '-A');
+    runGit(dir, 'commit', '-q', '-m', `${base} init`);
+  };
+
+  /** A `git` that dies exactly like < 2.36 when it sees -s with --recurse-submodules. */
+  const installOldGitShim = () => {
+    const shimDir = path.join(tempDir, '.shim');
+    fs.mkdirSync(shimDir, { recursive: true });
+    const realGit = execFileSync('which', ['git']).toString().trim();
+    const shim = path.join(shimDir, 'git');
+    fs.writeFileSync(
+      shim,
+      [
+        '#!/bin/sh',
+        'for a in "$@"; do',
+        '  [ "$a" = "--recurse-submodules" ] && rs=1',
+        '  [ "$a" = "-s" ] && st=1',
+        'done',
+        'if [ -n "$rs" ] && [ -n "$st" ]; then',
+        '  echo "fatal: ls-files --recurse-submodules unsupported mode" >&2',
+        '  exit 128',
+        'fi',
+        `exec ${JSON.stringify(realGit)} "$@"`,
+      ].join('\n'),
+    );
+    fs.chmodSync(shim, 0o755);
+    originalPath = process.env.PATH;
+    process.env.PATH = `${shimDir}:${originalPath ?? ''}`;
+  };
+
+  beforeEach(() => {
+    tempDir = createTempDir();
+  });
+
+  afterEach(() => {
+    if (originalPath !== undefined) process.env.PATH = originalPath;
+    originalPath = undefined;
+  });
+
+  it('still honours includeIgnored when `ls-files --recurse-submodules` is unsupported', () => {
+    const root = path.join(tempDir, 'root');
+    makeRepo(root, 'a');
+    // An embedded repo that .gitignore excludes but codegraph.json opts back in.
+    makeRepo(path.join(root, 'dir_b'), 'b');
+    fs.writeFileSync(path.join(root, '.gitignore'), 'dir_b/\n');
+    fs.writeFileSync(
+      path.join(root, 'codegraph.json'),
+      JSON.stringify({ includeIgnored: ['dir_b/'] }),
+    );
+    runGit(root, 'add', '-A');
+    runGit(root, 'commit', '-q', '-m', 'ignore dir_b');
+
+    // Baseline: the real git resolves both files.
+    const withRealGit = scanDirectory(root);
+    expect(withRealGit).toContain('a.ts');
+    expect(withRealGit).toContain(path.join('dir_b', 'b.ts'));
+
+    installOldGitShim();
+
+    // The opted-in file must survive the unsupported-mode failure, not vanish.
+    const withOldGit = scanDirectory(root);
+    expect(withOldGit).toContain('a.ts');
+    expect(withOldGit).toContain(path.join('dir_b', 'b.ts'));
+  });
+});

src/extraction/index.ts

@@ -791,9 +791,7 @@ export function discoverEmbeddedRepoRoots(rootDir: string): string[] {
     // same way collectGitFiles does, keeping watcher scope == indexer scope.
     // (#1031, #1033)
     try {
-      const staged = execFileSync(
-        'git',
-        ['ls-files', '-z', '-s', '--recurse-submodules'],
+      const staged = lsFilesStaged(
         { cwd: repoAbs, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true }
       );
       const repoIgnore = buildDefaultIgnore(repoAbs);
@@ -925,6 +923,30 @@ function findIgnoredEmbeddedRepos(repoDir: string, includeIgnored: Ignore | null
  * found) is recorded in `embeddedRoots` so callers can exempt its files from the
  * parent's own gitignore rules.
  */
+/**
+ * `git ls-files -z -s`, expanding submodules where git allows it.
+ *
+ * `--recurse-submodules` could not be combined with `-s` before git 2.36:
+ * `builtin/ls-files.c` listed `show_stage` among the modes that die, and the
+ * check is unconditional — it does not look at whether the repo actually has
+ * submodules, so every call fails on older git. Ubuntu 22.04 LTS (2.34.1) and
+ * Debian 11 (2.30.2) are both below that line.
+ *
+ * Letting the throw escape cost far more than submodule expansion: it unwound
+ * the whole git-visible pass, so `includeIgnored`, gitlink recursion and the
+ * `codegraph.json` include allowlist silently stopped applying and files went
+ * missing from the index with no error (#1549). Retry without the flag instead
+ * — `-s` is the part that matters here, since gitlink detection reads the mode
+ * bits, and embedded repos are reached through the gitlink recursion anyway.
+ */
+function lsFilesStaged(gitOpts: Parameters<typeof execFileSync>[2]): string {
+  try {
+    return execFileSync('git', ['ls-files', '-z', '-s', '--recurse-submodules'], gitOpts) as unknown as string;
+  } catch {
+    return execFileSync('git', ['ls-files', '-z', '-s'], gitOpts) as unknown as string;
+  }
+}
+
 function collectGitFiles(repoDir: string, prefix: string, files: Set<string>, embeddedRoots?: Set<string>, includeIgnored: Ignore | null = null): void {
   const gitOpts = { cwd: repoDir, encoding: 'utf-8' as const, timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'] as ['pipe', 'pipe', 'pipe'], windowsHide: true };
 
@@ -951,7 +973,7 @@ function collectGitFiles(repoDir: string, prefix: string, files: Set<string>, em
   // on disk → those files are silently dropped from the index. (#541) With -s the
   // path follows a TAB after the `<mode> <object> <stage>` prefix.
   const gitlinkRels: string[] = [];
-  const tracked = execFileSync('git', ['ls-files', '-z', '-s', '--recurse-submodules'], gitOpts);
+  const tracked = lsFilesStaged(gitOpts);
   for (const entry of tracked.split('\0')) {
     if (!entry) continue;
     const tab = entry.indexOf('\t');