From 099e3a00095fb71fe78c6ae0462d90562556af72 Mon Sep 17 00:00:00 2001 From: Max Hsu Date: Sun, 23 Aug 2026 23:36:22 +0800 Subject: [PATCH] fix(extraction): keep the git-aware scan working on git older than 2.36 `git ls-files -s --recurse-submodules` dies before git 2.36: builtin/ls-files.c listed show_stage among the modes it rejects, and the check never looks at whether the repo has submodules, so every call fails on Ubuntu 22.04 LTS (2.34.1), Debian 11 (2.30.2) and older. The throw escaped to getGitVisibleFiles' catch, which returns null and drops the caller into the plain filesystem walk. That walk knows nothing of includeIgnored, gitlink recursion, or the codegraph.json include allowlist, so all three silently stopped applying and files vanished from the index with no error. Retry without --recurse-submodules rather than without -s: the mode bits are what gitlink detection reads, and embedded repos are reached through that recursion anyway. A PATH shim makes the regression testable on any git version. Co-authored-by: newshowardz777 --- CHANGELOG.md | 4 + __tests__/extraction-old-git.test.ts | 107 +++++++++++++++++++++++++++ src/extraction/index.ts | 30 +++++++- 3 files changed, 137 insertions(+), 4 deletions(-) create mode 100644 __tests__/extraction-old-git.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ca5b9ff1..6bda759fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Fixes + +- Files opted back in through `includeIgnored` are no longer dropped on older git. On git before 2.36 — which includes Ubuntu 22.04 LTS and Debian 11 — the command CodeGraph used to list tracked files refused to run at all, and the failure quietly took the whole git-aware scan with it: `includeIgnored`, submodule and embedded-repo recursion, and the `include` allowlist in `codegraph.json` all stopped applying, so files went missing from the index with no error and no warning. CodeGraph now falls back to a form those versions accept, and every one of those settings works there as it does on newer git. Thanks @newshowardz777. (#1549) + ## [1.6.0] - 2026-08-26 diff --git a/__tests__/extraction-old-git.test.ts b/__tests__/extraction-old-git.test.ts new file mode 100644 index 000000000..38f62395c --- /dev/null +++ b/__tests__/extraction-old-git.test.ts @@ -0,0 +1,107 @@ +/** + * Regression: git older than 2.36 rejects `ls-files -s --recurse-submodules` (#1549). + * + * Kept in its own file rather than appended to extraction.test.ts: that suite + * loads every tree-sitter grammar in `beforeAll`, and running a git-scan case + * after it pushed the worker past its memory ceiling. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import { execFileSync } from 'child_process'; +import { scanDirectory } from '../src/extraction'; + +function createTempDir(): string { + return fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-test-')); +} + +// git < 2.36 rejects `ls-files -s --recurse-submodules` outright: the guard in +// builtin/ls-files.c listed `show_stage` among the modes that die, and it was +// only dropped in 2.36. The die is unconditional — it does not check whether the +// repo has submodules — so on Ubuntu 22.04 (git 2.34.1), Debian 11 (2.30.2) and +// older, every call threw, `getGitVisibleFiles` swallowed it, and the whole +// git-visible path went with it: `includeIgnored`, gitlink recursion and the +// `codegraph.json` `include` allowlist all silently stopped applying (#1549). +// +// A PATH shim reproduces that on any git version, which is what makes this +// testable in CI at all. +describe('Old git without `ls-files -s --recurse-submodules` support (#1549)', () => { + let tempDir: string; + let originalPath: string | undefined; + + const runGit = (cwd: string, ...args: string[]) => + execFileSync('git', args, { cwd, stdio: 'pipe' }); + + const makeRepo = (dir: string, base: string) => { + fs.mkdirSync(dir, { recursive: true }); + runGit(dir, 'init', '-q'); + runGit(dir, 'config', 'user.email', 'test@test.com'); + runGit(dir, 'config', 'user.name', 'Test'); + fs.writeFileSync(path.join(dir, `${base}.ts`), `export const ${base} = 1;`); + runGit(dir, 'add', '-A'); + runGit(dir, 'commit', '-q', '-m', `${base} init`); + }; + + /** A `git` that dies exactly like < 2.36 when it sees -s with --recurse-submodules. */ + const installOldGitShim = () => { + const shimDir = path.join(tempDir, '.shim'); + fs.mkdirSync(shimDir, { recursive: true }); + const realGit = execFileSync('which', ['git']).toString().trim(); + const shim = path.join(shimDir, 'git'); + fs.writeFileSync( + shim, + [ + '#!/bin/sh', + 'for a in "$@"; do', + ' [ "$a" = "--recurse-submodules" ] && rs=1', + ' [ "$a" = "-s" ] && st=1', + 'done', + 'if [ -n "$rs" ] && [ -n "$st" ]; then', + ' echo "fatal: ls-files --recurse-submodules unsupported mode" >&2', + ' exit 128', + 'fi', + `exec ${JSON.stringify(realGit)} "$@"`, + ].join('\n'), + ); + fs.chmodSync(shim, 0o755); + originalPath = process.env.PATH; + process.env.PATH = `${shimDir}:${originalPath ?? ''}`; + }; + + beforeEach(() => { + tempDir = createTempDir(); + }); + + afterEach(() => { + if (originalPath !== undefined) process.env.PATH = originalPath; + originalPath = undefined; + }); + + it('still honours includeIgnored when `ls-files --recurse-submodules` is unsupported', () => { + const root = path.join(tempDir, 'root'); + makeRepo(root, 'a'); + // An embedded repo that .gitignore excludes but codegraph.json opts back in. + makeRepo(path.join(root, 'dir_b'), 'b'); + fs.writeFileSync(path.join(root, '.gitignore'), 'dir_b/\n'); + fs.writeFileSync( + path.join(root, 'codegraph.json'), + JSON.stringify({ includeIgnored: ['dir_b/'] }), + ); + runGit(root, 'add', '-A'); + runGit(root, 'commit', '-q', '-m', 'ignore dir_b'); + + // Baseline: the real git resolves both files. + const withRealGit = scanDirectory(root); + expect(withRealGit).toContain('a.ts'); + expect(withRealGit).toContain(path.join('dir_b', 'b.ts')); + + installOldGitShim(); + + // The opted-in file must survive the unsupported-mode failure, not vanish. + const withOldGit = scanDirectory(root); + expect(withOldGit).toContain('a.ts'); + expect(withOldGit).toContain(path.join('dir_b', 'b.ts')); + }); +}); diff --git a/src/extraction/index.ts b/src/extraction/index.ts index 93be48352..4fa90e2a0 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -791,9 +791,7 @@ export function discoverEmbeddedRepoRoots(rootDir: string): string[] { // same way collectGitFiles does, keeping watcher scope == indexer scope. // (#1031, #1033) try { - const staged = execFileSync( - 'git', - ['ls-files', '-z', '-s', '--recurse-submodules'], + const staged = lsFilesStaged( { cwd: repoAbs, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true } ); const repoIgnore = buildDefaultIgnore(repoAbs); @@ -925,6 +923,30 @@ function findIgnoredEmbeddedRepos(repoDir: string, includeIgnored: Ignore | null * found) is recorded in `embeddedRoots` so callers can exempt its files from the * parent's own gitignore rules. */ +/** + * `git ls-files -z -s`, expanding submodules where git allows it. + * + * `--recurse-submodules` could not be combined with `-s` before git 2.36: + * `builtin/ls-files.c` listed `show_stage` among the modes that die, and the + * check is unconditional — it does not look at whether the repo actually has + * submodules, so every call fails on older git. Ubuntu 22.04 LTS (2.34.1) and + * Debian 11 (2.30.2) are both below that line. + * + * Letting the throw escape cost far more than submodule expansion: it unwound + * the whole git-visible pass, so `includeIgnored`, gitlink recursion and the + * `codegraph.json` include allowlist silently stopped applying and files went + * missing from the index with no error (#1549). Retry without the flag instead + * — `-s` is the part that matters here, since gitlink detection reads the mode + * bits, and embedded repos are reached through the gitlink recursion anyway. + */ +function lsFilesStaged(gitOpts: Parameters[2]): string { + try { + return execFileSync('git', ['ls-files', '-z', '-s', '--recurse-submodules'], gitOpts) as unknown as string; + } catch { + return execFileSync('git', ['ls-files', '-z', '-s'], gitOpts) as unknown as string; + } +} + function collectGitFiles(repoDir: string, prefix: string, files: Set, embeddedRoots?: Set, includeIgnored: Ignore | null = null): void { const gitOpts = { cwd: repoDir, encoding: 'utf-8' as const, timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'] as ['pipe', 'pipe', 'pipe'], windowsHide: true }; @@ -951,7 +973,7 @@ function collectGitFiles(repoDir: string, prefix: string, files: Set, em // on disk → those files are silently dropped from the index. (#541) With -s the // path follows a TAB after the ` ` prefix. const gitlinkRels: string[] = []; - const tracked = execFileSync('git', ['ls-files', '-z', '-s', '--recurse-submodules'], gitOpts); + const tracked = lsFilesStaged(gitOpts); for (const entry of tracked.split('\0')) { if (!entry) continue; const tab = entry.indexOf('\t');