From 5e0dbaaeb617cbc3cbd1361df32ef95836d2c06b Mon Sep 17 00:00:00 2001 From: Carsten Date: Sat, 4 Jul 2026 10:27:37 +0200 Subject: [PATCH] feat(context): populate ContextPack.symbols via a symbol index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fill in the symbol slices left empty by the Workspace cache: InProcessWorkspace now extracts top-level declarations (function/class/interface/type/enum/variable) from supported TS/JS source files and includes them in the context pack, capped at 200 files per build and cached by HEAD SHA like the rest of the pack. Implementation note: uses the TypeScript compiler API (pure JS, no native/wasm) rather than tree-sitter, to keep the runtime and Docker image dependency-light for this TypeScript-centric tool — `npm prune --omit=dev` + wasm grammar bundling made tree-sitter a real packaging risk. Multi-language indexing via tree-sitter or LSP remains the documented escalation behind the same SymbolSlice shape. typescript is promoted from devDependencies to dependencies accordingly. Co-Authored-By: Claude Opus 4.8 (1M context) --- package-lock.json | 3 +- package.json | 2 +- src/context/in-process-workspace.ts | 21 +++++++++- src/context/symbol-index.ts | 45 +++++++++++++++++++++ tests/unit/context/symbol-index.test.ts | 53 +++++++++++++++++++++++++ tests/unit/context/workspace.test.ts | 21 ++++++++++ 6 files changed, 140 insertions(+), 5 deletions(-) create mode 100644 src/context/symbol-index.ts create mode 100644 tests/unit/context/symbol-index.test.ts diff --git a/package-lock.json b/package-lock.json index 285ce67..d66fb9e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ "cors": "^2.8.5", "express": "^4.21.2", "jose": "^5.10.0", + "typescript": "^5.8.3", "uuid": "^11.1.0", "zod": "^3.25.0" }, @@ -35,7 +36,6 @@ "eventsource": "^3.0.5", "supertest": "^7.1.0", "tsx": "^4.19.3", - "typescript": "^5.8.3", "vitest": "^3.1.3" }, "engines": { @@ -7209,7 +7209,6 @@ "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "dev": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", diff --git a/package.json b/package.json index 0c4fa09..abbf5bc 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "cors": "^2.8.5", "express": "^4.21.2", "jose": "^5.10.0", + "typescript": "^5.8.3", "uuid": "^11.1.0", "zod": "^3.25.0" }, @@ -51,7 +52,6 @@ "eventsource": "^3.0.5", "supertest": "^7.1.0", "tsx": "^4.19.3", - "typescript": "^5.8.3", "vitest": "^3.1.3" } } diff --git a/src/context/in-process-workspace.ts b/src/context/in-process-workspace.ts index 39616ca..6a1aa93 100644 --- a/src/context/in-process-workspace.ts +++ b/src/context/in-process-workspace.ts @@ -1,5 +1,9 @@ import { execFileSync } from 'node:child_process'; -import type { Conventions, ContextPack, Workspace } from './workspace.js'; +import type { Conventions, ContextPack, SymbolSlice, Workspace } from './workspace.js'; +import { extractSymbols, isSupportedSource } from './symbol-index.js'; + +/** Cap on source files parsed per build, to bound cost on large repos. */ +const MAX_SYMBOL_FILES = 200; /** Runs a git subcommand in the repo and returns stdout. Injectable for testing. */ export type GitRunner = (args: string[]) => string; @@ -57,7 +61,20 @@ export class InProcessWorkspace implements Workspace { private _build(): ContextPack { const files = this._run(['ls-tree', '-r', '--name-only', 'HEAD']) .split('\n').map((s) => s.trim()).filter(Boolean); - return { files, conventions: this._conventions(files), symbols: [], truncated: false }; + return { files, conventions: this._conventions(files), symbols: this._symbols(files), truncated: false }; + } + + /** Extracts symbols from up to {@link MAX_SYMBOL_FILES} supported source files. */ + private _symbols(files: string[]): SymbolSlice[] { + const symbols: SymbolSlice[] = []; + let parsed = 0; + for (const file of files) { + if (parsed >= MAX_SYMBOL_FILES) break; + if (!isSupportedSource(file)) continue; + parsed += 1; + symbols.push(...extractSymbols(file, this._show(file))); + } + return symbols; } private _conventions(files: string[]): Conventions { diff --git a/src/context/symbol-index.ts b/src/context/symbol-index.ts new file mode 100644 index 0000000..2139784 --- /dev/null +++ b/src/context/symbol-index.ts @@ -0,0 +1,45 @@ +import ts from 'typescript'; +import type { SymbolSlice } from './workspace.js'; + +/** Matches TypeScript/JavaScript source extensions: .ts .tsx .mts .cts .js .jsx .mjs .cjs */ +const SUPPORTED_SOURCE = /\.(?:m|c)?[jt]sx?$/; + +/** Whether a repo path is a source file the symbol indexer can parse. */ +export function isSupportedSource(path: string): boolean { + return SUPPORTED_SOURCE.test(path); +} + +/** + * Extracts top-level declarations (functions, classes, interfaces, type aliases, enums, and + * variable bindings) from a TS/JS source string via the TypeScript compiler API. + * + * Uses the compiler (pure JS, no native/wasm) rather than tree-sitter to keep the runtime and + * Docker image dependency-light for this TypeScript-centric tool; multi-language indexing via + * tree-sitter/LSP remains the documented escalation behind this same {@link SymbolSlice} shape. + */ +export function extractSymbols(filePath: string, source: string): SymbolSlice[] { + const sourceFile = ts.createSourceFile(filePath, source, ts.ScriptTarget.Latest, false); + const symbols: SymbolSlice[] = []; + const add = (kind: string, name: string): void => { + symbols.push({ name, kind, file: filePath }); + }; + + for (const statement of sourceFile.statements) { + if (ts.isFunctionDeclaration(statement) && statement.name) { + add('function', statement.name.text); + } else if (ts.isClassDeclaration(statement) && statement.name) { + add('class', statement.name.text); + } else if (ts.isInterfaceDeclaration(statement)) { + add('interface', statement.name.text); + } else if (ts.isTypeAliasDeclaration(statement)) { + add('type', statement.name.text); + } else if (ts.isEnumDeclaration(statement)) { + add('enum', statement.name.text); + } else if (ts.isVariableStatement(statement)) { + for (const declaration of statement.declarationList.declarations) { + if (ts.isIdentifier(declaration.name)) add('variable', declaration.name.text); + } + } + } + return symbols; +} diff --git a/tests/unit/context/symbol-index.test.ts b/tests/unit/context/symbol-index.test.ts new file mode 100644 index 0000000..47c2f6a --- /dev/null +++ b/tests/unit/context/symbol-index.test.ts @@ -0,0 +1,53 @@ +import { describe, it, expect } from 'vitest'; +import { extractSymbols, isSupportedSource } from '../../../src/context/symbol-index.js'; + +describe('isSupportedSource', () => { + it('accepts TS/JS extensions', () => { + for (const p of ['a.ts', 'a.tsx', 'a.mts', 'a.cts', 'a.js', 'a.jsx', 'a.mjs', 'a.cjs']) { + expect(isSupportedSource(p)).toBe(true); + } + }); + it('rejects non-source files', () => { + for (const p of ['README.md', 'package.json', 'a.py', 'a.txt']) { + expect(isSupportedSource(p)).toBe(false); + } + }); +}); + +describe('extractSymbols', () => { + it('extracts each top-level declaration kind', () => { + const src = ` + export function foo() {} + export class Bar {} + export interface Baz {} + export type Qux = string; + export enum Color { Red } + export const answer = 42; + let mutable = 1; + `; + const symbols = extractSymbols('src/a.ts', src); + const byKind = (kind: string) => symbols.filter((s) => s.kind === kind).map((s) => s.name); + expect(byKind('function')).toEqual(['foo']); + expect(byKind('class')).toEqual(['Bar']); + expect(byKind('interface')).toEqual(['Baz']); + expect(byKind('type')).toEqual(['Qux']); + expect(byKind('enum')).toEqual(['Color']); + expect(byKind('variable')).toEqual(['answer', 'mutable']); + expect(symbols.every((s) => s.file === 'src/a.ts')).toBe(true); + }); + + it('skips anonymous default function/class declarations (no name)', () => { + expect(extractSymbols('a.ts', 'export default function () {}')).toEqual([]); + expect(extractSymbols('a.ts', 'export default class {}')).toEqual([]); + }); + + it('skips destructuring bindings (non-identifier names)', () => { + const symbols = extractSymbols('a.ts', 'const { x, y } = point;'); + expect(symbols).toEqual([]); + }); + + it('returns nothing for empty or declaration-free source', () => { + expect(extractSymbols('a.ts', '')).toEqual([]); + expect(extractSymbols('a.ts', 'foo(); bar();')).toEqual([]); + }); +}); diff --git a/tests/unit/context/workspace.test.ts b/tests/unit/context/workspace.test.ts index 8503f27..0c2e655 100644 --- a/tests/unit/context/workspace.test.ts +++ b/tests/unit/context/workspace.test.ts @@ -95,6 +95,27 @@ describe('InProcessWorkspace', () => { expect(calls.filter((c) => c[0] === 'ls-tree').length).toBe(2); }); + it('extracts symbols from supported source files', async () => { + const { git } = fakeGit({ + files: ['src/a.ts', 'README.md'], + show: { 'src/a.ts': 'export function hello() {}\nexport const x = 1;' }, + }); + const pack = await new InProcessWorkspace('/repo', git).getContextPack(); + expect(pack.symbols).toEqual([ + { name: 'hello', kind: 'function', file: 'src/a.ts' }, + { name: 'x', kind: 'variable', file: 'src/a.ts' }, + ]); + }); + + it('caps symbol extraction at 200 source files', async () => { + const files = Array.from({ length: 201 }, (_, i) => `f${i}.ts`); + const show: Record = {}; + for (const f of files) show[f] = `export const s = 1;`; + const { git } = fakeGit({ files, show }); + const pack = await new InProcessWorkspace('/repo', git).getContextPack(); + expect(pack.symbols).toHaveLength(200); // one symbol per parsed file, capped + }); + it('uses the default git runner (execFileSync) when none is injected', async () => { // execFileSync is mocked to return 'sha\n' for every call → empty tree, empty conventions. const pack = await new InProcessWorkspace('/repo').getContextPack();