From 73914b62c98317012124f9e05ccfb801c4ed659d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 09:07:21 +0000 Subject: [PATCH] fix(index): exclude large trees and stop dialogue record from loading them Huge workspaces were indexed in full, then every dialogue record read the whole file graph into the JS heap. graphPolicy.excludeGlobs, maxFileSizeBytes, and .graphflowignore narrow the walk. On a file store over 512MB, dialogue record streams only dialogue nodes and appends a delta; sqlite lists those rows with SQL instead of readSnapshot. Fixes #56 --- CHANGELOG.md | 7 + src/config/loader.ts | 28 ++ src/config/schema.ts | 12 + src/graph/client-factory.ts | 20 ++ src/graph/dialogue-node-match.ts | 20 ++ src/graph/file-indexer-cache.ts | 3 +- src/graph/file-indexer-walker.ts | 24 +- src/graph/file-indexer.ts | 19 +- src/graph/graph-store-json-chunks.ts | 40 ++- src/graph/graphify-file-client.ts | 141 +++++++++- src/graph/sqlite-client.ts | 24 ++ src/graph/workspace-ignore.ts | 123 +++++++++ src/learning/dialogue-thread.ts | 9 + src/surfaces/cli/runtime/graph.ts | 4 + src/surfaces/cli/runtime/routing.ts | 10 +- src/surfaces/cli/runtime/selfcheck.ts | 4 + ...orkspace-exclude-and-dialogue-read.test.ts | 254 ++++++++++++++++++ 17 files changed, 724 insertions(+), 18 deletions(-) create mode 100644 src/graph/dialogue-node-match.ts create mode 100644 src/graph/workspace-ignore.ts create mode 100644 tests/m-workspace-exclude-and-dialogue-read.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index a45e46c6..b40e34c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to this project are documented in this file. ## [Unreleased] +### Fixed — large workspaces no longer have to be fully indexed, and dialogue record no longer materializes a huge file graph + +- `graphPolicy.excludeGlobs` and `graphPolicy.maxFileSizeBytes` are honored by workspace index, single-file index, and the pending-index check. `/.graphflowignore` uses the same rules (`name/` at any depth, anchored `path/to/dir/`, basename globs, path globs). Negation and nested ignore files are not supported. Markdown stays on the 200KB source cap; the 5MB document cap applies to office/PDF only. +- `dialogue record` on a file store larger than 512MB streams just the dialogue nodes and appends a delta line. It does not `JSON.parse` the code graph. SQLite `listDialogueNodes()` returns the same rows with a SQL filter instead of `readSnapshot()`. + +## [Unreleased] + ### Fixed — 第 0 步性能与健壮性(迁移 Rust 前的架构债清偿,复审登记项全闭环) - **file 传输读路径共享缓存(P2-8)**:`GraphifyFileClient.peekStore()` 静态共享读入口(走进程内 `graphifyFileStoreCache`,stat 校验+delta 已应用,不开句柄不写);`graphStoreNeedsIndexing` 与 `readFileGraphStore`/`resolveGraphStoreAfterIndex` 不再各自 readFileSync+JSON.parse 整个 9.5MB store——每 preview 省 2 次全量读+解析。消费方 mutation 全审计(全只读),peek 结果仍做浅拷贝防御未来调用方。 diff --git a/src/config/loader.ts b/src/config/loader.ts index 716db986..acd5303a 100644 --- a/src/config/loader.ts +++ b/src/config/loader.ts @@ -92,6 +92,19 @@ export function validateConfigDetailed(path = "graphflow.config.json"): ConfigVa issues.push({ severity: "error", field: "graphPolicy.includeExtensions", message: "Extensions must start with '.'" }); } } + if (parsed.graphPolicy.excludeGlobs !== undefined) { + const globs = parsed.graphPolicy.excludeGlobs; + const invalid = !Array.isArray(globs) || globs.some((rule) => typeof rule !== "string" || rule.trim() === ""); + if (invalid) { + issues.push({ severity: "error", field: "graphPolicy.excludeGlobs", message: "excludeGlobs entries must be non-empty strings" }); + } + } + if (parsed.graphPolicy.maxFileSizeBytes !== undefined) { + const size = parsed.graphPolicy.maxFileSizeBytes; + if (typeof size !== "number" || !Number.isFinite(size) || size < 1) { + issues.push({ severity: "error", field: "graphPolicy.maxFileSizeBytes", message: "maxFileSizeBytes must be a positive number" }); + } + } if (parsed.graphPolicy.workspaceRoot) { issues.push({ severity: "warning", field: "graphPolicy.workspaceRoot", message: "workspaceRoot in config is deprecated; resolved from process.cwd() at runtime" }); } @@ -342,6 +355,21 @@ export function validateConfig(input: GraphFlowConfig): GraphFlowConfig { } } + if (input.graphPolicy.excludeGlobs !== undefined) { + const globs = input.graphPolicy.excludeGlobs; + const invalid = !Array.isArray(globs) || globs.some((rule) => typeof rule !== "string" || rule.trim() === ""); + if (invalid) { + throw new Error("Invalid config: graphPolicy.excludeGlobs entries must be non-empty strings."); + } + } + + if (input.graphPolicy.maxFileSizeBytes !== undefined) { + const size = input.graphPolicy.maxFileSizeBytes; + if (typeof size !== "number" || !Number.isFinite(size) || size < 1) { + throw new Error("Invalid config: graphPolicy.maxFileSizeBytes must be a positive number."); + } + } + // learningPolicy is optional at the input boundary: every sub-field has a // default, so a minimal config (providers + tiers + graphPolicy) must be // valid. Requiring the bare object made minimal configs fail validation and diff --git a/src/config/schema.ts b/src/config/schema.ts index 8f068867..e6b61b6b 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -169,6 +169,18 @@ export interface GraphFlowConfig { enableDialogueThread?: boolean; workspaceRoot?: string; includeExtensions?: string[]; + /** + * Workspace-relative paths and globs the indexer must skip, unioned with + * `/.graphflowignore`. Syntax: `name/` (any directory of that + * name), `path/to/dir/` (anchored), `*.ext` (basename), path globs where + * `*` does not cross `/` and a double-star does. No gitignore negation. + */ + excludeGlobs?: string[]; + /** + * Skip source files larger than this many bytes. Office/PDF conversion + * still uses its own higher cap. Default 200_000. + */ + maxFileSizeBytes?: number; /** * Skip files git ignores when indexing (exact semantics via `git ls-files * --exclude-standard`). Default true; ignored outside git checkouts. diff --git a/src/graph/client-factory.ts b/src/graph/client-factory.ts index da5198fc..e077e12b 100644 --- a/src/graph/client-factory.ts +++ b/src/graph/client-factory.ts @@ -10,6 +10,7 @@ import { GraphifySqliteClient } from "./sqlite-client"; import { mergeSiblingJsonStoreIntoSqlite } from "./store-migration"; import { CACHE_FILE, SQLITE_INDEX_MANIFEST } from "./file-indexer-cache"; import { existsSync } from "node:fs"; +import { isDialogueRecordNode } from "./dialogue-node-match"; export interface GraphStoreSnapshot { nodes: GraphNode[]; @@ -33,6 +34,16 @@ export interface GraphClient { upsertGraph?(batch: { nodes?: GraphNode[]; edges?: GraphEdge[] }): Promise; queryByKeyword(query: string): Promise; readSnapshot?(): GraphStoreSnapshot; + /** + * Dialogue turns and sessions without materializing the rest of the graph. + * Backends that cannot project fall back to `readSnapshot` at the call site. + */ + listDialogueNodes?(): Promise; + /** + * When true, `upsertEdges` is idempotent and callers must not `readSnapshot` + * solely to drop duplicate edges (that read is the large-graph OOM). + */ + edgesAreIdempotent?(): boolean; getNodesByIds?(ids: string[]): Promise; getNeighbors?( nodeIds: string[], @@ -113,6 +124,10 @@ class MutationAwareGraphClient implements GraphClient { return this.inner.indexManifestName; } + edgesAreIdempotent(): boolean { + return this.inner.edgesAreIdempotent?.() === true; + } + async upsertNodes(nodes: GraphNode[]): Promise { await this.inner.upsertNodes(nodes); markGraphMutated(nodes.map((n) => n.id)); @@ -150,6 +165,11 @@ class MutationAwareGraphClient implements GraphClient { return this.inner.queryByKeyword(query); } + async listDialogueNodes(): Promise { + if (this.inner.listDialogueNodes) return this.inner.listDialogueNodes(); + return this.readSnapshot().nodes.filter((node) => isDialogueRecordNode(node)); + } + readSnapshot(): GraphStoreSnapshot { // 所有被包装的后端（file/sqlite/memory/mcp）均实现 readSnapshot， // `?.` 仅作防御；接口此处为可选方法，调用方先做存在性检查。 diff --git a/src/graph/dialogue-node-match.ts b/src/graph/dialogue-node-match.ts new file mode 100644 index 00000000..970d73c2 --- /dev/null +++ b/src/graph/dialogue-node-match.ts @@ -0,0 +1,20 @@ +/** + * Dialogue records live in the same store as code nodes. Callers that only + * need the thread must recognize them without assuming a particular backend. + * + * Id prefixes are authoritative (`dialogue::`, `dialogue-session:`). + * Metadata kinds cover records written before the id scheme, but code nodes + * (File / Module / Symbol) are never dialogue even if a kind field collides. + */ +export function isDialogueRecordNode(node: { + id?: unknown; + type?: unknown; + metadata?: { kind?: unknown } | null; +}): boolean { + if (typeof node.id !== "string" || node.id.length === 0) return false; + if (node.id.startsWith("dialogue-session:") || node.id.startsWith("dialogue:")) return true; + const type = typeof node.type === "string" ? node.type : ""; + if (type === "File" || type === "Module" || type === "Symbol") return false; + const kind = node.metadata && typeof node.metadata === "object" ? node.metadata.kind : undefined; + return kind === "dialogue-turn" || kind === "dialogue-session"; +} diff --git a/src/graph/file-indexer-cache.ts b/src/graph/file-indexer-cache.ts index 740431ff..ac275c39 100644 --- a/src/graph/file-indexer-cache.ts +++ b/src/graph/file-indexer-cache.ts @@ -160,7 +160,7 @@ export function hasPendingGraphIndexWork( rootDir: string, options?: Pick< FileIndexerOptions, - "includeExtensions" | "maxFileSizeBytes" | "forceReindex" | "respectGitIgnore" + "includeExtensions" | "maxFileSizeBytes" | "forceReindex" | "respectGitIgnore" | "excludeGlobs" > & { manifestName?: string | undefined } ): boolean { const includeExtensions = options?.includeExtensions ?? DEFAULT_EXTENSIONS; @@ -173,6 +173,7 @@ export function hasPendingGraphIndexWork( const cacheState = loadCacheStateCached(indexManifestPath(rootDir, options?.manifestName)); const scanned = walkScannableFiles(rootDir, includeExtensions, maxFileSizeBytes, { ...(options?.respectGitIgnore === false ? { respectGitIgnore: false } : {}), + ...(options?.excludeGlobs?.length ? { excludeGlobs: options.excludeGlobs } : {}), }); const currentRelPaths = new Set(scanned.map((file) => file.relPath)); diff --git a/src/graph/file-indexer-walker.ts b/src/graph/file-indexer-walker.ts index 7efd63cb..8bbf73cb 100644 --- a/src/graph/file-indexer-walker.ts +++ b/src/graph/file-indexer-walker.ts @@ -15,6 +15,7 @@ import { OFFICE_DOCUMENT_EXTENSIONS, } from "./document-convert.js"; import { safeReaddirSync, safeStatSync } from "../utils/safe-fs.js"; +import { loadWorkspaceIgnore } from "./workspace-ignore.js"; import type { EmbeddingProvider } from "../learning/embeddings.js"; @@ -53,6 +54,11 @@ export interface FileIndexerOptions { * the machine's cores. `GRAPHFLOW_INDEX_WORKERS=0` disables it globally. */ indexWorkers?: number; + /** + * Extra exclude rules, same syntax as `/.graphflowignore`. + * The ignore file is always loaded; these rules are unioned with it. + */ + excludeGlobs?: string[]; } export interface ScannedFile { @@ -189,7 +195,7 @@ export function walkScannableFiles( rootDir: string, includeExtensions: string[], maxFileSizeBytes: number, - options: { respectGitIgnore?: boolean } = {} + options: { respectGitIgnore?: boolean; excludeGlobs?: string[] } = {} ): ScannedFile[] { const files = walkFiles(rootDir, includeExtensions, options); const scanned: ScannedFile[] = []; @@ -225,11 +231,15 @@ export function walkScannableFiles( export function walkFiles( rootDir: string, includeExtensions: string[], - options: { respectGitIgnore?: boolean } = {} + options: { respectGitIgnore?: boolean; excludeGlobs?: string[] } = {} ): string[] { const files: string[] = []; const dirStack: string[] = [rootDir]; const gitVisible = options.respectGitIgnore === false ? undefined : readGitVisibleFiles(rootDir); + const ignore = loadWorkspaceIgnore( + rootDir, + options.excludeGlobs && options.excludeGlobs.length > 0 ? { excludeGlobs: options.excludeGlobs } : undefined + ); while (dirStack.length > 0) { const current = dirStack.pop()!; @@ -245,6 +255,10 @@ export function walkFiles( if (IGNORED_DIRS.has(entry.name)) { continue; } + const relDir = normalizePath(relative(rootDir, full)); + if (ignore?.(relDir, true)) { + continue; + } // A directory holding only ignored/untracked files cannot contribute. if (gitVisible && !directoryMayContainVisibleFiles(rootDir, full, gitVisible)) { continue; @@ -259,7 +273,11 @@ export function walkFiles( if (isGeneratedOrLockFile(entry.name)) { continue; } - if (gitVisible && !gitVisible.has(normalizePath(relative(rootDir, full)))) { + const relFile = normalizePath(relative(rootDir, full)); + if (ignore?.(relFile, false)) { + continue; + } + if (gitVisible && !gitVisible.has(relFile)) { continue; } files.push(full); diff --git a/src/graph/file-indexer.ts b/src/graph/file-indexer.ts index 84fa3b96..f6de0f5d 100644 --- a/src/graph/file-indexer.ts +++ b/src/graph/file-indexer.ts @@ -44,6 +44,7 @@ export { resolveCallerAtLine } from "./file-indexer-nodes.js"; // ── Internal imports ───────────────────────────────────────────────── import { DEFAULT_EXTENSIONS, DEFAULT_MAX_FILE_SIZE, normalizePath, extOf, walkScannableFiles } from "./file-indexer-walker.js"; +import { loadWorkspaceIgnore } from "./workspace-ignore.js"; import type { FileIndexerOptions } from "./file-indexer-walker.js"; import { indexManifestPath, loadCacheState, saveCacheState } from "./file-indexer-cache.js"; import type { @@ -262,6 +263,7 @@ export async function indexWorkspaceFiles( const scanned = walkScannableFiles(rootDir, includeExtensions, maxFileSizeBytes, { ...(options?.respectGitIgnore === false ? { respectGitIgnore: false } : {}), + ...(options?.excludeGlobs?.length ? { excludeGlobs: options.excludeGlobs } : {}), }); const currentRelPaths = new Set(scanned.map((file) => file.relPath)); @@ -470,6 +472,7 @@ export async function indexSingleFile( FileIndexerOptions, | "includeExtensions" | "maxFileSizeBytes" + | "excludeGlobs" | "embeddingProvider" | "referenceEdgeMaxDefinitionFiles" | "referenceEdgeMaxPerFile" @@ -507,6 +510,21 @@ export async function indexSingleFile( }; } + const relPath = normalizePath(relative(rootDir, absPath)); + const ignore = loadWorkspaceIgnore( + rootDir, + options?.excludeGlobs && options.excludeGlobs.length > 0 ? { excludeGlobs: options.excludeGlobs } : undefined + ); + if (ignore?.(relPath, false)) { + return { + indexedFiles: 0, + indexedSymbols: 0, + indexedReferences: 0, + skipped: true, + reason: "excluded by .graphflowignore or graphPolicy.excludeGlobs", + }; + } + const officeDoc = isOfficeDocumentPath(absPath); const sizeLimit = officeDoc ? Math.max(maxFileSizeBytes, DEFAULT_DOCUMENT_MAX_FILE_SIZE) @@ -528,7 +546,6 @@ export async function indexSingleFile( return { indexedFiles: 0, indexedSymbols: 0, indexedReferences: 0, skipped: true, reason: "file exceeds maxFileSizeBytes" }; } - const relPath = normalizePath(relative(rootDir, absPath)); const mtimeMs = stat.mtimeMs; let content = ""; let currentHash = ""; diff --git a/src/graph/graph-store-json-chunks.ts b/src/graph/graph-store-json-chunks.ts index 228cf7aa..5c8410a7 100644 --- a/src/graph/graph-store-json-chunks.ts +++ b/src/graph/graph-store-json-chunks.ts @@ -22,6 +22,17 @@ export interface ChunkedGraphStore { edges: unknown[]; } +export interface ChunkedGraphStoreReadOptions { + /** + * Drop nodes for which this returns false. The element string is parsed + * and then discarded, so a projection (dialogue records, a handful of ids) + * does not retain the rest of the graph. + */ + keepNode?: (node: unknown) => boolean; + /** Scan the edges array without retaining it. */ + skipEdges?: boolean; +} + /** Flush the accumulated element buffer at this size while scanning. */ const DEFAULT_CHUNK_BYTES = 8 * 1024 * 1024; @@ -32,7 +43,10 @@ class GraphStoreChunkParser { private buffer = ""; private index = 0; - constructor(private readonly pull: () => string | null) {} + constructor( + private readonly pull: () => string | null, + private readonly options: ChunkedGraphStoreReadOptions = {} + ) {} private fill(): boolean { if (this.index > 0) { @@ -135,7 +149,7 @@ class GraphStoreChunkParser { } } - private readElements(target: unknown[]): void { + private readElements(target: unknown[] | null, keep?: (node: unknown) => boolean): void { while (true) { this.skipWhitespace(); const ch = this.peek(); @@ -156,7 +170,10 @@ class GraphStoreChunkParser { if (!complete) { throw new Error("Truncated graph store: incomplete element"); } - target.push(JSON.parse(text)); + if (target === null) continue; + const parsed: unknown = JSON.parse(text); + if (keep && !keep(parsed)) continue; + target.push(parsed); } } @@ -195,7 +212,11 @@ class GraphStoreChunkParser { throw new Error(`Truncated graph store: expected an array for "${name}"`); } this.index += 1; - this.readElements(name === "nodes" ? store.nodes : store.edges); + const discard = name === "edges" && this.options.skipEdges === true; + this.readElements( + discard ? null : name === "nodes" ? store.nodes : store.edges, + name === "nodes" ? this.options.keepNode : undefined + ); continue; } const { complete } = this.readValue(); @@ -209,14 +230,17 @@ class GraphStoreChunkParser { } /** Parse a graph store document from a sequence of text chunks. */ -export function readGraphStoreFromChunks(pull: () => string | null): ChunkedGraphStore { - return new GraphStoreChunkParser(pull).parse(); +export function readGraphStoreFromChunks( + pull: () => string | null, + options: ChunkedGraphStoreReadOptions = {} +): ChunkedGraphStore { + return new GraphStoreChunkParser(pull, options).parse(); } /** Read a graph store file in bounded chunks (used above the single-string cap). */ export function readGraphStoreFileChunked( filePath: string, - options: { chunkBytes?: number } = {} + options: { chunkBytes?: number } & ChunkedGraphStoreReadOptions = {} ): ChunkedGraphStore { const chunkBytes = Math.max(64 * 1024, options.chunkBytes ?? DEFAULT_CHUNK_BYTES); const fd = openSync(filePath, "r"); @@ -228,7 +252,7 @@ export function readGraphStoreFileChunked( return readGraphStoreFromChunks(() => { const read = readSync(fd, buffer, 0, chunkBytes, null); return read > 0 ? decoder.write(buffer.subarray(0, read)) : null; - }); + }, options); } finally { closeSync(fd); } diff --git a/src/graph/graphify-file-client.ts b/src/graph/graphify-file-client.ts index 9b0f7aec..2380c334 100644 --- a/src/graph/graphify-file-client.ts +++ b/src/graph/graphify-file-client.ts @@ -17,6 +17,7 @@ import type { GraphEdge, GraphNode } from "../core/types"; import { logger } from "../utils/logger"; import { tokenizeForIndex, nodeSearchableText, nodeRecallText } from "./graph-utils"; import { readGraphStoreFileChunked } from "./graph-store-json-chunks"; +import { isDialogueRecordNode } from "./dialogue-node-match"; interface GraphStore { nodes: GraphNode[]; @@ -238,13 +239,29 @@ const GRAPH_STORE_WRITE_CHUNK_BYTES = 4 * 1024 * 1024; */ export const GRAPH_STORE_MAX_READ_BYTES = 512 * 1024 * 1024; +/** + * Stores larger than this are not materialized for a small read or write + * (dialogue record, id lookup). Default equals the single-string cap: below + * it, `JSON.parse` of the whole file is the historical path. Tests lower it. + */ +let hugeStoreMinBytes = GRAPH_STORE_MAX_READ_BYTES; + +/** Small batches on a huge store append a delta line instead of rewriting the base. */ +export const GRAPH_STORE_BLIND_DELTA_MAX_ELEMENTS = 256; + +/** Test hook: treat stores larger than `bytes` as too big to materialize. */ +export function setGraphStoreHugeReadBytesForTests(bytes?: number): void { + hugeStoreMinBytes = bytes ?? GRAPH_STORE_MAX_READ_BYTES; +} + /** Actionable error for a store too large to read as one string (shared). */ export function graphStoreTooLargeError(storePath: string, sizeBytes: number): Error { const sizeMb = Math.round(sizeBytes / (1024 * 1024)); const limitMb = Math.round(GRAPH_STORE_MAX_READ_BYTES / (1024 * 1024)); return new Error( `Graph store is too large to read (${sizeMb} MB at ${storePath}, limit ${limitMb} MB). ` + - "Remove the file and run a rebuild, or narrow graphPolicy.includeExtensions." + "Remove the file and run a rebuild, or narrow what gets indexed " + + "(graphPolicy.excludeGlobs, graphPolicy.includeExtensions, or a .graphflowignore file)." ); } @@ -492,7 +509,55 @@ function readStoreCacheEntry(storePath: string): StoreCacheEntry { return entry; } +function asGraphNode(value: unknown): GraphNode | undefined { + if (!value || typeof value !== "object") return undefined; + const node = value as GraphNode; + if (typeof node.id !== "string" || typeof node.type !== "string" || typeof node.content !== "string") { + return undefined; + } + return node; +} + +/** Apply delta upserts/deletes onto an already-projected node list. */ +function overlayDeltaNodes( + nodes: GraphNode[], + deltaPath: string, + deltaStat: FileStat | null, + keep: (node: GraphNode) => boolean +): GraphNode[] { + if (deltaStat === null || deltaStat.size === 0) return nodes; + let contents: string; + try { + contents = readFileSync(deltaPath, "utf8"); + } catch { + return nodes; + } + const map = new Map(nodes.map((node) => [node.id, node])); + for (const line of contents.split("\n")) { + if (!line.trim()) continue; + let op: DeltaOp; + try { + op = JSON.parse(line) as DeltaOp; + } catch { + continue; + } + if (op.op === "upsert") { + for (const node of op.nodes ?? []) { + if (keep(node)) map.set(node.id, node); + } + } else if (op.op === "delete") { + for (const id of op.nodeIds ?? []) map.delete(id); + } + } + return Array.from(map.values()); +} + export class GraphifyFileClient { + /** Edge upserts dedupe on apply / in upsertGraph, so callers need not readSnapshot first. */ + edgesAreIdempotent(): boolean { + return true; + } + constructor( private readonly storePath: string, private readonly options: { @@ -502,6 +567,51 @@ export class GraphifyFileClient { } = {} ) {} + private storeAbsPath(): string { + return resolve(this.storePath); + } + + /** True when the base file is over the materialization cap and not already cached. */ + private isHugeUncached(): boolean { + const absPath = this.storeAbsPath(); + const current = statIfExists(absPath); + if (current === null || current.size <= hugeStoreMinBytes) return false; + const cached = graphifyFileStoreCache.get(absPath); + if (!cached) return true; + const deltaStat = statIfExists(resolve(deltaPathFor(absPath))); + return !sameStat(cached.stat, current) || !sameStat(cached.deltaStat, deltaStat); + } + + /** + * Append one upsert without reading the base. Compacting the delta would + * materialize the huge file — the failure mode dialogue record is avoiding — + * so an oversized log is left in place for the next full rebuild. + */ + private appendBlindDelta(op: DeltaUpsertOp): void { + const absPath = this.storeAbsPath(); + const deltaPath = resolve(deltaPathFor(absPath)); + mkdirSync(dirname(deltaPath), { recursive: true }); + appendFileSync(deltaPath, `${JSON.stringify(op)}\n`, "utf8"); + graphifyFileStoreCache.delete(absPath); + } + + /** Stream the base file, keep `predicate` nodes, then overlay the delta log. */ + private readProjectedNodes(predicate: (node: GraphNode) => boolean): GraphNode[] { + const absPath = this.storeAbsPath(); + const current = statIfExists(absPath); + if (current === null) return []; + const chunked = readGraphStoreFileChunked(absPath, { + skipEdges: true, + keepNode: (value) => { + const node = asGraphNode(value); + return node !== undefined && predicate(node); + }, + }); + const nodes = chunked.nodes.map((value) => asGraphNode(value)).filter((node): node is GraphNode => node !== undefined); + const deltaPath = resolve(deltaPathFor(absPath)); + return overlayDeltaNodes(nodes, deltaPath, statIfExists(deltaPath), predicate); + } + /** * Merge nodes and edges into the store with a SINGLE read + write. * @@ -527,6 +637,18 @@ export class GraphifyFileClient { return; } + if ( + this.isHugeUncached() && + incomingNodes.length + incomingEdges.length <= GRAPH_STORE_BLIND_DELTA_MAX_ELEMENTS + ) { + this.appendBlindDelta({ + op: "upsert", + ...(incomingNodes.length > 0 ? { nodes: incomingNodes } : {}), + ...(incomingEdges.length > 0 ? { edges: incomingEdges } : {}), + }); + return; + } + const entry = this.readStoreEntry(); const store = entry.store; const nodeMap = new Map(store.nodes.map((node) => [node.id, node])); @@ -689,11 +811,26 @@ export class GraphifyFileClient { } async getNodesByIds(ids: string[]): Promise { - const store = this.readStore(); + if (ids.length === 0) return []; const want = new Set(ids); + if (this.isHugeUncached()) { + return this.readProjectedNodes((node) => want.has(node.id)); + } + const store = this.readStore(); return store.nodes.filter((n) => want.has(n.id)); } + /** + * Dialogue turns and sessions only. On a huge file store this streams the + * document and drops every other node instead of building the full graph. + */ + async listDialogueNodes(): Promise { + if (this.isHugeUncached()) { + return this.readProjectedNodes((node) => isDialogueRecordNode(node)); + } + return this.readStore().nodes.filter((node) => isDialogueRecordNode(node)); + } + async getNeighbors( nodeIds: string[], relations?: GraphEdge["relation"][], diff --git a/src/graph/sqlite-client.ts b/src/graph/sqlite-client.ts index e72566d6..f2ddff7a 100644 --- a/src/graph/sqlite-client.ts +++ b/src/graph/sqlite-client.ts @@ -6,6 +6,7 @@ import type { GraphClient } from "./client-factory"; import { SQLITE_INDEX_MANIFEST } from "./file-indexer-cache"; import { tokenizeForIndex, containsCJK, nodeRecallText } from "./graph-utils"; import { requireFromOptionalDeps, resolveSqliteDepsRoot } from "../utils/optional-deps"; +import { isDialogueRecordNode } from "./dialogue-node-match"; const requireFn = createRequire(__filename); @@ -204,6 +205,10 @@ const SNAPSHOT_ROWS_IDLE_MS = 15_000; export class GraphifySqliteClient implements GraphClient { readonly indexManifestName = SQLITE_INDEX_MANIFEST; + /** INSERT OR IGNORE makes a repeated edge a no-op, so callers skip the snapshot pre-read. */ + edgesAreIdempotent(): boolean { + return true; + } private readonly db: import("better-sqlite3").Database; /** * One preview reads the whole store many times. Raw rows are reused while @@ -454,6 +459,25 @@ export class GraphifySqliteClient implements GraphClient { return rows.map(rowToNode); } + /** + * Dialogue rows only. A full `readSnapshot()` pulls every code node into JS; + * this scan stays in SQLite and returns the thread. + */ + async listDialogueNodes(): Promise { + const rows = this.db + .prepare( + `SELECT id, type, content, metadata FROM nodes + WHERE id LIKE 'dialogue-session:%' + OR id LIKE 'dialogue:%' + OR ( + type NOT IN ('File', 'Module', 'Symbol') + AND json_extract(metadata, '$.kind') IN ('dialogue-turn', 'dialogue-session') + )` + ) + .all() as NodeRow[]; + return rows.map(rowToNode).filter((node) => isDialogueRecordNode(node)); + } + async getNodesByIds(ids: string[]): Promise { if (ids.length === 0) return []; const placeholders = ids.map(() => "?").join(", "); diff --git a/src/graph/workspace-ignore.ts b/src/graph/workspace-ignore.ts new file mode 100644 index 00000000..aa5b11f0 --- /dev/null +++ b/src/graph/workspace-ignore.ts @@ -0,0 +1,123 @@ +/** + * Workspace exclude rules for indexing. + * + * Two sources, same syntax, unioned: + * - `graphPolicy.excludeGlobs` + * - `/.graphflowignore` + * + * This is intentionally smaller than `.gitignore`: no negation (`!`), no + * nested ignore files, no character classes. A rule that does not match is + * skipped rather than treated as a path. + * + * name/ skip any directory named `name` at any depth + * path/to/dir/ skip that workspace-relative directory (anchored) + * *.ext skip files whose basename matches the glob + * a path glob `*` does not cross `/`; a double-star does + * + * Inline ` # comments` are stripped. `*` does not cross `/`; `**` does. + */ +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; + +export const WORKSPACE_IGNORE_FILE = ".graphflowignore"; + +export type IgnoreMatch = (relPath: string, isDir: boolean) => boolean; + +function globToRegexSource(glob: string): string { + let out = ""; + for (let i = 0; i < glob.length; i += 1) { + const ch = glob[i]!; + if (ch === "*") { + if (glob[i + 1] === "*") { + i += 1; + out += ".*"; + } else { + out += "[^/]*"; + } + continue; + } + if (ch === "?") { + out += "[^/]"; + continue; + } + out += ch.replace(/[.+^${}()|[\]\\]/g, "\\$&"); + } + return out; +} + +/** Parse ignore text. `undefined` when every line is blank or a comment. */ +export function parseIgnoreRules(text: string): IgnoreMatch | undefined { + const dirNames = new Set(); + const relPrefixes: string[] = []; + const basenameRes: RegExp[] = []; + const relRes: RegExp[] = []; + + for (const rawLine of String(text).split(/\r?\n/)) { + let rule = rawLine.trim().replace(/\\/g, "/"); + if (!rule || rule.startsWith("#") || rule.startsWith("!")) continue; + rule = rule.replace(/\s+#.*$/, "").trim(); + if (!rule) continue; + if (rule.startsWith("./")) rule = rule.slice(2); + if (rule.startsWith("/")) rule = rule.slice(1); + if (rule.endsWith("/")) rule = rule.slice(0, -1); + if (!rule) continue; + if (!/[*?]/.test(rule)) { + if (rule.includes("/")) relPrefixes.push(rule); + else dirNames.add(rule); + continue; + } + const source = `^${globToRegexSource(rule)}$`; + const re = new RegExp(source); + if (rule.includes("/")) relRes.push(re); + else basenameRes.push(re); + } + + if (dirNames.size === 0 && relPrefixes.length === 0 && basenameRes.length === 0 && relRes.length === 0) { + return undefined; + } + + return function ignoreMatch(relPath: string, isDir: boolean): boolean { + const rel = String(relPath).replace(/\\/g, "/"); + if (!rel || rel === ".") return false; + const parts = rel.split("/"); + const dirParts = isDir ? parts : parts.slice(0, -1); + for (const part of dirParts) { + if (dirNames.has(part)) return true; + } + for (const prefix of relPrefixes) { + if (rel === prefix || rel.startsWith(`${prefix}/`)) return true; + } + if (!isDir) { + const base = parts[parts.length - 1] ?? ""; + for (const re of basenameRes) { + if (re.test(base)) return true; + } + } + for (const re of relRes) { + if (re.test(rel)) return true; + } + return false; + }; +} + +/** + * Load `.graphflowignore` plus `options.excludeGlobs`. Unreadable or absent + * file is the same as no file: indexing stays unchanged. + */ +export function loadWorkspaceIgnore( + rootDir: string, + options?: { excludeGlobs?: readonly string[] } +): IgnoreMatch | undefined { + const extra = Array.isArray(options?.excludeGlobs) ? options.excludeGlobs : []; + let text = extra.length > 0 ? `${extra.join("\n")}\n` : ""; + try { + const filePath = join(rootDir, WORKSPACE_IGNORE_FILE); + if (existsSync(filePath)) { + text += `${readFileSync(filePath, "utf8")}\n`; + } + } catch { + // Missing or unreadable: same as an unpatched walker. + } + if (!text.trim()) return undefined; + return parseIgnoreRules(text); +} diff --git a/src/learning/dialogue-thread.ts b/src/learning/dialogue-thread.ts index 25d97b9d..509c7f70 100644 --- a/src/learning/dialogue-thread.ts +++ b/src/learning/dialogue-thread.ts @@ -842,6 +842,9 @@ async function loadSession( } async function collectDialogueNodes(client: GraphClient): Promise { + if (client.listDialogueNodes) { + return client.listDialogueNodes(); + } const snapshot = client.readSnapshot?.(); if (snapshot) { return snapshot.nodes.filter((node) => isDialogueTurnNode(node) || isDialogueSessionNode(node)); @@ -887,6 +890,12 @@ export async function listOwnedDialogueSessionIds( async function upsertUniqueEdges(client: GraphClient, edges: GraphEdge[]): Promise { if (edges.length === 0) return; + // File and sqlite upserts are idempotent. Reading the snapshot only to drop + // duplicates forces every dialogue record to materialize the code graph. + if (client.edgesAreIdempotent?.() === true) { + await client.upsertEdges(edges); + return; + } const snapshot = client.readSnapshot?.(); const existing = new Set( (snapshot?.edges ?? []).map((edge) => `${edge.from}|${edge.relation}|${edge.to}`) diff --git a/src/surfaces/cli/runtime/graph.ts b/src/surfaces/cli/runtime/graph.ts index 97854869..186d4029 100644 --- a/src/surfaces/cli/runtime/graph.ts +++ b/src/surfaces/cli/runtime/graph.ts @@ -356,6 +356,8 @@ function buildContextPressureBlock(params: { */ function buildIndexOptions(config: GraphFlowConfig): { includeExtensions?: string[]; + excludeGlobs?: string[]; + maxFileSizeBytes?: number; respectGitIgnore?: boolean; referenceEdgeMaxDefinitionFiles?: number; referenceEdgeMaxPerFile?: number; @@ -366,6 +368,8 @@ function buildIndexOptions(config: GraphFlowConfig): { const embeddingProvider = createEmbeddingProviderFromConfig(config); return { ...(graphPolicy.includeExtensions ? { includeExtensions: graphPolicy.includeExtensions } : {}), + ...(graphPolicy.excludeGlobs?.length ? { excludeGlobs: graphPolicy.excludeGlobs } : {}), + ...(typeof graphPolicy.maxFileSizeBytes === "number" ? { maxFileSizeBytes: graphPolicy.maxFileSizeBytes } : {}), ...(graphPolicy.respectGitIgnore === false ? { respectGitIgnore: false } : {}), ...(typeof graphPolicy.referenceEdgeMaxDefinitionFiles === "number" ? { referenceEdgeMaxDefinitionFiles: graphPolicy.referenceEdgeMaxDefinitionFiles } diff --git a/src/surfaces/cli/runtime/routing.ts b/src/surfaces/cli/runtime/routing.ts index 4ca78a91..3c65cc69 100644 --- a/src/surfaces/cli/runtime/routing.ts +++ b/src/surfaces/cli/runtime/routing.ts @@ -181,9 +181,13 @@ export async function runTaskResult( const graphClient = createGraphClient(config); if (config.graphPolicy.autoIndexOnRun) { const root = config.graphPolicy.workspaceRoot ?? process.cwd(); - const indexOptions = config.graphPolicy.includeExtensions - ? { includeExtensions: config.graphPolicy.includeExtensions } - : undefined; + const indexOptions = { + ...(config.graphPolicy.includeExtensions ? { includeExtensions: config.graphPolicy.includeExtensions } : {}), + ...(config.graphPolicy.excludeGlobs?.length ? { excludeGlobs: config.graphPolicy.excludeGlobs } : {}), + ...(typeof config.graphPolicy.maxFileSizeBytes === "number" + ? { maxFileSizeBytes: config.graphPolicy.maxFileSizeBytes } + : {}), + }; const storeIncomplete = indexedStoreIsIncomplete( root, graphClient.indexManifestName, diff --git a/src/surfaces/cli/runtime/selfcheck.ts b/src/surfaces/cli/runtime/selfcheck.ts index c34e3c0a..35ac5683 100644 --- a/src/surfaces/cli/runtime/selfcheck.ts +++ b/src/surfaces/cli/runtime/selfcheck.ts @@ -107,6 +107,10 @@ export async function runSelfcheck( try { const pending = hasPendingGraphIndexWork(root, { ...(config.graphPolicy.includeExtensions ? { includeExtensions: config.graphPolicy.includeExtensions } : {}), + ...(config.graphPolicy.excludeGlobs?.length ? { excludeGlobs: config.graphPolicy.excludeGlobs } : {}), + ...(typeof config.graphPolicy.maxFileSizeBytes === "number" + ? { maxFileSizeBytes: config.graphPolicy.maxFileSizeBytes } + : {}), manifestName: resolveIndexManifestName(config), }); items.push({ diff --git a/tests/m-workspace-exclude-and-dialogue-read.test.ts b/tests/m-workspace-exclude-and-dialogue-read.test.ts new file mode 100644 index 00000000..579ce127 --- /dev/null +++ b/tests/m-workspace-exclude-and-dialogue-read.test.ts @@ -0,0 +1,254 @@ +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, relative } from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { validateConfig } from "../src/config/loader"; +import { isDialogueRecordNode } from "../src/graph/dialogue-node-match"; +import { createGraphClient } from "../src/graph/client-factory"; +import { + getGraphifyFileStoreParseCount, + graphifyFileStoreCache, + graphStoreDeltaPath, + GraphifyFileClient, + resetGraphifyFileStoreCacheForTests, + setGraphStoreHugeReadBytesForTests, + writeGraphStoreFile, +} from "../src/graph/graphify-file-client"; +import { readGraphStoreFileChunked } from "../src/graph/graph-store-json-chunks"; +import { indexSingleFile, indexWorkspaceFiles } from "../src/graph/file-indexer"; +import { walkFiles } from "../src/graph/file-indexer-walker"; +import { parseIgnoreRules } from "../src/graph/workspace-ignore"; +import { recordDialogueTurn } from "../src/learning/dialogue-thread"; +import { GraphifySqliteClient } from "../src/graph/sqlite-client"; +import type { GraphNode } from "../src/core/types"; + +const dirs: string[] = []; + +afterEach(() => { + setGraphStoreHugeReadBytesForTests(undefined); + resetGraphifyFileStoreCacheForTests(); + while (dirs.length > 0) { + const dir = dirs.pop(); + if (dir) rmSync(dir, { recursive: true, force: true }); + } +}); + +function tempDir(prefix: string): string { + const dir = mkdtempSync(join(tmpdir(), prefix)); + dirs.push(dir); + return dir; +} + +const rel = (root: string, path: string): string => relative(root, path).replace(/\\/g, "/"); + +function memoryClient() { + return createGraphClient(validateConfig({ + providers: {}, + tiers: { + smart: { provider: "openai", model: "gpt-4.1" }, + economy: { provider: "openai", model: "gpt-4.1-mini" }, + }, + budgetPolicy: { runTokenCap: 2000 }, + graphPolicy: { + enableAutoBuild: false, + transport: "memory", + maxContextTokens: 1500, + }, + learningPolicy: { enableFlywheel: false, trainingCadence: "nightly", exportPath: "graphflow-out/learning-dataset.jsonl" }, + })); +} + +describe("workspace exclude rules", () => { + it("parses directory, basename, path-glob, and inline-comment rules", () => { + const match = parseIgnoreRules([ + "# full line", + "vendor/ # any depth", + "tools/cache/", + "*.secret.ts", + "docs/**/*.md", + "!not-supported/", + "", + ].join("\n")); + expect(match).toBeTypeOf("function"); + expect(match!("nested/vendor", true)).toBe(true); + expect(match!("nested/vendor/lib.ts", false)).toBe(true); + expect(match!("vendor.ts", false)).toBe(false); + expect(match!("tools/cache", true)).toBe(true); + expect(match!("tools/cache/a.ts", false)).toBe(true); + expect(match!("tools/other.ts", false)).toBe(false); + expect(match!("src/a.secret.ts", false)).toBe(true); + expect(match!("docs/a/b.md", false)).toBe(true); + // A double-star still needs the slash the pattern wrote; `docs/readme.md` + // is one segment under docs, so it does not match `docs/**/*.md`. + expect(match!("docs/readme.md", false)).toBe(false); + expect(match!("keep/file.ts", false)).toBe(false); + // Backslash rules are normalized the same way Windows paths are. + const win = parseIgnoreRules("tools\\cache/\n"); + expect(win!("tools/cache/a.ts", false)).toBe(true); + }); + + it("walkFiles honors .graphflowignore and excludeGlobs without dropping other files", () => { + const root = tempDir("gf-ignore-"); + mkdirSync(join(root, "src"), { recursive: true }); + mkdirSync(join(root, "skip", "vendor"), { recursive: true }); + mkdirSync(join(root, "nested", "vendor"), { recursive: true }); + writeFileSync(join(root, "src", "app.ts"), "export const a = 1;\n"); + writeFileSync(join(root, "src", "a.secret.ts"), "export const s = 1;\n"); + writeFileSync(join(root, "skip", "hidden.ts"), "export const h = 1;\n"); + writeFileSync(join(root, "nested", "vendor", "x.ts"), "export const x = 1;\n"); + writeFileSync(join(root, ".graphflowignore"), "skip/ # copies\nvendor/\n"); + + const found = walkFiles(root, [".ts"], { respectGitIgnore: false, excludeGlobs: ["*.secret.ts"] }) + .map((path) => rel(root, path)) + .sort(); + expect(found).toEqual(["src/app.ts"]); + }); + + it("index skips excluded files and files over maxFileSizeBytes", async () => { + const root = tempDir("gf-index-exclude-"); + mkdirSync(join(root, "src"), { recursive: true }); + mkdirSync(join(root, "skip"), { recursive: true }); + writeFileSync(join(root, "src", "app.ts"), "export const a = 1;\n"); + writeFileSync(join(root, "src", "big.ts"), `export const big = "${"x".repeat(80)}";\n`); + writeFileSync(join(root, "skip", "hidden.ts"), "export const h = 1;\n"); + writeFileSync(join(root, ".graphflowignore"), "skip/\n"); + + const client = memoryClient(); + const indexed = await indexWorkspaceFiles(client, root, { + includeExtensions: [".ts"], + respectGitIgnore: false, + maxFileSizeBytes: 40, + }); + expect(indexed.indexedFiles).toBe(1); + + const single = await indexSingleFile(client, root, join(root, "skip", "hidden.ts"), { + includeExtensions: [".ts"], + excludeGlobs: ["skip/"], + }); + expect(single.skipped).toBe(true); + expect(single.reason).toMatch(/graphflowignore|excludeGlobs/); + }); + + it("validateConfig keeps excludeGlobs and maxFileSizeBytes and rejects bad values", () => { + const config = validateConfig({ + providers: {}, + tiers: { + smart: { provider: "openai" }, + economy: { provider: "openai" }, + }, + budgetPolicy: { runTokenCap: 100 }, + graphPolicy: { + enableAutoBuild: false, + transport: "file", + maxContextTokens: 100, + excludeGlobs: ["工具/", "*.jsonl"], + maxFileSizeBytes: 50_000, + }, + learningPolicy: { enableFlywheel: false, trainingCadence: "weekly", exportPath: "out.jsonl" }, + }); + expect(config.graphPolicy.excludeGlobs).toEqual(["工具/", "*.jsonl"]); + expect(config.graphPolicy.maxFileSizeBytes).toBe(50_000); + + expect(() => validateConfig({ + ...config, + graphPolicy: { ...config.graphPolicy, maxFileSizeBytes: 0 }, + })).toThrow(/maxFileSizeBytes/); + expect(() => validateConfig({ + ...config, + graphPolicy: { ...config.graphPolicy, excludeGlobs: [""] }, + })).toThrow(/excludeGlobs/); + }); +}); + +describe("dialogue record does not materialize a huge file store", () => { + it("appends a delta and still reads the new turns back", async () => { + const root = tempDir("gf-huge-dialogue-"); + const storePath = join(root, "graphflow-graph.json"); + const filler: GraphNode = { id: "file:big.md", type: "File", content: "x".repeat(200) }; + writeGraphStoreFile(storePath, { nodes: [filler], edges: [] }); + setGraphStoreHugeReadBytesForTests(1); + resetGraphifyFileStoreCacheForTests(); + + const client = new GraphifyFileClient(storePath); + const first = await recordDialogueTurn(client, { + userQuery: "where is the indexer", + workspaceRoot: root, + now: 1_000, + }); + const second = await recordDialogueTurn(client, { + userQuery: "exclude the copies", + workspaceRoot: root, + now: 2_000, + }); + expect(first.recorded).toBe(true); + expect(second.turn?.seq).toBe(2); + expect(getGraphifyFileStoreParseCount()).toBe(0); + expect(graphifyFileStoreCache.has(storePath)).toBe(false); + + const base = readFileSync(storePath, "utf8"); + expect(base).toContain("file:big.md"); + expect(base).not.toContain("dialogue-session"); + const delta = readFileSync(graphStoreDeltaPath(storePath), "utf8"); + expect(delta).toContain("dialogue-session"); + expect(delta).toContain("exclude the copies"); + + setGraphStoreHugeReadBytesForTests(undefined); + const snapshot = client.readSnapshot(); + expect(snapshot.nodes.some((node) => node.id === "file:big.md")).toBe(true); + expect(snapshot.nodes.filter((node) => isDialogueRecordNode(node)).length).toBeGreaterThanOrEqual(3); + }); + + it("projects dialogue nodes out of a chunked store without keeping edges", () => { + const root = tempDir("gf-chunk-project-"); + const storePath = join(root, "graph.json"); + writeFileSync(storePath, JSON.stringify({ + nodes: [ + { id: "file:a.ts", type: "File", content: "code" }, + { id: "dialogue:1", type: "Decision", content: "turn", metadata: { kind: "dialogue-turn" } }, + ], + edges: [{ from: "file:a.ts", to: "dialogue:1", relation: "references" }], + })); + const full = readGraphStoreFileChunked(storePath); + expect(full.nodes).toHaveLength(2); + expect(full.edges).toHaveLength(1); + const projected = readGraphStoreFileChunked(storePath, { + skipEdges: true, + keepNode: (node) => isDialogueRecordNode(node as { id: string; type: string; metadata?: { kind?: string } }), + }); + expect(projected.edges).toEqual([]); + expect(projected.nodes.map((node) => (node as { id: string }).id)).toEqual(["dialogue:1"]); + }); + + it("sqlite lists dialogue rows without readSnapshot", async () => { + try { + require("better-sqlite3"); + } catch { + return; + } + const root = tempDir("gf-sql-dialogue-"); + const client = new GraphifySqliteClient(join(root, "graph.sqlite")); + let snap: { mockRestore: () => void } | undefined; + try { + const nodes: GraphNode[] = []; + for (let i = 0; i < 40; i += 1) { + nodes.push({ id: `file:${i}.ts`, type: "File", content: "code" }); + } + nodes.push({ + id: "dialogue:abc:0001", + type: "Decision", + content: "question", + metadata: { kind: "dialogue-turn" }, + }); + await client.upsertNodes(nodes); + snap = vi.spyOn(GraphifySqliteClient.prototype, "readSnapshot"); + const listed = await client.listDialogueNodes(); + expect(snap).not.toHaveBeenCalled(); + expect(listed.map((node) => node.id)).toEqual(["dialogue:abc:0001"]); + } finally { + snap?.mockRestore(); + client.close(); + } + }); +});