diff --git a/CHANGELOG.md b/CHANGELOG.md index a80ffb62..027865fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,11 @@ All notable changes to this project are documented in this file. - `importSkillsFromMarkdownRuntime`, `exportSkillsToMarkdownRuntime`, and `extractDialogueKnowledgeRuntime` now `close()` the graph client in `finally`. Default `auto` transport opens better-sqlite3; leaving that handle open makes a later `rmSync` of `graphflow-graph.sqlite` fail with `EBUSY` on Windows. +### Fixed — large workspaces no longer have to be fully indexed, and dialogue record no longer materializes a huge file graph + +- `graphPolicy.excludeGlobs` and `graphPolicy.maxFileSizeBytes` are honored by workspace index, single-file index, and the pending-index check. `/.graphflowignore` uses the same rules (`name/` at any depth, anchored `path/to/dir/`, basename globs, path globs). Negation and nested ignore files are not supported. Markdown stays on the 200KB source cap; the 5MB document cap applies to office/PDF only. +- `dialogue record` on a file store larger than 512MB streams just the dialogue nodes and appends a delta line. It does not `JSON.parse` the code graph. SQLite `listDialogueNodes()` returns the same rows with a SQL filter instead of `readSnapshot()`. + ## [Unreleased] ### Fixed — 第 0 步性能与健壮性(迁移 Rust 前的架构债清偿,复审登记项全闭环) diff --git a/src/config/loader.ts b/src/config/loader.ts index 716db986..acd5303a 100644 --- a/src/config/loader.ts +++ b/src/config/loader.ts @@ -92,6 +92,19 @@ export function validateConfigDetailed(path = "graphflow.config.json"): ConfigVa issues.push({ severity: "error", field: "graphPolicy.includeExtensions", message: "Extensions must start with '.'" }); } } + if (parsed.graphPolicy.excludeGlobs !== undefined) { + const globs = parsed.graphPolicy.excludeGlobs; + const invalid = !Array.isArray(globs) || globs.some((rule) => typeof rule !== "string" || rule.trim() === ""); + if (invalid) { + issues.push({ severity: "error", field: "graphPolicy.excludeGlobs", message: "excludeGlobs entries must be non-empty strings" }); + } + } + if (parsed.graphPolicy.maxFileSizeBytes !== undefined) { + const size = parsed.graphPolicy.maxFileSizeBytes; + if (typeof size !== "number" || !Number.isFinite(size) || size < 1) { + issues.push({ severity: "error", field: "graphPolicy.maxFileSizeBytes", message: "maxFileSizeBytes must be a positive number" }); + } + } if (parsed.graphPolicy.workspaceRoot) { issues.push({ severity: "warning", field: "graphPolicy.workspaceRoot", message: "workspaceRoot in config is deprecated; resolved from process.cwd() at runtime" }); } @@ -342,6 +355,21 @@ export function validateConfig(input: GraphFlowConfig): GraphFlowConfig { } } + if (input.graphPolicy.excludeGlobs !== undefined) { + const globs = input.graphPolicy.excludeGlobs; + const invalid = !Array.isArray(globs) || globs.some((rule) => typeof rule !== "string" || rule.trim() === ""); + if (invalid) { + throw new Error("Invalid config: graphPolicy.excludeGlobs entries must be non-empty strings."); + } + } + + if (input.graphPolicy.maxFileSizeBytes !== undefined) { + const size = input.graphPolicy.maxFileSizeBytes; + if (typeof size !== "number" || !Number.isFinite(size) || size < 1) { + throw new Error("Invalid config: graphPolicy.maxFileSizeBytes must be a positive number."); + } + } + // learningPolicy is optional at the input boundary: every sub-field has a // default, so a minimal config (providers + tiers + graphPolicy) must be // valid. Requiring the bare object made minimal configs fail validation and diff --git a/src/config/schema.ts b/src/config/schema.ts index 8f068867..e6b61b6b 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -169,6 +169,18 @@ export interface GraphFlowConfig { enableDialogueThread?: boolean; workspaceRoot?: string; includeExtensions?: string[]; + /** + * Workspace-relative paths and globs the indexer must skip, unioned with + * `/.graphflowignore`. Syntax: `name/` (any directory of that + * name), `path/to/dir/` (anchored), `*.ext` (basename), path globs where + * `*` does not cross `/` and a double-star does. No gitignore negation. + */ + excludeGlobs?: string[]; + /** + * Skip source files larger than this many bytes. Office/PDF conversion + * still uses its own higher cap. Default 200_000. + */ + maxFileSizeBytes?: number; /** * Skip files git ignores when indexing (exact semantics via `git ls-files * --exclude-standard`). Default true; ignored outside git checkouts. diff --git a/src/graph/client-factory.ts b/src/graph/client-factory.ts index da5198fc..e077e12b 100644 --- a/src/graph/client-factory.ts +++ b/src/graph/client-factory.ts @@ -10,6 +10,7 @@ import { GraphifySqliteClient } from "./sqlite-client"; import { mergeSiblingJsonStoreIntoSqlite } from "./store-migration"; import { CACHE_FILE, SQLITE_INDEX_MANIFEST } from "./file-indexer-cache"; import { existsSync } from "node:fs"; +import { isDialogueRecordNode } from "./dialogue-node-match"; export interface GraphStoreSnapshot { nodes: GraphNode[]; @@ -33,6 +34,16 @@ export interface GraphClient { upsertGraph?(batch: { nodes?: GraphNode[]; edges?: GraphEdge[] }): Promise; queryByKeyword(query: string): Promise; readSnapshot?(): GraphStoreSnapshot; + /** + * Dialogue turns and sessions without materializing the rest of the graph. + * Backends that cannot project fall back to `readSnapshot` at the call site. + */ + listDialogueNodes?(): Promise; + /** + * When true, `upsertEdges` is idempotent and callers must not `readSnapshot` + * solely to drop duplicate edges (that read is the large-graph OOM). + */ + edgesAreIdempotent?(): boolean; getNodesByIds?(ids: string[]): Promise; getNeighbors?( nodeIds: string[], @@ -113,6 +124,10 @@ class MutationAwareGraphClient implements GraphClient { return this.inner.indexManifestName; } + edgesAreIdempotent(): boolean { + return this.inner.edgesAreIdempotent?.() === true; + } + async upsertNodes(nodes: GraphNode[]): Promise { await this.inner.upsertNodes(nodes); markGraphMutated(nodes.map((n) => n.id)); @@ -150,6 +165,11 @@ class MutationAwareGraphClient implements GraphClient { return this.inner.queryByKeyword(query); } + async listDialogueNodes(): Promise { + if (this.inner.listDialogueNodes) return this.inner.listDialogueNodes(); + return this.readSnapshot().nodes.filter((node) => isDialogueRecordNode(node)); + } + readSnapshot(): GraphStoreSnapshot { // 所有被包装的后端(file/sqlite/memory/mcp)均实现 readSnapshot, // `?.` 仅作防御;接口此处为可选方法,调用方先做存在性检查。 diff --git a/src/graph/dialogue-node-match.ts b/src/graph/dialogue-node-match.ts new file mode 100644 index 00000000..970d73c2 --- /dev/null +++ b/src/graph/dialogue-node-match.ts @@ -0,0 +1,20 @@ +/** + * Dialogue records live in the same store as code nodes. Callers that only + * need the thread must recognize them without assuming a particular backend. + * + * Id prefixes are authoritative (`dialogue::`, `dialogue-session:`). + * Metadata kinds cover records written before the id scheme, but code nodes + * (File / Module / Symbol) are never dialogue even if a kind field collides. + */ +export function isDialogueRecordNode(node: { + id?: unknown; + type?: unknown; + metadata?: { kind?: unknown } | null; +}): boolean { + if (typeof node.id !== "string" || node.id.length === 0) return false; + if (node.id.startsWith("dialogue-session:") || node.id.startsWith("dialogue:")) return true; + const type = typeof node.type === "string" ? node.type : ""; + if (type === "File" || type === "Module" || type === "Symbol") return false; + const kind = node.metadata && typeof node.metadata === "object" ? node.metadata.kind : undefined; + return kind === "dialogue-turn" || kind === "dialogue-session"; +} diff --git a/src/graph/file-indexer-cache.ts b/src/graph/file-indexer-cache.ts index 740431ff..ac275c39 100644 --- a/src/graph/file-indexer-cache.ts +++ b/src/graph/file-indexer-cache.ts @@ -160,7 +160,7 @@ export function hasPendingGraphIndexWork( rootDir: string, options?: Pick< FileIndexerOptions, - "includeExtensions" | "maxFileSizeBytes" | "forceReindex" | "respectGitIgnore" + "includeExtensions" | "maxFileSizeBytes" | "forceReindex" | "respectGitIgnore" | "excludeGlobs" > & { manifestName?: string | undefined } ): boolean { const includeExtensions = options?.includeExtensions ?? DEFAULT_EXTENSIONS; @@ -173,6 +173,7 @@ export function hasPendingGraphIndexWork( const cacheState = loadCacheStateCached(indexManifestPath(rootDir, options?.manifestName)); const scanned = walkScannableFiles(rootDir, includeExtensions, maxFileSizeBytes, { ...(options?.respectGitIgnore === false ? { respectGitIgnore: false } : {}), + ...(options?.excludeGlobs?.length ? { excludeGlobs: options.excludeGlobs } : {}), }); const currentRelPaths = new Set(scanned.map((file) => file.relPath)); diff --git a/src/graph/file-indexer-walker.ts b/src/graph/file-indexer-walker.ts index 7efd63cb..8bbf73cb 100644 --- a/src/graph/file-indexer-walker.ts +++ b/src/graph/file-indexer-walker.ts @@ -15,6 +15,7 @@ import { OFFICE_DOCUMENT_EXTENSIONS, } from "./document-convert.js"; import { safeReaddirSync, safeStatSync } from "../utils/safe-fs.js"; +import { loadWorkspaceIgnore } from "./workspace-ignore.js"; import type { EmbeddingProvider } from "../learning/embeddings.js"; @@ -53,6 +54,11 @@ export interface FileIndexerOptions { * the machine's cores. `GRAPHFLOW_INDEX_WORKERS=0` disables it globally. */ indexWorkers?: number; + /** + * Extra exclude rules, same syntax as `/.graphflowignore`. + * The ignore file is always loaded; these rules are unioned with it. + */ + excludeGlobs?: string[]; } export interface ScannedFile { @@ -189,7 +195,7 @@ export function walkScannableFiles( rootDir: string, includeExtensions: string[], maxFileSizeBytes: number, - options: { respectGitIgnore?: boolean } = {} + options: { respectGitIgnore?: boolean; excludeGlobs?: string[] } = {} ): ScannedFile[] { const files = walkFiles(rootDir, includeExtensions, options); const scanned: ScannedFile[] = []; @@ -225,11 +231,15 @@ export function walkScannableFiles( export function walkFiles( rootDir: string, includeExtensions: string[], - options: { respectGitIgnore?: boolean } = {} + options: { respectGitIgnore?: boolean; excludeGlobs?: string[] } = {} ): string[] { const files: string[] = []; const dirStack: string[] = [rootDir]; const gitVisible = options.respectGitIgnore === false ? undefined : readGitVisibleFiles(rootDir); + const ignore = loadWorkspaceIgnore( + rootDir, + options.excludeGlobs && options.excludeGlobs.length > 0 ? { excludeGlobs: options.excludeGlobs } : undefined + ); while (dirStack.length > 0) { const current = dirStack.pop()!; @@ -245,6 +255,10 @@ export function walkFiles( if (IGNORED_DIRS.has(entry.name)) { continue; } + const relDir = normalizePath(relative(rootDir, full)); + if (ignore?.(relDir, true)) { + continue; + } // A directory holding only ignored/untracked files cannot contribute. if (gitVisible && !directoryMayContainVisibleFiles(rootDir, full, gitVisible)) { continue; @@ -259,7 +273,11 @@ export function walkFiles( if (isGeneratedOrLockFile(entry.name)) { continue; } - if (gitVisible && !gitVisible.has(normalizePath(relative(rootDir, full)))) { + const relFile = normalizePath(relative(rootDir, full)); + if (ignore?.(relFile, false)) { + continue; + } + if (gitVisible && !gitVisible.has(relFile)) { continue; } files.push(full); diff --git a/src/graph/file-indexer.ts b/src/graph/file-indexer.ts index 84fa3b96..f6de0f5d 100644 --- a/src/graph/file-indexer.ts +++ b/src/graph/file-indexer.ts @@ -44,6 +44,7 @@ export { resolveCallerAtLine } from "./file-indexer-nodes.js"; // ── Internal imports ───────────────────────────────────────────────── import { DEFAULT_EXTENSIONS, DEFAULT_MAX_FILE_SIZE, normalizePath, extOf, walkScannableFiles } from "./file-indexer-walker.js"; +import { loadWorkspaceIgnore } from "./workspace-ignore.js"; import type { FileIndexerOptions } from "./file-indexer-walker.js"; import { indexManifestPath, loadCacheState, saveCacheState } from "./file-indexer-cache.js"; import type { @@ -262,6 +263,7 @@ export async function indexWorkspaceFiles( const scanned = walkScannableFiles(rootDir, includeExtensions, maxFileSizeBytes, { ...(options?.respectGitIgnore === false ? { respectGitIgnore: false } : {}), + ...(options?.excludeGlobs?.length ? { excludeGlobs: options.excludeGlobs } : {}), }); const currentRelPaths = new Set(scanned.map((file) => file.relPath)); @@ -470,6 +472,7 @@ export async function indexSingleFile( FileIndexerOptions, | "includeExtensions" | "maxFileSizeBytes" + | "excludeGlobs" | "embeddingProvider" | "referenceEdgeMaxDefinitionFiles" | "referenceEdgeMaxPerFile" @@ -507,6 +510,21 @@ export async function indexSingleFile( }; } + const relPath = normalizePath(relative(rootDir, absPath)); + const ignore = loadWorkspaceIgnore( + rootDir, + options?.excludeGlobs && options.excludeGlobs.length > 0 ? { excludeGlobs: options.excludeGlobs } : undefined + ); + if (ignore?.(relPath, false)) { + return { + indexedFiles: 0, + indexedSymbols: 0, + indexedReferences: 0, + skipped: true, + reason: "excluded by .graphflowignore or graphPolicy.excludeGlobs", + }; + } + const officeDoc = isOfficeDocumentPath(absPath); const sizeLimit = officeDoc ? Math.max(maxFileSizeBytes, DEFAULT_DOCUMENT_MAX_FILE_SIZE) @@ -528,7 +546,6 @@ export async function indexSingleFile( return { indexedFiles: 0, indexedSymbols: 0, indexedReferences: 0, skipped: true, reason: "file exceeds maxFileSizeBytes" }; } - const relPath = normalizePath(relative(rootDir, absPath)); const mtimeMs = stat.mtimeMs; let content = ""; let currentHash = ""; diff --git a/src/graph/graph-store-json-chunks.ts b/src/graph/graph-store-json-chunks.ts index 228cf7aa..5c8410a7 100644 --- a/src/graph/graph-store-json-chunks.ts +++ b/src/graph/graph-store-json-chunks.ts @@ -22,6 +22,17 @@ export interface ChunkedGraphStore { edges: unknown[]; } +export interface ChunkedGraphStoreReadOptions { + /** + * Drop nodes for which this returns false. The element string is parsed + * and then discarded, so a projection (dialogue records, a handful of ids) + * does not retain the rest of the graph. + */ + keepNode?: (node: unknown) => boolean; + /** Scan the edges array without retaining it. */ + skipEdges?: boolean; +} + /** Flush the accumulated element buffer at this size while scanning. */ const DEFAULT_CHUNK_BYTES = 8 * 1024 * 1024; @@ -32,7 +43,10 @@ class GraphStoreChunkParser { private buffer = ""; private index = 0; - constructor(private readonly pull: () => string | null) {} + constructor( + private readonly pull: () => string | null, + private readonly options: ChunkedGraphStoreReadOptions = {} + ) {} private fill(): boolean { if (this.index > 0) { @@ -135,7 +149,7 @@ class GraphStoreChunkParser { } } - private readElements(target: unknown[]): void { + private readElements(target: unknown[] | null, keep?: (node: unknown) => boolean): void { while (true) { this.skipWhitespace(); const ch = this.peek(); @@ -156,7 +170,10 @@ class GraphStoreChunkParser { if (!complete) { throw new Error("Truncated graph store: incomplete element"); } - target.push(JSON.parse(text)); + if (target === null) continue; + const parsed: unknown = JSON.parse(text); + if (keep && !keep(parsed)) continue; + target.push(parsed); } } @@ -195,7 +212,11 @@ class GraphStoreChunkParser { throw new Error(`Truncated graph store: expected an array for "${name}"`); } this.index += 1; - this.readElements(name === "nodes" ? store.nodes : store.edges); + const discard = name === "edges" && this.options.skipEdges === true; + this.readElements( + discard ? null : name === "nodes" ? store.nodes : store.edges, + name === "nodes" ? this.options.keepNode : undefined + ); continue; } const { complete } = this.readValue(); @@ -209,14 +230,17 @@ class GraphStoreChunkParser { } /** Parse a graph store document from a sequence of text chunks. */ -export function readGraphStoreFromChunks(pull: () => string | null): ChunkedGraphStore { - return new GraphStoreChunkParser(pull).parse(); +export function readGraphStoreFromChunks( + pull: () => string | null, + options: ChunkedGraphStoreReadOptions = {} +): ChunkedGraphStore { + return new GraphStoreChunkParser(pull, options).parse(); } /** Read a graph store file in bounded chunks (used above the single-string cap). */ export function readGraphStoreFileChunked( filePath: string, - options: { chunkBytes?: number } = {} + options: { chunkBytes?: number } & ChunkedGraphStoreReadOptions = {} ): ChunkedGraphStore { const chunkBytes = Math.max(64 * 1024, options.chunkBytes ?? DEFAULT_CHUNK_BYTES); const fd = openSync(filePath, "r"); @@ -228,7 +252,7 @@ export function readGraphStoreFileChunked( return readGraphStoreFromChunks(() => { const read = readSync(fd, buffer, 0, chunkBytes, null); return read > 0 ? decoder.write(buffer.subarray(0, read)) : null; - }); + }, options); } finally { closeSync(fd); } diff --git a/src/graph/graphify-file-client.ts b/src/graph/graphify-file-client.ts index 9b0f7aec..2380c334 100644 --- a/src/graph/graphify-file-client.ts +++ b/src/graph/graphify-file-client.ts @@ -17,6 +17,7 @@ import type { GraphEdge, GraphNode } from "../core/types"; import { logger } from "../utils/logger"; import { tokenizeForIndex, nodeSearchableText, nodeRecallText } from "./graph-utils"; import { readGraphStoreFileChunked } from "./graph-store-json-chunks"; +import { isDialogueRecordNode } from "./dialogue-node-match"; interface GraphStore { nodes: GraphNode[]; @@ -238,13 +239,29 @@ const GRAPH_STORE_WRITE_CHUNK_BYTES = 4 * 1024 * 1024; */ export const GRAPH_STORE_MAX_READ_BYTES = 512 * 1024 * 1024; +/** + * Stores larger than this are not materialized for a small read or write + * (dialogue record, id lookup). Default equals the single-string cap: below + * it, `JSON.parse` of the whole file is the historical path. Tests lower it. + */ +let hugeStoreMinBytes = GRAPH_STORE_MAX_READ_BYTES; + +/** Small batches on a huge store append a delta line instead of rewriting the base. */ +export const GRAPH_STORE_BLIND_DELTA_MAX_ELEMENTS = 256; + +/** Test hook: treat stores larger than `bytes` as too big to materialize. */ +export function setGraphStoreHugeReadBytesForTests(bytes?: number): void { + hugeStoreMinBytes = bytes ?? GRAPH_STORE_MAX_READ_BYTES; +} + /** Actionable error for a store too large to read as one string (shared). */ export function graphStoreTooLargeError(storePath: string, sizeBytes: number): Error { const sizeMb = Math.round(sizeBytes / (1024 * 1024)); const limitMb = Math.round(GRAPH_STORE_MAX_READ_BYTES / (1024 * 1024)); return new Error( `Graph store is too large to read (${sizeMb} MB at ${storePath}, limit ${limitMb} MB). ` + - "Remove the file and run a rebuild, or narrow graphPolicy.includeExtensions." + "Remove the file and run a rebuild, or narrow what gets indexed " + + "(graphPolicy.excludeGlobs, graphPolicy.includeExtensions, or a .graphflowignore file)." ); } @@ -492,7 +509,55 @@ function readStoreCacheEntry(storePath: string): StoreCacheEntry { return entry; } +function asGraphNode(value: unknown): GraphNode | undefined { + if (!value || typeof value !== "object") return undefined; + const node = value as GraphNode; + if (typeof node.id !== "string" || typeof node.type !== "string" || typeof node.content !== "string") { + return undefined; + } + return node; +} + +/** Apply delta upserts/deletes onto an already-projected node list. */ +function overlayDeltaNodes( + nodes: GraphNode[], + deltaPath: string, + deltaStat: FileStat | null, + keep: (node: GraphNode) => boolean +): GraphNode[] { + if (deltaStat === null || deltaStat.size === 0) return nodes; + let contents: string; + try { + contents = readFileSync(deltaPath, "utf8"); + } catch { + return nodes; + } + const map = new Map(nodes.map((node) => [node.id, node])); + for (const line of contents.split("\n")) { + if (!line.trim()) continue; + let op: DeltaOp; + try { + op = JSON.parse(line) as DeltaOp; + } catch { + continue; + } + if (op.op === "upsert") { + for (const node of op.nodes ?? []) { + if (keep(node)) map.set(node.id, node); + } + } else if (op.op === "delete") { + for (const id of op.nodeIds ?? []) map.delete(id); + } + } + return Array.from(map.values()); +} + export class GraphifyFileClient { + /** Edge upserts dedupe on apply / in upsertGraph, so callers need not readSnapshot first. */ + edgesAreIdempotent(): boolean { + return true; + } + constructor( private readonly storePath: string, private readonly options: { @@ -502,6 +567,51 @@ export class GraphifyFileClient { } = {} ) {} + private storeAbsPath(): string { + return resolve(this.storePath); + } + + /** True when the base file is over the materialization cap and not already cached. */ + private isHugeUncached(): boolean { + const absPath = this.storeAbsPath(); + const current = statIfExists(absPath); + if (current === null || current.size <= hugeStoreMinBytes) return false; + const cached = graphifyFileStoreCache.get(absPath); + if (!cached) return true; + const deltaStat = statIfExists(resolve(deltaPathFor(absPath))); + return !sameStat(cached.stat, current) || !sameStat(cached.deltaStat, deltaStat); + } + + /** + * Append one upsert without reading the base. Compacting the delta would + * materialize the huge file — the failure mode dialogue record is avoiding — + * so an oversized log is left in place for the next full rebuild. + */ + private appendBlindDelta(op: DeltaUpsertOp): void { + const absPath = this.storeAbsPath(); + const deltaPath = resolve(deltaPathFor(absPath)); + mkdirSync(dirname(deltaPath), { recursive: true }); + appendFileSync(deltaPath, `${JSON.stringify(op)}\n`, "utf8"); + graphifyFileStoreCache.delete(absPath); + } + + /** Stream the base file, keep `predicate` nodes, then overlay the delta log. */ + private readProjectedNodes(predicate: (node: GraphNode) => boolean): GraphNode[] { + const absPath = this.storeAbsPath(); + const current = statIfExists(absPath); + if (current === null) return []; + const chunked = readGraphStoreFileChunked(absPath, { + skipEdges: true, + keepNode: (value) => { + const node = asGraphNode(value); + return node !== undefined && predicate(node); + }, + }); + const nodes = chunked.nodes.map((value) => asGraphNode(value)).filter((node): node is GraphNode => node !== undefined); + const deltaPath = resolve(deltaPathFor(absPath)); + return overlayDeltaNodes(nodes, deltaPath, statIfExists(deltaPath), predicate); + } + /** * Merge nodes and edges into the store with a SINGLE read + write. * @@ -527,6 +637,18 @@ export class GraphifyFileClient { return; } + if ( + this.isHugeUncached() && + incomingNodes.length + incomingEdges.length <= GRAPH_STORE_BLIND_DELTA_MAX_ELEMENTS + ) { + this.appendBlindDelta({ + op: "upsert", + ...(incomingNodes.length > 0 ? { nodes: incomingNodes } : {}), + ...(incomingEdges.length > 0 ? { edges: incomingEdges } : {}), + }); + return; + } + const entry = this.readStoreEntry(); const store = entry.store; const nodeMap = new Map(store.nodes.map((node) => [node.id, node])); @@ -689,11 +811,26 @@ export class GraphifyFileClient { } async getNodesByIds(ids: string[]): Promise { - const store = this.readStore(); + if (ids.length === 0) return []; const want = new Set(ids); + if (this.isHugeUncached()) { + return this.readProjectedNodes((node) => want.has(node.id)); + } + const store = this.readStore(); return store.nodes.filter((n) => want.has(n.id)); } + /** + * Dialogue turns and sessions only. On a huge file store this streams the + * document and drops every other node instead of building the full graph. + */ + async listDialogueNodes(): Promise { + if (this.isHugeUncached()) { + return this.readProjectedNodes((node) => isDialogueRecordNode(node)); + } + return this.readStore().nodes.filter((node) => isDialogueRecordNode(node)); + } + async getNeighbors( nodeIds: string[], relations?: GraphEdge["relation"][], diff --git a/src/graph/sqlite-client.ts b/src/graph/sqlite-client.ts index e72566d6..f2ddff7a 100644 --- a/src/graph/sqlite-client.ts +++ b/src/graph/sqlite-client.ts @@ -6,6 +6,7 @@ import type { GraphClient } from "./client-factory"; import { SQLITE_INDEX_MANIFEST } from "./file-indexer-cache"; import { tokenizeForIndex, containsCJK, nodeRecallText } from "./graph-utils"; import { requireFromOptionalDeps, resolveSqliteDepsRoot } from "../utils/optional-deps"; +import { isDialogueRecordNode } from "./dialogue-node-match"; const requireFn = createRequire(__filename); @@ -204,6 +205,10 @@ const SNAPSHOT_ROWS_IDLE_MS = 15_000; export class GraphifySqliteClient implements GraphClient { readonly indexManifestName = SQLITE_INDEX_MANIFEST; + /** INSERT OR IGNORE makes a repeated edge a no-op, so callers skip the snapshot pre-read. */ + edgesAreIdempotent(): boolean { + return true; + } private readonly db: import("better-sqlite3").Database; /** * One preview reads the whole store many times. Raw rows are reused while @@ -454,6 +459,25 @@ export class GraphifySqliteClient implements GraphClient { return rows.map(rowToNode); } + /** + * Dialogue rows only. A full `readSnapshot()` pulls every code node into JS; + * this scan stays in SQLite and returns the thread. + */ + async listDialogueNodes(): Promise { + const rows = this.db + .prepare( + `SELECT id, type, content, metadata FROM nodes + WHERE id LIKE 'dialogue-session:%' + OR id LIKE 'dialogue:%' + OR ( + type NOT IN ('File', 'Module', 'Symbol') + AND json_extract(metadata, '$.kind') IN ('dialogue-turn', 'dialogue-session') + )` + ) + .all() as NodeRow[]; + return rows.map(rowToNode).filter((node) => isDialogueRecordNode(node)); + } + async getNodesByIds(ids: string[]): Promise { if (ids.length === 0) return []; const placeholders = ids.map(() => "?").join(", "); diff --git a/src/graph/workspace-ignore.ts b/src/graph/workspace-ignore.ts new file mode 100644 index 00000000..aa5b11f0 --- /dev/null +++ b/src/graph/workspace-ignore.ts @@ -0,0 +1,123 @@ +/** + * Workspace exclude rules for indexing. + * + * Two sources, same syntax, unioned: + * - `graphPolicy.excludeGlobs` + * - `/.graphflowignore` + * + * This is intentionally smaller than `.gitignore`: no negation (`!`), no + * nested ignore files, no character classes. A rule that does not match is + * skipped rather than treated as a path. + * + * name/ skip any directory named `name` at any depth + * path/to/dir/ skip that workspace-relative directory (anchored) + * *.ext skip files whose basename matches the glob + * a path glob `*` does not cross `/`; a double-star does + * + * Inline ` # comments` are stripped. `*` does not cross `/`; `**` does. + */ +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; + +export const WORKSPACE_IGNORE_FILE = ".graphflowignore"; + +export type IgnoreMatch = (relPath: string, isDir: boolean) => boolean; + +function globToRegexSource(glob: string): string { + let out = ""; + for (let i = 0; i < glob.length; i += 1) { + const ch = glob[i]!; + if (ch === "*") { + if (glob[i + 1] === "*") { + i += 1; + out += ".*"; + } else { + out += "[^/]*"; + } + continue; + } + if (ch === "?") { + out += "[^/]"; + continue; + } + out += ch.replace(/[.+^${}()|[\]\\]/g, "\\$&"); + } + return out; +} + +/** Parse ignore text. `undefined` when every line is blank or a comment. */ +export function parseIgnoreRules(text: string): IgnoreMatch | undefined { + const dirNames = new Set(); + const relPrefixes: string[] = []; + const basenameRes: RegExp[] = []; + const relRes: RegExp[] = []; + + for (const rawLine of String(text).split(/\r?\n/)) { + let rule = rawLine.trim().replace(/\\/g, "/"); + if (!rule || rule.startsWith("#") || rule.startsWith("!")) continue; + rule = rule.replace(/\s+#.*$/, "").trim(); + if (!rule) continue; + if (rule.startsWith("./")) rule = rule.slice(2); + if (rule.startsWith("/")) rule = rule.slice(1); + if (rule.endsWith("/")) rule = rule.slice(0, -1); + if (!rule) continue; + if (!/[*?]/.test(rule)) { + if (rule.includes("/")) relPrefixes.push(rule); + else dirNames.add(rule); + continue; + } + const source = `^${globToRegexSource(rule)}$`; + const re = new RegExp(source); + if (rule.includes("/")) relRes.push(re); + else basenameRes.push(re); + } + + if (dirNames.size === 0 && relPrefixes.length === 0 && basenameRes.length === 0 && relRes.length === 0) { + return undefined; + } + + return function ignoreMatch(relPath: string, isDir: boolean): boolean { + const rel = String(relPath).replace(/\\/g, "/"); + if (!rel || rel === ".") return false; + const parts = rel.split("/"); + const dirParts = isDir ? parts : parts.slice(0, -1); + for (const part of dirParts) { + if (dirNames.has(part)) return true; + } + for (const prefix of relPrefixes) { + if (rel === prefix || rel.startsWith(`${prefix}/`)) return true; + } + if (!isDir) { + const base = parts[parts.length - 1] ?? ""; + for (const re of basenameRes) { + if (re.test(base)) return true; + } + } + for (const re of relRes) { + if (re.test(rel)) return true; + } + return false; + }; +} + +/** + * Load `.graphflowignore` plus `options.excludeGlobs`. Unreadable or absent + * file is the same as no file: indexing stays unchanged. + */ +export function loadWorkspaceIgnore( + rootDir: string, + options?: { excludeGlobs?: readonly string[] } +): IgnoreMatch | undefined { + const extra = Array.isArray(options?.excludeGlobs) ? options.excludeGlobs : []; + let text = extra.length > 0 ? `${extra.join("\n")}\n` : ""; + try { + const filePath = join(rootDir, WORKSPACE_IGNORE_FILE); + if (existsSync(filePath)) { + text += `${readFileSync(filePath, "utf8")}\n`; + } + } catch { + // Missing or unreadable: same as an unpatched walker. + } + if (!text.trim()) return undefined; + return parseIgnoreRules(text); +} diff --git a/src/learning/dialogue-thread.ts b/src/learning/dialogue-thread.ts index 25d97b9d..509c7f70 100644 --- a/src/learning/dialogue-thread.ts +++ b/src/learning/dialogue-thread.ts @@ -842,6 +842,9 @@ async function loadSession( } async function collectDialogueNodes(client: GraphClient): Promise { + if (client.listDialogueNodes) { + return client.listDialogueNodes(); + } const snapshot = client.readSnapshot?.(); if (snapshot) { return snapshot.nodes.filter((node) => isDialogueTurnNode(node) || isDialogueSessionNode(node)); @@ -887,6 +890,12 @@ export async function listOwnedDialogueSessionIds( async function upsertUniqueEdges(client: GraphClient, edges: GraphEdge[]): Promise { if (edges.length === 0) return; + // File and sqlite upserts are idempotent. Reading the snapshot only to drop + // duplicates forces every dialogue record to materialize the code graph. + if (client.edgesAreIdempotent?.() === true) { + await client.upsertEdges(edges); + return; + } const snapshot = client.readSnapshot?.(); const existing = new Set( (snapshot?.edges ?? []).map((edge) => `${edge.from}|${edge.relation}|${edge.to}`) diff --git a/src/surfaces/cli/runtime/graph.ts b/src/surfaces/cli/runtime/graph.ts index 97854869..186d4029 100644 --- a/src/surfaces/cli/runtime/graph.ts +++ b/src/surfaces/cli/runtime/graph.ts @@ -356,6 +356,8 @@ function buildContextPressureBlock(params: { */ function buildIndexOptions(config: GraphFlowConfig): { includeExtensions?: string[]; + excludeGlobs?: string[]; + maxFileSizeBytes?: number; respectGitIgnore?: boolean; referenceEdgeMaxDefinitionFiles?: number; referenceEdgeMaxPerFile?: number; @@ -366,6 +368,8 @@ function buildIndexOptions(config: GraphFlowConfig): { const embeddingProvider = createEmbeddingProviderFromConfig(config); return { ...(graphPolicy.includeExtensions ? { includeExtensions: graphPolicy.includeExtensions } : {}), + ...(graphPolicy.excludeGlobs?.length ? { excludeGlobs: graphPolicy.excludeGlobs } : {}), + ...(typeof graphPolicy.maxFileSizeBytes === "number" ? { maxFileSizeBytes: graphPolicy.maxFileSizeBytes } : {}), ...(graphPolicy.respectGitIgnore === false ? { respectGitIgnore: false } : {}), ...(typeof graphPolicy.referenceEdgeMaxDefinitionFiles === "number" ? { referenceEdgeMaxDefinitionFiles: graphPolicy.referenceEdgeMaxDefinitionFiles } diff --git a/src/surfaces/cli/runtime/routing.ts b/src/surfaces/cli/runtime/routing.ts index 4ca78a91..3c65cc69 100644 --- a/src/surfaces/cli/runtime/routing.ts +++ b/src/surfaces/cli/runtime/routing.ts @@ -181,9 +181,13 @@ export async function runTaskResult( const graphClient = createGraphClient(config); if (config.graphPolicy.autoIndexOnRun) { const root = config.graphPolicy.workspaceRoot ?? process.cwd(); - const indexOptions = config.graphPolicy.includeExtensions - ? { includeExtensions: config.graphPolicy.includeExtensions } - : undefined; + const indexOptions = { + ...(config.graphPolicy.includeExtensions ? { includeExtensions: config.graphPolicy.includeExtensions } : {}), + ...(config.graphPolicy.excludeGlobs?.length ? { excludeGlobs: config.graphPolicy.excludeGlobs } : {}), + ...(typeof config.graphPolicy.maxFileSizeBytes === "number" + ? { maxFileSizeBytes: config.graphPolicy.maxFileSizeBytes } + : {}), + }; const storeIncomplete = indexedStoreIsIncomplete( root, graphClient.indexManifestName, diff --git a/src/surfaces/cli/runtime/selfcheck.ts b/src/surfaces/cli/runtime/selfcheck.ts index c34e3c0a..35ac5683 100644 --- a/src/surfaces/cli/runtime/selfcheck.ts +++ b/src/surfaces/cli/runtime/selfcheck.ts @@ -107,6 +107,10 @@ export async function runSelfcheck( try { const pending = hasPendingGraphIndexWork(root, { ...(config.graphPolicy.includeExtensions ? { includeExtensions: config.graphPolicy.includeExtensions } : {}), + ...(config.graphPolicy.excludeGlobs?.length ? { excludeGlobs: config.graphPolicy.excludeGlobs } : {}), + ...(typeof config.graphPolicy.maxFileSizeBytes === "number" + ? { maxFileSizeBytes: config.graphPolicy.maxFileSizeBytes } + : {}), manifestName: resolveIndexManifestName(config), }); items.push({ diff --git a/tests/m-workspace-exclude-and-dialogue-read.test.ts b/tests/m-workspace-exclude-and-dialogue-read.test.ts new file mode 100644 index 00000000..579ce127 --- /dev/null +++ b/tests/m-workspace-exclude-and-dialogue-read.test.ts @@ -0,0 +1,254 @@ +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, relative } from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { validateConfig } from "../src/config/loader"; +import { isDialogueRecordNode } from "../src/graph/dialogue-node-match"; +import { createGraphClient } from "../src/graph/client-factory"; +import { + getGraphifyFileStoreParseCount, + graphifyFileStoreCache, + graphStoreDeltaPath, + GraphifyFileClient, + resetGraphifyFileStoreCacheForTests, + setGraphStoreHugeReadBytesForTests, + writeGraphStoreFile, +} from "../src/graph/graphify-file-client"; +import { readGraphStoreFileChunked } from "../src/graph/graph-store-json-chunks"; +import { indexSingleFile, indexWorkspaceFiles } from "../src/graph/file-indexer"; +import { walkFiles } from "../src/graph/file-indexer-walker"; +import { parseIgnoreRules } from "../src/graph/workspace-ignore"; +import { recordDialogueTurn } from "../src/learning/dialogue-thread"; +import { GraphifySqliteClient } from "../src/graph/sqlite-client"; +import type { GraphNode } from "../src/core/types"; + +const dirs: string[] = []; + +afterEach(() => { + setGraphStoreHugeReadBytesForTests(undefined); + resetGraphifyFileStoreCacheForTests(); + while (dirs.length > 0) { + const dir = dirs.pop(); + if (dir) rmSync(dir, { recursive: true, force: true }); + } +}); + +function tempDir(prefix: string): string { + const dir = mkdtempSync(join(tmpdir(), prefix)); + dirs.push(dir); + return dir; +} + +const rel = (root: string, path: string): string => relative(root, path).replace(/\\/g, "/"); + +function memoryClient() { + return createGraphClient(validateConfig({ + providers: {}, + tiers: { + smart: { provider: "openai", model: "gpt-4.1" }, + economy: { provider: "openai", model: "gpt-4.1-mini" }, + }, + budgetPolicy: { runTokenCap: 2000 }, + graphPolicy: { + enableAutoBuild: false, + transport: "memory", + maxContextTokens: 1500, + }, + learningPolicy: { enableFlywheel: false, trainingCadence: "nightly", exportPath: "graphflow-out/learning-dataset.jsonl" }, + })); +} + +describe("workspace exclude rules", () => { + it("parses directory, basename, path-glob, and inline-comment rules", () => { + const match = parseIgnoreRules([ + "# full line", + "vendor/ # any depth", + "tools/cache/", + "*.secret.ts", + "docs/**/*.md", + "!not-supported/", + "", + ].join("\n")); + expect(match).toBeTypeOf("function"); + expect(match!("nested/vendor", true)).toBe(true); + expect(match!("nested/vendor/lib.ts", false)).toBe(true); + expect(match!("vendor.ts", false)).toBe(false); + expect(match!("tools/cache", true)).toBe(true); + expect(match!("tools/cache/a.ts", false)).toBe(true); + expect(match!("tools/other.ts", false)).toBe(false); + expect(match!("src/a.secret.ts", false)).toBe(true); + expect(match!("docs/a/b.md", false)).toBe(true); + // A double-star still needs the slash the pattern wrote; `docs/readme.md` + // is one segment under docs, so it does not match `docs/**/*.md`. + expect(match!("docs/readme.md", false)).toBe(false); + expect(match!("keep/file.ts", false)).toBe(false); + // Backslash rules are normalized the same way Windows paths are. + const win = parseIgnoreRules("tools\\cache/\n"); + expect(win!("tools/cache/a.ts", false)).toBe(true); + }); + + it("walkFiles honors .graphflowignore and excludeGlobs without dropping other files", () => { + const root = tempDir("gf-ignore-"); + mkdirSync(join(root, "src"), { recursive: true }); + mkdirSync(join(root, "skip", "vendor"), { recursive: true }); + mkdirSync(join(root, "nested", "vendor"), { recursive: true }); + writeFileSync(join(root, "src", "app.ts"), "export const a = 1;\n"); + writeFileSync(join(root, "src", "a.secret.ts"), "export const s = 1;\n"); + writeFileSync(join(root, "skip", "hidden.ts"), "export const h = 1;\n"); + writeFileSync(join(root, "nested", "vendor", "x.ts"), "export const x = 1;\n"); + writeFileSync(join(root, ".graphflowignore"), "skip/ # copies\nvendor/\n"); + + const found = walkFiles(root, [".ts"], { respectGitIgnore: false, excludeGlobs: ["*.secret.ts"] }) + .map((path) => rel(root, path)) + .sort(); + expect(found).toEqual(["src/app.ts"]); + }); + + it("index skips excluded files and files over maxFileSizeBytes", async () => { + const root = tempDir("gf-index-exclude-"); + mkdirSync(join(root, "src"), { recursive: true }); + mkdirSync(join(root, "skip"), { recursive: true }); + writeFileSync(join(root, "src", "app.ts"), "export const a = 1;\n"); + writeFileSync(join(root, "src", "big.ts"), `export const big = "${"x".repeat(80)}";\n`); + writeFileSync(join(root, "skip", "hidden.ts"), "export const h = 1;\n"); + writeFileSync(join(root, ".graphflowignore"), "skip/\n"); + + const client = memoryClient(); + const indexed = await indexWorkspaceFiles(client, root, { + includeExtensions: [".ts"], + respectGitIgnore: false, + maxFileSizeBytes: 40, + }); + expect(indexed.indexedFiles).toBe(1); + + const single = await indexSingleFile(client, root, join(root, "skip", "hidden.ts"), { + includeExtensions: [".ts"], + excludeGlobs: ["skip/"], + }); + expect(single.skipped).toBe(true); + expect(single.reason).toMatch(/graphflowignore|excludeGlobs/); + }); + + it("validateConfig keeps excludeGlobs and maxFileSizeBytes and rejects bad values", () => { + const config = validateConfig({ + providers: {}, + tiers: { + smart: { provider: "openai" }, + economy: { provider: "openai" }, + }, + budgetPolicy: { runTokenCap: 100 }, + graphPolicy: { + enableAutoBuild: false, + transport: "file", + maxContextTokens: 100, + excludeGlobs: ["工具/", "*.jsonl"], + maxFileSizeBytes: 50_000, + }, + learningPolicy: { enableFlywheel: false, trainingCadence: "weekly", exportPath: "out.jsonl" }, + }); + expect(config.graphPolicy.excludeGlobs).toEqual(["工具/", "*.jsonl"]); + expect(config.graphPolicy.maxFileSizeBytes).toBe(50_000); + + expect(() => validateConfig({ + ...config, + graphPolicy: { ...config.graphPolicy, maxFileSizeBytes: 0 }, + })).toThrow(/maxFileSizeBytes/); + expect(() => validateConfig({ + ...config, + graphPolicy: { ...config.graphPolicy, excludeGlobs: [""] }, + })).toThrow(/excludeGlobs/); + }); +}); + +describe("dialogue record does not materialize a huge file store", () => { + it("appends a delta and still reads the new turns back", async () => { + const root = tempDir("gf-huge-dialogue-"); + const storePath = join(root, "graphflow-graph.json"); + const filler: GraphNode = { id: "file:big.md", type: "File", content: "x".repeat(200) }; + writeGraphStoreFile(storePath, { nodes: [filler], edges: [] }); + setGraphStoreHugeReadBytesForTests(1); + resetGraphifyFileStoreCacheForTests(); + + const client = new GraphifyFileClient(storePath); + const first = await recordDialogueTurn(client, { + userQuery: "where is the indexer", + workspaceRoot: root, + now: 1_000, + }); + const second = await recordDialogueTurn(client, { + userQuery: "exclude the copies", + workspaceRoot: root, + now: 2_000, + }); + expect(first.recorded).toBe(true); + expect(second.turn?.seq).toBe(2); + expect(getGraphifyFileStoreParseCount()).toBe(0); + expect(graphifyFileStoreCache.has(storePath)).toBe(false); + + const base = readFileSync(storePath, "utf8"); + expect(base).toContain("file:big.md"); + expect(base).not.toContain("dialogue-session"); + const delta = readFileSync(graphStoreDeltaPath(storePath), "utf8"); + expect(delta).toContain("dialogue-session"); + expect(delta).toContain("exclude the copies"); + + setGraphStoreHugeReadBytesForTests(undefined); + const snapshot = client.readSnapshot(); + expect(snapshot.nodes.some((node) => node.id === "file:big.md")).toBe(true); + expect(snapshot.nodes.filter((node) => isDialogueRecordNode(node)).length).toBeGreaterThanOrEqual(3); + }); + + it("projects dialogue nodes out of a chunked store without keeping edges", () => { + const root = tempDir("gf-chunk-project-"); + const storePath = join(root, "graph.json"); + writeFileSync(storePath, JSON.stringify({ + nodes: [ + { id: "file:a.ts", type: "File", content: "code" }, + { id: "dialogue:1", type: "Decision", content: "turn", metadata: { kind: "dialogue-turn" } }, + ], + edges: [{ from: "file:a.ts", to: "dialogue:1", relation: "references" }], + })); + const full = readGraphStoreFileChunked(storePath); + expect(full.nodes).toHaveLength(2); + expect(full.edges).toHaveLength(1); + const projected = readGraphStoreFileChunked(storePath, { + skipEdges: true, + keepNode: (node) => isDialogueRecordNode(node as { id: string; type: string; metadata?: { kind?: string } }), + }); + expect(projected.edges).toEqual([]); + expect(projected.nodes.map((node) => (node as { id: string }).id)).toEqual(["dialogue:1"]); + }); + + it("sqlite lists dialogue rows without readSnapshot", async () => { + try { + require("better-sqlite3"); + } catch { + return; + } + const root = tempDir("gf-sql-dialogue-"); + const client = new GraphifySqliteClient(join(root, "graph.sqlite")); + let snap: { mockRestore: () => void } | undefined; + try { + const nodes: GraphNode[] = []; + for (let i = 0; i < 40; i += 1) { + nodes.push({ id: `file:${i}.ts`, type: "File", content: "code" }); + } + nodes.push({ + id: "dialogue:abc:0001", + type: "Decision", + content: "question", + metadata: { kind: "dialogue-turn" }, + }); + await client.upsertNodes(nodes); + snap = vi.spyOn(GraphifySqliteClient.prototype, "readSnapshot"); + const listed = await client.listDialogueNodes(); + expect(snap).not.toHaveBeenCalled(); + expect(listed.map((node) => node.id)).toEqual(["dialogue:abc:0001"]); + } finally { + snap?.mockRestore(); + client.close(); + } + }); +});