fix(index): exclude large trees and keep dialogue record off the full graph

CHANGELOG.md

@@ -13,6 +13,11 @@ All notable changes to this project are documented in this file.
 
 - `importSkillsFromMarkdownRuntime`, `exportSkillsToMarkdownRuntime`, and `extractDialogueKnowledgeRuntime` now `close()` the graph client in `finally`. Default `auto` transport opens better-sqlite3; leaving that handle open makes a later `rmSync` of `graphflow-graph.sqlite` fail with `EBUSY` on Windows.
 
+### Fixed — large workspaces no longer have to be fully indexed, and dialogue record no longer materializes a huge file graph
+
+- `graphPolicy.excludeGlobs` and `graphPolicy.maxFileSizeBytes` are honored by workspace index, single-file index, and the pending-index check. `<workspace>/.graphflowignore` uses the same rules (`name/` at any depth, anchored `path/to/dir/`, basename globs, path globs). Negation and nested ignore files are not supported. Markdown stays on the 200KB source cap; the 5MB document cap applies to office/PDF only.
+- `dialogue record` on a file store larger than 512MB streams just the dialogue nodes and appends a delta line. It does not `JSON.parse` the code graph. SQLite `listDialogueNodes()` returns the same rows with a SQL filter instead of `readSnapshot()`.
+
 ## [Unreleased]
 
 ### Fixed — 第 0 步性能与健壮性(迁移 Rust 前的架构债清偿,复审登记项全闭环)

src/config/loader.ts

@@ -92,6 +92,19 @@ export function validateConfigDetailed(path = "graphflow.config.json"): ConfigVa
         issues.push({ severity: "error", field: "graphPolicy.includeExtensions", message: "Extensions must start with '.'" });
       }
     }
+    if (parsed.graphPolicy.excludeGlobs !== undefined) {
+      const globs = parsed.graphPolicy.excludeGlobs;
+      const invalid = !Array.isArray(globs) || globs.some((rule) => typeof rule !== "string" || rule.trim() === "");
+      if (invalid) {
+        issues.push({ severity: "error", field: "graphPolicy.excludeGlobs", message: "excludeGlobs entries must be non-empty strings" });
+      }
+    }
+    if (parsed.graphPolicy.maxFileSizeBytes !== undefined) {
+      const size = parsed.graphPolicy.maxFileSizeBytes;
+      if (typeof size !== "number" || !Number.isFinite(size) || size < 1) {
+        issues.push({ severity: "error", field: "graphPolicy.maxFileSizeBytes", message: "maxFileSizeBytes must be a positive number" });
+      }
+    }
     if (parsed.graphPolicy.workspaceRoot) {
       issues.push({ severity: "warning", field: "graphPolicy.workspaceRoot", message: "workspaceRoot in config is deprecated; resolved from process.cwd() at runtime" });
     }
@@ -342,6 +355,21 @@ export function validateConfig(input: GraphFlowConfig): GraphFlowConfig {
     }
   }
 
+  if (input.graphPolicy.excludeGlobs !== undefined) {
+    const globs = input.graphPolicy.excludeGlobs;
+    const invalid = !Array.isArray(globs) || globs.some((rule) => typeof rule !== "string" || rule.trim() === "");
+    if (invalid) {
+      throw new Error("Invalid config: graphPolicy.excludeGlobs entries must be non-empty strings.");
+    }
+  }
+
+  if (input.graphPolicy.maxFileSizeBytes !== undefined) {
+    const size = input.graphPolicy.maxFileSizeBytes;
+    if (typeof size !== "number" || !Number.isFinite(size) || size < 1) {
+      throw new Error("Invalid config: graphPolicy.maxFileSizeBytes must be a positive number.");
+    }
+  }
+
   // learningPolicy is optional at the input boundary: every sub-field has a
   // default, so a minimal config (providers + tiers + graphPolicy) must be
   // valid. Requiring the bare object made minimal configs fail validation and

src/config/schema.ts

@@ -169,6 +169,18 @@ export interface GraphFlowConfig {
     enableDialogueThread?: boolean;
     workspaceRoot?: string;
     includeExtensions?: string[];
+    /**
+     * Workspace-relative paths and globs the indexer must skip, unioned with
+     * `<workspace>/.graphflowignore`. Syntax: `name/` (any directory of that
+     * name), `path/to/dir/` (anchored), `*.ext` (basename), path globs where
+     * `*` does not cross `/` and a double-star does. No gitignore negation.
+     */
+    excludeGlobs?: string[];
+    /**
+     * Skip source files larger than this many bytes. Office/PDF conversion
+     * still uses its own higher cap. Default 200_000.
+     */
+    maxFileSizeBytes?: number;
     /**
      * Skip files git ignores when indexing (exact semantics via `git ls-files
      * --exclude-standard`). Default true; ignored outside git checkouts.

src/graph/client-factory.ts

@@ -10,6 +10,7 @@ import { GraphifySqliteClient } from "./sqlite-client";
 import { mergeSiblingJsonStoreIntoSqlite } from "./store-migration";
 import { CACHE_FILE, SQLITE_INDEX_MANIFEST } from "./file-indexer-cache";
 import { existsSync } from "node:fs";
+import { isDialogueRecordNode } from "./dialogue-node-match";
 
 export interface GraphStoreSnapshot {
   nodes: GraphNode[];
@@ -33,6 +34,16 @@ export interface GraphClient {
   upsertGraph?(batch: { nodes?: GraphNode[]; edges?: GraphEdge[] }): Promise<void>;
   queryByKeyword(query: string): Promise<GraphNode[]>;
   readSnapshot?(): GraphStoreSnapshot;
+  /**
+   * Dialogue turns and sessions without materializing the rest of the graph.
+   * Backends that cannot project fall back to `readSnapshot` at the call site.
+   */
+  listDialogueNodes?(): Promise<GraphNode[]>;
+  /**
+   * When true, `upsertEdges` is idempotent and callers must not `readSnapshot`
+   * solely to drop duplicate edges (that read is the large-graph OOM).
+   */
+  edgesAreIdempotent?(): boolean;
   getNodesByIds?(ids: string[]): Promise<GraphNode[]>;
   getNeighbors?(
     nodeIds: string[],
@@ -113,6 +124,10 @@ class MutationAwareGraphClient implements GraphClient {
     return this.inner.indexManifestName;
   }
 
+  edgesAreIdempotent(): boolean {
+    return this.inner.edgesAreIdempotent?.() === true;
+  }
+
   async upsertNodes(nodes: GraphNode[]): Promise<void> {
     await this.inner.upsertNodes(nodes);
     markGraphMutated(nodes.map((n) => n.id));
@@ -150,6 +165,11 @@ class MutationAwareGraphClient implements GraphClient {
     return this.inner.queryByKeyword(query);
   }
 
+  async listDialogueNodes(): Promise<GraphNode[]> {
+    if (this.inner.listDialogueNodes) return this.inner.listDialogueNodes();
+    return this.readSnapshot().nodes.filter((node) => isDialogueRecordNode(node));
+  }
+
   readSnapshot(): GraphStoreSnapshot {
     // 所有被包装的后端（file/sqlite/memory/mcp）均实现 readSnapshot，
     // `?.` 仅作防御；接口此处为可选方法，调用方先做存在性检查。

src/graph/dialogue-node-match.ts

@@ -0,0 +1,20 @@
+/**
+ * Dialogue records live in the same store as code nodes. Callers that only
+ * need the thread must recognize them without assuming a particular backend.
+ *
+ * Id prefixes are authoritative (`dialogue:<hash>:<seq>`, `dialogue-session:<hash>`).
+ * Metadata kinds cover records written before the id scheme, but code nodes
+ * (File / Module / Symbol) are never dialogue even if a kind field collides.
+ */
+export function isDialogueRecordNode(node: {
+  id?: unknown;
+  type?: unknown;
+  metadata?: { kind?: unknown } | null;
+}): boolean {
+  if (typeof node.id !== "string" || node.id.length === 0) return false;
+  if (node.id.startsWith("dialogue-session:") || node.id.startsWith("dialogue:")) return true;
+  const type = typeof node.type === "string" ? node.type : "";
+  if (type === "File" || type === "Module" || type === "Symbol") return false;
+  const kind = node.metadata && typeof node.metadata === "object" ? node.metadata.kind : undefined;
+  return kind === "dialogue-turn" || kind === "dialogue-session";
+}

src/graph/file-indexer-cache.ts

@@ -160,7 +160,7 @@ export function hasPendingGraphIndexWork(
   rootDir: string,
   options?: Pick<
     FileIndexerOptions,
-    "includeExtensions" | "maxFileSizeBytes" | "forceReindex" | "respectGitIgnore"
+    "includeExtensions" | "maxFileSizeBytes" | "forceReindex" | "respectGitIgnore" | "excludeGlobs"
   > & { manifestName?: string | undefined }
 ): boolean {
   const includeExtensions = options?.includeExtensions ?? DEFAULT_EXTENSIONS;
@@ -173,6 +173,7 @@ export function hasPendingGraphIndexWork(
   const cacheState = loadCacheStateCached(indexManifestPath(rootDir, options?.manifestName));
   const scanned = walkScannableFiles(rootDir, includeExtensions, maxFileSizeBytes, {
     ...(options?.respectGitIgnore === false ? { respectGitIgnore: false } : {}),
+    ...(options?.excludeGlobs?.length ? { excludeGlobs: options.excludeGlobs } : {}),
   });
   const currentRelPaths = new Set(scanned.map((file) => file.relPath));
 

src/graph/file-indexer-walker.ts

@@ -15,6 +15,7 @@ import {
   OFFICE_DOCUMENT_EXTENSIONS,
 } from "./document-convert.js";
 import { safeReaddirSync, safeStatSync } from "../utils/safe-fs.js";
+import { loadWorkspaceIgnore } from "./workspace-ignore.js";
 
 import type { EmbeddingProvider } from "../learning/embeddings.js";
 
@@ -53,6 +54,11 @@ export interface FileIndexerOptions {
    * the machine's cores. `GRAPHFLOW_INDEX_WORKERS=0` disables it globally.
    */
   indexWorkers?: number;
+  /**
+   * Extra exclude rules, same syntax as `<workspace>/.graphflowignore`.
+   * The ignore file is always loaded; these rules are unioned with it.
+   */
+  excludeGlobs?: string[];
 }
 
 export interface ScannedFile {
@@ -189,7 +195,7 @@ export function walkScannableFiles(
   rootDir: string,
   includeExtensions: string[],
   maxFileSizeBytes: number,
-  options: { respectGitIgnore?: boolean } = {}
+  options: { respectGitIgnore?: boolean; excludeGlobs?: string[] } = {}
 ): ScannedFile[] {
   const files = walkFiles(rootDir, includeExtensions, options);
   const scanned: ScannedFile[] = [];
@@ -225,11 +231,15 @@ export function walkScannableFiles(
 export function walkFiles(
   rootDir: string,
   includeExtensions: string[],
-  options: { respectGitIgnore?: boolean } = {}
+  options: { respectGitIgnore?: boolean; excludeGlobs?: string[] } = {}
 ): string[] {
   const files: string[] = [];
   const dirStack: string[] = [rootDir];
   const gitVisible = options.respectGitIgnore === false ? undefined : readGitVisibleFiles(rootDir);
+  const ignore = loadWorkspaceIgnore(
+    rootDir,
+    options.excludeGlobs && options.excludeGlobs.length > 0 ? { excludeGlobs: options.excludeGlobs } : undefined
+  );
 
   while (dirStack.length > 0) {
     const current = dirStack.pop()!;
@@ -245,6 +255,10 @@ export function walkFiles(
         if (IGNORED_DIRS.has(entry.name)) {
           continue;
         }
+        const relDir = normalizePath(relative(rootDir, full));
+        if (ignore?.(relDir, true)) {
+          continue;
+        }
         // A directory holding only ignored/untracked files cannot contribute.
         if (gitVisible && !directoryMayContainVisibleFiles(rootDir, full, gitVisible)) {
           continue;
@@ -259,7 +273,11 @@ export function walkFiles(
       if (isGeneratedOrLockFile(entry.name)) {
         continue;
       }
-      if (gitVisible && !gitVisible.has(normalizePath(relative(rootDir, full)))) {
+      const relFile = normalizePath(relative(rootDir, full));
+      if (ignore?.(relFile, false)) {
+        continue;
+      }
+      if (gitVisible && !gitVisible.has(relFile)) {
         continue;
       }
       files.push(full);

src/graph/file-indexer.ts

@@ -44,6 +44,7 @@ export { resolveCallerAtLine } from "./file-indexer-nodes.js";
 
 // ── Internal imports ─────────────────────────────────────────────────
 import { DEFAULT_EXTENSIONS, DEFAULT_MAX_FILE_SIZE, normalizePath, extOf, walkScannableFiles } from "./file-indexer-walker.js";
+import { loadWorkspaceIgnore } from "./workspace-ignore.js";
 import type { FileIndexerOptions } from "./file-indexer-walker.js";
 import { indexManifestPath, loadCacheState, saveCacheState } from "./file-indexer-cache.js";
 import type {
@@ -262,6 +263,7 @@ export async function indexWorkspaceFiles(
 
   const scanned = walkScannableFiles(rootDir, includeExtensions, maxFileSizeBytes, {
     ...(options?.respectGitIgnore === false ? { respectGitIgnore: false } : {}),
+    ...(options?.excludeGlobs?.length ? { excludeGlobs: options.excludeGlobs } : {}),
   });
   const currentRelPaths = new Set(scanned.map((file) => file.relPath));
 
@@ -470,6 +472,7 @@ export async function indexSingleFile(
     FileIndexerOptions,
     | "includeExtensions"
     | "maxFileSizeBytes"
+    | "excludeGlobs"
     | "embeddingProvider"
     | "referenceEdgeMaxDefinitionFiles"
     | "referenceEdgeMaxPerFile"
@@ -507,6 +510,21 @@ export async function indexSingleFile(
     };
   }
 
+  const relPath = normalizePath(relative(rootDir, absPath));
+  const ignore = loadWorkspaceIgnore(
+    rootDir,
+    options?.excludeGlobs && options.excludeGlobs.length > 0 ? { excludeGlobs: options.excludeGlobs } : undefined
+  );
+  if (ignore?.(relPath, false)) {
+    return {
+      indexedFiles: 0,
+      indexedSymbols: 0,
+      indexedReferences: 0,
+      skipped: true,
+      reason: "excluded by .graphflowignore or graphPolicy.excludeGlobs",
+    };
+  }
+
   const officeDoc = isOfficeDocumentPath(absPath);
   const sizeLimit = officeDoc
     ? Math.max(maxFileSizeBytes, DEFAULT_DOCUMENT_MAX_FILE_SIZE)
@@ -528,7 +546,6 @@ export async function indexSingleFile(
     return { indexedFiles: 0, indexedSymbols: 0, indexedReferences: 0, skipped: true, reason: "file exceeds maxFileSizeBytes" };
   }
 
-  const relPath = normalizePath(relative(rootDir, absPath));
   const mtimeMs = stat.mtimeMs;
   let content = "";
   let currentHash = "";

src/graph/graph-store-json-chunks.ts

@@ -22,6 +22,17 @@ export interface ChunkedGraphStore {
   edges: unknown[];
 }
 
+export interface ChunkedGraphStoreReadOptions {
+  /**
+   * Drop nodes for which this returns false. The element string is parsed
+   * and then discarded, so a projection (dialogue records, a handful of ids)
+   * does not retain the rest of the graph.
+   */
+  keepNode?: (node: unknown) => boolean;
+  /** Scan the edges array without retaining it. */
+  skipEdges?: boolean;
+}
+
 /** Flush the accumulated element buffer at this size while scanning. */
 const DEFAULT_CHUNK_BYTES = 8 * 1024 * 1024;
 
@@ -32,7 +43,10 @@ class GraphStoreChunkParser {
   private buffer = "";
   private index = 0;
 
-  constructor(private readonly pull: () => string | null) {}
+  constructor(
+    private readonly pull: () => string | null,
+    private readonly options: ChunkedGraphStoreReadOptions = {}
+  ) {}
 
   private fill(): boolean {
     if (this.index > 0) {
@@ -135,7 +149,7 @@ class GraphStoreChunkParser {
     }
   }
 
-  private readElements(target: unknown[]): void {
+  private readElements(target: unknown[] | null, keep?: (node: unknown) => boolean): void {
     while (true) {
       this.skipWhitespace();
       const ch = this.peek();
@@ -156,7 +170,10 @@ class GraphStoreChunkParser {
       if (!complete) {
         throw new Error("Truncated graph store: incomplete element");
       }
-      target.push(JSON.parse(text));
+      if (target === null) continue;
+      const parsed: unknown = JSON.parse(text);
+      if (keep && !keep(parsed)) continue;
+      target.push(parsed);
     }
   }
 
@@ -195,7 +212,11 @@ class GraphStoreChunkParser {
           throw new Error(`Truncated graph store: expected an array for "${name}"`);
         }
         this.index += 1;
-        this.readElements(name === "nodes" ? store.nodes : store.edges);
+        const discard = name === "edges" && this.options.skipEdges === true;
+        this.readElements(
+          discard ? null : name === "nodes" ? store.nodes : store.edges,
+          name === "nodes" ? this.options.keepNode : undefined
+        );
         continue;
       }
       const { complete } = this.readValue();
@@ -209,14 +230,17 @@ class GraphStoreChunkParser {
 }
 
 /** Parse a graph store document from a sequence of text chunks. */
-export function readGraphStoreFromChunks(pull: () => string | null): ChunkedGraphStore {
-  return new GraphStoreChunkParser(pull).parse();
+export function readGraphStoreFromChunks(
+  pull: () => string | null,
+  options: ChunkedGraphStoreReadOptions = {}
+): ChunkedGraphStore {
+  return new GraphStoreChunkParser(pull, options).parse();
 }
 
 /** Read a graph store file in bounded chunks (used above the single-string cap). */
 export function readGraphStoreFileChunked(
   filePath: string,
-  options: { chunkBytes?: number } = {}
+  options: { chunkBytes?: number } & ChunkedGraphStoreReadOptions = {}
 ): ChunkedGraphStore {
   const chunkBytes = Math.max(64 * 1024, options.chunkBytes ?? DEFAULT_CHUNK_BYTES);
   const fd = openSync(filePath, "r");
@@ -228,7 +252,7 @@ export function readGraphStoreFileChunked(
     return readGraphStoreFromChunks(() => {
       const read = readSync(fd, buffer, 0, chunkBytes, null);
       return read > 0 ? decoder.write(buffer.subarray(0, read)) : null;
-    });
+    }, options);
   } finally {
     closeSync(fd);
   }