feat(index): per-vault SQLite FTS5 index + scan + reindex (M2a)

src/cli.ts

@@ -1,12 +1,52 @@
 #!/usr/bin/env node
 
+import { spawnSync } from 'node:child_process';
+import { createRequire } from 'node:module';
 import { Command } from 'commander';
 import { registerSetup } from './commands/setup.js';
 import { registerStatus } from './commands/status.js';
 import { registerUpdate } from './commands/update.js';
 import { registerVault } from './commands/vault.js';
 import { VERSION } from './utils/version.js';
 
+// Silence Node's "SQLite is an experimental feature" warning: it prints on every index
+// command and the user cannot act on it.
+const hushSqliteWarning = (): void => {
+  const emit = process.emitWarning.bind(process);
+  process.emitWarning = ((warning: string | Error, ...rest: unknown[]): void => {
+    const message = typeof warning === 'string' ? warning : warning.message;
+    if (message.includes('SQLite is an experimental feature')) return;
+    (emit as (w: string | Error, ...r: unknown[]) => void)(warning, ...rest);
+  }) as typeof process.emitWarning;
+};
+
+// The vault index uses node:sqlite, which is behind --experimental-sqlite on Node < 23.4.
+// For index-touching commands, re-exec once with the flag when the module can't load, so
+// the CLI works across the whole Node >= 22 range. Non-index commands never pay for this.
+// (Command modules load node:sqlite lazily, so importing them above stays flag-free.)
+const bootstrapSqlite = (): void => {
+  if (!process.argv.slice(2).some((a) => a === 'memory' || a === 'reindex')) return;
+  hushSqliteWarning(); // install before requiring node:sqlite - the warning fires at load time
+  const req = createRequire(import.meta.url);
+  const loadable = ((): boolean => {
+    try {
+      req('node:sqlite');
+      return true;
+    } catch {
+      return false;
+    }
+  })();
+  if (loadable) return;
+  if (process.execArgv.includes('--experimental-sqlite')) return;
+  const result = spawnSync(
+    process.execPath,
+    ['--experimental-sqlite', process.argv[1] as string, ...process.argv.slice(2)],
+    { stdio: 'inherit' }
+  );
+  process.exit(result.status ?? 1);
+};
+bootstrapSqlite();
+
 const program = new Command();
 
 program.name('agentage').description('The agentage CLI').version(VERSION);

src/commands/reindex.test.ts

@@ -0,0 +1,48 @@
+import { describe, expect, it, vi } from 'vitest';
+import { VaultsConfig } from '../lib/vaults.schema.js';
+import { runReindex, type ReindexDeps } from './reindex.js';
+
+const cfg = (names: string[]): VaultsConfig =>
+  VaultsConfig.parse({
+    version: 1,
+    vaults: names.map((n) => ({ name: n, type: 'local', path: `~/vaults/${n}` })),
+  });
+
+const makeDeps = (config: VaultsConfig) => {
+  const logs: string[] = [];
+  const reindex = vi.fn(async () => ({ added: 1, modified: 0, removed: 0, unchanged: 2 }));
+  const deps: ReindexDeps = {
+    load: () => ({ config, source: null }),
+    reindex,
+    log: (m) => logs.push(m),
+  };
+  return { deps, logs, reindex };
+};
+
+describe('runReindex', () => {
+  it('reindexes every vault when no name is given', async () => {
+    const h = makeDeps(cfg(['a', 'b']));
+    await runReindex(undefined, h.deps);
+    expect(h.reindex).toHaveBeenCalledTimes(2);
+    expect(h.logs.join()).toContain("Reindexed 'a'");
+  });
+
+  it('reindexes only the named vault, passing its path', async () => {
+    const h = makeDeps(cfg(['a', 'b']));
+    await runReindex('b', h.deps);
+    expect(h.reindex).toHaveBeenCalledTimes(1);
+    expect(h.reindex).toHaveBeenCalledWith('b', '~/vaults/b');
+  });
+
+  it('throws when the named vault is unknown', async () => {
+    const h = makeDeps(cfg(['a']));
+    await expect(runReindex('nope', h.deps)).rejects.toThrow(/not found/);
+  });
+
+  it('reports when there are no vaults', async () => {
+    const h = makeDeps(cfg([]));
+    await runReindex(undefined, h.deps);
+    expect(h.logs.join()).toContain('No vaults');
+    expect(h.reindex).not.toHaveBeenCalled();
+  });
+});

src/commands/reindex.ts

@@ -0,0 +1,32 @@
+import { reindexNamedVault, type ReindexStats } from '../lib/vault-scan.js';
+import { loadVaultsConfig, type LoadedVaults } from '../lib/vaults.js';
+
+export interface ReindexDeps {
+  load: () => LoadedVaults;
+  reindex: (name: string, vaultPath: string) => Promise<ReindexStats>;
+  log: (msg: string) => void;
+}
+
+const defaultDeps: ReindexDeps = {
+  load: loadVaultsConfig,
+  reindex: reindexNamedVault,
+  log: (msg) => console.log(msg),
+};
+
+// Rebuild one vault's index (or every vault's when name is omitted) from its markdown.
+export const runReindex = async (
+  name: string | undefined,
+  deps: ReindexDeps = defaultDeps
+): Promise<void> => {
+  const { vaults } = deps.load().config;
+  const targets = name ? vaults.filter((v) => v.name === name) : vaults;
+  if (name && targets.length === 0) throw new Error(`vault '${name}' not found`);
+  if (targets.length === 0) {
+    deps.log('No vaults to reindex.');
+    return;
+  }
+  for (const v of targets) {
+    const s = await deps.reindex(v.name, v.path);
+    deps.log(`Reindexed '${v.name}': +${s.added} ~${s.modified} -${s.removed} (=${s.unchanged})`);
+  }
+};

src/commands/vault.ts

@@ -9,6 +9,7 @@ import {
 } from '../lib/vault-registry.js';
 import { loadVaultsConfig, saveVaultsConfig, type LoadedVaults } from '../lib/vaults.js';
 import { type VaultsConfig } from '../lib/vaults.schema.js';
+import { runReindex } from './reindex.js';
 
 export interface VaultDeps {
   load: () => LoadedVaults;
@@ -92,6 +93,15 @@ const guard = (fn: () => void): void => {
   }
 };
 
+const guardAsync = async (fn: () => Promise<void>): Promise<void> => {
+  try {
+    await fn();
+  } catch (err) {
+    console.error(chalk.red(err instanceof Error ? err.message : String(err)));
+    process.exitCode = 1;
+  }
+};
+
 export const registerVault = (program: Command): void => {
   const vault = program.command('vault').description('Manage local memory vaults');
 
@@ -114,4 +124,9 @@ export const registerVault = (program: Command): void => {
     .command('remove <name>')
     .description('Unregister a vault and drop its index (files stay)')
     .action((name: string) => guard(() => runVaultRemove(name)));
+
+  vault
+    .command('reindex [name]')
+    .description('Rebuild a vault index from its markdown (all vaults if omitted)')
+    .action((name: string | undefined) => guardAsync(() => runReindex(name)));
 };

src/lib/vault-index.test.ts

@@ -0,0 +1,80 @@
+import { describe, expect, it } from 'vitest';
+import { openIndex, type FileChange } from './vault-index.js';
+
+const change = (path: string, content: string): FileChange => ({
+  path,
+  content,
+  sha256: `sha-${path}-${content.length}`,
+  size: content.length,
+  mtime: 1,
+});
+
+describe('vault index (node:sqlite FTS5)', () => {
+  it('reconciles adds and searches by content', () => {
+    const idx = openIndex(':memory:');
+    idx.reconcile({
+      added: [change('a.md', 'the quick brown fox'), change('b.md', 'lazy dog sleeps')],
+      modified: [],
+      removed: [],
+    });
+    expect(idx.fileCount()).toBe(2);
+    const hits = idx.search('fox');
+    expect(hits).toHaveLength(1);
+    expect(hits[0]?.path).toBe('a.md');
+    expect(hits[0]?.snippet).toContain('fox');
+    idx.close();
+  });
+
+  it('stat returns metadata or null', () => {
+    const idx = openIndex(':memory:');
+    idx.reconcile({ added: [change('a.md', 'hi')], modified: [], removed: [] });
+    expect(idx.stat('a.md')).toMatchObject({ path: 'a.md', size: 2 });
+    expect(idx.stat('missing.md')).toBeNull();
+    idx.close();
+  });
+
+  it('list respects prefix + path order', () => {
+    const idx = openIndex(':memory:');
+    idx.reconcile({
+      added: [change('notes/b.md', 'y'), change('notes/a.md', 'x'), change('other/c.md', 'z')],
+      modified: [],
+      removed: [],
+    });
+    expect(idx.list({ prefix: 'notes/' }).map((e) => e.path)).toEqual(['notes/a.md', 'notes/b.md']);
+    expect(idx.list()).toHaveLength(3);
+    idx.close();
+  });
+
+  it('modified replaces content; removed drops from search + files', () => {
+    const idx = openIndex(':memory:');
+    idx.reconcile({ added: [change('a.md', 'apple')], modified: [], removed: [] });
+    idx.reconcile({ added: [], modified: [change('a.md', 'banana')], removed: [] });
+    expect(idx.search('apple')).toHaveLength(0);
+    expect(idx.search('banana')).toHaveLength(1);
+    idx.reconcile({ added: [], modified: [], removed: ['a.md'] });
+    expect(idx.fileCount()).toBe(0);
+    expect(idx.search('banana')).toHaveLength(0);
+    idx.close();
+  });
+
+  it('returns [] for an empty/whitespace query', () => {
+    const idx = openIndex(':memory:');
+    expect(idx.search('   ')).toEqual([]);
+    idx.close();
+  });
+
+  it('treats punctuation as literal, not FTS syntax', () => {
+    const idx = openIndex(':memory:');
+    idx.reconcile({ added: [change('a.md', 'node:sqlite is great')], modified: [], removed: [] });
+    expect(idx.search('node:sqlite')).toHaveLength(1);
+    idx.close();
+  });
+
+  it('records indexedAt after reconcile', () => {
+    const idx = openIndex(':memory:', () => new Date('2026-07-06T00:00:00Z'));
+    expect(idx.indexedAt()).toBeNull();
+    idx.reconcile({ added: [change('a.md', 'x')], modified: [], removed: [] });
+    expect(idx.indexedAt()).toBe('2026-07-06T00:00:00.000Z');
+    idx.close();
+  });
+});