diff --git a/src/cli.ts b/src/cli.ts index 895cd29..9c111f4 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,5 +1,7 @@ #!/usr/bin/env node +import { spawnSync } from 'node:child_process'; +import { createRequire } from 'node:module'; import { Command } from 'commander'; import { registerSetup } from './commands/setup.js'; import { registerStatus } from './commands/status.js'; @@ -7,6 +9,44 @@ import { registerUpdate } from './commands/update.js'; import { registerVault } from './commands/vault.js'; import { VERSION } from './utils/version.js'; +// Silence Node's "SQLite is an experimental feature" warning: it prints on every index +// command and the user cannot act on it. +const hushSqliteWarning = (): void => { + const emit = process.emitWarning.bind(process); + process.emitWarning = ((warning: string | Error, ...rest: unknown[]): void => { + const message = typeof warning === 'string' ? warning : warning.message; + if (message.includes('SQLite is an experimental feature')) return; + (emit as (w: string | Error, ...r: unknown[]) => void)(warning, ...rest); + }) as typeof process.emitWarning; +}; + +// The vault index uses node:sqlite, which is behind --experimental-sqlite on Node < 23.4. +// For index-touching commands, re-exec once with the flag when the module can't load, so +// the CLI works across the whole Node >= 22 range. Non-index commands never pay for this. +// (Command modules load node:sqlite lazily, so importing them above stays flag-free.) +const bootstrapSqlite = (): void => { + if (!process.argv.slice(2).some((a) => a === 'memory' || a === 'reindex')) return; + hushSqliteWarning(); // install before requiring node:sqlite - the warning fires at load time + const req = createRequire(import.meta.url); + const loadable = ((): boolean => { + try { + req('node:sqlite'); + return true; + } catch { + return false; + } + })(); + if (loadable) return; + if (process.execArgv.includes('--experimental-sqlite')) return; + const result = spawnSync( + process.execPath, + ['--experimental-sqlite', process.argv[1] as string, ...process.argv.slice(2)], + { stdio: 'inherit' } + ); + process.exit(result.status ?? 1); +}; +bootstrapSqlite(); + const program = new Command(); program.name('agentage').description('The agentage CLI').version(VERSION); diff --git a/src/commands/reindex.test.ts b/src/commands/reindex.test.ts new file mode 100644 index 0000000..1b65e7f --- /dev/null +++ b/src/commands/reindex.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it, vi } from 'vitest'; +import { VaultsConfig } from '../lib/vaults.schema.js'; +import { runReindex, type ReindexDeps } from './reindex.js'; + +const cfg = (names: string[]): VaultsConfig => + VaultsConfig.parse({ + version: 1, + vaults: names.map((n) => ({ name: n, type: 'local', path: `~/vaults/${n}` })), + }); + +const makeDeps = (config: VaultsConfig) => { + const logs: string[] = []; + const reindex = vi.fn(async () => ({ added: 1, modified: 0, removed: 0, unchanged: 2 })); + const deps: ReindexDeps = { + load: () => ({ config, source: null }), + reindex, + log: (m) => logs.push(m), + }; + return { deps, logs, reindex }; +}; + +describe('runReindex', () => { + it('reindexes every vault when no name is given', async () => { + const h = makeDeps(cfg(['a', 'b'])); + await runReindex(undefined, h.deps); + expect(h.reindex).toHaveBeenCalledTimes(2); + expect(h.logs.join()).toContain("Reindexed 'a'"); + }); + + it('reindexes only the named vault, passing its path', async () => { + const h = makeDeps(cfg(['a', 'b'])); + await runReindex('b', h.deps); + expect(h.reindex).toHaveBeenCalledTimes(1); + expect(h.reindex).toHaveBeenCalledWith('b', '~/vaults/b'); + }); + + it('throws when the named vault is unknown', async () => { + const h = makeDeps(cfg(['a'])); + await expect(runReindex('nope', h.deps)).rejects.toThrow(/not found/); + }); + + it('reports when there are no vaults', async () => { + const h = makeDeps(cfg([])); + await runReindex(undefined, h.deps); + expect(h.logs.join()).toContain('No vaults'); + expect(h.reindex).not.toHaveBeenCalled(); + }); +}); diff --git a/src/commands/reindex.ts b/src/commands/reindex.ts new file mode 100644 index 0000000..8ce568b --- /dev/null +++ b/src/commands/reindex.ts @@ -0,0 +1,32 @@ +import { reindexNamedVault, type ReindexStats } from '../lib/vault-scan.js'; +import { loadVaultsConfig, type LoadedVaults } from '../lib/vaults.js'; + +export interface ReindexDeps { + load: () => LoadedVaults; + reindex: (name: string, vaultPath: string) => Promise; + log: (msg: string) => void; +} + +const defaultDeps: ReindexDeps = { + load: loadVaultsConfig, + reindex: reindexNamedVault, + log: (msg) => console.log(msg), +}; + +// Rebuild one vault's index (or every vault's when name is omitted) from its markdown. +export const runReindex = async ( + name: string | undefined, + deps: ReindexDeps = defaultDeps +): Promise => { + const { vaults } = deps.load().config; + const targets = name ? vaults.filter((v) => v.name === name) : vaults; + if (name && targets.length === 0) throw new Error(`vault '${name}' not found`); + if (targets.length === 0) { + deps.log('No vaults to reindex.'); + return; + } + for (const v of targets) { + const s = await deps.reindex(v.name, v.path); + deps.log(`Reindexed '${v.name}': +${s.added} ~${s.modified} -${s.removed} (=${s.unchanged})`); + } +}; diff --git a/src/commands/vault.ts b/src/commands/vault.ts index f4bdee2..fb49418 100644 --- a/src/commands/vault.ts +++ b/src/commands/vault.ts @@ -9,6 +9,7 @@ import { } from '../lib/vault-registry.js'; import { loadVaultsConfig, saveVaultsConfig, type LoadedVaults } from '../lib/vaults.js'; import { type VaultsConfig } from '../lib/vaults.schema.js'; +import { runReindex } from './reindex.js'; export interface VaultDeps { load: () => LoadedVaults; @@ -92,6 +93,15 @@ const guard = (fn: () => void): void => { } }; +const guardAsync = async (fn: () => Promise): Promise => { + try { + await fn(); + } catch (err) { + console.error(chalk.red(err instanceof Error ? err.message : String(err))); + process.exitCode = 1; + } +}; + export const registerVault = (program: Command): void => { const vault = program.command('vault').description('Manage local memory vaults'); @@ -114,4 +124,9 @@ export const registerVault = (program: Command): void => { .command('remove ') .description('Unregister a vault and drop its index (files stay)') .action((name: string) => guard(() => runVaultRemove(name))); + + vault + .command('reindex [name]') + .description('Rebuild a vault index from its markdown (all vaults if omitted)') + .action((name: string | undefined) => guardAsync(() => runReindex(name))); }; diff --git a/src/lib/vault-index.test.ts b/src/lib/vault-index.test.ts new file mode 100644 index 0000000..e63fb21 --- /dev/null +++ b/src/lib/vault-index.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from 'vitest'; +import { openIndex, type FileChange } from './vault-index.js'; + +const change = (path: string, content: string): FileChange => ({ + path, + content, + sha256: `sha-${path}-${content.length}`, + size: content.length, + mtime: 1, +}); + +describe('vault index (node:sqlite FTS5)', () => { + it('reconciles adds and searches by content', () => { + const idx = openIndex(':memory:'); + idx.reconcile({ + added: [change('a.md', 'the quick brown fox'), change('b.md', 'lazy dog sleeps')], + modified: [], + removed: [], + }); + expect(idx.fileCount()).toBe(2); + const hits = idx.search('fox'); + expect(hits).toHaveLength(1); + expect(hits[0]?.path).toBe('a.md'); + expect(hits[0]?.snippet).toContain('fox'); + idx.close(); + }); + + it('stat returns metadata or null', () => { + const idx = openIndex(':memory:'); + idx.reconcile({ added: [change('a.md', 'hi')], modified: [], removed: [] }); + expect(idx.stat('a.md')).toMatchObject({ path: 'a.md', size: 2 }); + expect(idx.stat('missing.md')).toBeNull(); + idx.close(); + }); + + it('list respects prefix + path order', () => { + const idx = openIndex(':memory:'); + idx.reconcile({ + added: [change('notes/b.md', 'y'), change('notes/a.md', 'x'), change('other/c.md', 'z')], + modified: [], + removed: [], + }); + expect(idx.list({ prefix: 'notes/' }).map((e) => e.path)).toEqual(['notes/a.md', 'notes/b.md']); + expect(idx.list()).toHaveLength(3); + idx.close(); + }); + + it('modified replaces content; removed drops from search + files', () => { + const idx = openIndex(':memory:'); + idx.reconcile({ added: [change('a.md', 'apple')], modified: [], removed: [] }); + idx.reconcile({ added: [], modified: [change('a.md', 'banana')], removed: [] }); + expect(idx.search('apple')).toHaveLength(0); + expect(idx.search('banana')).toHaveLength(1); + idx.reconcile({ added: [], modified: [], removed: ['a.md'] }); + expect(idx.fileCount()).toBe(0); + expect(idx.search('banana')).toHaveLength(0); + idx.close(); + }); + + it('returns [] for an empty/whitespace query', () => { + const idx = openIndex(':memory:'); + expect(idx.search(' ')).toEqual([]); + idx.close(); + }); + + it('treats punctuation as literal, not FTS syntax', () => { + const idx = openIndex(':memory:'); + idx.reconcile({ added: [change('a.md', 'node:sqlite is great')], modified: [], removed: [] }); + expect(idx.search('node:sqlite')).toHaveLength(1); + idx.close(); + }); + + it('records indexedAt after reconcile', () => { + const idx = openIndex(':memory:', () => new Date('2026-07-06T00:00:00Z')); + expect(idx.indexedAt()).toBeNull(); + idx.reconcile({ added: [change('a.md', 'x')], modified: [], removed: [] }); + expect(idx.indexedAt()).toBe('2026-07-06T00:00:00.000Z'); + idx.close(); + }); +}); diff --git a/src/lib/vault-index.ts b/src/lib/vault-index.ts new file mode 100644 index 0000000..485dc1d --- /dev/null +++ b/src/lib/vault-index.ts @@ -0,0 +1,195 @@ +import { mkdirSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { dirname } from 'node:path'; + +// A per-vault SQLite FTS5 index (~/.agentage/index/.db). The index is a rebuildable +// cache: the markdown files are canonical, this is derived and can be dropped + rebuilt. +// Ported from the archived v0.24 module to node:sqlite (native sqlite addons are forbidden). + +const require = createRequire(import.meta.url); + +export interface Hit { + path: string; + score: number; + snippet: string; +} +export interface FileEntry { + path: string; + size: number; + mtime: number; + sha256: string; +} +export interface FileChange extends FileEntry { + content: string; +} +export interface DiskDiff { + added: FileChange[]; + modified: FileChange[]; + removed: string[]; +} +export interface SearchOptions { + limit?: number; + offset?: number; +} +export interface ListOptions { + prefix?: string; + limit?: number; + offset?: number; +} + +// Minimal shape of the node:sqlite surface we use (avoids a hard type dep on the module). +interface SqliteStatement { + get(...params: unknown[]): unknown; + all(...params: unknown[]): unknown[]; + run(...params: unknown[]): unknown; +} +interface SqliteDb { + exec(sql: string): void; + prepare(sql: string): SqliteStatement; + close(): void; +} +interface SqliteModule { + DatabaseSync: new (path: string) => SqliteDb; +} + +// Lazy: importing node:sqlite at module top would throw on Node versions where it is still +// flagged, breaking every command. cli.ts re-execs index commands with the flag when needed. +export const loadSqlite = (): SqliteModule | null => { + try { + return require('node:sqlite') as SqliteModule; + } catch { + return null; + } +}; + +const requireSqlite = (): SqliteModule => { + const mod = loadSqlite(); + if (!mod) + throw new Error( + 'the vault index needs node:sqlite (Node >= 22.5 with --experimental-sqlite, or Node >= 23.4)' + ); + return mod; +}; + +const SCHEMA_VERSION = 1; +const SCHEMA = ` +PRAGMA journal_mode = WAL; + +CREATE TABLE IF NOT EXISTS files ( + path TEXT PRIMARY KEY, + size INTEGER NOT NULL, + mtime INTEGER NOT NULL, + sha256 TEXT NOT NULL +); + +CREATE VIRTUAL TABLE IF NOT EXISTS files_fts USING fts5( + path UNINDEXED, + content, + tokenize='porter unicode61' +); + +CREATE TABLE IF NOT EXISTS meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); +`; + +// Quote each term so FTS5 treats punctuation as literal, not query syntax. +const sanitizeFtsQuery = (query: string): string | null => { + const tokens = query + .trim() + .split(/\s+/) + .filter((t) => t.length > 0) + .map((t) => `"${t.replace(/"/g, '""')}"`); + return tokens.length === 0 ? null : tokens.join(' '); +}; + +const transaction = (db: SqliteDb, fn: () => void): void => { + db.exec('BEGIN'); + try { + fn(); + db.exec('COMMIT'); + } catch (err) { + db.exec('ROLLBACK'); + throw err; + } +}; + +export interface VaultIndex { + search(query: string, opts?: SearchOptions): Hit[]; + stat(path: string): FileEntry | null; + list(opts?: ListOptions): FileEntry[]; + reconcile(diff: DiskDiff): void; + fileCount(): number; + indexedAt(): string | null; + close(): void; +} + +export const openIndex = (dbPath: string, now: () => Date = () => new Date()): VaultIndex => { + const { DatabaseSync } = requireSqlite(); + if (dbPath !== ':memory:') mkdirSync(dirname(dbPath), { recursive: true }); + const db = new DatabaseSync(dbPath); + db.exec(SCHEMA); + if (!db.prepare('SELECT value FROM meta WHERE key = ?').get('schema_version')) + db.prepare('INSERT INTO meta (key, value) VALUES (?, ?)').run( + 'schema_version', + String(SCHEMA_VERSION) + ); + + return { + search: (query, opts = {}) => { + const sanitized = sanitizeFtsQuery(query); + if (sanitized === null) return []; + return db + .prepare( + `SELECT path, snippet(files_fts, 1, '<<', '>>', '...', 32) AS snippet, bm25(files_fts) AS score + FROM files_fts WHERE files_fts MATCH ? ORDER BY score LIMIT ? OFFSET ?` + ) + .all(sanitized, opts.limit ?? 20, opts.offset ?? 0) as Hit[]; + }, + stat: (path) => + (db.prepare('SELECT path, size, mtime, sha256 FROM files WHERE path = ?').get(path) as + FileEntry | undefined) ?? null, + list: (opts = {}) => { + const limit = opts.limit ?? -1; + const offset = opts.offset ?? 0; + if (opts.prefix !== undefined) + return db + .prepare( + 'SELECT path, size, mtime, sha256 FROM files WHERE path LIKE ? ORDER BY path LIMIT ? OFFSET ?' + ) + .all(`${opts.prefix}%`, limit, offset) as FileEntry[]; + return db + .prepare('SELECT path, size, mtime, sha256 FROM files ORDER BY path LIMIT ? OFFSET ?') + .all(limit, offset) as FileEntry[]; + }, + reconcile: (diff) => { + const upsertFile = db.prepare( + 'INSERT OR REPLACE INTO files (path, size, mtime, sha256) VALUES (?, ?, ?, ?)' + ); + const deleteFts = db.prepare('DELETE FROM files_fts WHERE path = ?'); + const insertFts = db.prepare('INSERT INTO files_fts (path, content) VALUES (?, ?)'); + const deleteFile = db.prepare('DELETE FROM files WHERE path = ?'); + const setMeta = db.prepare('INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)'); + transaction(db, () => { + for (const f of [...diff.added, ...diff.modified]) { + upsertFile.run(f.path, f.size, f.mtime, f.sha256); + deleteFts.run(f.path); + insertFts.run(f.path, f.content); + } + for (const path of diff.removed) { + deleteFile.run(path); + deleteFts.run(path); + } + setMeta.run('indexed_at', now().toISOString()); + }); + }, + fileCount: () => (db.prepare('SELECT COUNT(*) AS n FROM files').get() as { n: number }).n, + indexedAt: () => + ( + db.prepare('SELECT value FROM meta WHERE key = ?').get('indexed_at') as + { value: string } | undefined + )?.value ?? null, + close: () => db.close(), + }; +}; diff --git a/src/lib/vault-scan.test.ts b/src/lib/vault-scan.test.ts new file mode 100644 index 0000000..0867e2e --- /dev/null +++ b/src/lib/vault-scan.test.ts @@ -0,0 +1,77 @@ +import { mkdirSync, mkdtempSync, rmSync, unlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { openIndex } from './vault-index.js'; +import { reindexVault } from './vault-scan.js'; + +describe('reindexVault', () => { + let dir: string; + let vault: string; + let db: string; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'agentage-scan-')); + vault = join(dir, 'vault'); + mkdirSync(vault, { recursive: true }); + db = join(dir, 'index.db'); + }); + afterEach(() => rmSync(dir, { recursive: true, force: true })); + + const put = (rel: string, body: string): void => { + const full = join(vault, rel); + mkdirSync(dirname(full), { recursive: true }); + writeFileSync(full, body); + }; + const searchPaths = (query: string): string[] => { + const idx = openIndex(db); + const paths = idx.search(query).map((h) => h.path); + idx.close(); + return paths; + }; + + it('builds an index that search can query, walking subfolders', async () => { + put('a.md', 'alpha content'); + put('sub/b.md', 'beta content'); + const stats = await reindexVault(vault, db); + expect(stats.added).toBe(2); + expect(searchPaths('beta')).toEqual(['sub/b.md']); + }); + + it('drop the db + reindex = identical results (index is a pure cache)', async () => { + put('a.md', 'findme unique'); + await reindexVault(vault, db); + const first = searchPaths('findme'); + unlinkSync(db); + await reindexVault(vault, db); + expect(searchPaths('findme')).toEqual(first); + expect(first).toEqual(['a.md']); + }); + + it('detects modified + removed on a second pass', async () => { + put('a.md', 'one'); + put('b.md', 'two'); + await reindexVault(vault, db); + writeFileSync(join(vault, 'a.md'), 'one changed'); + unlinkSync(join(vault, 'b.md')); + const stats = await reindexVault(vault, db); + expect(stats.modified).toBe(1); + expect(stats.removed).toBe(1); + expect(searchPaths('two')).toHaveLength(0); + }); + + it('leaves unchanged files untouched on a re-run', async () => { + put('a.md', 'stable'); + await reindexVault(vault, db); + const stats = await reindexVault(vault, db); + expect(stats).toMatchObject({ added: 0, modified: 0, removed: 0, unchanged: 1 }); + }); + + it('skips dot-dirs like .obsidian and .git', async () => { + put('a.md', 'keep'); + mkdirSync(join(vault, '.obsidian'), { recursive: true }); + writeFileSync(join(vault, '.obsidian', 'x.md'), 'ignore me'); + const stats = await reindexVault(vault, db); + expect(stats.added).toBe(1); + }); +}); diff --git a/src/lib/vault-scan.ts b/src/lib/vault-scan.ts new file mode 100644 index 0000000..05283e4 --- /dev/null +++ b/src/lib/vault-scan.ts @@ -0,0 +1,101 @@ +import { createHash } from 'node:crypto'; +import { readdir, readFile, stat } from 'node:fs/promises'; +import { join, relative } from 'node:path'; +import { expandHome, indexDbPath } from './vault-registry.js'; +import { openIndex, type DiskDiff, type FileChange, type FileEntry } from './vault-index.js'; + +// Reconcile a vault's markdown tree against its index: walk the folder, diff by mtime/size +// then sha256, and apply added/modified/removed. Dot-dirs and node_modules are skipped. + +export interface ReindexStats { + added: number; + modified: number; + removed: number; + unchanged: number; +} + +const isMarkdown = (name: string): boolean => /\.md$/i.test(name); +const sha256 = (buf: Buffer): string => createHash('sha256').update(buf).digest('hex'); + +interface WalkResult { + unchanged: number; + added: FileChange[]; + modified: FileChange[]; + seen: Set; +} + +const walkMarkdown = async (root: string, inIndex: Map): Promise => { + const added: FileChange[] = []; + const modified: FileChange[] = []; + const seen = new Set(); + let unchanged = 0; + const queue = [root]; + + while (queue.length > 0) { + const dir = queue.shift() as string; + let entries; + try { + entries = await readdir(dir, { withFileTypes: true }); + } catch { + continue; + } + for (const entry of entries) { + if (entry.name.startsWith('.') || entry.name === 'node_modules') continue; + const full = join(dir, entry.name); + if (entry.isDirectory()) { + queue.push(full); + continue; + } + if (!entry.isFile() || !isMarkdown(entry.name)) continue; + + const relPath = relative(root, full); + seen.add(relPath); + const st = await stat(full); + const existing = inIndex.get(relPath); + if (existing && existing.mtime === st.mtimeMs && existing.size === st.size) { + unchanged++; + continue; + } + const buf = await readFile(full); + const change: FileChange = { + path: relPath, + content: buf.toString('utf-8'), + sha256: sha256(buf), + size: st.size, + mtime: st.mtimeMs, + }; + if (!existing) added.push(change); + else if (existing.sha256 !== change.sha256) modified.push(change); + else unchanged++; + } + } + return { unchanged, added, modified, seen }; +}; + +const buildDiff = (walk: WalkResult, inIndex: Map): DiskDiff => { + const removed = [...inIndex.keys()].filter((path) => !walk.seen.has(path)); + return { added: walk.added, modified: walk.modified, removed }; +}; + +// Open the index at dbPath, reconcile it against the markdown tree at vaultPath, close. +export const reindexVault = async (vaultPath: string, dbPath: string): Promise => { + const index = openIndex(dbPath); + try { + const inIndex = new Map(index.list().map((e) => [e.path, e])); + const walk = await walkMarkdown(expandHome(vaultPath), inIndex); + const diff = buildDiff(walk, inIndex); + index.reconcile(diff); + return { + added: diff.added.length, + modified: diff.modified.length, + removed: diff.removed.length, + unchanged: walk.unchanged, + }; + } finally { + index.close(); + } +}; + +// Resolve the index db path for a named vault (its markdown dir is `vaultPath`). +export const reindexNamedVault = (name: string, vaultPath: string): Promise => + reindexVault(vaultPath, indexDbPath(name)); diff --git a/vitest.config.ts b/vitest.config.ts index 2352ebf..3f50d20 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -3,6 +3,9 @@ import { defineConfig } from 'vitest/config'; export default defineConfig({ test: { include: ['src/**/*.test.ts'], + // node:sqlite sits behind --experimental-sqlite on Node < 23.4 (the vault index needs + // it); pass the flag to the fork workers. Harmless (accepted) on newer Node. + poolOptions: { forks: { execArgv: ['--experimental-sqlite'] } }, coverage: { provider: 'v8', include: ['src/**/*.ts'],