diff --git a/e2e/memory-offline.test.ts b/e2e/memory-offline.test.ts new file mode 100644 index 0000000..508c13b --- /dev/null +++ b/e2e/memory-offline.test.ts @@ -0,0 +1,79 @@ +import { expect, test } from '@playwright/test'; +import { assertCliBuilt, createCliMachine } from './helpers.js'; + +// The 6 memory verbs make zero network calls (DirectClient = fs + SQLite), so this is the +// full offline CRUD round trip against the built binary. @p0 +test.describe('offline memory CRUD @p0', () => { + test.beforeAll(() => assertCliBuilt()); + + test('write -> search -> read -> edit -> list -> delete, no network', async () => { + const m = createCliMachine(); + try { + const add = await m.exec([ + 'vault', + 'add', + 'main', + '--local', + '--path', + `${m.configDir}/main`, + ]); + expect(add.code, add.stderr).toBe(0); + + const write = await m.exec([ + 'memory', + 'write', + '@main/notes/q.md', + '--body', + 'the quokka is a marsupial', + ]); + expect(write.code, write.stderr).toBe(0); + + const search = await m.exec(['memory', 'search', 'quokka', '--json']); + expect(search.code, search.stderr).toBe(0); + expect(JSON.parse(search.stdout).results.map((r: { path: string }) => r.path)).toEqual([ + 'notes/q.md', + ]); + + const read = await m.exec(['memory', 'read', '@main/notes/q.md']); + expect(read.stdout).toContain('marsupial'); + + const edit = await m.exec([ + 'memory', + 'edit', + '@main/notes/q.md', + '--old', + 'quokka', + '--new', + 'wombat', + ]); + expect(edit.code, edit.stderr).toBe(0); + expect((await m.exec(['memory', 'read', '@main/notes/q.md'])).stdout).toContain('wombat'); + + const list = await m.exec(['memory', 'list', '--json']); + expect(JSON.parse(list.stdout).entries.map((e: { path: string }) => e.path)).toEqual([ + 'notes/q.md', + ]); + + const del = await m.exec(['memory', 'delete', '@main/notes/q.md']); + expect(del.code, del.stderr).toBe(0); + expect(JSON.parse((await m.exec(['memory', 'list', '--json'])).stdout).entries).toHaveLength( + 0 + ); + } finally { + m.cleanup(); + } + }); + + test('a duplicate str_replace target reports the canonical error', async () => { + const m = createCliMachine(); + try { + await m.exec(['vault', 'add', 'main', '--local', '--path', `${m.configDir}/main`]); + await m.exec(['memory', 'write', '@main/d.md', '--body', 'x x']); + const res = await m.exec(['memory', 'edit', '@main/d.md', '--old', 'x', '--new', 'y']); + expect(res.code).not.toBe(0); + expect(res.stderr + res.stdout).toContain('Multiple occurrences'); + } finally { + m.cleanup(); + } + }); +}); diff --git a/src/cli.ts b/src/cli.ts index 9c111f4..d458b17 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -3,6 +3,7 @@ import { spawnSync } from 'node:child_process'; import { createRequire } from 'node:module'; import { Command } from 'commander'; +import { registerMemory } from './commands/memory.js'; import { registerSetup } from './commands/setup.js'; import { registerStatus } from './commands/status.js'; import { registerUpdate } from './commands/update.js'; @@ -54,6 +55,7 @@ program.name('agentage').description('The agentage CLI').version(VERSION); registerSetup(program); registerStatus(program); registerVault(program); +registerMemory(program); registerUpdate(program); program.parseAsync().catch((err: unknown) => { diff --git a/src/commands/memory.test.ts b/src/commands/memory.test.ts new file mode 100644 index 0000000..9bcbf7d --- /dev/null +++ b/src/commands/memory.test.ts @@ -0,0 +1,104 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { MemoryClient } from '../lib/memory-client.js'; +import { runDelete, runEdit, runList, runRead, runSearch, runWrite } from './memory.js'; + +const client = (): MemoryClient => ({ + search: vi.fn(async () => ({ vault: 'v', results: [{ path: 'a.md', snippet: 's', score: -1 }] })), + read: vi.fn(async () => ({ + vault: 'v', + path: 'a.md', + title: 'A', + frontmatter: {}, + body: 'hi', + tags: [], + updated: 'now', + size: 2, + truncated: false, + })), + write: vi.fn(async () => ({ vault: 'v', path: 'a.md', bytesWritten: 5 })), + edit: vi.fn(async () => ({ vault: 'v', path: 'a.md', bytesWritten: 3 })), + list: vi.fn(async () => ({ + vault: 'v', + folder: '', + entries: [{ path: 'a.md', updated: 'now' }], + })), + delete: vi.fn(async () => ({ vault: 'v', path: 'a.md', trashedTo: '.trash/a.md' })), +}); + +let logs: string[]; +beforeEach(() => { + logs = []; + vi.spyOn(console, 'log').mockImplementation((m: unknown) => void logs.push(String(m))); + vi.spyOn(console, 'error').mockImplementation((m: unknown) => void logs.push(String(m))); +}); +afterEach(() => vi.restoreAllMocks()); + +describe('memory command wiring', () => { + it('search maps --limit and prints hits', async () => { + const c = client(); + await runSearch('quokka', { limit: '5' }, c); + expect(c.search).toHaveBeenCalledWith('quokka', { vault: undefined, limit: 5 }); + expect(logs.join()).toContain('a.md'); + }); + + it('search --json emits JSON', async () => { + const c = client(); + await runSearch('q', { json: true }, c); + expect(JSON.parse(logs[0] ?? '{}')).toMatchObject({ vault: 'v' }); + }); + + it('write passes an inline --body and parsed --frontmatter', async () => { + const c = client(); + await runWrite('a.md', { body: 'hello', frontmatter: '{"title":"T"}' }, c); + expect(c.write).toHaveBeenCalledWith('a.md', 'hello', { + vault: undefined, + frontmatter: { title: 'T' }, + }); + }); + + it('edit maps --old/--new to a str_replace op', async () => { + const c = client(); + await runEdit('a.md', { old: 'x', new: 'y' }, c); + expect(c.edit).toHaveBeenCalledWith( + 'a.md', + { oldStr: 'x', newStr: 'y', body: undefined, mode: 'replace' }, + { vault: undefined } + ); + }); + + it('edit --append selects append mode', async () => { + const c = client(); + await runEdit('a.md', { body: 'more', append: true }, c); + expect(c.edit).toHaveBeenCalledWith( + 'a.md', + { oldStr: undefined, newStr: undefined, body: 'more', mode: 'append' }, + { vault: undefined } + ); + }); + + it('read warns when the output was truncated', async () => { + const c = client(); + c.read = vi.fn(async () => ({ + vault: 'v', + path: 'a.md', + title: 'A', + frontmatter: {}, + body: 'big', + tags: [], + updated: 'now', + size: 3, + truncated: true, + })); + await runRead('a.md', {}, c); + expect(logs.join()).toContain('truncated'); + }); + + it('list and delete call through to the client', async () => { + const c = client(); + await runList('notes', { vault: 'v' }, c); + expect(c.list).toHaveBeenCalledWith('notes', { vault: 'v' }); + await runDelete('a.md', {}, c); + expect(c.delete).toHaveBeenCalledWith('a.md', { vault: undefined }); + expect(logs.join()).toContain('.trash/a.md'); + }); +}); diff --git a/src/commands/memory.ts b/src/commands/memory.ts new file mode 100644 index 0000000..653a20a --- /dev/null +++ b/src/commands/memory.ts @@ -0,0 +1,173 @@ +import chalk from 'chalk'; +import { type Command } from 'commander'; +import { createDirectClient, type MemoryClient } from '../lib/memory-client.js'; +import { loadVaultsConfig } from '../lib/vaults.js'; + +const defaultClient = (): MemoryClient => createDirectClient(loadVaultsConfig().config); + +const readStdin = async (): Promise => { + const chunks: Buffer[] = []; + for await (const chunk of process.stdin) chunks.push(chunk as Buffer); + return Buffer.concat(chunks).toString('utf-8'); +}; + +const emit = (json: boolean, data: unknown, human: () => void): void => { + if (json) console.log(JSON.stringify(data, null, 2)); + else human(); +}; + +interface CommonOpts { + vault?: string; + json?: boolean; +} + +export const runSearch = async ( + query: string, + opts: CommonOpts & { limit?: string }, + client: MemoryClient = defaultClient() +): Promise => { + const out = await client.search(query, { + vault: opts.vault, + limit: opts.limit ? Number.parseInt(opts.limit, 10) : undefined, + }); + emit(opts.json ?? false, out, () => { + if (out.results.length === 0) console.log('No matches.'); + for (const h of out.results) console.log(`${h.path}\n ${chalk.gray(h.snippet)}`); + }); +}; + +export const runRead = async ( + ref: string, + opts: CommonOpts, + client: MemoryClient = defaultClient() +): Promise => { + const doc = await client.read(ref, { vault: opts.vault }); + emit(opts.json ?? false, doc, () => { + console.log(doc.body); + if (doc.truncated) console.error(chalk.yellow('(output truncated)')); + }); +}; + +export const runWrite = async ( + ref: string, + opts: CommonOpts & { body?: string; frontmatter?: string }, + client: MemoryClient = defaultClient() +): Promise => { + const body = opts.body !== undefined && opts.body !== '-' ? opts.body : await readStdin(); + const frontmatter = opts.frontmatter + ? (JSON.parse(opts.frontmatter) as Record) + : undefined; + const out = await client.write(ref, body, { vault: opts.vault, frontmatter }); + emit(opts.json ?? false, out, () => + console.log(chalk.green(`Wrote @${out.vault}/${out.path} (${out.bytesWritten} bytes)`)) + ); +}; + +export const runEdit = async ( + ref: string, + opts: CommonOpts & { old?: string; new?: string; body?: string; append?: boolean }, + client: MemoryClient = defaultClient() +): Promise => { + const out = await client.edit( + ref, + { + oldStr: opts.old, + newStr: opts.new, + body: opts.body, + mode: opts.append ? 'append' : 'replace', + }, + { vault: opts.vault } + ); + emit(opts.json ?? false, out, () => console.log(chalk.green(`Edited @${out.vault}/${out.path}`))); +}; + +export const runList = async ( + folder: string | undefined, + opts: CommonOpts, + client: MemoryClient = defaultClient() +): Promise => { + const out = await client.list(folder, { vault: opts.vault }); + emit(opts.json ?? false, out, () => { + if (out.entries.length === 0) console.log('No documents.'); + for (const e of out.entries) console.log(e.path); + }); +}; + +export const runDelete = async ( + ref: string, + opts: CommonOpts, + client: MemoryClient = defaultClient() +): Promise => { + const out = await client.delete(ref, { vault: opts.vault }); + emit(opts.json ?? false, out, () => + console.log(`Deleted @${out.vault}/${out.path} (recoverable in ${out.trashedTo})`) + ); +}; + +const guard = (fn: () => Promise): Promise => + fn().catch((err: unknown) => { + console.error(chalk.red(err instanceof Error ? err.message : String(err))); + process.exitCode = 1; + }); + +export const registerMemory = (program: Command): void => { + const memory = program.command('memory').description('Read + write memory, offline'); + + memory + .command('search ') + .description('Full-text search a vault (FTS5)') + .option('--vault ', 'target vault') + .option('--limit ', 'max hits (default 20)') + .option('--json', 'machine-readable output') + .action((query: string[], opts: CommonOpts & { limit?: string }) => + guard(() => runSearch(query.join(' '), opts)) + ); + + memory + .command('read ') + .description('Print a document (@/ or --vault)') + .option('--vault ', 'target vault') + .option('--json', 'machine-readable output') + .action((ref: string, opts: CommonOpts) => guard(() => runRead(ref, opts))); + + memory + .command('write ') + .description('Create or overwrite a document (body from --body or stdin)') + .option('--vault ', 'target vault') + .option('--body ', 'document body (omit or "-" to read stdin)') + .option('--frontmatter ', 'frontmatter as a JSON object') + .option('--json', 'machine-readable output') + .action((ref: string, opts: CommonOpts & { body?: string; frontmatter?: string }) => + guard(() => runWrite(ref, opts)) + ); + + memory + .command('edit ') + .description('Edit a document: str_replace (--old/--new) or --body (--append)') + .option('--vault ', 'target vault') + .option('--old ', 'exact, unique substring to replace') + .option('--new ', 'replacement (omit to delete the match)') + .option('--body ', 'replace the whole body') + .option('--append', 'append --body instead of replacing') + .option('--json', 'machine-readable output') + .action( + ( + ref: string, + opts: CommonOpts & { old?: string; new?: string; body?: string; append?: boolean } + ) => guard(() => runEdit(ref, opts)) + ); + + memory + .command('list [folder]') + .description('List documents in a vault (optionally under a folder)') + .option('--vault ', 'target vault') + .option('--json', 'machine-readable output') + .action((folder: string | undefined, opts: CommonOpts) => guard(() => runList(folder, opts))); + + memory + .command('delete ') + .description('Soft-delete a document (moved to .trash/, recoverable)') + .option('--vault ', 'target vault') + .option('--json', 'machine-readable output') + .action((ref: string, opts: CommonOpts) => guard(() => runDelete(ref, opts))); +}; diff --git a/src/lib/memory-client.test.ts b/src/lib/memory-client.test.ts new file mode 100644 index 0000000..31dfafb --- /dev/null +++ b/src/lib/memory-client.test.ts @@ -0,0 +1,115 @@ +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { createDirectClient } from './memory-client.js'; +import { VaultsConfig, type VaultType } from './vaults.schema.js'; + +// Each test gets its own config dir (for the index) + vault dir(s). +const configFor = (vaults: { name: string; path: string; type?: VaultType }[]): VaultsConfig => + VaultsConfig.parse({ + version: 1, + vaults: vaults.map((v) => ({ + name: v.name, + path: v.path, + ...(v.type === 'couchdb' + ? { type: 'couchdb', server: 'agentage' } + : v.type === 'git' + ? { type: 'git', remote: 'git@x:y.git' } + : { type: 'local' }), + })), + }); + +describe('DirectClient', () => { + let root: string; + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'agentage-mc-')); + process.env['AGENTAGE_CONFIG_DIR'] = join(root, 'cfg'); + }); + afterEach(() => { + delete process.env['AGENTAGE_CONFIG_DIR']; + rmSync(root, { recursive: true, force: true }); + }); + + const single = () => { + const vault = join(root, 'v'); + return { client: createDirectClient(configFor([{ name: 'main', path: vault }])), vault }; + }; + + it('write -> read round-trips', async () => { + const { client } = single(); + await client.write('a.md', 'hello memory'); + const doc = await client.read('a.md'); + expect(doc.body).toBe('hello memory'); + expect(doc.vault).toBe('main'); + }); + + it('search finds a written doc after its index refresh', async () => { + const { client } = single(); + await client.write('notes/x.md', 'the quokka is a marsupial'); + const out = await client.search('quokka'); + expect(out.results.map((r) => r.path)).toEqual(['notes/x.md']); + }); + + it('list reflects writes and deletes; delete is soft', async () => { + const { client } = single(); + await client.write('a.md', 'one'); + await client.write('b.md', 'two'); + expect((await client.list(undefined)).entries.map((e) => e.path).sort()).toEqual([ + 'a.md', + 'b.md', + ]); + await client.delete('a.md'); + expect((await client.list(undefined)).entries.map((e) => e.path)).toEqual(['b.md']); + await expect(client.read('a.md')).rejects.toThrow(/not found/); + }); + + it('edits via str_replace through the client', async () => { + const { client } = single(); + await client.write('a.md', 'alpha beta'); + await client.edit('a.md', { oldStr: 'beta', newStr: 'BETA' }); + expect((await client.read('a.md')).body).toBe('alpha BETA'); + }); + + it('routes by @vault/ prefix and by --vault', async () => { + const work = join(root, 'work'); + const notes = join(root, 'notes'); + const client = createDirectClient( + configFor([ + { name: 'work', path: work }, + { name: 'notes', path: notes }, + ]) + ); + await client.write('@work/a.md', 'in work'); + await client.write('a.md', 'in notes', { vault: 'notes' }); + expect((await client.read('@work/a.md')).body).toBe('in work'); + expect((await client.read('a.md', { vault: 'notes' })).body).toBe('in notes'); + }); + + it('errors on an ambiguous or unknown vault', async () => { + const client = createDirectClient( + configFor([ + { name: 'a', path: join(root, 'a') }, + { name: 'b', path: join(root, 'b') }, + ]) + ); + await expect(client.read('x.md')).rejects.toThrow(/multiple vaults/); + await expect(client.read('x.md', { vault: 'nope' })).rejects.toThrow(/unknown vault/); + }); + + it('enforces the 8 MB cap on account-synced (couchdb) vaults only', async () => { + const client = createDirectClient( + configFor([{ name: 'cloud', path: join(root, 'c'), type: 'couchdb' }]) + ); + const tooBig = 'x'.repeat(8 * 1024 * 1024 + 1); + await expect(client.write('big.md', tooBig)).rejects.toThrow(/8 MB/); + }); + + it('bounds read output with a truncation flag', async () => { + const { client } = single(); + await client.write('big.md', 'y'.repeat(1_000_050)); + const doc = await client.read('big.md'); + expect(doc.truncated).toBe(true); + expect(doc.body.length).toBe(1_000_000); + }); +}); diff --git a/src/lib/memory-client.ts b/src/lib/memory-client.ts new file mode 100644 index 0000000..6eaff52 --- /dev/null +++ b/src/lib/memory-client.ts @@ -0,0 +1,171 @@ +import { expandHome, indexDbPath } from './vault-registry.js'; +import { openIndex, type Hit, type VaultIndex } from './vault-index.js'; +import { reconcileIndex } from './vault-scan.js'; +import { + deleteDoc, + editDoc, + readDoc, + writeDoc, + type DocView, + type WriteReceipt, +} from './vault-store.js'; +import { type EditOp } from './memory-edit.js'; +import { type VaultsConfig, type VaultType } from './vaults.schema.js'; + +// The one seam every index-touching verb goes through (daemon-index-ownership V10). This is +// the DirectClient (in-process) impl; the DaemonClient lands with the daemon (M2.5). Verbs +// never touch SQLite or the store directly - they call these six methods. + +const MAX_SYNCED_DOC = 8 * 1024 * 1024; // ADR-011 WI-4: 8 MB per-doc cap on account vaults +const READ_MAX = 1_000_000; // bounded read output +const SEARCH_LIMIT_MAX = 100; // bounded search page + +export interface SearchOutput { + vault: string; + results: Hit[]; +} +export interface ListOutput { + vault: string; + folder: string; + entries: { path: string; updated: string }[]; +} +export interface DeleteOutput { + vault: string; + path: string; + trashedTo: string; +} +export type ReadOutput = DocView & { vault: string; truncated: boolean }; + +export interface VerbOptions { + vault?: string; + folder?: string; + limit?: number; +} + +export interface MemoryClient { + search(query: string, opts?: VerbOptions): Promise; + read(ref: string, opts?: VerbOptions): Promise; + write( + ref: string, + body: string, + opts?: VerbOptions & { frontmatter?: Record } + ): Promise; + edit( + ref: string, + op: Omit, + opts?: VerbOptions + ): Promise; + list(folder: string | undefined, opts?: VerbOptions): Promise; + delete(ref: string, opts?: VerbOptions): Promise; +} + +interface Resolved { + name: string; + path: string; + type: VaultType; + relPath: string; +} + +const parseRef = (ref: string): { vaultName?: string; relPath: string } => { + const m = /^@([A-Za-z0-9_-]{1,64})\/(.+)$/.exec(ref); + return m ? { vaultName: m[1]!, relPath: m[2]! } : { relPath: ref }; +}; + +export const createDirectClient = (config: VaultsConfig): MemoryClient => { + const resolve = (ref: string, vaultOpt?: string): Resolved => { + const { vaultName, relPath } = parseRef(ref); + const want = vaultName ?? vaultOpt; + const vaults = config.vaults; + if (vaults.length === 0) throw new Error('no vaults registered - run `agentage vault add`'); + const vault = want + ? vaults.find((v) => v.name === want) + : vaults.length === 1 + ? vaults[0] + : undefined; + if (!vault) + throw new Error( + want ? `unknown vault: ${want}` : 'multiple vaults - use --vault or @/' + ); + return { name: vault.name, path: expandHome(vault.path), type: vault.type, relPath }; + }; + + // A folder-only ref (no doc path) resolves the vault without requiring a relPath. + const resolveVault = (vaultOpt?: string): Omit => { + const r = resolve('_', vaultOpt); + return { name: r.name, path: r.path, type: r.type }; + }; + + const refreshed = async ( + v: Omit, + use: (i: VaultIndex) => T + ): Promise => { + const index = openIndex(indexDbPath(v.name)); + try { + await reconcileIndex(index, v.path); + return use(index); + } finally { + index.close(); + } + }; + + const capForSync = (v: Resolved, body: string): void => { + if (v.type === 'couchdb' && Buffer.byteLength(body, 'utf-8') > MAX_SYNCED_DOC) + throw new Error(`document exceeds the 8 MB limit for account-synced vaults`); + }; + + return { + async search(query, opts = {}) { + const v = resolveVault(opts.vault); + const limit = Math.min(opts.limit ?? 20, SEARCH_LIMIT_MAX); + const results = await refreshed(v, (i) => i.search(query, { limit })); + return { vault: v.name, results }; + }, + + async read(ref, opts = {}) { + const v = resolve(ref, opts.vault); + const doc = await readDoc(v.path, v.relPath); + const truncated = doc.body.length > READ_MAX; + return { + ...doc, + vault: v.name, + truncated, + body: truncated ? doc.body.slice(0, READ_MAX) : doc.body, + }; + }, + + async write(ref, body, opts = {}) { + const v = resolve(ref, opts.vault); + capForSync(v, body); + const receipt = await writeDoc(v.path, v.relPath, body, opts.frontmatter); + await refreshed(v, () => undefined); + return { ...receipt, vault: v.name }; + }, + + async edit(ref, op, opts = {}) { + const v = resolve(ref, opts.vault); + const receipt = await editDoc(v.path, { ...op, path: v.relPath }); + await refreshed(v, () => undefined); + return { ...receipt, vault: v.name }; + }, + + async list(folder, opts = {}) { + const v = resolveVault(opts.vault); + const prefix = + folder && folder.length > 0 ? (folder.endsWith('/') ? folder : `${folder}/`) : undefined; + const entries = await refreshed(v, (i) => + i.list(prefix ? { prefix } : {}).map((e) => ({ + path: e.path, + updated: new Date(e.mtime).toISOString(), + })) + ); + return { vault: v.name, folder: folder ?? '', entries }; + }, + + async delete(ref, opts = {}) { + const v = resolve(ref, opts.vault); + const { trashedTo } = await deleteDoc(v.path, v.relPath); + await refreshed(v, () => undefined); + return { vault: v.name, path: v.relPath, trashedTo }; + }, + }; +}; diff --git a/src/lib/memory-edit.test.ts b/src/lib/memory-edit.test.ts new file mode 100644 index 0000000..cee2b6d --- /dev/null +++ b/src/lib/memory-edit.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from 'vitest'; +import { resolveEdit, strReplace } from './memory-edit.js'; + +describe('strReplace', () => { + it('replaces a unique occurrence', () => { + expect(strReplace('alpha beta gamma', 'f.md', 'beta', 'BETA')).toBe('alpha BETA gamma'); + }); + + it('deletes the match when new_str is empty', () => { + expect(strReplace('a b c', 'f.md', 'b ', '')).toBe('a c'); + }); + + it('throws the canonical error when old_str is absent', () => { + expect(() => strReplace('abc', 'f.md', 'xyz', 'q')).toThrow( + /No replacement was performed, old_str `xyz` did not appear verbatim in f\.md\./ + ); + }); + + it('throws the canonical error (with lines) on multiple occurrences', () => { + expect(() => strReplace('a\na', 'f.md', 'a', 'X')).toThrow( + /Multiple occurrences of old_str `a` in f\.md \(lines: 1, 2\)\. Please ensure it is unique\./ + ); + }); +}); + +describe('resolveEdit', () => { + it('applies a str_replace', () => { + expect(resolveEdit('a b', { path: 'f', oldStr: 'b', newStr: 'X' })).toBe('a X'); + }); + + it('appends with a newline guard', () => { + expect(resolveEdit('x', { path: 'f', body: 'more', mode: 'append' })).toBe('x\nmore'); + expect(resolveEdit('x\n', { path: 'f', body: 'more', mode: 'append' })).toBe('x\nmore'); + }); + + it('replaces the whole body', () => { + expect(resolveEdit('old', { path: 'f', body: 'new' })).toBe('new'); + }); + + it('rejects combining str_replace with a body', () => { + expect(() => resolveEdit('x', { path: 'f', oldStr: 'x', body: 'y' })).toThrow(/cannot combine/); + }); + + it('rejects an edit with neither --old nor --body', () => { + expect(() => resolveEdit('x', { path: 'f' })).toThrow(/either --old.*or --body/); + }); +}); diff --git a/src/lib/memory-edit.ts b/src/lib/memory-edit.ts new file mode 100644 index 0000000..eb47d6f --- /dev/null +++ b/src/lib/memory-edit.ts @@ -0,0 +1,42 @@ +// str_replace + edit-body resolution, mirroring @agentage/memory-core's local-ops verbatim +// (exact + unique match, canonical error strings) so the local CLI stays contract-faithful. + +const lineOf = (text: string, index: number): number => text.slice(0, index).split('\n').length; + +// Exact, unique substring replacement. new_str '' deletes the match. +export const strReplace = (body: string, path: string, oldStr: string, newStr: string): string => { + const starts: number[] = []; + for (let i = body.indexOf(oldStr); i !== -1; i = body.indexOf(oldStr, i + 1)) starts.push(i); + if (starts.length === 0) + throw new Error( + `No replacement was performed, old_str \`${oldStr}\` did not appear verbatim in ${path}.` + ); + if (starts.length > 1) + throw new Error( + `Multiple occurrences of old_str \`${oldStr}\` in ${path} (lines: ${starts + .map((s) => lineOf(body, s)) + .join(', ')}). Please ensure it is unique.` + ); + return body.slice(0, starts[0]) + newStr + body.slice(starts[0]! + oldStr.length); +}; + +export interface EditOp { + path: string; + oldStr?: string; + newStr?: string; + body?: string; + mode?: 'append' | 'replace'; +} + +// Resolve the new body for an edit. str_replace (old_str present) and a whole-body edit are +// mutually exclusive; a str_replace with no new_str deletes the matched text. +export const resolveEdit = (existing: string, op: EditOp): string => { + const hasStrReplace = op.oldStr !== undefined; + const hasBody = op.body !== undefined; + if (hasStrReplace && hasBody) + throw new Error('cannot combine a str_replace (--old/--new) with a --body edit'); + if (hasStrReplace) return strReplace(existing, op.path, op.oldStr!, op.newStr ?? ''); + if (!hasBody) throw new Error('edit needs either --old (str_replace) or --body'); + if (op.mode === 'append') return `${existing}${existing.endsWith('\n') ? '' : '\n'}${op.body!}`; + return op.body!; +}; diff --git a/src/lib/path-safety.ts b/src/lib/path-safety.ts new file mode 100644 index 0000000..8fba7aa --- /dev/null +++ b/src/lib/path-safety.ts @@ -0,0 +1,11 @@ +import { isAbsolute, relative, resolve } from 'node:path'; + +// Resolve a vault-relative path to an absolute path, refusing anything that escapes the +// vault root (absolute inputs, `..` traversal). Ported from the v0.24 module. +export const safeJoin = (vaultPath: string, vaultRelPath: string): string => { + if (isAbsolute(vaultRelPath)) throw new Error('path must be vault-relative, not absolute'); + const full = resolve(vaultPath, vaultRelPath); + const rel = relative(vaultPath, full); + if (rel.startsWith('..') || isAbsolute(rel)) throw new Error('path escapes the vault root'); + return full; +}; diff --git a/src/lib/vault-scan.ts b/src/lib/vault-scan.ts index 05283e4..9663a8d 100644 --- a/src/lib/vault-scan.ts +++ b/src/lib/vault-scan.ts @@ -2,7 +2,13 @@ import { createHash } from 'node:crypto'; import { readdir, readFile, stat } from 'node:fs/promises'; import { join, relative } from 'node:path'; import { expandHome, indexDbPath } from './vault-registry.js'; -import { openIndex, type DiskDiff, type FileChange, type FileEntry } from './vault-index.js'; +import { + openIndex, + type DiskDiff, + type FileChange, + type FileEntry, + type VaultIndex, +} from './vault-index.js'; // Reconcile a vault's markdown tree against its index: walk the folder, diff by mtime/size // then sha256, and apply added/modified/removed. Dot-dirs and node_modules are skipped. @@ -77,20 +83,29 @@ const buildDiff = (walk: WalkResult, inIndex: Map): DiskDiff return { added: walk.added, modified: walk.modified, removed }; }; +// Reconcile an already-open index against the markdown tree at vaultPath (no open/close). +// Lets a caller keep the index open to query right after refreshing it. +export const reconcileIndex = async ( + index: VaultIndex, + vaultPath: string +): Promise => { + const inIndex = new Map(index.list().map((e) => [e.path, e])); + const walk = await walkMarkdown(expandHome(vaultPath), inIndex); + const diff = buildDiff(walk, inIndex); + index.reconcile(diff); + return { + added: diff.added.length, + modified: diff.modified.length, + removed: diff.removed.length, + unchanged: walk.unchanged, + }; +}; + // Open the index at dbPath, reconcile it against the markdown tree at vaultPath, close. export const reindexVault = async (vaultPath: string, dbPath: string): Promise => { const index = openIndex(dbPath); try { - const inIndex = new Map(index.list().map((e) => [e.path, e])); - const walk = await walkMarkdown(expandHome(vaultPath), inIndex); - const diff = buildDiff(walk, inIndex); - index.reconcile(diff); - return { - added: diff.added.length, - modified: diff.modified.length, - removed: diff.removed.length, - unchanged: walk.unchanged, - }; + return await reconcileIndex(index, vaultPath); } finally { index.close(); } diff --git a/src/lib/vault-store.test.ts b/src/lib/vault-store.test.ts new file mode 100644 index 0000000..4e8762e --- /dev/null +++ b/src/lib/vault-store.test.ts @@ -0,0 +1,60 @@ +import { existsSync, mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { deleteDoc, docExists, editDoc, readDoc, writeDoc } from './vault-store.js'; + +describe('vault store', () => { + let vault: string; + beforeEach(() => { + vault = mkdtempSync(join(tmpdir(), 'agentage-store-')); + }); + afterEach(() => rmSync(vault, { recursive: true, force: true })); + + it('round-trips a plain body', async () => { + await writeDoc(vault, 'a.md', 'hello world'); + const doc = await readDoc(vault, 'a.md'); + expect(doc.body).toBe('hello world'); + expect(doc.frontmatter).toEqual({}); + }); + + it('writes + parses frontmatter, deriving title + tags', async () => { + await writeDoc(vault, 'n.md', 'body text', { title: 'My Note', tags: ['x', 'y'] }); + const doc = await readDoc(vault, 'n.md'); + expect(doc.frontmatter).toMatchObject({ title: 'My Note', tags: ['x', 'y'] }); + expect(doc.body).toBe('body text'); + expect(doc.title).toBe('My Note'); + expect(doc.tags).toEqual(['x', 'y']); + }); + + it('falls back to the first heading, then the filename, for the title', async () => { + await writeDoc(vault, 'h.md', '# Heading Title\n\nbody'); + expect((await readDoc(vault, 'h.md')).title).toBe('Heading Title'); + await writeDoc(vault, 'plain.md', 'no heading'); + expect((await readDoc(vault, 'plain.md')).title).toBe('plain'); + }); + + it('edits via str_replace, preserving frontmatter', async () => { + await writeDoc(vault, 'a.md', 'one two three', { title: 'T' }); + await editDoc(vault, { path: 'a.md', oldStr: 'two', newStr: 'TWO' }); + const doc = await readDoc(vault, 'a.md'); + expect(doc.body).toBe('one TWO three'); + expect(doc.frontmatter).toMatchObject({ title: 'T' }); + }); + + it('soft-deletes into .trash/, keeping the file recoverable', async () => { + await writeDoc(vault, 'sub/a.md', 'keep me'); + const { trashedTo } = await deleteDoc(vault, 'sub/a.md'); + expect(trashedTo).toBe('.trash/sub/a.md'); + expect(docExists(vault, 'sub/a.md')).toBe(false); + expect(existsSync(join(vault, '.trash/sub/a.md'))).toBe(true); + }); + + it('refuses a path that escapes the vault root', async () => { + await expect(readDoc(vault, '../escape.md')).rejects.toThrow(/escapes the vault root/); + }); + + it('errors reading a missing file', async () => { + await expect(readDoc(vault, 'nope.md')).rejects.toThrow(/not found/); + }); +}); diff --git a/src/lib/vault-store.ts b/src/lib/vault-store.ts new file mode 100644 index 0000000..9b0fe29 --- /dev/null +++ b/src/lib/vault-store.ts @@ -0,0 +1,108 @@ +import { existsSync } from 'node:fs'; +import { mkdir, readFile, rename, stat, writeFile } from 'node:fs/promises'; +import { basename, dirname } from 'node:path'; +import { parse as parseYaml, stringify as stringifyYaml } from 'yaml'; +import { resolveEdit, type EditOp } from './memory-edit.js'; +import { safeJoin } from './path-safety.js'; + +// Markdown CRUD over a vault directory. The file is canonical; the SQLite index is derived. +// Frontmatter is a leading `---` YAML block (parsed/emitted with the `yaml` dep). + +export interface DocView { + path: string; + title: string; + frontmatter: Record; + body: string; + tags: string[]; + updated: string; + size: number; +} + +export interface WriteReceipt { + path: string; + bytesWritten: number; +} + +const FRONTMATTER = /^---\r?\n([\s\S]*?)\r?\n---\r?\n?/; + +const parseDoc = (raw: string): { frontmatter: Record; body: string } => { + const m = FRONTMATTER.exec(raw); + if (!m) return { frontmatter: {}, body: raw }; + let frontmatter: Record = {}; + try { + const parsed: unknown = parseYaml(m[1]!); + if (parsed && typeof parsed === 'object') frontmatter = parsed as Record; + } catch { + // a malformed frontmatter block is treated as body, not a hard error + return { frontmatter: {}, body: raw }; + } + return { frontmatter, body: raw.slice(m[0].length) }; +}; + +const composeDoc = (body: string, frontmatter?: Record): string => + frontmatter && Object.keys(frontmatter).length > 0 + ? `---\n${stringifyYaml(frontmatter)}---\n${body}` + : body; + +const titleOf = (relPath: string, fm: Record, body: string): string => { + if (typeof fm['title'] === 'string' && fm['title'].length > 0) return fm['title']; + const heading = /^#\s+(.+)$/m.exec(body); + return heading ? heading[1]!.trim() : basename(relPath).replace(/\.md$/i, ''); +}; + +const tagsOf = (fm: Record): string[] => + Array.isArray(fm['tags']) ? fm['tags'].filter((t): t is string => typeof t === 'string') : []; + +export const docExists = (vaultPath: string, relPath: string): boolean => + existsSync(safeJoin(vaultPath, relPath)); + +export const readDoc = async (vaultPath: string, relPath: string): Promise => { + const full = safeJoin(vaultPath, relPath); + if (!existsSync(full)) throw new Error(`not found: ${relPath}`); + const raw = await readFile(full, 'utf-8'); + const st = await stat(full); + const { frontmatter, body } = parseDoc(raw); + return { + path: relPath, + title: titleOf(relPath, frontmatter, body), + frontmatter, + body, + tags: tagsOf(frontmatter), + updated: new Date(st.mtimeMs).toISOString(), + size: st.size, + }; +}; + +export const writeDoc = async ( + vaultPath: string, + relPath: string, + body: string, + frontmatter?: Record +): Promise => { + const full = safeJoin(vaultPath, relPath); + const payload = composeDoc(body, frontmatter); + await mkdir(dirname(full), { recursive: true }); + await writeFile(full, payload, 'utf-8'); + return { path: relPath, bytesWritten: Buffer.byteLength(payload, 'utf-8') }; +}; + +export const editDoc = async (vaultPath: string, op: EditOp): Promise => { + const current = await readDoc(vaultPath, op.path); + const nextBody = resolveEdit(current.body, op); + return writeDoc(vaultPath, op.path, nextBody, current.frontmatter); +}; + +// Soft delete: move the file into the vault's `.trash/` (dot-prefixed, so the scanner skips +// it), preserving its relative path. Recoverable; the index drops it on the next reconcile. +export const deleteDoc = async ( + vaultPath: string, + relPath: string +): Promise<{ path: string; trashedTo: string }> => { + const full = safeJoin(vaultPath, relPath); + if (!existsSync(full)) throw new Error(`not found: ${relPath}`); + const trashRel = `.trash/${relPath}`; + const trashFull = safeJoin(vaultPath, trashRel); + await mkdir(dirname(trashFull), { recursive: true }); + await rename(full, trashFull); + return { path: relPath, trashedTo: trashRel }; +}; diff --git a/vitest.config.ts b/vitest.config.ts index 3f50d20..8c4fb54 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -4,8 +4,9 @@ export default defineConfig({ test: { include: ['src/**/*.test.ts'], // node:sqlite sits behind --experimental-sqlite on Node < 23.4 (the vault index needs - // it); pass the flag to the fork workers. Harmless (accepted) on newer Node. - poolOptions: { forks: { execArgv: ['--experimental-sqlite'] } }, + // it); pass the flag to the workers. Harmless (accepted) on newer Node. (Vitest 4: + // execArgv is a top-level test option; the old poolOptions.forks form was removed.) + execArgv: ['--experimental-sqlite'], coverage: { provider: 'v8', include: ['src/**/*.ts'],