diff --git a/packages/runtime/src/__tests__/session-import.test.ts b/packages/runtime/src/__tests__/session-import.test.ts index 0d569fc818..e91036689f 100644 --- a/packages/runtime/src/__tests__/session-import.test.ts +++ b/packages/runtime/src/__tests__/session-import.test.ts @@ -18,7 +18,7 @@ */ import assert from 'node:assert/strict'; -import { mkdir, mkdtemp, readdir, rm, writeFile } from 'node:fs/promises'; +import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { basename, dirname, join } from 'node:path'; import { DatabaseSync } from 'node:sqlite'; @@ -1118,3 +1118,289 @@ test('clears the collection candidate of a blob the import references again', as await rm(target.root, { recursive: true, force: true }); } }); + +test('refuses a bundle whose context references a Session it does not carry', async () => { + const source = await makeWorkspace('maka-import-foreign-ref-source'); + const target = await makeWorkspace('maka-import-foreign-ref-target'); + try { + const sessionId = await createSession(source.workspaceRoot); + await seedHistory(source.workspaceRoot, sessionId); + await seedContext(source.workspaceRoot, sessionId, 'ABC'); + await createSession(target.workspaceRoot, 'Unrelated'); + + // An export only keeps refs for the Sessions it carries, but a bundle can + // be assembled rather than exported, and its digest still checks out. A + // reference owned by a Session that never arrives can never be released: + // that happens when its Session is retired. + const tamper = new DatabaseSync(join(source.workspaceRoot, 'context-offload.sqlite')); + try { + // Usage rows move with it. Left inconsistent, the snapshot validator + // catches the tampering first and this guard is never reached -- the + // test would pass while proving the wrong thing. + tamper.prepare('UPDATE context_refs SET session_id = ?').run('a-session-not-in-this-bundle'); + tamper + .prepare('UPDATE context_session_usage SET session_id = ?') + .run('a-session-not-in-this-bundle'); + } finally { + tamper.close(); + } + + await assert.rejects( + () => importState(target.workspaceRoot, source.workspaceRoot), + /references a Session it does not carry/, + ); + } finally { + await rm(source.root, { recursive: true, force: true }); + await rm(target.root, { recursive: true, force: true }); + } +}); + +test('refuses a bundle carrying collection state from the workspace it left', async () => { + const source = await makeWorkspace('maka-import-stale-gc-source'); + const target = await makeWorkspace('maka-import-stale-gc-target'); + try { + const sessionId = await createSession(source.workspaceRoot); + await seedHistory(source.workspaceRoot, sessionId); + await seedContext(source.workspaceRoot, sessionId, 'ABC'); + await createSession(target.workspaceRoot, 'Unrelated'); + + // The export empties this queue on its private copy. One that survives is a + // decision about a moment in another workspace, and a fresh target adopts + // the bundle's database whole -- so the candidate names a blob the target + // now references, which collection treats as corruption from then on. + const tamper = new DatabaseSync(join(source.workspaceRoot, 'context-offload.sqlite')); + try { + tamper.exec('INSERT INTO context_gc_candidates SELECT blob_id, 0 FROM context_blobs'); + } finally { + tamper.close(); + } + + await assert.rejects( + () => importState(target.workspaceRoot, source.workspaceRoot), + /collection state from the workspace it left/, + ); + } finally { + await rm(source.root, { recursive: true, force: true }); + await rm(target.root, { recursive: true, force: true }); + } +}); + +test('refuses a payload path that is a symlink, even onto matching bytes', async () => { + const source = await makeWorkspace('maka-import-payload-link-source'); + const target = await makeWorkspace('maka-import-payload-link-target'); + try { + const sessionId = await createSession(source.workspaceRoot); + await seedHistory(source.workspaceRoot, sessionId); + const seeded = await seedContext(source.workspaceRoot, sessionId, 'ABC'); + const targetSession = await createSession(target.workspaceRoot, 'Unrelated'); + await seedContext(target.workspaceRoot, targetSession, 'ZZ'); + + // Matching content read through a link is not the same fact as matching + // content at the path: the Context Store refuses to read through one and + // reports the payload corrupt. Accepting it here imports a tree that the + // Store cannot use. + const { symlink } = await import('node:fs/promises'); + const destination = join(target.workspaceRoot, 'context-offload-values', seeded.relativePath); + const decoy = join(target.root, 'decoy'); + await writeFile(decoy, 'ABC'); + await mkdir(dirname(destination), { recursive: true }); + await symlink(decoy, destination); + + await assert.rejects( + () => importState(target.workspaceRoot, source.workspaceRoot), + /Context payload already names different content/, + ); + } finally { + await rm(source.root, { recursive: true, force: true }); + await rm(target.root, { recursive: true, force: true }); + } +}); + +test('reports a payload path occupied by a directory as a conflict', async () => { + const source = await makeWorkspace('maka-import-payload-dir-source'); + const target = await makeWorkspace('maka-import-payload-dir-target'); + try { + const sessionId = await createSession(source.workspaceRoot); + await seedHistory(source.workspaceRoot, sessionId); + const seeded = await seedContext(source.workspaceRoot, sessionId, 'ABC'); + const targetSession = await createSession(target.workspaceRoot, 'Unrelated'); + await seedContext(target.workspaceRoot, targetSession, 'ZZ'); + + // `O_NOFOLLOW` answers the symlink; it says nothing about the other things + // a path can be. A directory opens fine and only fails at the read, which + // surfaces a raw `EISDIR` instead of saying what is actually wrong. + await mkdir(join(target.workspaceRoot, 'context-offload-values', seeded.relativePath), { + recursive: true, + }); + + await assert.rejects( + () => importState(target.workspaceRoot, source.workspaceRoot), + /Context payload already names different content/, + ); + } finally { + await rm(source.root, { recursive: true, force: true }); + await rm(target.root, { recursive: true, force: true }); + } +}); + +test('refuses a bundle carrying a payload nothing references', async () => { + const source = await makeWorkspace('maka-import-orphan-blob-source'); + const target = await makeWorkspace('maka-import-orphan-blob-target'); + try { + const sessionId = await createSession(source.workspaceRoot); + await seedHistory(source.workspaceRoot, sessionId); + await seedContext(source.workspaceRoot, sessionId, 'ABC'); + await createSession(target.workspaceRoot, 'Unrelated'); + + // An export drops blobs nothing references. One that survives is quota the + // target can never reclaim: collection only ever considers a blob after a + // reference releases it, and this one never had a reference to release. + const tamper = new DatabaseSync(join(source.workspaceRoot, 'context-offload.sqlite')); + try { + tamper.exec(` + DELETE FROM context_refs; + DELETE FROM context_session_usage; + `); + } finally { + tamper.close(); + } + + await assert.rejects( + () => importState(target.workspaceRoot, source.workspaceRoot), + /payload nothing references/, + ); + } finally { + await rm(source.root, { recursive: true, force: true }); + await rm(target.root, { recursive: true, force: true }); + } +}); + +test('refuses a bundle carrying usage for a Session it does not hold', async () => { + const source = await makeWorkspace('maka-import-surplus-usage-source'); + const target = await makeWorkspace('maka-import-surplus-usage-target'); + try { + const sessionId = await createSession(source.workspaceRoot); + await seedHistory(source.workspaceRoot, sessionId); + await seedContext(source.workspaceRoot, sessionId, 'ABC'); + await createSession(target.workspaceRoot, 'Unrelated'); + + // Surplus in the direction the aggregate comparison cannot see: every real + // Session's numbers still add up, and this row is simply extra. A fresh + // target adopts it, and the next write for that Session fails against a + // count that describes a store it never had. + const tamper = new DatabaseSync(join(source.workspaceRoot, 'context-offload.sqlite')); + try { + tamper.prepare('INSERT INTO context_session_usage VALUES (?, 0, 0)').run('a-ghost-session'); + } finally { + tamper.close(); + } + + await assert.rejects( + () => importState(target.workspaceRoot, source.workspaceRoot), + /usage for a Session it does not hold/, + ); + } finally { + await rm(source.root, { recursive: true, force: true }); + await rm(target.root, { recursive: true, force: true }); + } +}); + +test('never replaces a context database that is already there', async () => { + const source = await makeWorkspace('maka-import-no-replace-source'); + const target = await makeWorkspace('maka-import-no-replace-target'); + try { + const sessionId = await createSession(source.workspaceRoot); + await seedHistory(source.workspaceRoot, sessionId); + const seeded = await seedContext(source.workspaceRoot, sessionId, 'ABC'); + const targetSession = await createSession(target.workspaceRoot, 'Unrelated'); + const kept = await seedContext(target.workspaceRoot, targetSession, 'ZZ'); + + // Publication creates; it never replaces. What it would replace is a + // database a Context Store may hold open, and on POSIX that Store then + // keeps writing to an unlinked inode -- writes that survive until the next + // restart and are then simply gone. + const imported = await importState(target.workspaceRoot, source.workspaceRoot); + assert.equal(imported.contextRefs, 1); + + const after = new DatabaseSync(join(target.workspaceRoot, 'context-offload.sqlite'), { + readOnly: true, + }); + try { + const rows = after + .prepare('SELECT session_id FROM context_refs ORDER BY session_id') + .all() as Array<{ session_id?: unknown }>; + assert.deepEqual( + rows.map((row) => String(row.session_id)).sort(), + [sessionId, targetSession].sort(), + 'the Session that was already here keeps its reference', + ); + } finally { + after.close(); + } + const values = join(target.workspaceRoot, 'context-offload-values'); + assert.equal(await readFile(join(values, seeded.relativePath), 'utf8'), 'ABC'); + assert.equal(await readFile(join(values, kept.relativePath), 'utf8'), 'ZZ'); + } finally { + await rm(source.root, { recursive: true, force: true }); + await rm(target.root, { recursive: true, force: true }); + } +}); + +test('publishes only the payloads the bundle declares', async () => { + const source = await makeWorkspace('maka-import-undeclared-source'); + const target = await makeWorkspace('maka-import-undeclared-target'); + try { + const sessionId = await createSession(source.workspaceRoot); + await seedHistory(source.workspaceRoot, sessionId); + const seeded = await seedContext(source.workspaceRoot, sessionId, 'ABC'); + await createSession(target.workspaceRoot, 'Unrelated'); + + // Bytes no row names. The validator only ever looks at rows, so a bundle + // can be assembled with these and they arrive charged to nothing and + // reachable by nothing: usage counts blobs, and collection starts from a + // blob whose references were released. + const stowaway = join( + source.workspaceRoot, + 'context-offload-values', + 'sha256', + 'ff', + 'f'.repeat(64), + ); + await mkdir(dirname(stowaway), { recursive: true }); + await writeFile(stowaway, 'NOT DECLARED'); + + const imported = await importState(target.workspaceRoot, source.workspaceRoot); + assert.equal(imported.contextRefs, 1); + + const values = join(target.workspaceRoot, 'context-offload-values'); + assert.equal(await readFile(join(values, seeded.relativePath), 'utf8'), 'ABC'); + assert.equal( + await readFile(join(values, 'sha256', 'ff', 'f'.repeat(64)), 'utf8').catch(() => undefined), + undefined, + 'the undeclared payload did not travel', + ); + } finally { + await rm(source.root, { recursive: true, force: true }); + await rm(target.root, { recursive: true, force: true }); + } +}); + +test('refuses a bundle that declares a payload it does not carry', async () => { + const source = await makeWorkspace('maka-import-missing-payload-source'); + const target = await makeWorkspace('maka-import-missing-payload-target'); + try { + const sessionId = await createSession(source.workspaceRoot); + await seedHistory(source.workspaceRoot, sessionId); + const seeded = await seedContext(source.workspaceRoot, sessionId, 'ABC'); + await createSession(target.workspaceRoot, 'Unrelated'); + + // A row whose bytes are absent. Publishing the rest and committing the + // Session would leave a reference to content nobody can read back. + await rm(join(source.workspaceRoot, 'context-offload-values', seeded.relativePath)); + + await assert.rejects(() => importState(target.workspaceRoot, source.workspaceRoot)); + } finally { + await rm(source.root, { recursive: true, force: true }); + await rm(target.root, { recursive: true, force: true }); + } +}); diff --git a/packages/storage/src/context-offload-snapshot.ts b/packages/storage/src/context-offload-snapshot.ts index ae2ad30cb6..db7f5e0364 100644 --- a/packages/storage/src/context-offload-snapshot.ts +++ b/packages/storage/src/context-offload-snapshot.ts @@ -217,7 +217,26 @@ export async function planContextSnapshotFiles( } /** Verifies both payload integrity and typed ledger/message references before publication. */ -export async function validateContextSnapshot(root: string): Promise { +/** + * Asserts a context tree is the exact shape a snapshot writes. + * + * `copyContextSnapshot` is the only thing that produces one, and it settles + * every transient state before it finishes: the collection queue, the deletion + * queue, blobs nothing references, and usage rows that do not match what is + * there. Checking the payload hashes without checking those leaves a tree that + * decodes but cannot be adopted -- a fresh target takes a snapshot database + * whole, so a surviving deletion queue drains bytes the target never had and a + * forged usage row fails its next write. + * + * `sessionIds`, when given, additionally requires every reference to belong to + * one of them. A bundle is the case where that matters: a reference owned by a + * Session the tree does not carry can never be released, because releasing one + * happens when its Session is retired. + */ +export async function validateContextSnapshot( + root: string, + sessionIds?: readonly string[], +): Promise { let context: DatabaseSync | undefined; const contextPath = join(root, CONTEXT_OFFLOAD_DATABASE_NAME); try { @@ -274,6 +293,48 @@ export async function validateContextSnapshot(root: string): Promise { .get() ) throw new Error('Context snapshot Session usage mismatch'); + // A usage row for a Session with no references is surplus in the other + // direction, which the EXCEPT above cannot see. + if ( + context + .prepare(`SELECT 1 FROM context_session_usage u + WHERE NOT EXISTS (SELECT 1 FROM context_refs r WHERE r.session_id = u.session_id) + LIMIT 1`) + .get() + ) + throw new Error('Context snapshot carries usage for a Session it does not hold'); + // Transient state a snapshot settles. Left behind, a fresh target adopts + // it: the deletion queue drains bytes that were never there, and a blob + // nothing references is quota nothing will reclaim. + for (const [table, described] of [ + ['context_gc_candidates', 'collection state'], + ['context_file_deletions', 'a deletion queue'], + ] as const) { + if (context.prepare(`SELECT 1 FROM ${table} LIMIT 1`).get()) { + throw new Error(`Context snapshot carries ${described} from the workspace it left`); + } + } + if ( + context + .prepare(`SELECT 1 FROM context_blobs b + WHERE NOT EXISTS (SELECT 1 FROM context_refs r WHERE r.blob_id = b.blob_id) + LIMIT 1`) + .get() + ) + throw new Error('Context snapshot carries a payload nothing references'); + if (sessionIds !== undefined) { + const placeholders = sessionIds.map(() => '?').join(', '); + const foreign = context + .prepare( + `SELECT session_id FROM context_refs WHERE session_id NOT IN (${placeholders}) LIMIT 1`, + ) + .get(...sessionIds) as { session_id?: unknown } | undefined; + if (foreign) { + throw new Error( + `Context snapshot references a Session it does not carry: ${String(foreign.session_id)}`, + ); + } + } } validateLedgerContextRefs(root, context); } finally { diff --git a/packages/storage/src/session-bundle-policy.ts b/packages/storage/src/session-bundle-policy.ts index fed3948778..18fb1e9e2d 100644 --- a/packages/storage/src/session-bundle-policy.ts +++ b/packages/storage/src/session-bundle-policy.ts @@ -41,7 +41,7 @@ import { withArtifactWriterLock, withLeaseBoundArtifactWriterLock, } from './artifact-writer-lock.js'; -import { syncDirectoryChain } from './stable-storage.js'; +import { readStableBoundedFile, syncDirectoryChain } from './stable-storage.js'; import { prepareArtifactWriterLockAuthorityForLease, type StorageRootLease, @@ -1184,11 +1184,45 @@ async function copyBundleArtifacts( return { copied, created }; } +/** + * Compares two files without following a symlink at either path. + * + * A payload path is content-addressed, so `EEXIST` there is normally the same + * bytes arriving twice. A symlink planted at that exact path pointing at + * matching content compares equal through an ordinary read, and the import + * accepts a payload tree the Context Store will later reject as corrupt -- + * it refuses to read through a link. Opened no-follow, the planted link is a + * different content instead of the same content. + */ async function sameFileContent(left: string, right: string): Promise { - const [a, b] = await Promise.all([readFile(left), readFile(right)]); - return a.equals(b); + const expected = await readFile(left); + try { + // The repository's reader rather than an open of our own: it is + // non-blocking, so a FIFO planted at the path cannot hang the import, and + // it compares the opened file against `lstat` of the path, so a path that + // stops naming the same file mid-read reads as different content. + // + // On POSIX that also refuses a symlink, because the open carries + // `O_NOFOLLOW`. On Windows the flag is absent and `lstat` does not reliably + // report a file symlink as one, so the symlink refusal there rests on what + // the platform makes visible -- less than this reader gives on POSIX. Only + // `left` is trusted: it is a bundle entry, and the walk that reaches it + // admits `isFile()` directory entries, never a link. + const actual = await readStableBoundedFile({ + path: right, + maxBytes: expected.length, + invalidFile: () => new NotTheSamePayloadError(), + }); + return actual.equals(expected); + } catch (error) { + if (error instanceof NotTheSamePayloadError) return false; + throw error; + } } +/** Internal: the stable reader reports every refusal through one error. */ +class NotTheSamePayloadError extends Error {} + /** * Copy every table the bundle has, in one transaction. * @@ -1305,11 +1339,11 @@ async function mergeBundleContext( // An archive digest authenticates the archive, not the state inside it: it // says the bytes arrived as sent, and nothing about whether a row claiming a - // hash names a file that actually hashes to it. Validated here, against the - // hydrated copy, before anything is written to the target -- a payload that - // fails is a bundle nobody can use, and importing it publishes a reference to - // content that cannot be read back. - await validateContextSnapshot(bundleStateRoot); + // hash names a file that hashes to it, or whether the tree describes more + // than the Sessions it carries. Both are the snapshot's own shape, so one + // validator states it, against the hydrated copy, before anything reaches + // the target -- a second, bundle-only check could only ever drift from it. + await validateContextSnapshot(bundleStateRoot, sessionIds); // Everything below is one turn in the Storage Root's context mutation queue, // shared with the Context Store's own publication and collection. Those @@ -1322,13 +1356,7 @@ async function mergeBundleContext( // Managed payloads live at `sha256//`, so a copy that visited // only immediate children saw one directory, skipped it, and reported a // successful import whose referenced bytes were all absent. - const destination = resolveInside(stateRoot, CONTEXT_OFFLOAD_VALUES_DIRECTORY_NAME); - await mkdir(destination, { recursive: true, mode: 0o700 }); - await copyContextValueTree( - resolveInside(bundleStateRoot, CONTEXT_OFFLOAD_VALUES_DIRECTORY_NAME), - destination, - stateRoot, - ); + await copyDeclaredContextValues(bundleContext, bundleStateRoot, stateRoot); return mergeBundleContextDatabase(bundleContext, stateRoot, sessionIds); }); } @@ -1339,8 +1367,41 @@ async function mergeBundleContextDatabase( sessionIds: readonly string[], ): Promise { const targetContext = resolveInside(stateRoot, CONTEXT_OFFLOAD_DATABASE_NAME); - if (!(await pathExists(targetContext))) { - await copyFile(bundleContext, targetContext); + // One publication path, and the filesystem decides which case this is. + // + // Asking first whether the database exists and branching on the answer is a + // decision that can be stale by the time it is acted on: a Context Store + // initialising under the same lease creates that file, and an import that + // already decided "absent" would then REPLACE it. On POSIX the Store keeps + // writing to the now-unlinked inode while every later open reads the new + // one, so its writes are invisible and gone at the next restart. + // + // `copyFile` also fills its destination progressively, and this destination + // is the path a Store opens to decide whether the workspace has a store at + // all. Staged and linked, it is absent or complete, never partly there. + const staging = `${targetContext}.${process.pid}.${randomUUID()}.tmp`; + let created = false; + try { + await copyFile(bundleContext, staging, constants.COPYFILE_EXCL); + // Synced before it is named, and the directory synced after: the Session + // rows are committed later, and a power loss between the two must not leave + // a Session whose context database is a name with nothing behind it, or no + // name at all. Same ordering the managed payloads use. + const handle = await open(staging, 'r+'); + try { + await handle.sync(); + } finally { + await handle.close(); + } + await link(staging, targetContext); + created = true; + await syncDirectoryChain(dirname(targetContext), stateRoot); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + } finally { + await rm(staging, { force: true }).catch(() => {}); + } + if (created) { return countContextRefs(targetContext, sessionIds); } const database = new DatabaseSync(targetContext); @@ -1423,27 +1484,58 @@ async function mergeBundleContextDatabase( * Content-addressed names mean an existing file is the same file, so an * already-present payload is left alone rather than treated as a conflict. */ -async function copyContextValueTree( - source: string, - destination: string, +/** + * Publishes exactly the payloads the bundle's database declares. + * + * Walking the tree and copying every regular file publishes whatever is there, + * and the validator only ever looks at rows: a hand-built bundle can carry + * bytes no row names, and those arrive charged to nothing and reachable by + * nothing -- not by usage, which counts blobs, and not by collection, which + * starts from a blob whose references were released. The locators are the one + * list both sides agree on, so they are what gets copied. + * + * Reading them raw is safe because the validator has already run: it derives + * each locator from its blob id and refuses anything that is not exactly + * `sha256//`, so nothing here can name a path of its own + * choosing. Moving this before that check would remove that guarantee. + */ +async function copyDeclaredContextValues( + bundleContext: string, + bundleStateRoot: string, stateRoot: string, ): Promise { - if (!(await pathExists(source))) return; - await assertManagedDestinationDirectory(destination, stateRoot); - for (const entry of await readdir(source, { withFileTypes: true })) { - const from = resolveInside(source, entry.name); - const to = resolveInside(destination, entry.name); - if (entry.isDirectory()) { - await mkdir(to, { recursive: true, mode: 0o700 }); - await copyContextValueTree(from, to, stateRoot); - continue; + const locators: string[] = []; + const database = new DatabaseSync(bundleContext, { readOnly: true }); + try { + for (const row of database + .prepare("SELECT payload FROM context_blobs WHERE storage_kind = 'managed_file'") + .iterate() as Iterable<{ payload?: unknown }>) { + const payload = row.payload; + if (!(payload instanceof Uint8Array)) { + throw new SessionBundleImportError('invalid_root', 'Bundle context locator is unreadable'); + } + locators.push(Buffer.from(payload).toString('utf8')); } - if (!entry.isFile()) { + } finally { + database.close(); + } + + const sourceRoot = resolveInside(bundleStateRoot, CONTEXT_OFFLOAD_VALUES_DIRECTORY_NAME); + const destinationRoot = resolveInside(stateRoot, CONTEXT_OFFLOAD_VALUES_DIRECTORY_NAME); + await mkdir(destinationRoot, { recursive: true, mode: 0o700 }); + await assertManagedDestinationDirectory(destinationRoot, stateRoot); + for (const locator of locators) { + const from = resolveInside(sourceRoot, locator); + const to = resolveInside(destinationRoot, locator); + const entry = await lstat(from).catch(() => undefined); + if (!entry?.isFile()) { throw new SessionBundleImportError( - 'io_failed', - `Bundle context payload is not a regular file: ${entry.name}`, + 'invalid_root', + `Bundle context declares a payload it does not carry: ${locator}`, ); } + await mkdir(dirname(to), { recursive: true, mode: 0o700 }); + await assertManagedDestinationDirectory(dirname(to), stateRoot); await publishContextValue(from, to, stateRoot); } }