diff --git a/architecture.md b/architecture.md index e91b2e364..8c8149ccb 100644 --- a/architecture.md +++ b/architecture.md @@ -6361,6 +6361,28 @@ projection detaches every value it retains and freezes it, so nothing below holds a `DurableEvent` and nothing above can change one. Route resolution is pure and answers with the exact objects that model holds. +### Source ownership in retained entries + +A source position identifies authored bytes, not which invocation used them. +Repeated calls therefore retain `executablemd.source-owner` on the operations +the REPL attributes: the coroutine ID and zero-based yield index of the prior +successful source import. Canonical import admission captures this coordinate, +and Core binds the internal `SourceOwner` Context, named +`executablemd.source-owner`, around the source body. Branches and iterations +inherit the binding. A component body uses its own import; projected caller +content restores its captured caller binding. Attribution writers read it before +handing work to a separately owned task, so copying a public code-block context +preserves attribution. It grants no permission and creates no effect, coroutine +or identity allocator. + +Projection resolves the exact import and checks its source, prior order and +coroutine ancestry. An invalid explicit reference refuses; it never falls back +to a path guess. Legacy histories use the existing unique-owner inference and +retain their ambiguity refusals. Continuation can reference a legacy import +because both live and replayed yields advance the existing durable position. +Neither reading nor continuation rewrites earlier records to add ownership. +Generated-fragment attribution keeps its existing admission identity. + ### Sequential entries over one physical journal One execution is one append-only stream, and entries are ranges of it. A segment diff --git a/packages/cli/src/repl/model.ts b/packages/cli/src/repl/model.ts index fb42234a1..01d4847ca 100644 --- a/packages/cli/src/repl/model.ts +++ b/packages/cli/src/repl/model.ts @@ -61,6 +61,7 @@ export const ENTRY_SCOPE = entryKey(1); const ROOT_PROPS_BINDING = "props"; const SOURCE_POSITION_FIELD = "executablemd.source-position"; +const SOURCE_OWNER_FIELD = "executablemd.source-owner"; /** What the REPL could not read, and what it was reading when it stopped. */ export class ReplProjectionError extends Error { @@ -741,7 +742,16 @@ function buildEntry(segment: EntrySegment, markers: readonly string[]): Result(); + const sources = { paths: new Map(), imports: new Map() }; + const yields = new Map(); + const coordinates = events.map((event) => { + if (event.type !== "yield") { + return undefined; + } + const index = yields.get(event.coroutineId) ?? 0; + yields.set(event.coroutineId, index + 1); + return `${event.coroutineId}:${index}`; + }); /** Every generated fragment this prefix admitted, by the identity it carries. */ const fragments: GeneratedOwner[] = []; /** @@ -780,7 +790,9 @@ function buildEntry(segment: EntrySegment, markers: readonly string[]): Result, + index: { paths: Map; imports: Map }, fragments: readonly GeneratedOwner[], announced: ReadonlySet, position: ReplPosition | undefined, @@ -1459,10 +1466,47 @@ function ownerOf( ), ); } + if (where.sourceOwner !== undefined) { + return explicitOwnerOf(index.imports, where.sourceOwner, position, where, subject); + } if (position.path === undefined) { return generatedOwnerOf(fragments, announced, position, where, subject); } - return pathOwnerOf(index.get(position.path) ?? [], position, where, subject); + return pathOwnerOf(index.paths.get(position.path) ?? [], position, where, subject); +} + +function explicitOwnerOf( + imports: ReadonlyMap, + reference: Json, + position: ReplPosition, + where: EffectSite, + subject: string, +): Result<{ scope: ScopeDraft; position: ReplPosition }> { + if ( + !isJsonObject(reference) || + Object.keys(reference).length !== 2 || + typeof reference.coroutineId !== "string" || + typeof reference.index !== "number" || + !Number.isSafeInteger(reference.index) || + reference.index < 0 + ) { + return Err(new ReplProjectionError(`${subject} carries an unreadable source owner.`)); + } + const admission = imports.get(`${reference.coroutineId}:${reference.index}`); + if (admission === undefined || admission.order >= where.order) { + return Err(new ReplProjectionError(`${subject} names no earlier admitted source owner.`)); + } + if (position.path === undefined || admission.scope.path !== position.path) { + return Err( + new ReplProjectionError(`${subject} names an owner admitted from a different source.`), + ); + } + if (!descendsFrom(admission.coroutine, where.coroutine)) { + return Err( + new ReplProjectionError(`${subject} names a source owner on work it is not part of.`), + ); + } + return Ok({ scope: admission.scope, position }); } /** diff --git a/packages/cli/tests/fixtures/repl/OwnerAnswer.ts b/packages/cli/tests/fixtures/repl/OwnerAnswer.ts new file mode 100644 index 000000000..57582015e --- /dev/null +++ b/packages/cli/tests/fixtures/repl/OwnerAnswer.ts @@ -0,0 +1,29 @@ +import { exists, readTextFile, writeTextFile } from "@effectionx/fs"; +import { content, Elicitation } from "@executablemd/core"; +import type { Json } from "@executablemd/core"; +import { cwd } from "@executablemd/runtime"; +import type { Operation } from "effection"; +import { join } from "node:path"; + +export const props = { + type: "object", + properties: { path: { type: "string" } }, + required: ["path"], + additionalProperties: false, +}; + +export default function* OwnerAnswer(elementProps: Record): Operation { + if (typeof elementProps.path !== "string") { + throw new Error("the fixture requires its counter path"); + } + const path = join(yield* cwd(), elementProps.path); + yield* Elicitation.around({ + *elicit([request]) { + const before = (yield* exists(path)) ? yield* readTextFile(path) : ""; + const answer = request.message.trim(); + yield* writeTextFile(path, `${before}${answer}\n`); + return answer; + }, + }); + return yield* content(); +} diff --git a/packages/cli/tests/fixtures/repl/OwnerContent.md b/packages/cli/tests/fixtures/repl/OwnerContent.md new file mode 100644 index 000000000..772dbb91c --- /dev/null +++ b/packages/cli/tests/fixtures/repl/OwnerContent.md @@ -0,0 +1 @@ + diff --git a/packages/cli/tests/fixtures/repl/OwnerEval.md b/packages/cli/tests/fixtures/repl/OwnerEval.md new file mode 100644 index 000000000..ef6a17946 --- /dev/null +++ b/packages/cli/tests/fixtures/repl/OwnerEval.md @@ -0,0 +1,14 @@ +--- +required: [input] +props: + input: + type: string +returns: + type: string +--- + +```js eval +const value = props.input; +``` + + diff --git a/packages/cli/tests/fixtures/repl/OwnerLeaf.md b/packages/cli/tests/fixtures/repl/OwnerLeaf.md new file mode 100644 index 000000000..8d204507f --- /dev/null +++ b/packages/cli/tests/fixtures/repl/OwnerLeaf.md @@ -0,0 +1,11 @@ +--- +required: [input] +props: + input: + type: string +returns: + type: string +--- + +{props.input} + diff --git a/packages/cli/tests/fixtures/repl/OwnerWrapper.md b/packages/cli/tests/fixtures/repl/OwnerWrapper.md new file mode 100644 index 000000000..f4124f6d6 --- /dev/null +++ b/packages/cli/tests/fixtures/repl/OwnerWrapper.md @@ -0,0 +1,11 @@ +--- +required: [input] +props: + input: + type: string +returns: + type: string +--- + + + diff --git a/packages/cli/tests/fixtures/repl/source-ownership.ts b/packages/cli/tests/fixtures/repl/source-ownership.ts new file mode 100644 index 000000000..9ee3e5d55 --- /dev/null +++ b/packages/cli/tests/fixtures/repl/source-ownership.ts @@ -0,0 +1,47 @@ +import { collect, Elicitation, inlineSource } from "@executablemd/core"; +import { executeInstalled } from "@executablemd/core/host"; +import { InMemoryStream } from "@executablemd/durable-streams"; +import type { DurableEvent } from "@executablemd/durable-streams"; +import { scoped } from "effection"; +import type { Operation } from "effection"; +import { fileURLToPath } from "node:url"; + +export const OWNER_DIRECTORY = fileURLToPath(new URL("./", import.meta.url)); +export const OWNER_ENTRY = [ + '', + '', + "{first}/{second}", +].join("\n"); +export const OWNER_FIELD = "executablemd.source-owner"; + +export function ownerRun( + source: string = OWNER_ENTRY, + stream: InMemoryStream = new InMemoryStream(), + answer: (message: string) => Operation = function* (message) { + return message.trim(); + }, +): Operation<{ output: string; events: DurableEvent[] }> { + return scoped(function* () { + yield* Elicitation.around({ + *elicit([request]) { + return yield* answer(request.message); + }, + }); + const execution = yield* executeInstalled( + { ...inlineSource(source), stream, includes: [OWNER_DIRECTORY] }, + [], + ); + const output = String(yield* collect(execution)); + return { output, events: yield* stream.readAll() }; + }); +} + +export function legacyOwners(events: readonly DurableEvent[]): DurableEvent[] { + return events.map((event) => { + if (event.type !== "yield") { + return event; + } + const { [OWNER_FIELD]: _owner, ...description } = event.description; + return { ...event, description }; + }); +} diff --git a/packages/cli/tests/repl-include-pty.test.ts b/packages/cli/tests/repl-include-pty.test.ts index 24b4fa1d5..2c3bd606c 100644 --- a/packages/cli/tests/repl-include-pty.test.ts +++ b/packages/cli/tests/repl-include-pty.test.ts @@ -1,3 +1,4 @@ +import { OWNER_DIRECTORY, OWNER_ENTRY } from "./fixtures/repl/source-ownership.ts"; import { describe, it } from "@executablemd/test-support/bdd"; import { expect } from "@executablemd/test-support/expect"; import { @@ -11,8 +12,9 @@ import { ensureDir, exists, readTextFile, rm, writeTextFile } from "@effectionx/ import { parseDurableEvent } from "@executablemd/durable-streams"; import { scoped, sleep, until } from "effection"; import type { Operation } from "effection"; -import { readdir } from "node:fs/promises"; +import { readdir, symlink } from "node:fs/promises"; import { join, sep } from "node:path"; +import { fileURLToPath } from "node:url"; import { platform } from "node:os"; import { platformDataRoot } from "../src/repl-assembly.ts"; import { projectRepl } from "../src/repl/model.ts"; @@ -82,24 +84,19 @@ function modelOf(bytes: string) { } function* completed(pty: Pty, history: string, count: number): Operation { - const deadline = Date.now() + ACT_MS; - while (true) { - const bytes = yield* readTextFile(history); - const model = modelOf(bytes); - if ( - model.entries.length === count && - model.entries.every((entry) => entry.terminal?.status === "ok") - ) { - yield* showing(pty, `Ready for Entry ${count + 1}`, ACT_MS); - return bytes; - } - if (Date.now() > deadline) { - throw new Error( - `entries never closed. history=${bytes} screen=${JSON.stringify(yield* pty.shows())}`, - ); - } - yield* sleep(150); + // Ready requires the journal close and the entry's joined teardown. + yield* showing(pty, `Ready for Entry ${count + 1}`, ACT_MS); + const bytes = yield* readTextFile(history); + const model = modelOf(bytes); + if ( + model.entries.length !== count || + model.entries.some((entry) => entry.terminal?.status !== "ok") + ) { + throw new Error( + `entries never closed. history=${bytes} screen=${JSON.stringify(yield* pty.shows())}`, + ); } + return bytes; } function* leave(pty: Pty): Operation { @@ -261,3 +258,141 @@ describe("REPL includes through the production source terminal", () => { } }); }); + +describe("REPL sequential source owners through a real terminal (#976)", () => { + it("SO3–SO4: inspect both nested calls, then reopen completed work in another process without repeating its provider", function* () { + const root = yield* useTempDirectory("xmd-o-"); + const workspace = join(root, "workspace"); + const includes = join(root, "components"); + const home = join(root, "home"); + for (const directory of [workspace, includes, home]) { + yield* ensureDir(directory); + } + yield* writeTextFile(join(root, "package.json"), JSON.stringify({ type: "module" })); + yield* until( + symlink( + fileURLToPath(new URL("../../../node_modules", import.meta.url)), + join(root, "node_modules"), + "dir", + ), + ); + for (const file of ["OwnerWrapper.md", "OwnerLeaf.md", "OwnerAnswer.ts"]) { + yield* writeTextFile(join(includes, file), yield* readTextFile(join(OWNER_DIRECTORY, file))); + } + let cache = ""; + if (cliRuntime() === "deno") { + const info: unknown = JSON.parse((yield* runShell("deno info --json").expect()).stdout); + if ( + typeof info !== "object" || + info === null || + !("denoDir" in info) || + typeof info.denoDir !== "string" + ) { + throw new Error("deno info supplied no prepared cache path"); + } + cache = `DENO_DIR=${shellQuote(info.denoDir)} `; + } + const line = (args: string[]): string => + `cd ${shellQuote(workspace)} && ${cache}${cliShellCommand(["repl", ...args, "--include", includes, "--deny-all"])}`; + const source = `\n${OWNER_ENTRY}\n`; + let location = ""; + let history = ""; + let retained = ""; + function* activateKey(pty: Pty, label: string, occurrence = 0): Operation { + for (let step = 0; step < 160; step++) { + const screen = yield* pty.shows(); + const matching = screen.filter((line) => { + const at = line.indexOf(label); + return at >= 0 && (!label.startsWith("component ") || at < 36); + }); + const row = matching[occurrence]; + const focused = label.startsWith("component ") + ? row !== undefined && row.slice(0, row.indexOf(label)).includes(">") + : screen.some((line) => { + for (let at = line.indexOf(">"); at !== -1; at = line.indexOf(">", at + 1)) { + if ( + line + .slice(at + 1) + .trimStart() + .replace(/^\* /, "") + .startsWith(label) + ) { + return true; + } + } + return false; + }); + if (focused) { + yield* pty.type("\r"); + yield* sleep(150); + return; + } + yield* pty.type("\t"); + yield* sleep(40); + } + throw new Error( + `focus never reached ${label} (${occurrence}): ${JSON.stringify(yield* pty.shows())}`, + ); + } + function* inspect(pty: Pty): Operation { + for (const [index, expected] of ["first", "second"].entries()) { + yield* activateKey(pty, "component OwnerLeaf", index); + yield* showingAcross(pty, ' { + yield* activateKey(pty, "[exit]"); + const deadline = Date.now() + ACT_MS; + while ((yield* pty.status()) === undefined) { + if (Date.now() > deadline) { + throw new Error("the terminal journey did not exit"); + } + yield* sleep(150); + } + expect(yield* pty.status()).toBe(0); + const screen = (yield* pty.shows()).join("\n"); + const found = screen.match(/xmd repl '(xmd:\/\/repl\/[^']+)'/); + if (found?.[1] === undefined) { + throw new Error(`no printed location: ${screen}`); + } + return found[1]; + } + yield* scoped(function* () { + const pty = yield* allocate(line([]), home); + yield* showing(pty, "Ready for Entry 1", REACH_MS); + history = yield* historyPath(home); + yield* submit(pty, source); + retained = yield* completed(pty, history, 1); + expect(modelOf(retained).terminal?.output?.trim()).toBe("first/second"); + expect(yield* readTextFile(join(workspace, "provider-calls.txt"))).toBe("first\nsecond\n"); + yield* inspect(pty); + location = yield* finish(pty); + }); + yield* rm(includes, { recursive: true }); + yield* scoped(function* () { + const pty = yield* allocate(line([location]), home); + yield* showing(pty, "Ready for Entry 2", REACH_MS); + yield* inspect(pty); + yield* finish(pty); + }); + expect(yield* readTextFile(history)).toBe(retained); + expect(yield* readTextFile(join(workspace, "provider-calls.txt"))).toBe("first\nsecond\n"); + expect(modelOf(yield* readTextFile(history))).toEqual(modelOf(retained)); + }); +}); diff --git a/packages/cli/tests/repl-model.test.ts b/packages/cli/tests/repl-model.test.ts index 6e38a5dbc..77d0c718c 100644 --- a/packages/cli/tests/repl-model.test.ts +++ b/packages/cli/tests/repl-model.test.ts @@ -1,3 +1,11 @@ +import { scoped, spawn, suspend, withResolvers } from "effection"; +import { InMemoryStream } from "@executablemd/durable-streams"; +import { + legacyOwners, + OWNER_ENTRY, + OWNER_FIELD, + ownerRun, +} from "./fixtures/repl/source-ownership.ts"; /** * What the REPL knows, and where it learned it (#848 M1, M2). * @@ -12,7 +20,7 @@ import { beforeAll, describe, it } from "@executablemd/test-support/bdd"; import { expect } from "@executablemd/test-support/expect"; -import { useTempFileCompiler } from "@executablemd/core"; +import { Component, useTempFileCompiler } from "@executablemd/core"; import { parseDurableEvent, serializeDurableEvent } from "@executablemd/durable-streams"; import type { DurableEvent, Json } from "@executablemd/durable-streams"; @@ -414,7 +422,7 @@ describe("REPL model: the histories it refuses", () => { column: 1, }, })); - expect(refusal(elsewhere)).toContain("never admitted"); + expect(refusal(elsewhere)).toContain("different source"); } }); @@ -851,7 +859,7 @@ describe("REPL model: the Agent histories it refuses", () => { ...description, "executablemd.source-position": { path: "somewhere-else.md", offset: 0, line: 1, column: 1 }, })); - expect(refusal(elsewhere)).toContain("never admitted"); + expect(refusal(elsewhere)).toContain("different source"); const corrupted = withDescription(events, at, (description) => ({ ...description, @@ -1406,3 +1414,237 @@ describe("REPL model: which call of a repeated source owns its work", () => { yield* useTempFileCompiler(); }); }); + +describe("REPL source ownership by import coordinate (#976)", () => { + beforeAll(() => useTempFileCompiler()); + + it("SO1: sequential nested calls keep their own answers at every prefix and after serialization", function* () { + const run = yield* ownerRun(); + expect(run.output.trim()).toBe("first/second"); + for (let end = 1; end <= run.events.length; end++) { + const events = run.events.slice(0, end); + const model = projected(events); + expect(projected(copied(events))).toEqual(model); + const wrappers = only(model)?.scopes ?? []; + for (const [index, wrapper] of wrappers.entries()) { + const leaf = wrapper.scopes[0]; + expect(leaf?.name ?? "OwnerLeaf").toBe("OwnerLeaf"); + for (const question of leaf?.elicitations ?? []) { + expect(question).toMatchObject({ answer: index === 0 ? "first" : "second" }); + } + } + } + const leafImports = run.events.filter( + (event) => event.type === "yield" && event.description.name === "OwnerLeaf", + ); + expect( + leafImports.map((event) => + event.type === "yield" ? event.description[OWNER_FIELD] : undefined, + ), + ).toEqual([ + { coroutineId: "root", index: 1 }, + { coroutineId: "root", index: 5 }, + ]); + const entry = only(projected(run.events)); + expect(entry?.scopes.map((wrapper) => wrapper.scopes.map((leaf) => leaf.name))).toEqual([ + ["OwnerLeaf"], + ["OwnerLeaf"], + ]); + expect(entry?.bindings).toEqual([ + { name: "first", value: "first" }, + { name: "second", value: "second" }, + ]); + }); + + it("SO2: repeated loop sites and caller Content keep their lexical source owner", function* () { + const source = + '\n'; + const run = yield* ownerRun(source); + const entry = only(projected(run.events)); + expect(entry?.scopes.map((scope) => scope.name)).toEqual([ + "OwnerWrapper", + "OwnerWrapper", + "OwnerContent", + "OwnerWrapper", + ]); + expect(entry?.scopes[2].scopes).toEqual([]); + expect( + entry?.scopes + .filter((scope) => scope.name === "OwnerWrapper") + .map((scope) => scope.scopes[0]?.elicitations[0]), + ).toMatchObject([{ answer: "same" }, { answer: "same" }, { answer: "caller" }]); + const wrappers = run.events.filter( + (event) => event.type === "yield" && event.description.name === "OwnerWrapper", + ); + expect(wrappers).toHaveLength(3); + const yields = run.events.filter((event) => event.type === "yield"); + const wrapperCoordinates = yields.flatMap((event, index) => + event.description.name === "OwnerWrapper" ? [{ coroutineId: event.coroutineId, index }] : [], + ); + const leafOwners = yields + .filter((event) => event.description.name === "OwnerLeaf") + .map((event) => event.description[OWNER_FIELD]); + expect(leafOwners).toEqual(wrapperCoordinates); + expect( + wrappers[0].type === "yield" && wrappers[0].description["executablemd.source-position"], + ).toEqual( + wrappers[1].type === "yield" && wrappers[1].description["executablemd.source-position"], + ); + + for (const event of wrappers) { + if (event.type === "yield") { + expect(event.description[OWNER_FIELD]).toEqual({ coroutineId: "root", index: 0 }); + } + } + }); + + it("SO7: ordinary middleware copies preserve evaluated blocks' source ownership", function* () { + const source = [ + '', + '', + "", + "```js eval", + 'const callerValue = "caller";', + "```", + "", + "{first}/{second}", + ].join("\n"); + for (const copy of [false, true]) { + yield* scoped(function* () { + let forwarded = 0; + if (copy) { + yield* Component.around({ + *applyModifiers([modifiers, block], next) { + forwarded += 1; + return yield* next(modifiers, { ...block }); + }, + }); + } + const run = yield* ownerRun(source); + expect(run.output.trim()).toBe("first/second"); + expect(forwarded).toBe(copy ? 3 : 0); + const evaluations = run.events.filter( + (event) => event.type === "yield" && event.description.type === "eval", + ); + expect( + evaluations.map((event) => + event.type === "yield" ? event.description[OWNER_FIELD] : undefined, + ), + ).toEqual([ + { coroutineId: "root", index: 1 }, + { coroutineId: "root", index: 3 }, + { coroutineId: "root", index: 0 }, + ]); + for (let end = 1; end <= run.events.length; end++) { + const prefix = run.events.slice(0, end); + expect(projected(copied(prefix))).toEqual(projected(prefix)); + } + const entry = only(projected(copied(run.events))); + if (entry === undefined) { + throw new Error("the journal admits one entry"); + } + expect(entry.scopes.map((scope) => scope.bindings)).toEqual([ + [{ name: "value", value: "first" }], + [{ name: "value", value: "second" }], + [], + ]); + expect(binding(entry, "callerValue")).toBe("caller"); + }); + } + }); + + it("SO5: an interrupted legacy prefix continues with links to its existing import coordinates", function* () { + const reached = withResolvers(); + const source = 'ack\n' + OWNER_ENTRY; + const stream = new InMemoryStream(); + let performed = 0; + let prefix: DurableEvent[] = []; + yield* scoped(function* () { + const running = yield* spawn(() => + ownerRun(source, stream, function* (message) { + if (message.trim() === "first") { + reached.resolve(); + yield* suspend(); + } + performed += 1; + return message.trim(); + }), + ); + yield* reached.operation; + prefix = yield* stream.readAll(); + yield* running.halt(); + }); + expect(performed).toBe(1); + expect(prefix.every((event) => event.type !== "close")).toBe(true); + for (const old of [false, true]) { + const initial = old ? legacyOwners(prefix) : copied(prefix); + const continuing = new InMemoryStream(initial); + const continued = yield* ownerRun(source, continuing, function* (message) { + performed += 1; + return message.trim(); + }); + expect(continued.output.trim()).toBe("first/second"); + expect(continued.events.slice(0, initial.length)).toEqual(initial); + const model = projected(continued.events); + expect( + only(model)?.scopes.map((wrapper) => wrapper.scopes[0]?.elicitations[0]), + ).toMatchObject([{ answer: "first" }, { answer: "second" }]); + const suffix = continued.events.slice(initial.length); + const question = suffix.find( + (event) => event.type === "yield" && event.description.type === "elicit", + ); + if (question?.type !== "yield") { + throw new Error("the suffix records its second answer"); + } + const owner = question.description[OWNER_FIELD]; + expect(owner).toEqual({ coroutineId: "root", index: initial.length - 2 }); + } + expect(performed).toBe(5); + }); + + it("SO6: legacy uniqueness, legacy ambiguity and invalid explicit links retain their distinct refusals", function* () { + const reference = yield* referenceEvents(); + expect(projected(legacyOwners(reference))).toEqual(projected(reference)); + const run = yield* ownerRun(); + expect(refusal(legacyOwners(run.events))).toContain("more than once"); + const at = run.events.findIndex( + (event) => event.type === "yield" && event.description.name === "OwnerLeaf", + ); + const invalid: Json[] = [ + null, + {}, + { coroutineId: "root", index: -1 }, + { coroutineId: "root", index: 999 }, + { coroutineId: "root", index: 5 }, + { coroutineId: "missing", index: 0 }, + { coroutineId: "root", index: 0 }, + ]; + for (const reference of invalid) { + expect( + refusal( + withDescription(run.events, at, (description) => ({ + ...description, + [OWNER_FIELD]: reference, + })), + ), + ).toMatch(/owner/); + } + const legacy = [...entryAdmitted(), calledPlan("root.0"), askedInPlan("root.1")]; + expect( + refusal( + withDescription(legacy, 2, (description) => ({ + ...description, + [OWNER_FIELD]: { coroutineId: "root.0", index: 0 }, + })), + ), + ).toContain("not part of"); + expect( + refusal([ + ...entryAdmitted(), + calledPlan("root.0"), + calledPlan("root.0"), + askedInPlan("root.0"), + ]), + ).toContain("more than once"); + }); +}); diff --git a/packages/core/src/agent/journal.ts b/packages/core/src/agent/journal.ts index 8c6ea877c..ddc303118 100644 --- a/packages/core/src/agent/journal.ts +++ b/packages/core/src/agent/journal.ts @@ -46,7 +46,7 @@ * associates nothing. */ -import { createDurableOperation, serializeError } from "@executablemd/durable-streams"; +import { createDurableOperation, ephemeral, serializeError } from "@executablemd/durable-streams"; import type { ActivateDurabilityFailure, DurableStream, @@ -73,7 +73,7 @@ import type { SerializedPromptFailure } from "./errors.ts"; import { AgentInternal } from "./internal.ts"; import { HostObservationFailure } from "./publication.ts"; import type { AgentPromptAssociation, AgentPromptHandle } from "./publication.ts"; -import { sourceDescription } from "../source-position.ts"; +import { SourceOwner, sourceOwnerDescription } from "../source-owner.ts"; import type { SourcePosition } from "../types.ts"; /** The durable effect type every journaled Agent Prompt is recorded under. */ @@ -419,17 +419,22 @@ function permissionDecision(outcome: PermissionOutcome): PermissionOutcome | und } export function* persistPrompt( - identity: { name: string; input: string; position?: Readonly }, + identity: { + name: string; + input: string; + position?: Readonly; + }, live: () => Operation, association: () => AgentPromptAssociation | undefined = () => undefined, begun: () => AgentPromptHandle = () => undefined, ): Workflow { + const sourceOwner = yield* ephemeral(SourceOwner.get()); const stored = yield createDurableOperation( { type: AGENT_PROMPT, name: identity.name, input: identity.input, - ...sourceDescription(identity.position), + ...sourceOwnerDescription(identity.position, sourceOwner), }, function* (): Operation { return serializePromptRecord(yield* live()); diff --git a/packages/core/src/elicit-journal.ts b/packages/core/src/elicit-journal.ts index 668384c92..9d171a96e 100644 --- a/packages/core/src/elicit-journal.ts +++ b/packages/core/src/elicit-journal.ts @@ -26,6 +26,7 @@ import { createDurableOperation, + ephemeral, ReplayGuard, StaleInputError, } from "@executablemd/durable-streams"; @@ -38,7 +39,7 @@ import type { Operation } from "effection"; import { canonicalFingerprint } from "./canonical.ts"; import { isJsonObject, parseJson } from "./json.ts"; -import { sourceDescription } from "./source-position.ts"; +import { SourceOwner, sourceOwnerDescription } from "./source-owner.ts"; import type { Json, JsonObject, SourcePosition } from "./types.ts"; const ELICIT = "elicit"; @@ -148,13 +149,14 @@ export function* persistElicitation( identity: ElicitationIdentity, live: () => Operation, ): Workflow { + const sourceOwner = yield* ephemeral(SourceOwner.get()); const stored = yield createDurableOperation( { type: ELICIT, name: recordName(identity.location), input: identity.fingerprint, [ELICITATION_SCHEMA_FIELD]: identity.schema, - ...sourceDescription(identity.position), + ...sourceOwnerDescription(identity.position, sourceOwner), }, function* (): Operation { return parseJson(yield* live()); diff --git a/packages/core/src/eval-handler.ts b/packages/core/src/eval-handler.ts index f09b7673d..2286e712b 100644 --- a/packages/core/src/eval-handler.ts +++ b/packages/core/src/eval-handler.ts @@ -26,7 +26,7 @@ import { validateLiveOverlay, } from "./live-env.ts"; import { EphemeralEval, EphemeralEvalOutputError } from "./modifiers/ephemeral.ts"; -import { sourceDescription } from "./source-position.ts"; +import { SourceOwner, sourceOwnerDescription } from "./source-owner.ts"; import type { CodeBlockContext, CodeBlockResult, EvalEnv } from "./types.ts"; function* runEphemeralEval( @@ -117,6 +117,7 @@ function runBlock(blockId: string, block: () => Operation): Operation { export const evalFactory: ModifierFactory = (_params) => (_args, _next) => (function* () { const ctx = yield* useCodeBlock(); + const sourceOwner = yield* ephemeral(SourceOwner.get()); const evalEnv = yield* ephemeral(env); if (!evalEnv) { throw new Error( @@ -155,7 +156,7 @@ export const evalFactory: ModifierFactory = (_params) => (_args, _next) => type: "eval", name: `eval:${ctx.blockId}`, ...(ctx.language ? { language: ctx.language } : {}), - ...sourceDescription(ctx.position), + ...sourceOwnerDescription(ctx.position, sourceOwner), }, function* (): Operation { Object.assign(evalEnv.values, bindings); diff --git a/packages/core/src/execute.ts b/packages/core/src/execute.ts index 5856e4847..347987dbb 100644 --- a/packages/core/src/execute.ts +++ b/packages/core/src/execute.ts @@ -95,8 +95,17 @@ import { filesFatalFailure, useSegmentCauses, } from "./errors.ts"; -import { Component, importComponent, isMissingImportProvider, raise } from "./component-api.ts"; +import { + Component, + importComponent, + importThroughTerminal, + isMissingImportProvider, + raise, +} from "./component-api.ts"; import { sourceDescription } from "./source-position.ts"; +import { SourceOwner, sourceOwnerDescription } from "./source-owner.ts"; +import { durablePosition } from "@executablemd/durable-streams"; +import type { DurablePosition } from "@executablemd/durable-streams"; import { emissions, exactly, renderSegment } from "./render.ts"; import { createExactSource } from "./output/exact-source.ts"; import type { ExactSource as ExactSourceRecord } from "./output/exact-source.ts"; @@ -905,15 +914,22 @@ function* durableImportComponent( root: RootDocumentSource | undefined, position: Readonly | undefined, inputs: ImportInputs, + admitted: ((owner: DurablePosition) => void) | undefined, ): Workflow { // Taken before the durable operation and outside it, because the offer is // canonical core's own and a replay has to reach this the same way the live // run did: the element asking is inside the declaration's body, or it is not. const claimed = name === "__root__" ? undefined : inputs.declared?.claim(name); + const sourceOwner = yield* ephemeral(SourceOwner.get()); + const coordinate = yield* ephemeral(durablePosition()); const recorded = yield createDurableOperation( // The root import is the run's own entry rather than an authored element, // so it carries no source however it was reached. - { type: "import_component", name, ...(root ? {} : sourceDescription(position)) }, + { + type: "import_component", + name, + ...(root ? {} : sourceOwnerDescription(position, sourceOwner)), + }, () => selectImport(name, root, inputs, claimed), ); @@ -925,7 +941,9 @@ function* durableImportComponent( throw new Error(name === "__root__" ? UNREADABLE_ROOT_RECORD : UNREADABLE_IMPORT_RECORD); } - return yield* ephemeral(materializeImport(name, selection, inputs, claimed)); + const definition = yield* ephemeral(materializeImport(name, selection, inputs, claimed)); + admitted?.(coordinate); + return definition; } function isFunctionComponent(value: unknown): value is FunctionComponent { @@ -2185,9 +2203,16 @@ function* documentWorkflow( // installed by execute maps "__root__" to the run's root document source. // The ephemeral() wrapper bridges typing only — the import inside remains a // durable, journaled operation. + let sourceOwner: DurablePosition | undefined; + let rootAdmissions = 0; const imported = yield* ephemeral( (function* (): Operation { - const imported = yield* importComponent("__root__"); + const imported = yield* importThroughTerminal("__root__", undefined, (name, position) => + importTerminal(name, position, undefined, (owner) => { + rootAdmissions += 1; + sourceOwner = rootAdmissions === 1 ? owner : undefined; + }), + ); const imports = environment.componentResolution; // Asked only when a tier actually closes this name, exactly as an // ordinary import is. The environment used to be absent altogether for a @@ -2269,6 +2294,7 @@ function* documentWorkflow( const rootPath = ""; const scopedExpansion: Operation = scoped(function* () { + yield* SourceOwner.set(sourceOwner); // Before any authored work, and in this scope rather than outside // `durableRun`: this is the scope `Execution.document` has already // installed the document's runtime services in — its Agent provider, its @@ -2949,6 +2975,7 @@ function* executeDocument( name: string, position: Readonly | undefined, selection: ImportSelection | undefined, + admitted?: (owner: DurablePosition) => void, ): Operation { // Read per import, in the invoking scope, so a component registered // by a nested scope is visible to what that scope expands. @@ -2965,6 +2992,7 @@ function* executeDocument( catalog, guarded: identity.protected, }, + admitted, ); // Canonical selection, recorded into the frame of the import that asked // for it. This is the only thing that puts an invocation in one of this diff --git a/packages/core/src/expand.ts b/packages/core/src/expand.ts index 8d27a2042..f7e641726 100644 --- a/packages/core/src/expand.ts +++ b/packages/core/src/expand.ts @@ -141,7 +141,8 @@ import { ephemeral, StaleInputError, } from "@executablemd/durable-streams"; -import type { Workflow } from "@executablemd/durable-streams"; +import type { DurablePosition, Workflow } from "@executablemd/durable-streams"; +import { SourceOwner } from "./source-owner.ts"; import { SchemaValidationError, validateProps, validateReturnValue } from "./validate.ts"; import { isJsonObject, parseJson } from "./json.ts"; import { healSegment } from "./heal.ts"; @@ -201,6 +202,7 @@ export type ComponentImportTerminal = ( name: string, position: Readonly | undefined, selection: ImportSelection | undefined, + admitted?: (owner: DurablePosition) => void, ) => Operation; export function createBlockCounter(): BlockCounter { @@ -385,6 +387,7 @@ interface ProjectionState { } interface ProjectionFrame { + sourceOwner: DurablePosition | undefined; env: EvalEnv | undefined; meta: Record; props: Record; @@ -468,6 +471,7 @@ function createProjectionHandle(state: ProjectionState): ProjectionHandle { inner: ProjectionHandle | undefined; loop: LoopFrame | undefined; returnFrame: ReturnBody | undefined; + sourceOwner: DurablePosition | undefined; errors: Segment[]; /** * The caller's region, when this projection renders into one. Structural @@ -505,6 +509,7 @@ function createProjectionHandle(state: ProjectionState): ProjectionHandle { const task = contentScope.scope.run(function* () { try { yield* ErrorMode.set(options.mode); + yield* SourceOwner.set(options.sourceOwner); if (state.narrowedSyntax !== undefined) { yield* useNarrowedSyntax(state.narrowedSyntax); } @@ -607,6 +612,7 @@ function createProjectionHandle(state: ProjectionState): ProjectionHandle { const task = contentScope.scope.run(function* () { try { yield* ErrorMode.set(mode); + yield* SourceOwner.set(frame.sourceOwner); if (state.narrowedSyntax !== undefined) { yield* useNarrowedSyntax(state.narrowedSyntax); } @@ -684,6 +690,7 @@ function createProjectionHandle(state: ProjectionState): ProjectionHandle { inner: state.enclosing, loop: state.callerLoop, returnFrame: state.callerReturn, + sourceOwner: state.caller.sourceOwner, errors: [], owner, path: extendPath(elementPath, { f: "proj" }), @@ -2806,6 +2813,12 @@ function* expandComponent( } let imported: ComponentDefinition | FunctionComponentDefinition; + let importedOwner: DurablePosition | undefined; + let admissions = 0; + const admitted = (coordinate: DurablePosition): void => { + admissions += 1; + importedOwner = admissions === 1 ? coordinate : undefined; + }; // Opened before the ask and settled the moment it answers, however it // answered: what canonical resolution selected here is what decides whether // this invocation is in one of this execution's identity domains, and nothing @@ -2852,7 +2865,7 @@ function* expandComponent( imports === undefined ? yield* importComponent(name, position) : yield* importThroughTerminal(name, position, (asked, at) => - imports(asked, at, selection), + imports(asked, at, selection, admitted), ); } finally { offered?.close(); @@ -3064,6 +3077,7 @@ function* expandComponent( // Use the caller's eval env for the same reason: expression props (e.g. // {pr}) resolve against the scope where the JSX was written. const capturedCallerEnv = callerEvalEnv ?? componentEnv; + const callerSourceOwner = yield* SourceOwner.get(); // A body eval block can render errors away through a string projection // (renderChildren, render, useContent); the buffer records them so an // `as=` capture can be refused (§6.5). @@ -3094,12 +3108,14 @@ function* expandComponent( children, narrowedSyntax: undefined, caller: { + sourceOwner: callerSourceOwner, env: capturedCallerEnv, meta: callerMeta, props: callerProps, hideSet, }, authored: { + sourceOwner: importedOwner, env: componentEnv, meta: definition.meta, props: validatedProps, @@ -3171,26 +3187,28 @@ function* expandComponent( if (returns !== undefined) { let value: Json; try { - value = yield* withInvocation(function* (invocation) { - yield* installInvocation(invocation); - return yield* expandValueBody( - name, - returns, - definition.bodySegments, - children, - definition.meta, - validatedProps, - newHideSet, - counter, - callerEvalEnv ?? undefined, - claimProjection, - path, - checkedFailures, - bodyEnvironment, - imports, - returnBody, - ); - }, observed.settled); + value = yield* SourceOwner.with(importedOwner, () => + withInvocation(function* (invocation) { + yield* installInvocation(invocation); + return yield* expandValueBody( + name, + returns, + definition.bodySegments, + children, + definition.meta, + validatedProps, + newHideSet, + counter, + callerEvalEnv ?? undefined, + claimProjection, + path, + checkedFailures, + bodyEnvironment, + imports, + returnBody, + ); + }, observed.settled), + ); } catch (error) { // Body fail-fast propagates unchanged; a return-value failure is the // component's own printed error and follows the caller's error mode. @@ -3221,25 +3239,27 @@ function* expandComponent( // Where this body's own segments start, so what it renders can be told apart // from what the caller had already produced into the same array. const renderedFrom = bodyOwner?.length ?? 0; - const expanded = yield* withInvocation(function* (invocation) { - yield* installInvocation(invocation); - return yield* expandBody( - definition.bodySegments, - children, - definition.meta, - validatedProps, - newHideSet, - counter, - callerEvalEnv ?? undefined, - claimProjection, - bodyOwner, - path, - checkedFailures, - bodyEnvironment, - imports, - returnBody, - ); - }, observed.settled); + const expanded = yield* SourceOwner.with(importedOwner, () => + withInvocation(function* (invocation) { + yield* installInvocation(invocation); + return yield* expandBody( + definition.bodySegments, + children, + definition.meta, + validatedProps, + newHideSet, + counter, + callerEvalEnv ?? undefined, + claimProjection, + bodyOwner, + path, + checkedFailures, + bodyEnvironment, + imports, + returnBody, + ); + }, observed.settled), + ); // Exact source is a provenance, and both halves of it are read here from // things no answer can write: that canonical execution authorized this import @@ -3534,6 +3554,7 @@ function* expandFunctionComponent( // the mode inside an invocation differ from the mode at its site — governs // the component and not the text somebody else put inside it (§6.9). const siteErrorMode = (yield* ErrorMode.get()) ?? "print"; + const sourceOwner = yield* SourceOwner.get(); const expansion = snapshot(path, name, position); @@ -3608,12 +3629,14 @@ function* expandFunctionComponent( children, narrowedSyntax: undefined, caller: { + sourceOwner, env: undefined, meta: callerMeta, props: callerProps, hideSet, }, authored: { + sourceOwner, env: undefined, meta: callerMeta, props: callerProps, diff --git a/packages/core/src/generated-xmd.ts b/packages/core/src/generated-xmd.ts index 48a4a4689..f3bf128cd 100644 --- a/packages/core/src/generated-xmd.ts +++ b/packages/core/src/generated-xmd.ts @@ -139,7 +139,7 @@ import { GeneratedDataExpressions, validateDataExpression } from "./generated-ex import { capturedBinding } from "./invocation-rules.ts"; import { renderSegments } from "./render.ts"; import { scanSegments } from "./scanner.ts"; -import { sourceDescription } from "./source-position.ts"; +import { SourceOwner, sourceOwnerDescription } from "./source-owner.ts"; import { RESERVED_STRUCTURAL } from "./structural.ts"; // The ordinary source rules, read rather than reimplemented: a construct means // one thing whether a person wrote it or an Agent did. @@ -2232,12 +2232,13 @@ function* persistAdmission( marker: RecordMarker, language: AdmissionLanguage, ): Operation { + const sourceOwner = yield* SourceOwner.get(); const stored = yield createDurableOperation( { type: GENERATED_XMD, name: `generated:${id}`, input: policyRecord(policy, marker), - ...sourceDescription(position), + ...sourceOwnerDescription(position, sourceOwner), }, () => admitSource(id, source, table, policy, marker, language), ); @@ -2325,6 +2326,7 @@ function expand( owner: Segment[] | undefined, ): Operation { return scoped(function* () { + yield* SourceOwner.set(undefined); // A root selects what it renders, so a failure inside it renders as the // document's own partial output rather than throwing the whole evaluation // away. A fragment is admitted *into* a document and keeps throwing. diff --git a/packages/core/src/source-owner.ts b/packages/core/src/source-owner.ts new file mode 100644 index 000000000..2fb72d850 --- /dev/null +++ b/packages/core/src/source-owner.ts @@ -0,0 +1,19 @@ +import type { DurablePosition } from "@executablemd/durable-streams"; +import { createContext } from "effection"; +import type { Json, SourcePosition } from "./types.ts"; +import { sourceDescription } from "./source-position.ts"; + +/** Attribution follows modifier execution even when middleware copies block inputs. */ +export const SourceOwner = createContext("executablemd.source-owner"); + +export function sourceOwnerDescription( + position: Readonly | undefined, + owner: DurablePosition | undefined, +): Record { + return { + ...sourceDescription(position), + ...(position?.path === undefined || owner === undefined + ? {} + : { "executablemd.source-owner": { coroutineId: owner.coroutineId, index: owner.index } }), + }; +} diff --git a/packages/core/tests/journal-source-position.test.ts b/packages/core/tests/journal-source-position.test.ts index 87c086558..5f4efeb7a 100644 --- a/packages/core/tests/journal-source-position.test.ts +++ b/packages/core/tests/journal-source-position.test.ts @@ -107,6 +107,18 @@ describe("Tier JSP — authored source in the journal", () => { yield* collect(yield* execute({ path, stream })); const events = yield* stream.readAll(); + for (const event of events) { + if ( + event.type === "yield" && + ["eval", "import_component"].includes(event.description.type) && + event.description.name !== "__root__" + ) { + expect(event.description["executablemd.source-owner"]).toEqual({ + coroutineId: "root", + index: 0, + }); + } + } expect(sourceOf(events, "import_component", "Mark")).toEqual({ path, diff --git a/specs/executable-mdx-spec.md b/specs/executable-mdx-spec.md index bc65bb4ce..86389ccf4 100644 --- a/specs/executable-mdx-spec.md +++ b/specs/executable-mdx-spec.md @@ -11658,6 +11658,28 @@ Workflow history parses the optional field as a `SourcePosition` and never reconstructs it from an expansion ID or current source. Root, Close and trusted-host events may have no authored source. +For file-positioned imports, evaluated blocks, elicitations, Agent prompts and +generated-source admissions, `"executablemd.source-owner"` identifies the +successful import of the source containing the operation. Its value is +`{ coroutineId, index }`, with `index` the zero-based yield index on that +coroutine within the entry. The reference is descriptive metadata, not effect +identity or execution permission; it adds no operation or coroutine. + +Canonical import admission captures its coordinate. Core binds it in a scoped +Context around the source body: a component body uses its own import, structural +regions inherit their source, and projected `` restores the caller's +source binding. Attribution is independent of the public code-block object, so +middleware forwarding a shallow copy preserves it. +Generated-fragment positions keep their existing admission ID. + +The REPL follows an explicit reference to exactly one prior successful source +import and requires matching source and coroutine ancestry. A malformed, +missing, forward, wrong-source or unrelated-branch reference refuses without +inference. Without this field, existing unambiguous histories remain readable +and ambiguous histories remain refused. Reading changes no retained bytes and +performs no completed effect. Continuation counts replayed yields, so a new +suffix can reference an import in a legacy prefix at its original coordinate. + A position names **one** source. A file states `path`. An operation inside admitted generated source states `generatedSource`: the id that fragment was admitted under, which is the `generated_xmd` record's own `generated:{id}` name. diff --git a/specs/repl-spec.md b/specs/repl-spec.md index bf697ba98..ad9555921 100644 --- a/specs/repl-spec.md +++ b/specs/repl-spec.md @@ -362,6 +362,13 @@ files change or disappear; inspecting them reads no live component source, compiles no eval and asks no Agent. Imported component bytes remain retained for replay. New entries use this command's selected directories. +A component called twice keeps two source readings. Each nested source and +recorded answer belongs to the call that produced it, while caller content +projected through `` keeps its caller's source. Current records name +the owning source import directly. Older records remain readable when their +ownership is unambiguous; an older ambiguous history still refuses without +repairing its bytes or repeating completed work. + ## The packaged Plan, in the REPL's own drawer `` is the same packaged Component `xmd plan` runs, declared by this command