From 2601d5ae4e78611d65ec5a5ad55b0b7e3f9cd6c2 Mon Sep 17 00:00:00 2001 From: "duanjialing.777" Date: Wed, 30 Sep 2026 13:19:43 +0800 Subject: [PATCH 1/3] fix(goals): fence source turn settlement during recreation Signed-off-by: duanjialing.777 --- ...nstance-identity-and-orphan-recovery-v0.md | 27 + ...e-identity-and-orphan-recovery-v0.zh-CN.md | 24 + .../control_plane/effect_runtime_handlers.ts | 11 + .../goals/first_party_host_admission.py | 130 ++- .../goals/source_session_lifetime.ts | 483 ++++++++ .../goals/source_session_recreation.py | 382 +++++-- .../goals/source_session_turn_effects.py | 544 +++++++++ loopx/control_plane/turn_driver/executor.py | 146 ++- loopx/control_plane/turn_driver/settlement.py | 204 +++- .../goal_instance_binding_inventory_v1.json | 25 +- .../project_registry_io_manifest_v1.json | 32 +- .../test_source_session_registry_denial.py | 2 + .../source_session_lifetime.test.ts | 228 ++++ tests/test_loopx_turn_executor.py | 1013 +++++++++++++++-- 14 files changed, 3013 insertions(+), 238 deletions(-) create mode 100644 loopx/control_plane/goals/source_session_turn_effects.py diff --git a/docs/architecture/rfcs/goal-instance-identity-and-orphan-recovery-v0.md b/docs/architecture/rfcs/goal-instance-identity-and-orphan-recovery-v0.md index 23b84bf36e..44ca863aed 100644 --- a/docs/architecture/rfcs/goal-instance-identity-and-orphan-recovery-v0.md +++ b/docs/architecture/rfcs/goal-instance-identity-and-orphan-recovery-v0.md @@ -840,6 +840,33 @@ promotion retain their own acceptance. No new paid cohort or soak is authorized. drain, unsupported/warm binary coverage, or any other M3 row. `execution_authority: false` and the overall activation hold remain. +### 2026-09-30: M3 source Turn effect admission and drain candidate + +- **Baseline:** `3ec049e138917a8cce4f84197ba196d26445b2b0`. +- **Delivered:** Source-profile Turn settlement now records a per-effect + admission before durable writeback, quota spend, or terminal closeout. The + existing alias guard covers admission plus the prepared journal checkpoint, + and later covers the committed or aborted checkpoint plus admission release. + The final provider admission remains held through scheduler apply and the + optional post-settlement observer. A durable `source_effect_hold` marker keeps + crashes and `scheduler_action_required` resumable, and the admission releases + only with the final journal checkpoint. Provider calls, readbacks, and tail + callbacks remain outside the alias guard. +- **Retirement:** Recreation first closes the exact Goal A gate. It returns + `drain_required` while an admitted effect still needs provider readback and + publishes Goal B only after the admission set is empty. A committed readback + checkpoints once, an absent readback aborts without provider re-execution, + and an unknown readback keeps Goal A current. +- **Evidence:** Deterministic thread and crash tests cover provider commit + before checkpoint, close versus next-step admission, recreation during + scheduler and post-settlement callbacks, and committed, absent, and unknown + readbacks. Existing source-session recreation and non-source Turn paths + retain their schemas and behavior. +- **Remaining hold:** This qualifies the built-in source Turn settlement slice + of `first_party_host_runtime`. Other Host effects, unsupported or warm + binaries, and every other partial inventory owner remain blocked. + `execution_authority: false` and the overall M3 activation hold remain. + ## Appendix B: Decision log | Date | Decision | Owner / approval | Alternatives | Normative sections changed | diff --git a/docs/architecture/rfcs/goal-instance-identity-and-orphan-recovery-v0.zh-CN.md b/docs/architecture/rfcs/goal-instance-identity-and-orphan-recovery-v0.zh-CN.md index 8ccb2d1c92..f1406631c5 100644 --- a/docs/architecture/rfcs/goal-instance-identity-and-orphan-recovery-v0.zh-CN.md +++ b/docs/architecture/rfcs/goal-instance-identity-and-orphan-recovery-v0.zh-CN.md @@ -761,6 +761,30 @@ service adoption、D1–D3 provider promotion 保留各自验收。不授权付 binary 或其他 M3 行已完成。`execution_authority: false` 和总 activation hold 保持不变。 +### 2026-09-30：M3 source Turn effect 准入与 drain 候选 + +- **基线：** `3ec049e138917a8cce4f84197ba196d26445b2b0`。 +- **已交付：** Source profile Turn settlement 在 durable writeback、quota spend + 或 terminal closeout 前持久化逐 effect admission。既有 alias guard 覆盖 + admission 与 prepared journal checkpoint，之后再覆盖 committed/aborted + checkpoint 与 admission release。最后一个 provider admission 会保持到 + scheduler apply 和可选 post-settlement observer 完成。持久化 + `source_effect_hold` 标记让 crash 和 `scheduler_action_required` 可恢复；该 + admission 只在最终 journal checkpoint 时释放。Provider 调用、readback 和 + tail callback 都不持 alias guard。 +- **Retirement：** Recreation 先关闭精确 Goal A 的 gate。仍有 effect 需要 + provider readback 时返回 `drain_required`；只有 admission 集合为空后才发布 + Goal B。Committed readback 只 checkpoint 一次；absent readback 直接 abort， + 不重新调用 provider；unknown readback 保持 Goal A 为当前实例。 +- **证据：** 确定性的线程与 crash 测试覆盖 provider commit 先于 checkpoint、 + close 与下一 settlement step admission 的竞争、scheduler 与 post-settlement + callback 期间的 recreation，以及 committed、absent、unknown 三类 readback。 + 既有 source-session recreation 与非 source Turn 的 schema 和行为保持不变。 +- **剩余 hold：** 本切片只资格化 `first_party_host_runtime` 中内置 source Turn + settlement 的部分。其他 Host effect、不支持或常驻 binary，以及其余 partial + inventory owner 仍处于 hold。`execution_authority: false` 和 M3 总 activation + hold 保持不变。 + ## 附录 B：决策日志 | 日期 | 决策 | Owner／批准 | 替代方案 | 变更的规范章节 | diff --git a/loopx/control_plane/effect_runtime_handlers.ts b/loopx/control_plane/effect_runtime_handlers.ts index 7ab9b47db8..dfa4efa6e0 100644 --- a/loopx/control_plane/effect_runtime_handlers.ts +++ b/loopx/control_plane/effect_runtime_handlers.ts @@ -135,6 +135,10 @@ import { decideGoalRecreation, decideProjectSessionBind, decideProjectSessionUnbind, + decideSourceTurnEffectAbsentResolution, + decideSourceTurnEffectAdmission, + decideSourceTurnEffectGate, + decideSourceTurnEffectRelease, } from "./goals/source_session_lifetime.ts"; import { decideFirstPartyHostRuntime } from "./goals/first_party_host_runtime.ts"; import { decideChatSessionLifecycle } from "./goals/chat_session_lifecycle.ts"; @@ -554,6 +558,13 @@ export function createEffectRuntimeHandlers( ["goal.source_session.bind.decide", decideProjectSessionBind], ["goal.source_session.unbind.decide", decideProjectSessionUnbind], ["goal.source_session.recreate.decide", decideGoalRecreation], + ["goal.source_session.turn_effect.admit", decideSourceTurnEffectAdmission], + [ + "goal.source_session.turn_effect.resolve_absent", + decideSourceTurnEffectAbsentResolution, + ], + ["goal.source_session.turn_effect.release", decideSourceTurnEffectRelease], + ["goal.source_session.turn_effect.gate", decideSourceTurnEffectGate], ["goal.first_party_host_runtime.decide", decideFirstPartyHostRuntime], ["goal.chat_session.lifecycle.decide", decideChatSessionLifecycle], ["goal.acceptance.inspect", inspectLocalGoalAcceptance], diff --git a/loopx/control_plane/goals/first_party_host_admission.py b/loopx/control_plane/goals/first_party_host_admission.py index ae315c30b4..c353230fee 100644 --- a/loopx/control_plane/goals/first_party_host_admission.py +++ b/loopx/control_plane/goals/first_party_host_admission.py @@ -15,11 +15,20 @@ SOURCE_SESSION_PROFILE_ID, load_project_registry, ) +from ..effect_program import SettlementStepKind from .source_session_registry_state import ( exact_goal_ref, guard_path, require_goal_id, ) +from .source_session_turn_effects import ( + JournalPersist, + SourceTurnEffect, + SourceTurnEffectRejected, + prepare_source_turn_effect, + release_source_turn_effect, + source_turn_effect_allows_absent_reexecute, +) T = TypeVar("T") @@ -33,6 +42,89 @@ def __init__(self, code: str) -> None: self.code = code +@dataclass(frozen=True, slots=True) +class FirstPartyHostTurnEffectAdmission: + goal_admission: FirstPartyHostGoalAdmission + turn_key: str + journal_path: Path + + def _effect( + self, + step_kind: SettlementStepKind, + effect_ref: str, + ) -> SourceTurnEffect: + goal_ref = self.goal_admission.planned_goal_ref + if not isinstance(goal_ref, Mapping): + raise FirstPartyHostRuntimeRejected("goal_instance_id_missing") + goal_id = goal_ref.get("goal_id") + goal_instance_id = goal_ref.get("goal_instance_id") + if not isinstance(goal_id, str) or not isinstance(goal_instance_id, str): + raise FirstPartyHostRuntimeRejected("goal_instance_id_missing") + return SourceTurnEffect( + goal_ref=exact_goal_ref(goal_id, goal_instance_id), + turn_key=self.turn_key, + step_kind=step_kind, + effect_ref=effect_ref, + journal_path=self.journal_path, + ) + + def prepare( + self, + step_kind: SettlementStepKind, + effect_ref: str, + persist_journal: JournalPersist, + ) -> None: + try: + prepare_source_turn_effect( + registry_path=self.goal_admission.registry_path, + goal_id=self.goal_admission.goal_id, + effect=self._effect(step_kind, effect_ref), + source_admission=self.goal_admission.source_journal_admission_locked, + persist_journal=persist_journal, + ) + except SourceTurnEffectRejected as exc: + raise FirstPartyHostRuntimeRejected(exc.code) from exc + + def hold( + self, + step_kind: SettlementStepKind, + effect_ref: str, + persist_journal: JournalPersist, + ) -> None: + self.prepare(step_kind, effect_ref, persist_journal) + + def release( + self, + step_kind: SettlementStepKind, + effect_ref: str, + persist_journal: JournalPersist, + ) -> None: + try: + release_source_turn_effect( + registry_path=self.goal_admission.registry_path, + goal_id=self.goal_admission.goal_id, + effect=self._effect(step_kind, effect_ref), + source_admission=self.goal_admission.source_journal_admission_locked, + persist_journal=persist_journal, + ) + except SourceTurnEffectRejected as exc: + raise FirstPartyHostRuntimeRejected(exc.code) from exc + + def allows_absent_reexecute( + self, + step_kind: SettlementStepKind, + effect_ref: str, + ) -> bool: + try: + return source_turn_effect_allows_absent_reexecute( + registry_path=self.goal_admission.registry_path, + goal_id=self.goal_admission.goal_id, + effect=self._effect(step_kind, effect_ref), + ) + except SourceTurnEffectRejected as exc: + raise FirstPartyHostRuntimeRejected(exc.code) from exc + + def _source_authority(registry_path: Path, goal_id: str) -> dict[str, Any]: if not registry_path.is_file(): return {"kind": "unavailable", "reason": "registry_missing"} @@ -215,17 +307,33 @@ def source_journal_admission( target, operation="first_party_host_journal_commit", ): - yield { - "schema_version": "loopx_turn_journal_source_admission_v0", - "profile_id": SOURCE_SESSION_PROFILE_ID, - "registry_path": str(self.registry_path), - "planned_goal_ref": self.planned_goal_ref, - "authority": _source_authority( - self.registry_path, - self.goal_id, - ), - "lock": cross_runtime_lock_witness(target), - } + yield self.source_journal_admission_locked() + + def source_journal_admission_locked(self) -> dict[str, Any]: + """Build a TS handoff while the caller holds this Goal's source guard.""" + + target = guard_path(self.registry_path, self.goal_id) + return { + "schema_version": "loopx_turn_journal_source_admission_v0", + "profile_id": SOURCE_SESSION_PROFILE_ID, + "registry_path": str(self.registry_path), + "planned_goal_ref": self.planned_goal_ref, + "authority": _source_authority( + self.registry_path, + self.goal_id, + ), + "lock": cross_runtime_lock_witness(target), + } + + def turn_effect_admission( + self, + *, + turn_key: str, + journal_path: Path, + ) -> FirstPartyHostTurnEffectAdmission | None: + if not self.source_profile: + return None + return FirstPartyHostTurnEffectAdmission(self, turn_key, journal_path) def accept_result(self, commit_result: Callable[[], T]) -> T: if not self.source_profile: diff --git a/loopx/control_plane/goals/source_session_lifetime.ts b/loopx/control_plane/goals/source_session_lifetime.ts index e2d78e827e..f52640cf26 100644 --- a/loopx/control_plane/goals/source_session_lifetime.ts +++ b/loopx/control_plane/goals/source_session_lifetime.ts @@ -11,6 +11,10 @@ export const SOURCE_SESSION_PROFILE_ID = "source_session_v1"; export const SOURCE_SESSION_BINDING_LIMIT = 256; export const SOURCE_SESSION_RECEIPT_LIMIT = 4096; export const SOURCE_SESSION_LIFETIME_RECEIPT_LIMIT = 1024; +export const SOURCE_TURN_EFFECT_GATE_SCHEMA_VERSION = + "loopx_source_turn_effect_gate_v1"; +export const SOURCE_TURN_EFFECT_ADMISSION_SCHEMA_VERSION = + "loopx_source_turn_effect_admission_v1"; type WireGoalRef = Readonly<{ goal_id: string; @@ -62,6 +66,88 @@ export type GoalRecreationDecision = code: GoalRecreationRejection; }>; +type SourceTurnEffectStep = + | "durable_writeback" + | "quota_spend" + | "terminal_closeout"; + +type SourceTurnEffectAdmission = Readonly<{ + schema_version: typeof SOURCE_TURN_EFFECT_ADMISSION_SCHEMA_VERSION; + goal_ref: WireGoalRef; + turn_key: string; + step_kind: SourceTurnEffectStep; + effect_ref: string; +}>; + +type SourceTurnEffectOpenGate = Readonly<{ + schema_version: typeof SOURCE_TURN_EFFECT_GATE_SCHEMA_VERSION; + state: "open"; + goal_ref: WireGoalRef; +}>; + +type SourceTurnEffectClosingGate = Readonly<{ + schema_version: typeof SOURCE_TURN_EFFECT_GATE_SCHEMA_VERSION; + state: "closing"; + retired_goal_ref: WireGoalRef; + new_goal_ref: WireGoalRef; + operation_id: string; + request_digest: string; +}>; + +type SourceTurnEffectGate = + | SourceTurnEffectOpenGate + | SourceTurnEffectClosingGate; + +export type SourceTurnEffectRejection = + | "unsupported_profile" + | "stale_goal_instance" + | "goal_retirement_in_progress" + | "effect_admission_conflict" + | "effect_drain_required" + | "recreation_operation_conflict"; + +type SourceTurnEffectReject = Readonly<{ + kind: "reject"; + code: SourceTurnEffectRejection; +}>; + +export type SourceTurnEffectAdmissionDecision = + | Readonly<{ + kind: "commit"; + gate: SourceTurnEffectGate; + admission: SourceTurnEffectAdmission; + }> + | Readonly<{ + kind: "replay"; + gate: SourceTurnEffectGate; + admission: SourceTurnEffectAdmission; + }> + | SourceTurnEffectReject; + +export type SourceTurnEffectReleaseDecision = + | Readonly<{ kind: "commit" }> + | Readonly<{ kind: "replay" }> + | SourceTurnEffectReject; + +export type SourceTurnEffectAbsentDecision = + | Readonly<{ kind: "execute" }> + | Readonly<{ kind: "abort" }> + | SourceTurnEffectReject; + +export type SourceTurnEffectGateDecision = + | Readonly<{ + kind: "commit"; + gate: SourceTurnEffectGate; + }> + | Readonly<{ + kind: "replay"; + gate: SourceTurnEffectGate; + }> + | Readonly<{ + kind: "preserve"; + }> + | SourceTurnEffectReject; + type SessionBinding = Readonly<{ sessionId: string; goalRef: ExactGoalRef; @@ -143,6 +229,403 @@ function goalRefsEqual(left: ExactGoalRef, right: ExactGoalRef): boolean { && left.goalInstanceId.value === right.goalInstanceId.value; } +function sourceTurnEffectStep(value: unknown): SourceTurnEffectStep { + if ( + value !== "durable_writeback" + && value !== "quota_spend" + && value !== "terminal_closeout" + ) { + throw new EffectRuntimeRequestError( + "step_kind must name a supported Turn settlement effect", + ); + } + return value; +} + +function sourceTurnEffectAdmission( + value: unknown, + label: string, +): SourceTurnEffectAdmission { + const admission = requiredObject(value, label); + if ( + admission.schema_version !== SOURCE_TURN_EFFECT_ADMISSION_SCHEMA_VERSION + ) { + throw new EffectRuntimeRequestError(`${label} schema_version is unsupported`); + } + return { + schema_version: SOURCE_TURN_EFFECT_ADMISSION_SCHEMA_VERSION, + goal_ref: wireGoalRef(exactGoalRef(admission.goal_ref, `${label}.goal_ref`)), + turn_key: requestDigest(admission.turn_key), + step_kind: sourceTurnEffectStep(admission.step_kind), + effect_ref: requiredString(admission.effect_ref, `${label}.effect_ref`), + }; +} + +function sourceTurnEffectGate(value: unknown): SourceTurnEffectGate | null { + if (value === null) return null; + const gate = requiredObject(value, "gate"); + if (gate.schema_version !== SOURCE_TURN_EFFECT_GATE_SCHEMA_VERSION) { + throw new EffectRuntimeRequestError("gate schema_version is unsupported"); + } + if (gate.state === "open") { + return { + schema_version: SOURCE_TURN_EFFECT_GATE_SCHEMA_VERSION, + state: "open", + goal_ref: wireGoalRef(exactGoalRef(gate.goal_ref, "gate.goal_ref")), + }; + } + if (gate.state === "closing") { + const retiredGoalRef = exactGoalRef( + gate.retired_goal_ref, + "gate.retired_goal_ref", + ); + const newGoalRef = exactGoalRef(gate.new_goal_ref, "gate.new_goal_ref"); + if ( + retiredGoalRef.goalId.value !== newGoalRef.goalId.value + || retiredGoalRef.goalInstanceId.value === newGoalRef.goalInstanceId.value + ) { + throw new EffectRuntimeRequestError( + "closing gate must reserve a new instance of the retired Goal", + ); + } + return { + schema_version: SOURCE_TURN_EFFECT_GATE_SCHEMA_VERSION, + state: "closing", + retired_goal_ref: wireGoalRef(retiredGoalRef), + new_goal_ref: wireGoalRef(newGoalRef), + operation_id: requiredString(gate.operation_id, "gate.operation_id"), + request_digest: requestDigest(gate.request_digest), + }; + } + throw new EffectRuntimeRequestError("gate state is unsupported"); +} + +function admissionEqual( + left: SourceTurnEffectAdmission, + right: SourceTurnEffectAdmission, +): boolean { + return left.schema_version === right.schema_version + && left.goal_ref.goal_id === right.goal_ref.goal_id + && left.goal_ref.goal_instance_id === right.goal_ref.goal_instance_id + && left.turn_key === right.turn_key + && left.step_kind === right.step_kind + && left.effect_ref === right.effect_ref; +} + +function openSourceTurnEffectGate(goalRef: ExactGoalRef): SourceTurnEffectOpenGate { + return { + schema_version: SOURCE_TURN_EFFECT_GATE_SCHEMA_VERSION, + state: "open", + goal_ref: wireGoalRef(goalRef), + }; +} + +export function decideSourceTurnEffectAdmission( + value: unknown, +): SourceTurnEffectAdmissionDecision { + const facts = requiredObject(value, "source Turn effect admission facts"); + if (requiredString(facts.profile_id, "profile_id") !== SOURCE_SESSION_PROFILE_ID) { + return { kind: "reject", code: "unsupported_profile" }; + } + const requestedGoalRef = exactGoalRef( + facts.requested_goal_ref, + "requested_goal_ref", + ); + const currentGoalRef = exactGoalRef(facts.current_goal_ref, "current_goal_ref"); + if (!goalRefsEqual(requestedGoalRef, currentGoalRef)) { + return { kind: "reject", code: "stale_goal_instance" }; + } + const admission = sourceTurnEffectAdmission(facts.admission, "admission"); + const admissionGoalRef = exactGoalRef(admission.goal_ref, "admission.goal_ref"); + if (!goalRefsEqual(admissionGoalRef, requestedGoalRef)) { + throw new EffectRuntimeRequestError( + "admission.goal_ref must match requested_goal_ref", + ); + } + const existing = facts.existing_admission === null + ? null + : sourceTurnEffectAdmission( + facts.existing_admission, + "existing_admission", + ); + const gate = sourceTurnEffectGate(facts.gate); + if (gate?.state === "closing") { + const retiredGoalRef = exactGoalRef( + gate.retired_goal_ref, + "gate.retired_goal_ref", + ); + if ( + existing !== null + && admissionEqual(existing, admission) + && goalRefsEqual(retiredGoalRef, requestedGoalRef) + ) { + return { kind: "replay", gate, admission }; + } + return { kind: "reject", code: "goal_retirement_in_progress" }; + } + if ( + gate?.state === "open" + && ( + gate.goal_ref.goal_id !== requestedGoalRef.goalId.value + || gate.goal_ref.goal_instance_id !== requestedGoalRef.goalInstanceId.value + ) + ) { + return { kind: "reject", code: "stale_goal_instance" }; + } + const openedGate = gate ?? openSourceTurnEffectGate(requestedGoalRef); + if (existing === null) { + return { kind: "commit", gate: openedGate, admission }; + } + return admissionEqual(existing, admission) + ? { kind: "replay", gate: openedGate, admission } + : { kind: "reject", code: "effect_admission_conflict" }; +} + +export function decideSourceTurnEffectRelease( + value: unknown, +): SourceTurnEffectReleaseDecision { + const facts = requiredObject(value, "source Turn effect release facts"); + if (requiredString(facts.profile_id, "profile_id") !== SOURCE_SESSION_PROFILE_ID) { + return { kind: "reject", code: "unsupported_profile" }; + } + const requested = sourceTurnEffectAdmission(facts.admission, "admission"); + const requestedGoalRef = exactGoalRef(requested.goal_ref, "admission.goal_ref"); + const currentGoalRef = exactGoalRef(facts.current_goal_ref, "current_goal_ref"); + if (!goalRefsEqual(requestedGoalRef, currentGoalRef)) { + return { kind: "reject", code: "stale_goal_instance" }; + } + const gate = sourceTurnEffectGate(facts.gate); + if (gate === null) { + throw new EffectRuntimeRequestError("source Turn effect gate is missing"); + } + const gateGoalRef = exactGoalRef( + gate.state === "open" ? gate.goal_ref : gate.retired_goal_ref, + "gate GoalRef", + ); + if (!goalRefsEqual(gateGoalRef, requestedGoalRef)) { + return { kind: "reject", code: "stale_goal_instance" }; + } + if (facts.existing_admission === null) { + return { kind: "replay" }; + } + const existing = sourceTurnEffectAdmission( + facts.existing_admission, + "existing_admission", + ); + return admissionEqual(existing, requested) + ? { kind: "commit" } + : { kind: "reject", code: "effect_admission_conflict" }; +} + +export function decideSourceTurnEffectAbsentResolution( + value: unknown, +): SourceTurnEffectAbsentDecision { + const facts = requiredObject( + value, + "source Turn effect absent-resolution facts", + ); + if (requiredString(facts.profile_id, "profile_id") !== SOURCE_SESSION_PROFILE_ID) { + return { kind: "reject", code: "unsupported_profile" }; + } + const requested = sourceTurnEffectAdmission(facts.admission, "admission"); + const existing = facts.existing_admission === null + ? null + : sourceTurnEffectAdmission( + facts.existing_admission, + "existing_admission", + ); + if (existing === null || !admissionEqual(existing, requested)) { + return { kind: "reject", code: "effect_admission_conflict" }; + } + const requestedGoalRef = exactGoalRef(requested.goal_ref, "admission.goal_ref"); + const currentGoalRef = exactGoalRef(facts.current_goal_ref, "current_goal_ref"); + if (!goalRefsEqual(requestedGoalRef, currentGoalRef)) { + return { kind: "reject", code: "stale_goal_instance" }; + } + const gate = sourceTurnEffectGate(facts.gate); + if (gate === null) { + throw new EffectRuntimeRequestError("source Turn effect gate is missing"); + } + const gateGoalRef = exactGoalRef( + gate.state === "open" ? gate.goal_ref : gate.retired_goal_ref, + "gate GoalRef", + ); + if (!goalRefsEqual(gateGoalRef, requestedGoalRef)) { + return { kind: "reject", code: "stale_goal_instance" }; + } + return gate.state === "closing" ? { kind: "abort" } : { kind: "execute" }; +} + +export function decideSourceTurnEffectGate( + value: unknown, +): SourceTurnEffectGateDecision { + const facts = requiredObject(value, "source Turn effect gate facts"); + if (requiredString(facts.profile_id, "profile_id") !== SOURCE_SESSION_PROFILE_ID) { + return { kind: "reject", code: "unsupported_profile" }; + } + const operation = requiredString(facts.operation, "operation"); + const requestedGoalRef = exactGoalRef( + facts.requested_goal_ref, + "requested_goal_ref", + ); + const currentGoalRef = exactGoalRef(facts.current_goal_ref, "current_goal_ref"); + const reservedGoalRef = exactGoalRef( + facts.reserved_goal_ref, + "reserved_goal_ref", + ); + if ( + requestedGoalRef.goalId.value !== reservedGoalRef.goalId.value + || requestedGoalRef.goalInstanceId.value + === reservedGoalRef.goalInstanceId.value + ) { + throw new EffectRuntimeRequestError( + "reserved_goal_ref must name a new instance of the requested Goal", + ); + } + const operationId = requiredString(facts.operation_id, "operation_id"); + const digest = requestDigest(facts.request_digest); + const gate = sourceTurnEffectGate(facts.gate); + const closingGate: SourceTurnEffectClosingGate = { + schema_version: SOURCE_TURN_EFFECT_GATE_SCHEMA_VERSION, + state: "closing", + retired_goal_ref: wireGoalRef(requestedGoalRef), + new_goal_ref: wireGoalRef(reservedGoalRef), + operation_id: operationId, + request_digest: digest, + }; + const sameClosingGate = gate?.state === "closing" + && gate.retired_goal_ref.goal_id === closingGate.retired_goal_ref.goal_id + && gate.retired_goal_ref.goal_instance_id + === closingGate.retired_goal_ref.goal_instance_id + && gate.new_goal_ref.goal_id === closingGate.new_goal_ref.goal_id + && gate.new_goal_ref.goal_instance_id + === closingGate.new_goal_ref.goal_instance_id + && gate.operation_id === operationId + && gate.request_digest === digest; + + if (operation === "close") { + if ( + !goalRefsEqual(requestedGoalRef, currentGoalRef) + && !(sameClosingGate && goalRefsEqual(reservedGoalRef, currentGoalRef)) + ) { + return { kind: "reject", code: "stale_goal_instance" }; + } + if (gate?.state === "closing") { + return sameClosingGate + ? { kind: "replay", gate: closingGate } + : { kind: "reject", code: "recreation_operation_conflict" }; + } + if ( + gate?.state === "open" + && ( + gate.goal_ref.goal_id !== requestedGoalRef.goalId.value + || gate.goal_ref.goal_instance_id + !== requestedGoalRef.goalInstanceId.value + ) + ) { + return { kind: "reject", code: "stale_goal_instance" }; + } + return { kind: "commit", gate: closingGate }; + } + const admissionCount = nonNegativeInteger( + facts.admission_count, + "admission_count", + ); + if (operation === "repair") { + if ( + sameClosingGate + && goalRefsEqual(reservedGoalRef, currentGoalRef) + ) { + return admissionCount === 0 + ? { + kind: "replay", + gate: openSourceTurnEffectGate(reservedGoalRef), + } + : { kind: "reject", code: "effect_drain_required" }; + } + if (gate?.state === "open") { + const gateGoalRef = exactGoalRef(gate.goal_ref, "gate.goal_ref"); + if (goalRefsEqual(gateGoalRef, requestedGoalRef)) { + return { kind: "reject", code: "recreation_operation_conflict" }; + } + return gateGoalRef.goalId.value === requestedGoalRef.goalId.value + && goalRefsEqual(gateGoalRef, currentGoalRef) + ? { kind: "preserve" } + : { kind: "reject", code: "recreation_operation_conflict" }; + } + if (gate?.state === "closing") { + const retiredGateRef = exactGoalRef( + gate.retired_goal_ref, + "gate.retired_goal_ref", + ); + const newGateRef = exactGoalRef(gate.new_goal_ref, "gate.new_goal_ref"); + if (goalRefsEqual(retiredGateRef, requestedGoalRef)) { + return { kind: "reject", code: "recreation_operation_conflict" }; + } + return ( + retiredGateRef.goalId.value === requestedGoalRef.goalId.value + && ( + goalRefsEqual(retiredGateRef, currentGoalRef) + || goalRefsEqual(newGateRef, currentGoalRef) + ) + ) + ? { kind: "preserve" } + : { kind: "reject", code: "recreation_operation_conflict" }; + } + if ( + gate === null + && admissionCount === 0 + && goalRefsEqual(reservedGoalRef, currentGoalRef) + ) { + return { + kind: "replay", + gate: openSourceTurnEffectGate(reservedGoalRef), + }; + } + return { kind: "reject", code: "recreation_operation_conflict" }; + } + if (operation !== "publish") { + throw new EffectRuntimeRequestError( + "operation must be close, publish, or repair", + ); + } + if ( + gate === null + && admissionCount === 0 + && goalRefsEqual(reservedGoalRef, currentGoalRef) + ) { + return { + kind: "replay", + gate: openSourceTurnEffectGate(reservedGoalRef), + }; + } + if (gate?.state === "open") { + return ( + gate.goal_ref.goal_id === reservedGoalRef.goalId.value + && gate.goal_ref.goal_instance_id === reservedGoalRef.goalInstanceId.value + && goalRefsEqual(reservedGoalRef, currentGoalRef) + ) + ? { kind: "replay", gate } + : { kind: "reject", code: "recreation_operation_conflict" }; + } + if (!sameClosingGate) { + return { kind: "reject", code: "recreation_operation_conflict" }; + } + if (admissionCount !== 0) { + return { kind: "reject", code: "effect_drain_required" }; + } + if ( + !goalRefsEqual(requestedGoalRef, currentGoalRef) + && !goalRefsEqual(reservedGoalRef, currentGoalRef) + ) { + return { kind: "reject", code: "stale_goal_instance" }; + } + return { + kind: "commit", + gate: openSourceTurnEffectGate(reservedGoalRef), + }; +} + function sessionBinding(value: unknown): SessionBinding | null { if (value === null) return null; const binding = requiredObject(value, "current_binding"); diff --git a/loopx/control_plane/goals/source_session_recreation.py b/loopx/control_plane/goals/source_session_recreation.py index 64512cb50c..36a27560cc 100644 --- a/loopx/control_plane/goals/source_session_recreation.py +++ b/loopx/control_plane/goals/source_session_recreation.py @@ -25,6 +25,13 @@ session_binding_records, write_journal, ) +from .source_session_turn_effects import ( + decide_source_turn_effect_close_locked, + decide_source_turn_effect_publish_locked, + decide_source_turn_effect_repair_locked, + drain_releasable_source_turn_effects, + write_source_turn_effect_gate_locked, +) @dataclass(frozen=True, slots=True) @@ -35,6 +42,18 @@ class RecreateGoalRequest: operation_id: str +@dataclass(frozen=True, slots=True) +class _RecreationState: + registry: dict[str, Any] + active_goal_ref: dict[str, str] + reserved_goal_ref: dict[str, str] + bindings: list[dict[str, Any]] + session_receipts: list[dict[str, Any]] + lifetime_receipts: list[dict[str, Any]] + retiring_bindings: list[dict[str, Any]] + decision: dict[str, Any] + + def _recreation_journal_path( registry_path: Path, *, @@ -120,8 +139,126 @@ def _recreation_result( } +def _evaluate_recreation( + registry: dict[str, Any], + request: RecreateGoalRequest, + *, + requested_goal_ref: dict[str, str], + request_digest: str, + journal: dict[str, Any] | None, +) -> _RecreationState: + active_goal_ref, _goal = current_goal_ref( + registry, + goal_id=request.goal_id, + ) + bindings = session_binding_records(registry) + session_receipts = required_list(registry, "session_receipts") + lifetime_receipts = required_list(registry, "lifetime_receipts") + prior_receipt = prior_operation_receipt( + lifetime_receipts, + operation_id=request.operation_id, + ) + session_operation_receipt = prior_operation_receipt( + session_receipts, + operation_id=request.operation_id, + ) + if session_operation_receipt is not None and ( + session_operation_receipt.get("schema_version") + != "loopx_source_session_retirement_receipt_v1" + or session_operation_receipt.get("request_digest") != request_digest + ): + raise ValueError( + "source-session operation_id was reused across lifecycle operations" + ) + if journal is not None: + reserved_goal_ref = copy.deepcopy(journal["new_goal_ref"]) + elif prior_receipt is not None: + candidate = prior_receipt.get("new_goal_ref") + if not isinstance(candidate, dict): + raise ValueError("Goal recreation receipt new_goal_ref is invalid") + reserved_goal_ref = copy.deepcopy(candidate) + else: + reserved_goal_ref = { + "goal_id": request.goal_id, + "goal_instance_id": f"ginst_{uuid4().hex}", + } + retiring_bindings = [ + binding + for binding in bindings + if binding.get("foreground_goal_ref") == requested_goal_ref + ] + decision = effect_runtime_result( + "goal.source_session.recreate.decide", + { + "profile_id": registry["profile_id"], + "operation_id": request.operation_id, + "request_digest": request_digest, + "requested_goal_ref": requested_goal_ref, + "current_goal_ref": active_goal_ref, + "reserved_goal_ref": reserved_goal_ref, + "prior_receipt": prior_receipt, + "lifetime_receipt_count": len(lifetime_receipts), + "session_receipt_count": len(session_receipts), + "retiring_binding_count": len(retiring_bindings), + }, + ) + if not isinstance(decision, dict): + raise RuntimeError("Goal recreation decision must be an object") + if decision.get("kind") == "reject": + raise ValueError(f"source-session recreation rejected: {decision.get('code')}") + if decision.get("kind") not in {"commit", "replay"}: + raise RuntimeError("Goal recreation decision kind is unsupported") + return _RecreationState( + registry=registry, + active_goal_ref=active_goal_ref, + reserved_goal_ref=reserved_goal_ref, + bindings=bindings, + session_receipts=session_receipts, + lifetime_receipts=lifetime_receipts, + retiring_bindings=retiring_bindings, + decision=decision, + ) + + +def _reserved_journal( + request: RecreateGoalRequest, + *, + requested_goal_ref: dict[str, str], + reserved_goal_ref: dict[str, str], + request_digest: str, +) -> dict[str, Any]: + return { + "schema_version": "loopx_goal_recreation_journal_v1", + "operation_id": request.operation_id, + "request_digest": request_digest, + "retired_goal_ref": copy.deepcopy(requested_goal_ref), + "new_goal_ref": copy.deepcopy(reserved_goal_ref), + "reserved_at": now_local_iso(), + "phase": "reserved", + } + + +def _drain_required_result( + request: RecreateGoalRequest, + *, + requested_goal_ref: dict[str, str], + pending_effects: list[dict[str, str]], +) -> dict[str, Any]: + return { + "ok": False, + "schema_version": "loopx_goal_recreation_v1", + "status": "drain_required", + "changed": False, + "replayed": False, + "registry": str(request.registry_path), + "retired_goal_ref": copy.deepcopy(requested_goal_ref), + "pending_effects": pending_effects, + "execution_authority": False, + } + + def recreate_goal_instance(request: RecreateGoalRequest) -> dict[str, Any]: - """Retire exact A and publish one reserved B under the alias guard.""" + """Close A's effect gate, drain admitted work, then publish reserved B.""" requested_goal_ref = exact_goal_ref( request.goal_id, @@ -134,9 +271,10 @@ def recreate_goal_instance(request: RecreateGoalRequest) -> dict[str, Any]: goal_id=request.goal_id, operation_id=request.operation_id, ) + with exclusive_cross_runtime_file_lock( guard, - operation="source_session_goal_lifetime", + operation="source_session_goal_lifetime_close", ): journal = _read_recreation_journal(journal_path) if journal is not None and ( @@ -145,139 +283,169 @@ def recreate_goal_instance(request: RecreateGoalRequest) -> dict[str, Any]: or journal["retired_goal_ref"] != requested_goal_ref ): raise ValueError("Goal recreation operation_id conflicts with its journal") - with source_session_registry_transaction( request.registry_path, - operation="source_session_goal_recreate", + operation="source_session_goal_recreate_prepare", ) as transaction: - registry = transaction.payload_copy() - active_goal_ref, _goal = current_goal_ref( - registry, - goal_id=request.goal_id, - ) - bindings = session_binding_records(registry) - session_receipts = required_list(registry, "session_receipts") - lifetime_receipts = required_list(registry, "lifetime_receipts") - prior_receipt = prior_operation_receipt( - lifetime_receipts, - operation_id=request.operation_id, + state = _evaluate_recreation( + transaction.payload_copy(), + request, + requested_goal_ref=requested_goal_ref, + request_digest=request_digest, + journal=journal, ) - session_operation_receipt = prior_operation_receipt( - session_receipts, + if state.decision["kind"] == "replay": + replay_receipt = state.decision.get("receipt") + if not isinstance(replay_receipt, dict): + raise RuntimeError("Goal recreation replay omitted its receipt") + next_gate = decide_source_turn_effect_repair_locked( + registry_path=request.registry_path, + goal_id=request.goal_id, + requested_goal_ref=requested_goal_ref, + current_goal_ref=state.active_goal_ref, + reserved_goal_ref=state.reserved_goal_ref, operation_id=request.operation_id, + request_digest=request_digest, ) - if session_operation_receipt is not None and ( - session_operation_receipt.get("schema_version") - != "loopx_source_session_retirement_receipt_v1" - or session_operation_receipt.get("request_digest") != request_digest - ): - raise ValueError( - "source-session operation_id was reused across lifecycle operations" + if next_gate is not None: + write_source_turn_effect_gate_locked( + registry_path=request.registry_path, + goal_id=request.goal_id, + gate=next_gate, ) - if journal is not None: - reserved_goal_ref = copy.deepcopy(journal["new_goal_ref"]) - elif prior_receipt is not None: - candidate = prior_receipt.get("new_goal_ref") - if not isinstance(candidate, dict): - raise ValueError("Goal recreation receipt new_goal_ref is invalid") - reserved_goal_ref = copy.deepcopy(candidate) - else: - reserved_goal_ref = { - "goal_id": request.goal_id, - "goal_instance_id": f"ginst_{uuid4().hex}", + if journal is None: + journal = { + **_reserved_journal( + request, + requested_goal_ref=requested_goal_ref, + reserved_goal_ref=state.reserved_goal_ref, + request_digest=request_digest, + ), + "reserved_at": replay_receipt["committed_at"], } + write_journal(journal_path, {**journal, "phase": "published"}) + return _recreation_result( + request, + receipt=replay_receipt, + replayed=True, + ) + closing_gate = decide_source_turn_effect_close_locked( + registry_path=request.registry_path, + goal_id=request.goal_id, + requested_goal_ref=requested_goal_ref, + current_goal_ref=state.active_goal_ref, + reserved_goal_ref=state.reserved_goal_ref, + operation_id=request.operation_id, + request_digest=request_digest, + ) + if journal is None: + journal = _reserved_journal( + request, + requested_goal_ref=requested_goal_ref, + reserved_goal_ref=state.reserved_goal_ref, + request_digest=request_digest, + ) + write_journal(journal_path, journal) + write_source_turn_effect_gate_locked( + registry_path=request.registry_path, + goal_id=request.goal_id, + gate=closing_gate, + ) - retiring_bindings = [ - binding - for binding in bindings - if binding.get("foreground_goal_ref") == requested_goal_ref - ] - decision = effect_runtime_result( - "goal.source_session.recreate.decide", - { - "profile_id": registry["profile_id"], - "operation_id": request.operation_id, - "request_digest": request_digest, - "requested_goal_ref": requested_goal_ref, - "current_goal_ref": active_goal_ref, - "reserved_goal_ref": reserved_goal_ref, - "prior_receipt": prior_receipt, - "lifetime_receipt_count": len(lifetime_receipts), - "session_receipt_count": len(session_receipts), - "retiring_binding_count": len(retiring_bindings), - }, + pending_effects = drain_releasable_source_turn_effects( + registry_path=request.registry_path, + goal_id=request.goal_id, + requested_goal_ref=requested_goal_ref, + ) + if pending_effects: + return _drain_required_result( + request, + requested_goal_ref=requested_goal_ref, + pending_effects=pending_effects, + ) + + with exclusive_cross_runtime_file_lock( + guard, + operation="source_session_goal_lifetime_publish", + ): + journal = _read_recreation_journal(journal_path) + if journal is None: + raise RuntimeError("Goal recreation reservation journal is missing") + with source_session_registry_transaction( + request.registry_path, + operation="source_session_goal_recreate", + ) as transaction: + state = _evaluate_recreation( + transaction.payload_copy(), + request, + requested_goal_ref=requested_goal_ref, + request_digest=request_digest, + journal=journal, ) - if not isinstance(decision, dict): - raise RuntimeError("Goal recreation decision must be an object") - if decision.get("kind") == "reject": - raise ValueError( - f"source-session recreation rejected: {decision.get('code')}" - ) - if decision.get("kind") == "replay": - replay_receipt = decision.get("receipt") + if state.decision["kind"] == "replay": + replay_receipt = state.decision.get("receipt") if not isinstance(replay_receipt, dict): raise RuntimeError("Goal recreation replay omitted its receipt") - if journal is None: - journal = { - "schema_version": "loopx_goal_recreation_journal_v1", - "operation_id": request.operation_id, - "request_digest": request_digest, - "retired_goal_ref": copy.deepcopy(requested_goal_ref), - "new_goal_ref": copy.deepcopy(replay_receipt["new_goal_ref"]), - "reserved_at": replay_receipt["committed_at"], - "phase": "published", - } - write_journal(journal_path, journal) - elif journal["phase"] != "published": - write_journal(journal_path, {**journal, "phase": "published"}) + next_gate = decide_source_turn_effect_repair_locked( + registry_path=request.registry_path, + goal_id=request.goal_id, + requested_goal_ref=requested_goal_ref, + current_goal_ref=state.active_goal_ref, + reserved_goal_ref=state.reserved_goal_ref, + operation_id=request.operation_id, + request_digest=request_digest, + ) + if next_gate is not None: + write_source_turn_effect_gate_locked( + registry_path=request.registry_path, + goal_id=request.goal_id, + gate=next_gate, + ) + write_journal(journal_path, {**journal, "phase": "published"}) return _recreation_result( request, receipt=replay_receipt, replayed=True, ) - if decision.get("kind") != "commit": - raise RuntimeError("Goal recreation decision kind is unsupported") - - if journal is None: - journal = { - "schema_version": "loopx_goal_recreation_journal_v1", - "operation_id": request.operation_id, - "request_digest": request_digest, - "retired_goal_ref": copy.deepcopy(requested_goal_ref), - "new_goal_ref": copy.deepcopy(reserved_goal_ref), - "reserved_at": now_local_iso(), - "phase": "reserved", - } - write_journal(journal_path, journal) + next_gate = decide_source_turn_effect_publish_locked( + registry_path=request.registry_path, + goal_id=request.goal_id, + requested_goal_ref=requested_goal_ref, + current_goal_ref=state.active_goal_ref, + reserved_goal_ref=state.reserved_goal_ref, + operation_id=request.operation_id, + request_digest=request_digest, + ) committed_at = now_local_iso() retired_session_ids = sorted( - str(binding["session_id"]) for binding in retiring_bindings + str(binding["session_id"]) for binding in state.retiring_bindings ) receipt = { "schema_version": "loopx_goal_recreation_receipt_v1", "operation_id": request.operation_id, "request_digest": request_digest, "retired_goal_ref": copy.deepcopy(requested_goal_ref), - "new_goal_ref": copy.deepcopy(reserved_goal_ref), + "new_goal_ref": copy.deepcopy(state.reserved_goal_ref), "retired_session_ids": retired_session_ids, "committed_at": committed_at, } - registry["goals"] = [ + state.registry["goals"] = [ { **candidate, - "goal_instance_id": reserved_goal_ref["goal_instance_id"], + "goal_instance_id": state.reserved_goal_ref["goal_instance_id"], "execution_authority": False, } if candidate.get("id") == request.goal_id else candidate - for candidate in required_list(registry, "goals") + for candidate in required_list(state.registry, "goals") ] - registry["session_bindings"] = [ + state.registry["session_bindings"] = [ binding - for binding in bindings + for binding in state.bindings if binding.get("foreground_goal_ref") != requested_goal_ref ] + session_receipts = state.session_receipts if retired_session_ids: session_receipts = [ *session_receipts, @@ -293,26 +461,34 @@ def recreate_goal_instance(request: RecreateGoalRequest) -> dict[str, Any]: "committed_at": committed_at, }, ] - registry["session_receipts"] = session_receipts - retired = registry.get("retired_goal_instances", []) + state.registry["session_receipts"] = session_receipts + retired = state.registry.get("retired_goal_instances", []) if not isinstance(retired, list) or any( not isinstance(item, dict) for item in retired ): raise ValueError( "source-session retired_goal_instances must be a list of objects" ) - registry["retired_goal_instances"] = [ + state.registry["retired_goal_instances"] = [ *retired, { "goal_ref": copy.deepcopy(requested_goal_ref), - "successor_goal_ref": copy.deepcopy(reserved_goal_ref), + "successor_goal_ref": copy.deepcopy(state.reserved_goal_ref), "operation_id": request.operation_id, "retired_at": committed_at, }, ] - registry["lifetime_receipts"] = [*lifetime_receipts, receipt] - registry["updated_at"] = committed_at - transaction.commit(registry) + state.registry["lifetime_receipts"] = [ + *state.lifetime_receipts, + receipt, + ] + state.registry["updated_at"] = committed_at + transaction.commit(state.registry) + write_source_turn_effect_gate_locked( + registry_path=request.registry_path, + goal_id=request.goal_id, + gate=next_gate, + ) write_journal(journal_path, {**journal, "phase": "published"}) return _recreation_result( request, diff --git a/loopx/control_plane/goals/source_session_turn_effects.py b/loopx/control_plane/goals/source_session_turn_effects.py new file mode 100644 index 0000000000..48f746a920 --- /dev/null +++ b/loopx/control_plane/goals/source_session_turn_effects.py @@ -0,0 +1,544 @@ +from __future__ import annotations + +from collections.abc import Callable, Mapping +from dataclasses import dataclass +import json +from pathlib import Path +from typing import Any + +from ...file_lock import ( + LockAcquireTimeoutError, + LockAcquisitionPolicy, + exclusive_cross_runtime_file_lock, + exclusive_file_lock, +) +from ..effect_program import SettlementStepKind +from ..effect_runtime import effect_runtime_result +from ..projects.registry_codec import ( + SOURCE_SESSION_PROFILE_ID, + load_project_registry, +) +from ..todos.active_state_editing import fsync_state_directory +from .source_session_registry_state import ( + alias_digest, + canonical_digest, + current_goal_ref, + guard_path, + lifetime_root, + write_journal, +) + + +_GATE_SCHEMA = "loopx_source_turn_effect_gate_v1" +_ADMISSION_SCHEMA = "loopx_source_turn_effect_admission_v1" +_HOLD_SCHEMA = "loopx_source_turn_effect_hold_v1" + +SourceAdmissionFactory = Callable[[], Mapping[str, Any]] +JournalPersist = Callable[[Mapping[str, Any]], None] + + +class SourceTurnEffectRejected(ValueError): + def __init__(self, code: str) -> None: + super().__init__(f"source Turn effect rejected: {code}") + self.code = code + + +@dataclass(frozen=True, slots=True) +class SourceTurnEffect: + goal_ref: Mapping[str, str] + turn_key: str + step_kind: SettlementStepKind + effect_ref: str + journal_path: Path + + def payload(self) -> dict[str, Any]: + return { + "schema_version": _ADMISSION_SCHEMA, + "goal_ref": dict(self.goal_ref), + "turn_key": self.turn_key, + "step_kind": self.step_kind.value, + "effect_ref": self.effect_ref, + } + + +def _gate_root(registry_path: Path, goal_id: str) -> Path: + return lifetime_root(registry_path) / "turn-settlement" / alias_digest(goal_id) + + +def source_turn_effect_gate_path(registry_path: Path, goal_id: str) -> Path: + return _gate_root(registry_path, goal_id) / "gate.json" + + +def _admission_directory(registry_path: Path, goal_id: str) -> Path: + return _gate_root(registry_path, goal_id) / "admissions" + + +def _admission_path( + registry_path: Path, + goal_id: str, + effect: SourceTurnEffect, +) -> Path: + digest = canonical_digest(effect.payload()).removeprefix("sha256:") + return _admission_directory(registry_path, goal_id) / f"{digest}.json" + + +def _read_object(path: Path, *, label: str) -> dict[str, Any] | None: + if not path.exists(): + return None + try: + value = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError: + return None + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"{label} is unreadable") from exc + if not isinstance(value, dict): + raise ValueError(f"{label} must be a JSON object") + return value + + +def _read_gate(registry_path: Path, goal_id: str) -> dict[str, Any] | None: + return _read_object( + source_turn_effect_gate_path(registry_path, goal_id), + label="source Turn effect gate", + ) + + +def _read_admission( + registry_path: Path, + goal_id: str, + effect: SourceTurnEffect, +) -> dict[str, Any] | None: + return _read_object( + _admission_path(registry_path, goal_id, effect), + label="source Turn effect admission", + ) + + +def _source_state( + registry_path: Path, + goal_id: str, +) -> tuple[dict[str, Any], dict[str, str]]: + registry = load_project_registry(registry_path) + if registry.get("profile_id") != SOURCE_SESSION_PROFILE_ID: + raise SourceTurnEffectRejected("unsupported_profile") + active_goal_ref, _goal = current_goal_ref(registry, goal_id=goal_id) + return registry, active_goal_ref + + +def _require_canonical_journal_path( + registry: Mapping[str, Any], + *, + goal_id: str, + effect: SourceTurnEffect, +) -> None: + runtime_root = Path(str(registry.get("common_runtime_root") or "")) + if not runtime_root.is_absolute() or runtime_root.resolve() != runtime_root: + raise ValueError("source-session common_runtime_root must be absolute") + digest = effect.turn_key.removeprefix("sha256:") + expected = runtime_root / "goals" / goal_id / "turns" / f"{digest}.json" + if effect.journal_path.resolve() != expected: + raise SourceTurnEffectRejected("journal_path_mismatch") + + +def _decision(method: str, facts: Mapping[str, Any]) -> dict[str, Any]: + decision = effect_runtime_result(method, dict(facts)) + if not isinstance(decision, dict): + raise RuntimeError("source Turn effect decision must be an object") + if decision.get("kind") == "reject": + raise SourceTurnEffectRejected(str(decision.get("code") or "invalid")) + if decision.get("kind") not in {"commit", "replay"}: + raise RuntimeError("source Turn effect decision kind is unsupported") + return decision + + +def _write_decision_payload( + path: Path, + decision: Mapping[str, Any], + field: str, +) -> None: + payload = decision.get(field) + if not isinstance(payload, dict): + raise RuntimeError(f"source Turn effect decision omitted {field}") + write_journal(path, payload) + + +def _remove_admission(path: Path) -> None: + path.unlink(missing_ok=True) + fsync_state_directory(path) + + +def prepare_source_turn_effect( + *, + registry_path: Path, + goal_id: str, + effect: SourceTurnEffect, + source_admission: SourceAdmissionFactory, + persist_journal: JournalPersist, +) -> None: + target = guard_path(registry_path, goal_id) + with exclusive_cross_runtime_file_lock( + target, + operation="source_turn_effect_admit", + ): + registry, active_goal_ref = _source_state(registry_path, goal_id) + _require_canonical_journal_path( + registry, + goal_id=goal_id, + effect=effect, + ) + admission_path = _admission_path(registry_path, goal_id, effect) + decision = _decision( + "goal.source_session.turn_effect.admit", + { + "profile_id": SOURCE_SESSION_PROFILE_ID, + "requested_goal_ref": dict(effect.goal_ref), + "current_goal_ref": active_goal_ref, + "gate": _read_gate(registry_path, goal_id), + "admission": effect.payload(), + "existing_admission": _read_admission( + registry_path, + goal_id, + effect, + ), + }, + ) + _write_decision_payload( + source_turn_effect_gate_path(registry_path, goal_id), + decision, + "gate", + ) + _write_decision_payload(admission_path, decision, "admission") + persist_journal(source_admission()) + + +def release_source_turn_effect( + *, + registry_path: Path, + goal_id: str, + effect: SourceTurnEffect, + source_admission: SourceAdmissionFactory, + persist_journal: JournalPersist, +) -> None: + target = guard_path(registry_path, goal_id) + with exclusive_cross_runtime_file_lock( + target, + operation="source_turn_effect_release", + ): + registry, active_goal_ref = _source_state(registry_path, goal_id) + _require_canonical_journal_path( + registry, + goal_id=goal_id, + effect=effect, + ) + admission_path = _admission_path(registry_path, goal_id, effect) + _decision( + "goal.source_session.turn_effect.release", + { + "profile_id": SOURCE_SESSION_PROFILE_ID, + "current_goal_ref": active_goal_ref, + "gate": _read_gate(registry_path, goal_id), + "admission": effect.payload(), + "existing_admission": _read_admission( + registry_path, + goal_id, + effect, + ), + }, + ) + persist_journal(source_admission()) + _remove_admission(admission_path) + + +def source_turn_effect_allows_absent_reexecute( + *, + registry_path: Path, + goal_id: str, + effect: SourceTurnEffect, +) -> bool: + target = guard_path(registry_path, goal_id) + with exclusive_cross_runtime_file_lock( + target, + operation="source_turn_effect_resolve_absent", + ): + registry, active_goal_ref = _source_state(registry_path, goal_id) + _require_canonical_journal_path( + registry, + goal_id=goal_id, + effect=effect, + ) + decision = effect_runtime_result( + "goal.source_session.turn_effect.resolve_absent", + { + "profile_id": SOURCE_SESSION_PROFILE_ID, + "current_goal_ref": active_goal_ref, + "gate": _read_gate(registry_path, goal_id), + "admission": effect.payload(), + "existing_admission": _read_admission( + registry_path, + goal_id, + effect, + ), + }, + ) + if not isinstance(decision, dict): + raise RuntimeError("source Turn effect decision must be an object") + if decision.get("kind") == "execute": + return True + if decision.get("kind") == "abort": + return False + code = str(decision.get("code") or "invalid") + raise SourceTurnEffectRejected(code) + + +def decide_source_turn_effect_close_locked( + *, + registry_path: Path, + goal_id: str, + requested_goal_ref: Mapping[str, str], + current_goal_ref: Mapping[str, str], + reserved_goal_ref: Mapping[str, str], + operation_id: str, + request_digest: str, +) -> dict[str, Any]: + decision = _decision( + "goal.source_session.turn_effect.gate", + { + "profile_id": SOURCE_SESSION_PROFILE_ID, + "operation": "close", + "operation_id": operation_id, + "request_digest": request_digest, + "requested_goal_ref": dict(requested_goal_ref), + "current_goal_ref": dict(current_goal_ref), + "reserved_goal_ref": dict(reserved_goal_ref), + "gate": _read_gate(registry_path, goal_id), + }, + ) + gate = decision.get("gate") + if not isinstance(gate, dict): + raise RuntimeError("source Turn effect decision omitted gate") + return gate + + +def decide_source_turn_effect_publish_locked( + *, + registry_path: Path, + goal_id: str, + requested_goal_ref: Mapping[str, str], + current_goal_ref: Mapping[str, str], + reserved_goal_ref: Mapping[str, str], + operation_id: str, + request_digest: str, +) -> dict[str, Any]: + decision = _decision( + "goal.source_session.turn_effect.gate", + { + "profile_id": SOURCE_SESSION_PROFILE_ID, + "operation": "publish", + "operation_id": operation_id, + "request_digest": request_digest, + "requested_goal_ref": dict(requested_goal_ref), + "current_goal_ref": dict(current_goal_ref), + "reserved_goal_ref": dict(reserved_goal_ref), + "gate": _read_gate(registry_path, goal_id), + "admission_count": len(_admission_paths(registry_path, goal_id)), + }, + ) + gate = decision.get("gate") + if not isinstance(gate, dict): + raise RuntimeError("source Turn effect decision omitted gate") + return gate + + +def decide_source_turn_effect_repair_locked( + *, + registry_path: Path, + goal_id: str, + requested_goal_ref: Mapping[str, str], + current_goal_ref: Mapping[str, str], + reserved_goal_ref: Mapping[str, str], + operation_id: str, + request_digest: str, +) -> dict[str, Any] | None: + decision = effect_runtime_result( + "goal.source_session.turn_effect.gate", + { + "profile_id": SOURCE_SESSION_PROFILE_ID, + "operation": "repair", + "operation_id": operation_id, + "request_digest": request_digest, + "requested_goal_ref": dict(requested_goal_ref), + "current_goal_ref": dict(current_goal_ref), + "reserved_goal_ref": dict(reserved_goal_ref), + "gate": _read_gate(registry_path, goal_id), + "admission_count": len(_admission_paths(registry_path, goal_id)), + }, + ) + if not isinstance(decision, dict): + raise RuntimeError("source Turn effect decision must be an object") + if decision.get("kind") == "reject": + raise SourceTurnEffectRejected(str(decision.get("code") or "invalid")) + if decision.get("kind") == "preserve": + return None + if decision.get("kind") not in {"commit", "replay"}: + raise RuntimeError("source Turn effect decision kind is unsupported") + gate = decision.get("gate") + if not isinstance(gate, dict): + raise RuntimeError("source Turn effect decision omitted gate") + return gate + + +def write_source_turn_effect_gate_locked( + *, + registry_path: Path, + goal_id: str, + gate: dict[str, Any], +) -> None: + write_journal(source_turn_effect_gate_path(registry_path, goal_id), gate) + + +def _admission_paths(registry_path: Path, goal_id: str) -> list[Path]: + directory = _admission_directory(registry_path, goal_id) + if not directory.exists(): + return [] + if not directory.is_dir(): + raise ValueError("source Turn effect admissions path is not a directory") + return sorted(directory.glob("*.json")) + + +def _pending_projection( + admission: Mapping[str, Any], + *, + reason: str, +) -> dict[str, str]: + return { + "turn_key": str(admission.get("turn_key") or ""), + "step_kind": str(admission.get("step_kind") or ""), + "reason": reason, + } + + +def drain_releasable_source_turn_effects( + *, + registry_path: Path, + goal_id: str, + requested_goal_ref: Mapping[str, str], +) -> list[dict[str, str]]: + # turn_driver imports the Host admission owner, so defer this reverse edge. + from ..turn_driver.journal_store import load_turn_journal, turn_journal_path + + registry = load_project_registry(registry_path) + runtime_root = Path(str(registry.get("common_runtime_root") or "")) + if not runtime_root.is_absolute() or runtime_root.resolve() != runtime_root: + raise ValueError("source-session common_runtime_root must be absolute") + pending: list[dict[str, str]] = [] + for admission_path in _admission_paths(registry_path, goal_id): + admission = _read_object( + admission_path, + label="source Turn effect admission", + ) + if admission is None: + continue + if admission.get("goal_ref") != dict(requested_goal_ref): + pending.append(_pending_projection(admission, reason="goal_ref_mismatch")) + continue + turn_key = str(admission.get("turn_key") or "") + step_kind = str(admission.get("step_kind") or "") + effect_ref = str(admission.get("effect_ref") or "") + try: + journal_path = turn_journal_path( + runtime_root, + goal_id=goal_id, + turn_key=turn_key, + ) + except ValueError: + pending.append( + _pending_projection(admission, reason="journal_identity_invalid") + ) + continue + try: + with exclusive_file_lock( + journal_path, + policy=LockAcquisitionPolicy.SINGLE_FLIGHT, + operation="source_turn_effect_retirement_drain", + ): + try: + journal = load_turn_journal(journal_path) + except (OSError, TypeError, ValueError): + pending.append( + _pending_projection(admission, reason="journal_unreadable") + ) + continue + hold = ( + journal.get("source_effect_hold") + if isinstance(journal, Mapping) + else None + ) + if hold is not None: + expected_hold = { + "schema_version": _HOLD_SCHEMA, + "status": "held", + "step_kind": step_kind, + "effect_ref": effect_ref, + } + pending.append( + _pending_projection( + admission, + reason=( + "turn_tail_recovery_required" + if isinstance(hold, Mapping) + and dict(hold) == expected_hold + else "turn_tail_conflict" + ), + ) + ) + continue + attempts = ( + journal.get("effect_attempts") + if isinstance(journal, Mapping) + else None + ) + attempt = ( + attempts.get(step_kind) if isinstance(attempts, Mapping) else None + ) + if isinstance(attempt, Mapping): + if attempt.get("effect_ref") == effect_ref: + pending.append( + _pending_projection( + admission, + reason="provider_readback_required", + ) + ) + else: + pending.append( + _pending_projection( + admission, + reason="journal_effect_conflict", + ) + ) + continue + with exclusive_cross_runtime_file_lock( + guard_path(registry_path, goal_id), + operation="source_turn_effect_retirement_release", + ): + _registry, active_goal_ref = _source_state( + registry_path, + goal_id, + ) + existing = _read_object( + admission_path, + label="source Turn effect admission", + ) + _decision( + "goal.source_session.turn_effect.release", + { + "profile_id": SOURCE_SESSION_PROFILE_ID, + "current_goal_ref": active_goal_ref, + "gate": _read_gate(registry_path, goal_id), + "admission": admission, + "existing_admission": existing, + }, + ) + _remove_admission(admission_path) + except LockAcquireTimeoutError: + pending.append(_pending_projection(admission, reason="executor_active")) + return pending diff --git a/loopx/control_plane/turn_driver/executor.py b/loopx/control_plane/turn_driver/executor.py index 99e941cfc7..081908009a 100644 --- a/loopx/control_plane/turn_driver/executor.py +++ b/loopx/control_plane/turn_driver/executor.py @@ -58,7 +58,9 @@ build_host_recovery_record, ) from .settlement import ( + SourceJournalPersist, TurnEffectResolver, + TurnSettlementEffectAdmission, TurnSettlementJournalAdapter, completion_writeback_outcome, execute_turn_driver_settlement, @@ -667,21 +669,42 @@ def count_stderr(text: str) -> None: stderr_chars += len(text) try: - observed = run_host_process(argv, project=project, + observed = run_host_process( + argv, + project=project, input_text=json.dumps(request, ensure_ascii=False, separators=(",", ":")), - timeout_seconds=timeout_seconds, stdout_limit_bytes=HOST_RESULT_MAX_BYTES, - on_stdout=stdout.append, on_stderr=count_stderr) + timeout_seconds=timeout_seconds, + stdout_limit_bytes=HOST_RESULT_MAX_BYTES, + on_stdout=stdout.append, + on_stderr=count_stderr, + ) except (OSError, RuntimeError, ValueError) as exc: return {"ok": False, "reason": type(exc).__name__, "returncode": None} if observed["outcome"] == "output_limit": - return {"ok": False, "reason": "host stdout exceeded the result budget", "returncode": observed["returncode"]} + return { + "ok": False, + "reason": "host stdout exceeded the result budget", + "returncode": observed["returncode"], + } if observed["outcome"] != "exited": - return {"ok": False, "reason": "host process " + observed["outcome"], "returncode": observed["returncode"]} + return { + "ok": False, + "reason": "host process " + observed["outcome"], + "returncode": observed["returncode"], + } if not observed["output_complete"]: - return {"ok": False, "reason": "host output observation incomplete", "returncode": observed["returncode"]} + return { + "ok": False, + "reason": "host output observation incomplete", + "returncode": observed["returncode"], + } if observed["returncode"] != 0: - return {"ok": False, "reason": "host command returned non-zero", - "returncode": observed["returncode"], "stderr_chars": stderr_chars} + return { + "ok": False, + "reason": "host command returned non-zero", + "returncode": observed["returncode"], + "stderr_chars": stderr_chars, + } try: value = json.loads("".join(stdout)) except json.JSONDecodeError: @@ -780,7 +803,12 @@ def _host_result_stage( if confirm_start is not None: confirm_start() from ...usage_goal import observe_goal_execution - with observe_goal_execution(usage_runtime_root or project, usage_goal_id, host=str((plan.get("host") or {}).get("kind") or "unknown")): + + with observe_goal_execution( + usage_runtime_root or project, + usage_goal_id, + host=str((plan.get("host") or {}).get("kind") or "unknown"), + ): host_observation = ( _run_host_runner(request, runner=host_runner) if host_runner is not None @@ -995,8 +1023,7 @@ def _ensure_turn_settlement_plan( execution_mode=str(host_fields.get("execution_mode") or "isolated-headless"), session_action=str(host_fields.get("session_action") or "resume"), turn_instance_id=( - transaction_plan.get("turn_instance_id") - or transaction_plan.get("turn_key") + transaction_plan.get("turn_instance_id") or transaction_plan.get("turn_key") ), ) settlement_plan = built.get("settlement_plan") @@ -1020,6 +1047,8 @@ def _typed_settlement_stage( effect_resolvers: Mapping[SettlementStepKind, TurnEffectResolver], scheduler: Scheduler, post_settlement: PostSettlement | None, + source_effects: TurnSettlementEffectAdmission | None, + persist_source_journal: SourceJournalPersist | None, ) -> dict[str, Any]: transaction_plan = ( plan.get("transaction") if isinstance(plan.get("transaction"), Mapping) else {} @@ -1088,10 +1117,19 @@ def writeback_effect(effect_ref: str) -> Mapping[str, Any]: return invoke_result_effect(writeback, result, effect_ref) journal_adapter = TurnSettlementJournalAdapter( - journal, - effects, - lambda: persist_journal(journal), - _compact_callback, + journal=journal, + effects=effects, + persist=lambda: persist_journal(journal), + compact_payload=_compact_callback, + source_effects=source_effects, + persist_source=persist_source_journal, + deferred_release_step=( + SettlementStepKind.TERMINAL_CLOSEOUT + if source_effects is not None and terminal_closeout_required + else ( + SettlementStepKind.QUOTA_SPEND if source_effects is not None else None + ) + ), ) terminal_effect = None @@ -1130,7 +1168,9 @@ def writeback_effect(effect_ref: str) -> Mapping[str, Any]: terminal_closeout=terminal_effect, terminal_checkpoint=terminal_checkpoint, prepare=journal_adapter.prepare, + resume_prepare=journal_adapter.prepare if source_effects is not None else None, abort=journal_adapter.abort, + allow_absent_reexecute=journal_adapter.allows_absent_reexecute, effect_attempts=journal_adapter.effect_attempts, effect_resolvers=effect_resolvers, turn_result_kind=str(result.get("result_kind") or "") or None, @@ -1174,7 +1214,29 @@ def writeback_effect(effect_ref: str) -> Mapping[str, Any]: result = {**result, "result_kind": outcome["result_kind"]} completed_phases = [str(phase) for phase in outcome["completed_phases"]] spend_payload = dict(settlement_state.quota_spend) - persist_journal(journal) + tail_effect: tuple[SettlementStepKind, str] | None = None + if source_effects is None: + persist_journal(journal) + else: + effect_id = _journal_committed_effect_id(journal) + if effect_id is None or persist_source_journal is None: + raise RuntimeError("source Turn tail admission identity is unavailable") + tail_step = ( + SettlementStepKind.TERMINAL_CLOSEOUT + if terminal_closeout_required + else SettlementStepKind.QUOTA_SPEND + ) + tail_effect = (tail_step, f"{effect_id}#{tail_step.value}") + journal_adapter.hold_tail(tail_effect[0], tail_effect[1]) + + def persist_tail_checkpoint(*, release: bool) -> None: + if tail_effect is None: + persist_journal(journal) + return + if release: + journal_adapter.release_tail(tail_effect[0], tail_effect[1]) + else: + journal_adapter.hold_tail(tail_effect[0], tail_effect[1]) scheduler_payload = scheduler(spend_payload) journal["scheduler"] = scheduler_payload @@ -1190,7 +1252,7 @@ def writeback_effect(effect_ref: str) -> Mapping[str, Any]: status="scheduler_action_required", receipt=_receipt(plan, result, completed_phases=completed_phases), ) - persist_journal(journal) + persist_tail_checkpoint(release=False) return execution_payload( plan, journal, @@ -1206,7 +1268,7 @@ def writeback_effect(effect_ref: str) -> Mapping[str, Any]: completed_phases=completed_phases, receipt=_receipt(plan, result, completed_phases=completed_phases), ) - persist_journal(journal) + persist_tail_checkpoint(release=True) return execution_payload( plan, journal, @@ -1322,6 +1384,13 @@ def persist_journal(snapshot: Mapping[str, Any]) -> None: raise FirstPartyHostRuntimeRejected(exc.diagnostic_code) from exc raise + def persist_source_journal(source_admission: Mapping[str, Any]) -> None: + _write_journal( + journal_path, + journal, + source_admission=source_admission, + ) + with exclusive_file_lock(journal_path): journal = _load_journal(journal_path) recovery_decision: dict[str, Any] | None = None @@ -1373,24 +1442,31 @@ def persist_journal(snapshot: Mapping[str, Any]) -> None: and receipt.get("failed_phase") == "validation" and journal.get("validation_stage") != "task_postcondition" ) - needs_host = validation_reinvokes_host or journal is None or "typed_result" not in list( - journal.get("completed_phases") or [] + needs_host = ( + validation_reinvokes_host + or journal is None + or "typed_result" not in list(journal.get("completed_phases") or []) ) if needs_host and goal_admission is not None: goal_admission.require_current() admission = None if needs_host and admit_start is not None: - admission = admit_start({ - "turn_key": turn_key, - "attempt": int(journal.get("host_attempt_count") or 0) + 1 - if journal is not None else 1, - }) + admission = admit_start( + { + "turn_key": turn_key, + "attempt": int(journal.get("host_attempt_count") or 0) + 1 + if journal is not None + else 1, + } + ) if admission.get("admitted") is not True: waiting = { "status": "interval_wait", "host": host_projection, "completed_phases": [], - "reason": str(admission.get("reason") or "automatic start not admitted"), + "reason": str( + admission.get("reason") or "automatic start not admitted" + ), "admission": admission, } return execution_payload( @@ -1398,7 +1474,10 @@ def persist_journal(snapshot: Mapping[str, Any]) -> None: ) if journal is not None and recovery_decision is not None: journal["recovery_audit"] = build_turn_recovery_audit( - recovery_decision, journal, status="started", host_invoked=None, + recovery_decision, + journal, + status="started", + host_invoked=None, ) persist_journal(journal) @@ -1516,5 +1595,18 @@ def finish_recovery(payload: dict[str, Any]) -> dict[str, Any]: ), scheduler=scheduler, post_settlement=post_settlement, + source_effects=( + goal_admission.turn_effect_admission( + turn_key=turn_key, + journal_path=journal_path, + ) + if goal_admission is not None + else None + ), + persist_source_journal=( + persist_source_journal + if goal_admission is not None and goal_admission.enabled + else None + ), ) return finish_recovery(settled) diff --git a/loopx/control_plane/turn_driver/settlement.py b/loopx/control_plane/turn_driver/settlement.py index 1d2e01886b..560347bff8 100644 --- a/loopx/control_plane/turn_driver/settlement.py +++ b/loopx/control_plane/turn_driver/settlement.py @@ -5,7 +5,7 @@ import inspect from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass -from typing import Any +from typing import Any, Protocol from ..effect_program import ( SettlementResult, @@ -33,6 +33,36 @@ TerminalCloseoutCheckpoint = Callable[[Mapping[str, Any]], None] CompletionIntent = Callable[[Mapping[str, Any]], Mapping[str, Any]] +SourceJournalPersist = Callable[[Mapping[str, Any]], None] + + +class TurnSettlementEffectAdmission(Protocol): + def prepare( + self, + step_kind: SettlementStepKind, + effect_ref: str, + persist_journal: SourceJournalPersist, + ) -> None: ... + + def hold( + self, + step_kind: SettlementStepKind, + effect_ref: str, + persist_journal: SourceJournalPersist, + ) -> None: ... + + def release( + self, + step_kind: SettlementStepKind, + effect_ref: str, + persist_journal: SourceJournalPersist, + ) -> None: ... + + def allows_absent_reexecute( + self, + step_kind: SettlementStepKind, + effect_ref: str, + ) -> bool: ... @dataclass(frozen=True, slots=True) @@ -44,6 +74,7 @@ class TurnSettlementState: TURN_SETTLEMENT_TRANSACTION_SCHEMA_VERSION = "loopx_turn_settlement_transaction_v0" TURN_SETTLEMENT_REDUCTION_SCHEMA_VERSION = "loopx_turn_settlement_reduction_v0" +SOURCE_TURN_EFFECT_HOLD_SCHEMA_VERSION = "loopx_source_turn_effect_hold_v1" def _invoke_turn_effect(effect: TurnEffect, effect_ref: str) -> Mapping[str, Any]: @@ -119,6 +150,9 @@ class TurnSettlementJournalAdapter: effects: dict[str, bool] persist: Callable[[], None] compact_payload: Callable[[Mapping[str, Any]], Mapping[str, Any]] + source_effects: TurnSettlementEffectAdmission | None = None + persist_source: SourceJournalPersist | None = None + deferred_release_step: SettlementStepKind | None = None @property def effect_attempts(self) -> Mapping[str, Mapping[str, Any]]: @@ -133,11 +167,25 @@ def prepare(self, step_kind: SettlementStepKind, effect_ref: str) -> None: "status": "prepared", "effect_ref": effect_ref, } - self.persist() + if self.source_effects is None: + self.persist() + return + self.source_effects.prepare( + step_kind, + effect_ref, + self._require_source_persist(), + ) def abort(self, step_kind: SettlementStepKind, effect_ref: str) -> None: self._forget(step_kind, effect_ref) - self.persist() + if self.source_effects is None: + self.persist() + return + self.source_effects.release( + step_kind, + effect_ref, + self._require_source_persist(), + ) def checkpoint( self, @@ -145,6 +193,7 @@ def checkpoint( payload: Mapping[str, Any], phases: tuple[str, ...], ) -> None: + effect_ref = self._effect_ref(step_kind, payload) if step_kind is SettlementStepKind.DURABLE_WRITEBACK: self.effects["state_written"] = True self.journal["writeback"] = self._completion_payload(payload) @@ -152,10 +201,15 @@ def checkpoint( self.effects["quota_spent"] = True self.journal["quota_spend"] = dict(self.compact_payload(payload)) self.journal["completed_phases"] = list(phases) - self._forget(step_kind, str(payload.get("effect_ref") or ""), strict=False) - self.persist() + self._forget(step_kind, effect_ref) + if self.source_effects is None: + self.persist() + return + self._checkpoint_source_effect(step_kind, effect_ref) def checkpoint_terminal(self, payload: Mapping[str, Any]) -> None: + step_kind = SettlementStepKind.TERMINAL_CLOSEOUT + effect_ref = self._effect_ref(step_kind, payload) compact = self._completion_payload(payload) self.journal["terminal_closeout"] = compact writeback = self.journal.get("writeback") @@ -163,12 +217,88 @@ def checkpoint_terminal(self, payload: Mapping[str, Any]) -> None: **(dict(writeback) if isinstance(writeback, Mapping) else {}), "completion": compact.get("completion"), } - self._forget( - SettlementStepKind.TERMINAL_CLOSEOUT, - str(payload.get("effect_ref") or ""), - strict=False, + self._forget(step_kind, effect_ref) + if self.source_effects is None: + self.persist() + return + self._checkpoint_source_effect(step_kind, effect_ref) + + def allows_absent_reexecute( + self, + step_kind: SettlementStepKind, + effect_ref: str, + ) -> bool: + if self.source_effects is None: + return True + return self.source_effects.allows_absent_reexecute(step_kind, effect_ref) + + def hold_tail( + self, + step_kind: SettlementStepKind, + effect_ref: str, + ) -> None: + if self.source_effects is None: + raise RuntimeError("source Turn effect admission is unavailable") + self._set_tail_hold(step_kind, effect_ref) + self.source_effects.hold( + step_kind, + effect_ref, + self._require_source_persist(), + ) + + def release_tail( + self, + step_kind: SettlementStepKind, + effect_ref: str, + ) -> None: + expected = self._tail_hold(step_kind, effect_ref) + if self.journal.get("source_effect_hold") != expected: + raise RuntimeError("source Turn effect hold identity changed") + self.journal.pop("source_effect_hold") + if self.source_effects is None: + raise RuntimeError("source Turn effect admission is unavailable") + self.source_effects.release( + step_kind, + effect_ref, + self._require_source_persist(), ) - self.persist() + + def _checkpoint_source_effect( + self, + step_kind: SettlementStepKind, + effect_ref: str, + ) -> None: + if self.source_effects is None: + raise RuntimeError("source Turn effect admission is unavailable") + persist = self._require_source_persist() + if step_kind is self.deferred_release_step: + self._set_tail_hold(step_kind, effect_ref) + self.source_effects.hold(step_kind, effect_ref, persist) + return + self.source_effects.release(step_kind, effect_ref, persist) + + @staticmethod + def _tail_hold( + step_kind: SettlementStepKind, + effect_ref: str, + ) -> dict[str, str]: + return { + "schema_version": SOURCE_TURN_EFFECT_HOLD_SCHEMA_VERSION, + "status": "held", + "step_kind": step_kind.value, + "effect_ref": effect_ref, + } + + def _set_tail_hold( + self, + step_kind: SettlementStepKind, + effect_ref: str, + ) -> None: + expected = self._tail_hold(step_kind, effect_ref) + existing = self.journal.get("source_effect_hold") + if existing is not None and existing != expected: + raise RuntimeError("source Turn effect hold identity changed") + self.journal["source_effect_hold"] = expected def _completion_payload(self, payload: Mapping[str, Any]) -> dict[str, Any]: return { @@ -180,6 +310,29 @@ def _completion_payload(self, payload: Mapping[str, Any]) -> dict[str, Any]: ), } + def _effect_ref( + self, + step_kind: SettlementStepKind, + payload: Mapping[str, Any], + ) -> str: + payload_ref = str(payload.get("effect_ref") or "") + attempt = self.effect_attempts.get(step_kind.value) + attempt_ref = ( + str(attempt.get("effect_ref") or "") if isinstance(attempt, Mapping) else "" + ) + if attempt_ref: + if payload_ref and payload_ref != attempt_ref: + raise RuntimeError("Turn provider effect ref changed") + return attempt_ref + if payload_ref: + return payload_ref + raise RuntimeError("Turn journal prepared effect ref is missing") + + def _require_source_persist(self) -> SourceJournalPersist: + if self.persist_source is None: + raise RuntimeError("source Turn effect persistence is unavailable") + return self.persist_source + def _forget( self, step_kind: SettlementStepKind, @@ -288,6 +441,8 @@ def _resolve_prepared_effect( resolvers: Mapping[SettlementStepKind, TurnEffectResolver], step_kind: SettlementStepKind, effect_ref: str, + *, + allow_absent_reexecute: Callable[[SettlementStepKind, str], bool], ) -> tuple[bool, Mapping[str, Any] | None, Mapping[str, Any] | None]: """Return execute, committed payload, or a fail-closed observation.""" @@ -314,6 +469,16 @@ def _resolve_prepared_effect( ) kind = str(resolution.get("kind") or "unknown") if kind == "absent": + if not allow_absent_reexecute(step_kind, effect_ref): + return ( + False, + { + "ok": False, + "appended": False, + "reason": "Goal retirement closed effect re-execution", + }, + None, + ) return True, None, None if kind != "committed": observation = ( @@ -348,7 +513,9 @@ def execute_turn_driver_settlement( terminal_closeout: TurnEffect | None = None, terminal_checkpoint: TerminalCloseoutCheckpoint | None = None, prepare: TurnSettlementPrepare | None = None, + resume_prepare: TurnSettlementPrepare | None = None, abort: TurnSettlementAbort | None = None, + allow_absent_reexecute: Callable[[SettlementStepKind, str], bool] | None = None, effect_attempts: Mapping[str, Mapping[str, Any]] | None = None, effect_resolvers: Mapping[SettlementStepKind, TurnEffectResolver] | None = None, turn_result_kind: str | None = None, @@ -457,8 +624,17 @@ def reduce() -> Mapping[str, Any]: "effect_ref": effect_ref, } else: + if resume_prepare is not None: + resume_prepare(step_kind, effect_ref) should_execute, observed, observation = _resolve_prepared_effect( - resolvers, step_kind, effect_ref + resolvers, + step_kind, + effect_ref, + allow_absent_reexecute=( + allow_absent_reexecute + if allow_absent_reexecute is not None + else lambda _step, _ref: True + ), ) if observation is not None: observations[step_kind.value] = observation @@ -570,4 +746,8 @@ def turn_settlement_failure_outcome( raise RuntimeError( "TypeScript Turn settlement failure has unsupported result_kind" ) from exc - return result_kind, tuple(str(phase) for phase in outcome["completed_phases"]), failed_phase + return ( + result_kind, + tuple(str(phase) for phase in outcome["completed_phases"]), + failed_phase, + ) diff --git a/loopx/semantics/goal_instance_binding_inventory_v1.json b/loopx/semantics/goal_instance_binding_inventory_v1.json index 9a3a42ad93..f34c3e1d07 100644 --- a/loopx/semantics/goal_instance_binding_inventory_v1.json +++ b/loopx/semantics/goal_instance_binding_inventory_v1.json @@ -37,25 +37,42 @@ }, { "owner_id": "first_party_host_runtime", - "locator": "/goals//sessions and host runtime state", + "locator": "/goals//sessions and host runtime state; /.loopx/lifecycle/goal-instance/turn-settlement//{gate.json,admissions/*.json}", "revision_signal": "turn_instance_id", "content_digest_signal": "turn_key", "observed_reference": "goal_id + goal_instance_id for source_session_v1; goal_id otherwise", "producer_sites": [ "loopx/control_plane/goals/first_party_host_admission.py::capture_first_party_host_goal_ref", + "loopx/control_plane/goals/source_session_recreation.py::recreate_goal_instance", + "loopx/control_plane/goals/source_session_turn_effects.py::drain_releasable_source_turn_effects", + "loopx/control_plane/goals/source_session_turn_effects.py::prepare_source_turn_effect", + "loopx/control_plane/goals/source_session_turn_effects.py::release_source_turn_effect", + "loopx/control_plane/goals/source_session_turn_effects.py::write_source_turn_effect_gate_locked", "loopx/control_plane/turn_driver/codex_cli.py::codex_cli_session_binding", - "loopx/control_plane/turn_driver/driver.py::build_loopx_turn_plan" + "loopx/control_plane/turn_driver/driver.py::build_loopx_turn_plan", + "loopx/control_plane/turn_driver/settlement.py::TurnSettlementJournalAdapter.hold_tail", + "loopx/control_plane/turn_driver/settlement.py::TurnSettlementJournalAdapter.release_tail" ], "consumer_sites": [ "loopx/control_plane/goals/first_party_host_runtime.ts::decideFirstPartyHostRuntime", + "loopx/control_plane/goals/source_session_recreation.py::recreate_goal_instance", + "loopx/control_plane/goals/source_session_turn_effects.py::decide_source_turn_effect_close_locked", + "loopx/control_plane/goals/source_session_turn_effects.py::decide_source_turn_effect_publish_locked", + "loopx/control_plane/goals/source_session_turn_effects.py::decide_source_turn_effect_repair_locked", + "loopx/control_plane/goals/source_session_turn_effects.py::drain_releasable_source_turn_effects", + "loopx/control_plane/goals/source_session_turn_effects.py::prepare_source_turn_effect", + "loopx/control_plane/goals/source_session_turn_effects.py::release_source_turn_effect", + "loopx/control_plane/goals/source_session_turn_effects.py::source_turn_effect_allows_absent_reexecute", "loopx/control_plane/turn_driver/executor.py::run_loopx_turn_once", + "loopx/control_plane/turn_driver/settlement.py::TurnSettlementJournalAdapter.hold_tail", + "loopx/control_plane/turn_driver/settlement.py::TurnSettlementJournalAdapter.release_tail", "loopx/dsh_goal_mode/turn_host_adapter.py::run_dsh_host", "loopx/kunluncode_goal_mode/runtime.py::run_native_goal" ], - "effect_boundary": "loopx/control_plane/goals/first_party_host_admission.py::FirstPartyHostGoalAdmission.accept_result", + "effect_boundary": "loopx/control_plane/goals/source_session_turn_effects.py::prepare_source_turn_effect", "authority_role": "host_execution_binding", "current_identity_strength": "source_exact_partial", - "cleanup_support": "host_session_discard", + "cleanup_support": "host_session_discard_and_source_turn_effect_drain", "m1_disposition": "source_exact_partial_enforcement", "target_milestone": "M3" }, diff --git a/loopx/semantics/project_registry_io_manifest_v1.json b/loopx/semantics/project_registry_io_manifest_v1.json index 956f4c8b06..3f4567a78a 100644 --- a/loopx/semantics/project_registry_io_manifest_v1.json +++ b/loopx/semantics/project_registry_io_manifest_v1.json @@ -1279,7 +1279,7 @@ }, { "site": "loopx/control_plane/goals/first_party_host_admission.py::.FirstPartyHostGoalAdmission.for_plan::codec_read:load_project_registry#1", - "line": 124, + "line": 216, "column": 17, "kind": "codec_read", "api": "load_project_registry", @@ -1287,7 +1287,7 @@ }, { "site": "loopx/control_plane/goals/first_party_host_admission.py::._source_authority::codec_read:load_project_registry#1", - "line": 40, + "line": 132, "column": 20, "kind": "codec_read", "api": "load_project_registry", @@ -1295,7 +1295,7 @@ }, { "site": "loopx/control_plane/goals/first_party_host_admission.py::.capture_first_party_host_goal_ref::codec_read:load_project_registry#1", - "line": 75, + "line": 167, "column": 16, "kind": "codec_read", "api": "load_project_registry", @@ -1351,7 +1351,15 @@ }, { "site": "loopx/control_plane/goals/source_session_recreation.py::.recreate_goal_instance::codec_transaction:source_session_registry_transaction#1", - "line": 149, + "line": 286, + "column": 14, + "kind": "codec_transaction", + "api": "source_session_registry_transaction", + "classification": "codec_api" + }, + { + "site": "loopx/control_plane/goals/source_session_recreation.py::.recreate_goal_instance::codec_transaction:source_session_registry_transaction#2", + "line": 374, "column": 14, "kind": "codec_transaction", "api": "source_session_registry_transaction", @@ -1365,6 +1373,22 @@ "api": "source_session_registry_transaction", "classification": "codec_api" }, + { + "site": "loopx/control_plane/goals/source_session_turn_effects.py::._source_state::codec_read:load_project_registry#1", + "line": 121, + "column": 16, + "kind": "codec_read", + "api": "load_project_registry", + "classification": "codec_api" + }, + { + "site": "loopx/control_plane/goals/source_session_turn_effects.py::.drain_releasable_source_turn_effects::codec_read:load_project_registry#1", + "line": 429, + "column": 16, + "kind": "codec_read", + "api": "load_project_registry", + "classification": "codec_api" + }, { "site": "loopx/control_plane/goals/start_goal_todo_delta.py::._read_registry::codec_read:load_registry#1", "line": 229, diff --git a/tests/architecture/test_source_session_registry_denial.py b/tests/architecture/test_source_session_registry_denial.py index ba14096703..fade5d3b7d 100644 --- a/tests/architecture/test_source_session_registry_denial.py +++ b/tests/architecture/test_source_session_registry_denial.py @@ -19,6 +19,7 @@ "loopx/control_plane/collaboration/goal_instance_scope.py", "loopx/control_plane/collaboration/peers.py", "loopx/control_plane/goals/first_party_host_admission.py", + "loopx/control_plane/goals/source_session_turn_effects.py", "loopx/control_plane/coordination/runtime_shadow.py", "loopx/control_plane/coordination/shadow_goal_scope.py", "loopx/control_plane/projects/registry.py", @@ -52,6 +53,7 @@ def test_direct_project_registry_loaders_have_source_session_denial() -> None: "loopx/control_plane/collaboration/goal_instance_scope.py", "loopx/control_plane/collaboration/peers.py", "loopx/control_plane/goals/first_party_host_admission.py", + "loopx/control_plane/goals/source_session_turn_effects.py", "loopx/control_plane/coordination/runtime_shadow.py", "loopx/control_plane/coordination/shadow_goal_scope.py", "loopx/control_plane/projects/registry.py", diff --git a/tests/control_plane_ts/source_session_lifetime.test.ts b/tests/control_plane_ts/source_session_lifetime.test.ts index c9fd621899..51f41fb692 100644 --- a/tests/control_plane_ts/source_session_lifetime.test.ts +++ b/tests/control_plane_ts/source_session_lifetime.test.ts @@ -5,10 +5,15 @@ import { decideGoalRecreation, decideProjectSessionBind, decideProjectSessionUnbind, + decideSourceTurnEffectAbsentResolution, + decideSourceTurnEffectAdmission, + decideSourceTurnEffectGate, + decideSourceTurnEffectRelease, } from "../../loopx/control_plane/goals/source_session_lifetime.ts"; const INSTANCE_A = "ginst_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; const INSTANCE_B = "ginst_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; +const INSTANCE_C = "ginst_cccccccccccccccccccccccccccccccc"; function bindFacts(currentGoalInstanceId: string) { return { @@ -183,6 +188,229 @@ test("session capacity rejects new work after preserving exact replay", () => { ); }); +test("Turn effects admit once and release while retirement is closing", () => { + const goalRef = { + goal_id: "release", + goal_instance_id: INSTANCE_A, + }; + const admission = { + schema_version: "loopx_source_turn_effect_admission_v1", + goal_ref: goalRef, + turn_key: `sha256:${"c".repeat(64)}`, + step_kind: "durable_writeback", + effect_ref: `sha256:${"d".repeat(64)}`, + }; + const admitted = decideSourceTurnEffectAdmission({ + profile_id: "source_session_v1", + requested_goal_ref: goalRef, + current_goal_ref: goalRef, + gate: null, + admission, + existing_admission: null, + }); + assert.equal(admitted.kind, "commit"); + if (admitted.kind !== "commit") return; + assert.equal(admitted.gate.state, "open"); + assert.deepEqual( + decideSourceTurnEffectAdmission({ + profile_id: "source_session_v1", + requested_goal_ref: goalRef, + current_goal_ref: goalRef, + gate: admitted.gate, + admission, + existing_admission: admission, + }), + { + kind: "replay", + gate: admitted.gate, + admission, + }, + ); + + const closing = decideSourceTurnEffectGate({ + profile_id: "source_session_v1", + operation: "close", + operation_id: "recreate-release", + request_digest: `sha256:${"e".repeat(64)}`, + requested_goal_ref: goalRef, + current_goal_ref: goalRef, + reserved_goal_ref: { + goal_id: "release", + goal_instance_id: INSTANCE_B, + }, + gate: admitted.gate, + }); + assert.equal(closing.kind, "commit"); + if (closing.kind !== "commit") return; + assert.deepEqual( + decideSourceTurnEffectAdmission({ + profile_id: "source_session_v1", + requested_goal_ref: goalRef, + current_goal_ref: goalRef, + gate: closing.gate, + admission: { + ...admission, + step_kind: "quota_spend", + effect_ref: `sha256:${"f".repeat(64)}`, + }, + existing_admission: null, + }), + { kind: "reject", code: "goal_retirement_in_progress" }, + ); + assert.deepEqual( + decideSourceTurnEffectAbsentResolution({ + profile_id: "source_session_v1", + current_goal_ref: goalRef, + gate: closing.gate, + admission, + existing_admission: admission, + }), + { kind: "abort" }, + ); + assert.deepEqual( + decideSourceTurnEffectRelease({ + profile_id: "source_session_v1", + current_goal_ref: goalRef, + gate: closing.gate, + admission, + existing_admission: admission, + }), + { kind: "commit" }, + ); +}); + +test("Turn effect publication requires an empty matching closing gate", () => { + const facts = { + profile_id: "source_session_v1", + operation: "close", + operation_id: "recreate-release", + request_digest: `sha256:${"e".repeat(64)}`, + requested_goal_ref: { + goal_id: "release", + goal_instance_id: INSTANCE_A, + }, + current_goal_ref: { + goal_id: "release", + goal_instance_id: INSTANCE_A, + }, + reserved_goal_ref: { + goal_id: "release", + goal_instance_id: INSTANCE_B, + }, + gate: null, + }; + const closing = decideSourceTurnEffectGate(facts); + assert.equal(closing.kind, "commit"); + if (closing.kind !== "commit") return; + + assert.deepEqual( + decideSourceTurnEffectGate({ + ...facts, + operation: "publish", + gate: closing.gate, + admission_count: 1, + }), + { kind: "reject", code: "effect_drain_required" }, + ); + assert.deepEqual( + decideSourceTurnEffectGate({ + ...facts, + operation: "publish", + gate: closing.gate, + admission_count: 0, + }), + { + kind: "commit", + gate: { + schema_version: "loopx_source_turn_effect_gate_v1", + state: "open", + goal_ref: facts.reserved_goal_ref, + }, + }, + ); + assert.deepEqual( + decideSourceTurnEffectGate({ + ...facts, + operation: "publish", + current_goal_ref: facts.reserved_goal_ref, + gate: null, + admission_count: 0, + }), + { + kind: "replay", + gate: { + schema_version: "loopx_source_turn_effect_gate_v1", + state: "open", + goal_ref: facts.reserved_goal_ref, + }, + }, + ); +}); + +test("historical recreation replay preserves a later Goal gate", () => { + const repairFacts = { + profile_id: "source_session_v1", + operation: "repair", + operation_id: "recreate-release-b", + request_digest: `sha256:${"e".repeat(64)}`, + requested_goal_ref: { + goal_id: "release", + goal_instance_id: INSTANCE_A, + }, + reserved_goal_ref: { + goal_id: "release", + goal_instance_id: INSTANCE_B, + }, + admission_count: 1, + }; + const laterGate = { + schema_version: "loopx_source_turn_effect_gate_v1", + state: "closing", + retired_goal_ref: { + goal_id: "release", + goal_instance_id: INSTANCE_B, + }, + new_goal_ref: { + goal_id: "release", + goal_instance_id: INSTANCE_C, + }, + operation_id: "recreate-release-c", + request_digest: `sha256:${"f".repeat(64)}`, + }; + assert.deepEqual( + decideSourceTurnEffectGate({ + ...repairFacts, + current_goal_ref: laterGate.retired_goal_ref, + gate: laterGate, + }), + { kind: "preserve" }, + ); + assert.deepEqual( + decideSourceTurnEffectGate({ + ...repairFacts, + current_goal_ref: repairFacts.requested_goal_ref, + gate: { + schema_version: "loopx_source_turn_effect_gate_v1", + state: "open", + goal_ref: repairFacts.requested_goal_ref, + }, + }), + { kind: "reject", code: "recreation_operation_conflict" }, + ); + assert.deepEqual( + decideSourceTurnEffectGate({ + ...repairFacts, + current_goal_ref: laterGate.new_goal_ref, + gate: { + ...laterGate, + retired_goal_ref: repairFacts.requested_goal_ref, + operation_id: "conflicting-recreation", + }, + }), + { kind: "reject", code: "recreation_operation_conflict" }, + ); +}); + test("recreation publishes the reserved successor once and replays it exactly", () => { const facts = { profile_id: "source_session_v1", diff --git a/tests/test_loopx_turn_executor.py b/tests/test_loopx_turn_executor.py index d533eb111a..053297f25e 100644 --- a/tests/test_loopx_turn_executor.py +++ b/tests/test_loopx_turn_executor.py @@ -2,6 +2,7 @@ import json import sys +import threading from collections.abc import Mapping from pathlib import Path @@ -10,12 +11,21 @@ from loopx.cli_commands import turn_cadence from loopx.cli_commands.turn_cadence import ManagedCadenceStart, managed_cadence_start from loopx.control_plane.effect_runtime import effect_runtime_result +from loopx.control_plane.goals import source_session_turn_effects from loopx.control_plane.goals.first_party_host_admission import ( FirstPartyHostGoalAdmission, FirstPartyHostRuntimeRejected, ) -from loopx.control_plane.goals.source_session_registry_state import guard_path +from loopx.control_plane.goals.source_session_recreation import ( + RecreateGoalRequest, + recreate_goal_instance, +) +from loopx.control_plane.goals.source_session_registry_state import ( + alias_digest, + guard_path, +) from loopx.control_plane.projects.registry_codec import ( + load_project_registry, source_session_registry_transaction, ) from loopx.control_plane.turn_driver import executor as turn_executor @@ -183,9 +193,7 @@ def test_turn_journal_resolves_only_from_exact_settlement_identity( identity = settlement["identity"] assert isinstance(identity, dict) runtime_root = tmp_path / "runtime" - path = turn_journal_path( - runtime_root, goal_id="fixture-goal", turn_key=turn_key - ) + path = turn_journal_path(runtime_root, goal_id="fixture-goal", turn_key=turn_key) path.parent.mkdir(parents=True) path.write_text( json.dumps( @@ -201,20 +209,26 @@ def test_turn_journal_resolves_only_from_exact_settlement_identity( encoding="utf-8", ) - assert find_loopx_turn_key_by_settlement_identity( - runtime_root, - goal_id="fixture-goal", - agent_id="codex-fixture", - todo_id="todo_fixture0001", - turn_instance_id=str(identity["turn_instance_id"]), - ) == turn_key - assert find_loopx_turn_key_by_settlement_identity( - runtime_root, - goal_id="fixture-goal", - agent_id="other-agent", - todo_id="todo_fixture0001", - turn_instance_id=str(identity["turn_instance_id"]), - ) is None + assert ( + find_loopx_turn_key_by_settlement_identity( + runtime_root, + goal_id="fixture-goal", + agent_id="codex-fixture", + todo_id="todo_fixture0001", + turn_instance_id=str(identity["turn_instance_id"]), + ) + == turn_key + ) + assert ( + find_loopx_turn_key_by_settlement_identity( + runtime_root, + goal_id="fixture-goal", + agent_id="other-agent", + todo_id="todo_fixture0001", + turn_instance_id=str(identity["turn_instance_id"]), + ) + is None + ) def _adaptive_observation_plan( @@ -718,10 +732,7 @@ def test_child_receipt_schema_excludes_registered_peer_authority() -> None: item_schema["required"] ) assert properties["worker_ref"]["pattern"] == OPAQUE_REF_PATTERN - assert ( - properties["evidence_refs"]["items"]["pattern"] - == OPAQUE_REF_PATTERN - ) + assert properties["evidence_refs"]["items"]["pattern"] == OPAQUE_REF_PATTERN assert properties["evidence_refs"]["minItems"] == 1 @@ -828,9 +839,7 @@ def test_pre_spawn_rejection_remains_visible_without_blocking_parent() -> None: assert reconciliation["status"] == "guarded" assert reconciliation["counts"]["pre_spawn_rejected"] == 1 assert reconciliation["parent_blocked"] is False - assert reconciliation["pre_spawn_rejections"] == topology[ - "pre_spawn_rejections" - ] + assert reconciliation["pre_spawn_rejections"] == topology["pre_spawn_rejections"] def test_host_result_observes_missing_and_drifted_child_receipts() -> None: @@ -839,9 +848,9 @@ def test_host_result_observes_missing_and_drifted_child_receipts() -> None: assert missing["ok"] is True assert missing["result"]["subagent_reconciliation"]["status"] == "incomplete" - assert missing["result"]["subagent_reconciliation"]["lanes"][0][ - "reason_codes" - ] == ["worker_receipt_missing"] + assert missing["result"]["subagent_reconciliation"]["lanes"][0]["reason_codes"] == [ + "worker_receipt_missing" + ] drifted_result = _host_result(plan) drifted_result["child_execution_receipts"] = [ @@ -879,9 +888,7 @@ def test_host_result_observes_missing_and_drifted_child_receipts() -> None: "task_packet_digest": "sha256:" + "0" * 64, } ] - packet_mismatch = validate_loopx_turn_host_result( - plan, packet_mismatch_result - ) + packet_mismatch = validate_loopx_turn_host_result(plan, packet_mismatch_result) assert packet_mismatch["ok"] is True assert packet_mismatch["result"]["subagent_reconciliation"]["lanes"][0][ "reason_codes" @@ -894,9 +901,7 @@ def test_host_result_observes_missing_and_drifted_child_receipts() -> None: "context_mode": "forked_snapshot", } ] - context_mismatch = validate_loopx_turn_host_result( - plan, context_mismatch_result - ) + context_mismatch = validate_loopx_turn_host_result(plan, context_mismatch_result) assert context_mismatch["ok"] is True context_lane = context_mismatch["result"]["subagent_reconciliation"]["lanes"][0] assert context_lane["reason_codes"] == ["context_mode_mismatch"] @@ -1110,9 +1115,10 @@ def test_observation_only_reconciliation_does_not_change_settlement( assert committed["subagent_reconciliation"]["status"] == "incomplete" assert committed["subagent_reconciliation"]["settlement_enforced"] is False assert committed["subagent_reconciliation"]["parent_blocked"] is False - assert _journal(tmp_path / "runtime")["host_result"][ - "subagent_reconciliation" - ] == committed["subagent_reconciliation"] + assert ( + _journal(tmp_path / "runtime")["host_result"]["subagent_reconciliation"] + == committed["subagent_reconciliation"] + ) assert calls == {"writeback": 1, "spend": 1, "scheduler": 1} @@ -1153,20 +1159,32 @@ def host(_request: object) -> dict[str, object]: def wait(_identity: object) -> dict[str, object]: calls["admit"] += 1 - return {"admitted": False, "reason": "minimum_interval_wait", "next_eligible_at_ms": 9000} + return { + "admitted": False, + "reason": "minimum_interval_wait", + "next_eligible_at_ms": 9000, + } common = dict( - host_runner=host, project=tmp_path, runtime_root=tmp_path / "runtime", - goal_id="fixture-goal", timeout_seconds=5, execute=True, + host_runner=host, + project=tmp_path, + runtime_root=tmp_path / "runtime", + goal_id="fixture-goal", + timeout_seconds=5, + execute=True, task_validator=_passing_validator, - writeback=writeback, spend=spend, scheduler=scheduler, + writeback=writeback, + spend=spend, + scheduler=scheduler, ) denied = run_loopx_turn_once(plan, admit_start=wait, **common) assert denied["status"] == "interval_wait" assert denied["admission"]["next_eligible_at_ms"] == 9000 assert denied["effects"]["host_invoked"] is False assert calls == {"host": 0, "admit": 1, "writeback": 0, "spend": 0, "scheduler": 0} - assert not list((tmp_path / "runtime" / "goals" / "fixture-goal" / "turns").glob("*.json")) + assert not list( + (tmp_path / "runtime" / "goals" / "fixture-goal" / "turns").glob("*.json") + ) def allow(_identity: object) -> dict[str, object]: calls["admit"] += 1 @@ -1211,7 +1229,9 @@ def _cadence_starts(runtime_root: Path) -> list[dict[str, object]]: payload = json.loads(path.read_text(encoding="utf-8")) if ( isinstance(payload, dict) - and str(payload.get("schema_version", "")).startswith("automation_cadence_store") + and str(payload.get("schema_version", "")).startswith( + "automation_cadence_store" + ) and payload.get("goal_id") == "fixture-goal" ): stores.append(payload) @@ -1272,7 +1292,8 @@ def host(_request: object) -> dict[str, object]: def test_reserved_managed_start_recovers_after_death_before_the_first_journal_write( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, ) -> None: """A crash between the cadence reservation and the journal must not strand a Turn. @@ -1292,9 +1313,9 @@ def die_after_reservation(identity: Mapping[str, object]) -> dict[str, object]: with pytest.raises(RuntimeError, match="simulated process death"): run_loopx_turn_once(plan, admit_start=die_after_reservation, **common) - assert [(row["state"], row["request_id"]) for row in _cadence_starts(runtime_root)] == [ - ("reserved", f"{turn_key}:1") - ] + assert [ + (row["state"], row["request_id"]) for row in _cadence_starts(runtime_root) + ] == [("reserved", f"{turn_key}:1")] assert not list((runtime_root / "goals" / "fixture-goal" / "turns").glob("*.json")) assert calls == {"host": 0, "writeback": 0, "spend": 0, "scheduler": 0} @@ -1308,13 +1329,14 @@ def die_after_reservation(identity: Mapping[str, object]) -> dict[str, object]: assert recovered["status"] == "committed", recovered assert recovered["admission"]["resumed"] is True assert calls == {"host": 1, "writeback": 1, "spend": 1, "scheduler": 1} - assert [(row["state"], row["request_id"]) for row in _cadence_starts(runtime_root)] == [ - ("started", f"{turn_key}:1") - ] + assert [ + (row["state"], row["request_id"]) for row in _cadence_starts(runtime_root) + ] == [("started", f"{turn_key}:1")] def test_reserved_managed_start_recovers_after_death_before_the_attempt_record( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, ) -> None: """The same recovery holds when the journal exists without an attempt.""" @@ -1338,9 +1360,9 @@ def die_before_attempt_record(path: Path, journal: dict[str, object]) -> None: journal = _journal(runtime_root) assert "host_attempt_count" not in journal assert journal["admission"]["reserved"] is True - assert [(row["state"], row["request_id"]) for row in _cadence_starts(runtime_root)] == [ - ("reserved", f"{turn_key}:1") - ] + assert [ + (row["state"], row["request_id"]) for row in _cadence_starts(runtime_root) + ] == [("reserved", f"{turn_key}:1")] assert calls == {"host": 0, "writeback": 0, "spend": 0, "scheduler": 0} started_at_ms = int(_cadence_starts(runtime_root)[0]["started_at_ms"]) @@ -1352,9 +1374,9 @@ def die_before_attempt_record(path: Path, journal: dict[str, object]) -> None: assert recovered["status"] == "committed", recovered assert calls == {"host": 1, "writeback": 1, "spend": 1, "scheduler": 1} - assert [(row["state"], row["request_id"]) for row in _cadence_starts(runtime_root)] == [ - ("started", f"{turn_key}:1") - ] + assert [ + (row["state"], row["request_id"]) for row in _cadence_starts(runtime_root) + ] == [("started", f"{turn_key}:1")] def test_run_once_rejects_oversized_built_in_host_result(tmp_path: Path) -> None: @@ -1443,16 +1465,28 @@ def admit(identity: dict[str, object]) -> dict[str, object]: assert identity["attempt"] == (1 if calls["admit"] == 1 else 2) return {"admitted": calls["admit"] != 2, "reason": "minimum_interval_wait"} - common = dict(host_runner=host, admit_start=admit, project=tmp_path, - runtime_root=tmp_path / "runtime", goal_id="fixture-goal", timeout_seconds=5, - execute=True, task_validator=_passing_validator, writeback=writeback, - spend=spend, scheduler=scheduler) + common = dict( + host_runner=host, + admit_start=admit, + project=tmp_path, + runtime_root=tmp_path / "runtime", + goal_id="fixture-goal", + timeout_seconds=5, + execute=True, + task_validator=_passing_validator, + writeback=writeback, + spend=spend, + scheduler=scheduler, + ) first = run_loopx_turn_once(plan, **common) waiting = run_loopx_turn_once(plan, retry_failed=True, **common) recovered = run_loopx_turn_once(plan, retry_failed=True, **common) assert first["result_kind"] == "validation_failed" - assert waiting["status"] == "interval_wait" and waiting["effects"]["host_invoked"] is False + assert ( + waiting["status"] == "interval_wait" + and waiting["effects"]["host_invoked"] is False + ) assert recovered["status"] == "committed" assert calls == {"host": 2, "admit": 3, "writeback": 1, "spend": 1, "scheduler": 1} @@ -1710,8 +1744,7 @@ def host(_request: dict[str, object]) -> dict[str, object]: ) assert inspected["recovery_decision"]["action"] == "blocked" assert ( - inspected["recovery_decision"]["reason"] - == "session_binding_identity_mismatch" + inspected["recovery_decision"]["reason"] == "session_binding_identity_mismatch" ) with pytest.raises(ValueError, match="session binding does not match") as exc_info: run_loopx_turn_once(plan, retry_failed=True, **common) @@ -1765,12 +1798,18 @@ def test_run_once_commits_once_and_replays_without_duplicate_effects( transaction = plan["transaction"] assert isinstance(transaction, dict) turn_key = str(transaction["turn_key"]) - stored = json.loads(turn_journal_path( - tmp_path / "runtime", goal_id="fixture-goal", turn_key=turn_key, - ).read_text(encoding="utf-8")) + stored = json.loads( + turn_journal_path( + tmp_path / "runtime", + goal_id="fixture-goal", + turn_key=turn_key, + ).read_text(encoding="utf-8") + ) assert stored["plan"]["route"]["kind"] == "ready_for_host" resumed = load_loopx_turn_plan_from_journal( - tmp_path / "runtime", goal_id="fixture-goal", turn_key=turn_key, + tmp_path / "runtime", + goal_id="fixture-goal", + turn_key=turn_key, ) assert resumed["route"] == stored["plan"]["route"] @@ -1856,6 +1895,806 @@ def fail_before_writeback_checkpoint( assert "effect_attempts" not in _journal(tmp_path / "runtime") +def test_recreation_waits_for_admitted_provider_checkpoint( + tmp_path: Path, +) -> None: + registry = tmp_path / "project" / ".loopx" / "registry.json" + runtime_root = registry.parent + _write_source_registry(registry, INSTANCE_A) + plan = _source_plan() + admission = FirstPartyHostGoalAdmission.for_plan( + registry_path=registry, + goal_id="fixture-goal", + planned_goal_ref=plan["goal_ref"], + ) + provider_committed = threading.Event() + allow_provider_return = threading.Event() + turn_finished = threading.Event() + turn_errors: list[BaseException] = [] + calls = {"writeback": 0, "spend": 0, "scheduler": 0} + + def writeback( + _result: dict[str, object], + effect_ref: str, + ) -> dict[str, object]: + calls["writeback"] += 1 + provider_committed.set() + assert allow_provider_return.wait(timeout=5) + return {"ok": True, "appended": True, "effect_ref": effect_ref} + + def run_turn() -> None: + try: + run_loopx_turn_once( + plan, + host_runner=lambda _request: _host_result(plan), + project=tmp_path, + runtime_root=runtime_root, + goal_id="fixture-goal", + timeout_seconds=5, + execute=True, + task_validator=_passing_validator, + writeback=writeback, + spend=lambda: ( + calls.__setitem__("spend", calls["spend"] + 1) + or {"ok": True, "appended": True} + ), + scheduler=lambda _spend: ( + calls.__setitem__("scheduler", calls["scheduler"] + 1) + or {"completed": True} + ), + goal_admission=admission, + ) + except BaseException as exc: + turn_errors.append(exc) + finally: + turn_finished.set() + + turn_thread = threading.Thread(target=run_turn) + turn_thread.start() + assert provider_committed.wait(timeout=5), turn_errors + + request = RecreateGoalRequest( + registry_path=registry, + goal_id="fixture-goal", + goal_instance_id=INSTANCE_A, + operation_id="recreate-after-provider-commit", + ) + blocked = recreate_goal_instance(request) + + assert blocked["ok"] is False + assert blocked["status"] == "drain_required" + assert load_project_registry(registry)["goals"][0]["goal_instance_id"] == INSTANCE_A + gate_path = ( + registry.parent + / ".loopx" + / "lifecycle" + / "goal-instance" + / "turn-settlement" + / alias_digest("fixture-goal") + / "gate.json" + ) + gate = json.loads(gate_path.read_text(encoding="utf-8")) + assert gate["state"] == "closing" + + allow_provider_return.set() + assert turn_finished.wait(timeout=5) + turn_thread.join(timeout=5) + + assert len(turn_errors) == 1 + assert isinstance(turn_errors[0], FirstPartyHostRuntimeRejected) + assert turn_errors[0].code == "goal_retirement_in_progress" + journal = _journal(runtime_root) + assert journal["writeback"]["appended"] is True + assert "effect_attempts" not in journal + assert calls == {"writeback": 1, "spend": 0, "scheduler": 0} + + recreated = recreate_goal_instance(request) + assert recreated["ok"] is True + assert recreated["replayed"] is False + assert recreated["goal_ref"]["goal_instance_id"] != INSTANCE_A + assert ( + load_project_registry(registry)["goals"][0]["goal_instance_id"] + == (recreated["goal_ref"]["goal_instance_id"]) + ) + + +def test_source_provider_cannot_change_admitted_effect_ref( + tmp_path: Path, +) -> None: + registry = tmp_path / "project" / ".loopx" / "registry.json" + runtime_root = registry.parent + _write_source_registry(registry, INSTANCE_A) + plan = _source_plan() + admission = FirstPartyHostGoalAdmission.for_plan( + registry_path=registry, + goal_id="fixture-goal", + planned_goal_ref=plan["goal_ref"], + ) + + with pytest.raises(RuntimeError, match="provider effect ref changed"): + run_loopx_turn_once( + plan, + host_runner=lambda _request: _host_result(plan), + project=tmp_path, + runtime_root=runtime_root, + goal_id="fixture-goal", + timeout_seconds=5, + execute=True, + task_validator=_passing_validator, + writeback=lambda _result, _effect_ref: { + "ok": True, + "appended": True, + "effect_ref": "different-effect", + }, + spend=lambda: pytest.fail("quota must not run after identity drift"), + scheduler=lambda _spend: pytest.fail( + "scheduler must not run after identity drift" + ), + goal_admission=admission, + ) + + blocked = recreate_goal_instance( + RecreateGoalRequest( + registry_path=registry, + goal_id="fixture-goal", + goal_instance_id=INSTANCE_A, + operation_id="recreate-after-provider-identity-drift", + ) + ) + assert blocked["status"] == "drain_required" + assert blocked["pending_effects"][0]["reason"] == "provider_readback_required" + + +def test_recreation_waits_for_scheduler_and_post_settlement( + tmp_path: Path, +) -> None: + registry = tmp_path / "project" / ".loopx" / "registry.json" + runtime_root = registry.parent + _write_source_registry(registry, INSTANCE_A) + plan = _source_plan() + admission = FirstPartyHostGoalAdmission.for_plan( + registry_path=registry, + goal_id="fixture-goal", + planned_goal_ref=plan["goal_ref"], + ) + request = RecreateGoalRequest( + registry_path=registry, + goal_id="fixture-goal", + goal_instance_id=INSTANCE_A, + operation_id="recreate-during-settlement-tail", + ) + blocked: list[dict[str, object]] = [] + observed_instances: list[str] = [] + + def attempt_recreation() -> None: + blocked.append(recreate_goal_instance(request)) + observed_instances.append( + str(load_project_registry(registry)["goals"][0]["goal_instance_id"]) + ) + + def scheduler(_spend: dict[str, object]) -> dict[str, object]: + attempt_recreation() + return {"completed": True, "acknowledged": True} + + def post_settlement( + _plan: Mapping[str, object], + _result: Mapping[str, object], + _evidence: Mapping[str, object], + ) -> dict[str, object]: + attempt_recreation() + return {"status": "recorded"} + + settled = run_loopx_turn_once( + plan, + host_runner=lambda _request: _host_result(plan), + project=tmp_path, + runtime_root=runtime_root, + goal_id="fixture-goal", + timeout_seconds=5, + execute=True, + task_validator=_passing_validator, + writeback=lambda _result: {"ok": True, "appended": True}, + spend=lambda: {"ok": True, "appended": True}, + scheduler=scheduler, + post_settlement=post_settlement, + goal_admission=admission, + ) + + assert settled["status"] == "committed" + assert [result["status"] for result in blocked] == [ + "drain_required", + "drain_required", + ] + assert observed_instances == [INSTANCE_A, INSTANCE_A] + + recreated = recreate_goal_instance(request) + assert recreated["ok"] is True + assert recreated["goal_ref"]["goal_instance_id"] != INSTANCE_A + + +def test_recreation_cannot_publish_before_tail_hold_checkpoint( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + registry = tmp_path / "project" / ".loopx" / "registry.json" + runtime_root = registry.parent + _write_source_registry(registry, INSTANCE_A) + plan = _source_plan() + admission = FirstPartyHostGoalAdmission.for_plan( + registry_path=registry, + goal_id="fixture-goal", + planned_goal_ref=plan["goal_ref"], + ) + hold_started = threading.Event() + allow_hold = threading.Event() + turn_finished = threading.Event() + turn_errors: list[BaseException] = [] + original_hold = type( + admission.turn_effect_admission( + turn_key=str(plan["transaction"]["turn_key"]), + journal_path=turn_executor.turn_journal_path( + runtime_root, + goal_id="fixture-goal", + turn_key=str(plan["transaction"]["turn_key"]), + ), + ) + ).hold + + def blocked_hold( + self: object, + step_kind: object, + effect_ref: str, + persist_journal: object, + ) -> None: + hold_started.set() + assert allow_hold.wait(timeout=5) + original_hold(self, step_kind, effect_ref, persist_journal) + + monkeypatch.setattr( + "loopx.control_plane.goals.first_party_host_admission." + "FirstPartyHostTurnEffectAdmission.hold", + blocked_hold, + ) + + def run_turn() -> None: + try: + run_loopx_turn_once( + plan, + host_runner=lambda _request: _host_result(plan), + project=tmp_path, + runtime_root=runtime_root, + goal_id="fixture-goal", + timeout_seconds=5, + execute=True, + task_validator=_passing_validator, + writeback=lambda _result: {"ok": True, "appended": True}, + spend=lambda: {"ok": True, "appended": True}, + scheduler=lambda _spend: {"completed": True}, + goal_admission=admission, + ) + except BaseException as exc: + turn_errors.append(exc) + finally: + turn_finished.set() + + turn_thread = threading.Thread(target=run_turn) + turn_thread.start() + assert hold_started.wait(timeout=5), turn_errors + request = RecreateGoalRequest( + registry_path=registry, + goal_id="fixture-goal", + goal_instance_id=INSTANCE_A, + operation_id="recreate-before-tail-hold", + ) + + blocked = recreate_goal_instance(request) + + assert blocked["status"] == "drain_required" + assert load_project_registry(registry)["goals"][0]["goal_instance_id"] == INSTANCE_A + allow_hold.set() + assert turn_finished.wait(timeout=5) + turn_thread.join(timeout=5) + assert turn_errors == [] + assert recreate_goal_instance(request)["ok"] is True + + +@pytest.mark.parametrize( + ("result_kind", "tail_step"), + [ + ("validated_progress", "quota_spend"), + ("validated_completion", "terminal_closeout"), + ], +) +def test_final_provider_checkpoint_persists_tail_hold_before_confirmation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + result_kind: str, + tail_step: str, +) -> None: + registry = tmp_path / "project" / ".loopx" / "registry.json" + runtime_root = registry.parent + _write_source_registry(registry, INSTANCE_A) + plan = _source_plan() + admission = FirstPartyHostGoalAdmission.for_plan( + registry_path=registry, + goal_id="fixture-goal", + planned_goal_ref=plan["goal_ref"], + ) + + def crash_before_tail_confirmation( + _self: object, + _step_kind: object, + _effect_ref: str, + ) -> None: + raise RuntimeError("injected crash after final provider checkpoint") + + monkeypatch.setattr( + turn_executor.TurnSettlementJournalAdapter, + "hold_tail", + crash_before_tail_confirmation, + ) + completion_callbacks = ( + { + "completion_writeback": lambda _result: pytest.fail( + "terminal completion must not use the pre-spend lifecycle callback" + ), + "completion_intent": lambda _result: { + "todo_id": "todo_fixture0001", + "continuation": "no_followup", + }, + "terminal_closeout": lambda _result: { + "ok": True, + "appended": True, + "completion": { + "todo_id": "todo_fixture0001", + "continuation": "no_followup", + }, + }, + } + if result_kind == "validated_completion" + else {} + ) + + with pytest.raises( + RuntimeError, + match="injected crash after final provider checkpoint", + ): + run_loopx_turn_once( + plan, + host_runner=lambda _request: _host_result(plan, kind=result_kind), + project=tmp_path, + runtime_root=runtime_root, + goal_id="fixture-goal", + timeout_seconds=5, + execute=True, + task_validator=_passing_validator, + writeback=lambda _result: {"ok": True, "appended": True}, + spend=lambda: {"ok": True, "appended": True}, + scheduler=lambda _spend: pytest.fail( + "scheduler must not run before tail confirmation" + ), + goal_admission=admission, + **completion_callbacks, + ) + + journal = _journal(runtime_root) + hold = journal["source_effect_hold"] + assert hold["status"] == "held" + assert hold["step_kind"] == tail_step + assert "effect_attempts" not in journal + + blocked = recreate_goal_instance( + RecreateGoalRequest( + registry_path=registry, + goal_id="fixture-goal", + goal_instance_id=INSTANCE_A, + operation_id=f"recreate-after-{tail_step}-checkpoint", + ) + ) + assert blocked["status"] == "drain_required" + assert blocked["pending_effects"] == [ + { + "turn_key": str(plan["transaction"]["turn_key"]), + "step_kind": tail_step, + "reason": "turn_tail_recovery_required", + } + ] + assert load_project_registry(registry)["goals"][0]["goal_instance_id"] == INSTANCE_A + + +@pytest.mark.parametrize("first_scheduler_result", ["exception", "incomplete"]) +def test_recreation_waits_for_tail_recovery( + tmp_path: Path, + first_scheduler_result: str, +) -> None: + registry = tmp_path / "project" / ".loopx" / "registry.json" + runtime_root = registry.parent + _write_source_registry(registry, INSTANCE_A) + plan = _source_plan() + admission = FirstPartyHostGoalAdmission.for_plan( + registry_path=registry, + goal_id="fixture-goal", + planned_goal_ref=plan["goal_ref"], + ) + calls = {"host": 0, "writeback": 0, "spend": 0, "scheduler": 0} + + def host(_request: Mapping[str, object]) -> dict[str, object]: + calls["host"] += 1 + return _host_result(plan) + + def writeback(_result: Mapping[str, object]) -> dict[str, object]: + calls["writeback"] += 1 + return {"ok": True, "appended": True} + + def spend() -> dict[str, object]: + calls["spend"] += 1 + return {"ok": True, "appended": True} + + def first_scheduler(_spend: Mapping[str, object]) -> dict[str, object]: + calls["scheduler"] += 1 + if first_scheduler_result == "exception": + raise KeyboardInterrupt + return {"completed": False, "acknowledged": False} + + common = { + "host_runner": host, + "project": tmp_path, + "runtime_root": runtime_root, + "goal_id": "fixture-goal", + "timeout_seconds": 5, + "execute": True, + "task_validator": _passing_validator, + "writeback": writeback, + "spend": spend, + "goal_admission": admission, + } + if first_scheduler_result == "exception": + with pytest.raises(KeyboardInterrupt): + run_loopx_turn_once(plan, scheduler=first_scheduler, **common) + else: + first = run_loopx_turn_once(plan, scheduler=first_scheduler, **common) + assert first["status"] == "scheduler_action_required" + + request = RecreateGoalRequest( + registry_path=registry, + goal_id="fixture-goal", + goal_instance_id=INSTANCE_A, + operation_id=f"recreate-after-tail-{first_scheduler_result}", + ) + blocked = recreate_goal_instance(request) + assert blocked["status"] == "drain_required" + assert blocked["pending_effects"][0]["reason"] == ("turn_tail_recovery_required") + + def healthy_scheduler(_spend: Mapping[str, object]) -> dict[str, object]: + calls["scheduler"] += 1 + return {"completed": True, "acknowledged": True} + + recovered = run_loopx_turn_once(plan, scheduler=healthy_scheduler, **common) + assert recovered["status"] == "committed" + assert calls == {"host": 1, "writeback": 1, "spend": 1, "scheduler": 2} + assert recreate_goal_instance(request)["ok"] is True + + +@pytest.mark.parametrize( + ("readback_kind", "recreation_completes"), + [ + ("committed", True), + ("absent", True), + ("unknown", False), + ], +) +def test_recreation_reconciles_crashed_source_provider_without_reexecution( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + readback_kind: str, + recreation_completes: bool, +) -> None: + registry = tmp_path / "project" / ".loopx" / "registry.json" + runtime_root = registry.parent + _write_source_registry(registry, INSTANCE_A) + plan = _source_plan() + admission = FirstPartyHostGoalAdmission.for_plan( + registry_path=registry, + goal_id="fixture-goal", + planned_goal_ref=plan["goal_ref"], + ) + calls = {"writeback": 0, "spend": 0, "scheduler": 0} + provider_records: dict[str, dict[str, object]] = {} + + def writeback( + _result: dict[str, object], + effect_ref: str, + ) -> dict[str, object]: + calls["writeback"] += 1 + payload = {"ok": True, "appended": True, "effect_ref": effect_ref} + provider_records[effect_ref] = payload + return payload + + write_journal = turn_executor._write_journal + + def fail_before_writeback_checkpoint( + path: Path, + journal: Mapping[str, object], + **kwargs: object, + ) -> None: + if "writeback" in journal and "quota_spend" not in journal: + raise RuntimeError("injected source checkpoint crash") + write_journal(path, journal, **kwargs) + + monkeypatch.setattr( + turn_executor, + "_write_journal", + fail_before_writeback_checkpoint, + ) + with pytest.raises(RuntimeError, match="injected source checkpoint crash"): + run_loopx_turn_once( + plan, + host_runner=lambda _request: _host_result(plan), + project=tmp_path, + runtime_root=runtime_root, + goal_id="fixture-goal", + timeout_seconds=5, + execute=True, + task_validator=_passing_validator, + writeback=writeback, + spend=lambda: pytest.fail("quota must not run before writeback checkpoint"), + scheduler=lambda _spend: pytest.fail( + "scheduler must not run before writeback checkpoint" + ), + goal_admission=admission, + ) + monkeypatch.setattr(turn_executor, "_write_journal", write_journal) + + request = RecreateGoalRequest( + registry_path=registry, + goal_id="fixture-goal", + goal_instance_id=INSTANCE_A, + operation_id=f"recreate-after-{readback_kind}-readback", + ) + blocked = recreate_goal_instance(request) + assert blocked["status"] == "drain_required" + + effect_ref = next(iter(provider_records)) + + def resolver(_effect_ref: str) -> dict[str, object]: + assert _effect_ref == effect_ref + if readback_kind == "committed": + return {"kind": "committed", "payload": provider_records[effect_ref]} + if readback_kind == "absent": + return {"kind": "absent"} + return {"kind": "unknown", "reason": "provider unavailable"} + + def forbidden_writeback( + _result: dict[str, object], + _effect_ref: str, + ) -> dict[str, object]: + pytest.fail("retirement recovery must not re-execute the provider") + + try: + resumed = run_loopx_turn_once( + plan, + host_runner=lambda _request: pytest.fail( + "settlement recovery must not relaunch the Host" + ), + project=tmp_path, + runtime_root=runtime_root, + goal_id="fixture-goal", + timeout_seconds=5, + execute=True, + task_validator=_passing_validator, + writeback=forbidden_writeback, + writeback_resolver=resolver, + spend=lambda: ( + calls.__setitem__("spend", calls["spend"] + 1) + or {"ok": True, "appended": True} + ), + scheduler=lambda _spend: ( + calls.__setitem__("scheduler", calls["scheduler"] + 1) + or {"completed": True} + ), + goal_admission=admission, + ) + except FirstPartyHostRuntimeRejected as exc: + assert readback_kind == "committed" + assert exc.code == "goal_retirement_in_progress" + else: + assert resumed["status"] == "failed" + + assert calls == {"writeback": 1, "spend": 0, "scheduler": 0} + retried = recreate_goal_instance(request) + assert retried["ok"] is recreation_completes + if recreation_completes: + assert retried["goal_ref"]["goal_instance_id"] != INSTANCE_A + else: + assert retried["status"] == "drain_required" + assert load_project_registry(registry)["goals"][0]["goal_instance_id"] == ( + INSTANCE_A + ) + + +@pytest.mark.parametrize( + ("failure_point", "provider_calls"), + [ + ("prepared_journal", 0), + ("admission_release", 1), + ], +) +def test_recreation_repairs_partial_source_effect_admission( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + failure_point: str, + provider_calls: int, +) -> None: + registry = tmp_path / "project" / ".loopx" / "registry.json" + runtime_root = registry.parent + _write_source_registry(registry, INSTANCE_A) + plan = _source_plan() + admission = FirstPartyHostGoalAdmission.for_plan( + registry_path=registry, + goal_id="fixture-goal", + planned_goal_ref=plan["goal_ref"], + ) + calls = {"writeback": 0} + write_journal = turn_executor._write_journal + remove_admission = source_session_turn_effects._remove_admission + + if failure_point == "prepared_journal": + + def fail_prepared_journal( + path: Path, + journal: Mapping[str, object], + **kwargs: object, + ) -> None: + if "effect_attempts" in journal: + raise RuntimeError("injected prepared journal failure") + write_journal(path, journal, **kwargs) + + monkeypatch.setattr( + turn_executor, + "_write_journal", + fail_prepared_journal, + ) + expected_error = "injected prepared journal failure" + else: + + def fail_admission_release(_path: Path) -> None: + raise RuntimeError("injected admission release failure") + + monkeypatch.setattr( + source_session_turn_effects, + "_remove_admission", + fail_admission_release, + ) + expected_error = "injected admission release failure" + + def writeback( + _result: dict[str, object], + effect_ref: str, + ) -> dict[str, object]: + calls["writeback"] += 1 + return {"ok": True, "appended": True, "effect_ref": effect_ref} + + with pytest.raises(RuntimeError, match=expected_error): + run_loopx_turn_once( + plan, + host_runner=lambda _request: _host_result(plan), + project=tmp_path, + runtime_root=runtime_root, + goal_id="fixture-goal", + timeout_seconds=5, + execute=True, + task_validator=_passing_validator, + writeback=writeback, + spend=lambda: pytest.fail("quota must not run after the injected failure"), + scheduler=lambda _spend: pytest.fail( + "scheduler must not run after the injected failure" + ), + goal_admission=admission, + ) + + monkeypatch.setattr(turn_executor, "_write_journal", write_journal) + monkeypatch.setattr( + source_session_turn_effects, + "_remove_admission", + remove_admission, + ) + recreated = recreate_goal_instance( + RecreateGoalRequest( + registry_path=registry, + goal_id="fixture-goal", + goal_instance_id=INSTANCE_A, + operation_id=f"recreate-after-{failure_point}", + ) + ) + + assert calls["writeback"] == provider_calls + assert recreated["ok"] is True + assert recreated["goal_ref"]["goal_instance_id"] != INSTANCE_A + + +def test_recreation_replay_repairs_gate_after_registry_publication( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + registry = tmp_path / "project" / ".loopx" / "registry.json" + _write_source_registry(registry, INSTANCE_A) + request = RecreateGoalRequest( + registry_path=registry, + goal_id="fixture-goal", + goal_instance_id=INSTANCE_A, + operation_id="recreate-before-gate-reopen", + ) + write_sidecar = source_session_turn_effects.write_journal + failed_once = False + + def fail_gate_reopen(path: Path, payload: dict[str, object]) -> None: + nonlocal failed_once + if payload.get("state") == "open" and not failed_once: + failed_once = True + raise OSError("injected gate reopen failure") + write_sidecar(path, payload) + + monkeypatch.setattr( + source_session_turn_effects, + "write_journal", + fail_gate_reopen, + ) + with pytest.raises(OSError, match="injected gate reopen failure"): + recreate_goal_instance(request) + instance_b = load_project_registry(registry)["goals"][0]["goal_instance_id"] + assert instance_b != INSTANCE_A + + monkeypatch.setattr( + source_session_turn_effects, + "write_journal", + write_sidecar, + ) + replayed = recreate_goal_instance(request) + + assert replayed["ok"] is True + assert replayed["replayed"] is True + assert replayed["goal_ref"]["goal_instance_id"] == instance_b + + +def test_historical_recreation_replay_preserves_later_goal_gate( + tmp_path: Path, +) -> None: + registry = tmp_path / "project" / ".loopx" / "registry.json" + _write_source_registry(registry, INSTANCE_A) + first_request = RecreateGoalRequest( + registry_path=registry, + goal_id="fixture-goal", + goal_instance_id=INSTANCE_A, + operation_id="recreate-a-to-b", + ) + first = recreate_goal_instance(first_request) + instance_b = str(first["goal_ref"]["goal_instance_id"]) + second = recreate_goal_instance( + RecreateGoalRequest( + registry_path=registry, + goal_id="fixture-goal", + goal_instance_id=instance_b, + operation_id="recreate-b-to-c", + ) + ) + instance_c = str(second["goal_ref"]["goal_instance_id"]) + + replayed = recreate_goal_instance(first_request) + + assert replayed["ok"] is True + assert replayed["replayed"] is True + assert replayed["goal_ref"]["goal_instance_id"] == instance_b + assert load_project_registry(registry)["goals"][0]["goal_instance_id"] == instance_c + gate = json.loads( + source_session_turn_effects.source_turn_effect_gate_path( + registry, + "fixture-goal", + ).read_text(encoding="utf-8") + ) + assert gate["state"] == "open" + assert gate["goal_ref"]["goal_instance_id"] == instance_c + + def test_run_once_legacy_plan_without_settlement_plan_is_upgraded( tmp_path: Path, ) -> None: @@ -3141,7 +3980,9 @@ def test_run_once_fails_closed_when_the_managed_executor_cannot_launch(tmp_path) payload = run_loopx_turn_once( plan, - host_runner=lambda _request: pytest.fail("an unavailable executor must not run"), + host_runner=lambda _request: pytest.fail( + "an unavailable executor must not run" + ), project=tmp_path, runtime_root=runtime_root, goal_id="fixture-goal", @@ -3179,7 +4020,9 @@ def test_run_once_preview_reports_the_managed_executor_without_refusing(tmp_path assert payload["ok"] is True assert payload["status"] == "preview" assert payload["managed_executor"]["available"] is False - assert payload["managed_executor"]["unavailable_reason"] == "dsh_runtime_unavailable" + assert ( + payload["managed_executor"]["unavailable_reason"] == "dsh_runtime_unavailable" + ) def test_run_once_does_not_refuse_a_launchable_managed_executor(tmp_path): @@ -3190,7 +4033,9 @@ def test_run_once_does_not_refuse_a_launchable_managed_executor(tmp_path): with pytest.raises(ValueError, match="requires writeback, spend, and scheduler"): run_loopx_turn_once( plan, - host_runner=lambda _request: pytest.fail("host must not run without callbacks"), + host_runner=lambda _request: pytest.fail( + "host must not run without callbacks" + ), project=tmp_path, runtime_root=tmp_path / "runtime", goal_id="fixture-goal", @@ -3199,7 +4044,9 @@ def test_run_once_does_not_refuse_a_launchable_managed_executor(tmp_path): ) -def test_real_host_duration_is_observed_but_settlement_replay_is_not(tmp_path, monkeypatch): +def test_real_host_duration_is_observed_but_settlement_replay_is_not( + tmp_path, monkeypatch +): """Production Turn entrypoint, actual Host subprocess and detached TS state.""" import time from loopx import usage_ping @@ -3212,13 +4059,25 @@ def test_real_host_duration_is_observed_but_settlement_replay_is_not(tmp_path, m usage_ping.control("enable") plan = _plan() host_file = tmp_path / "host.py" - host_file.write_text("import json, sys, time\njson.load(sys.stdin)\ntime.sleep(0.08)\nprint(" + repr(json.dumps(_host_result(plan))) + ")\n") + host_file.write_text( + "import json, sys, time\njson.load(sys.stdin)\ntime.sleep(0.08)\nprint(" + + repr(json.dumps(_host_result(plan))) + + ")\n" + ) calls = {"writeback": 0, "spend": 0, "scheduler": 0} writeback, spend, scheduler = _callbacks(calls) - options = dict(host_argv=[sys.executable, str(host_file)], project=tmp_path, - runtime_root=tmp_path / "runtime", goal_id="fixture-goal", - timeout_seconds=5, execute=True, task_validator=_passing_validator, - writeback=writeback, spend=spend, scheduler=scheduler) + options = dict( + host_argv=[sys.executable, str(host_file)], + project=tmp_path, + runtime_root=tmp_path / "runtime", + goal_id="fixture-goal", + timeout_seconds=5, + execute=True, + task_validator=_passing_validator, + writeback=writeback, + spend=spend, + scheduler=scheduler, + ) result = run_loopx_turn_once(plan, **options) assert result["ok"], result local = machine / "usage-ping.json.goals" From 73f937907ae343efa7fa1cf6fff6bd483beb2d0d Mon Sep 17 00:00:00 2001 From: "duanjialing.777" Date: Wed, 30 Sep 2026 16:11:44 +0800 Subject: [PATCH 2/3] fix(goals): make recreation drain recovery explicit Signed-off-by: duanjialing.777 --- loopx/cli_commands/project.py | 20 +++- loopx/control_plane/effect_program.py | 26 ++++- loopx/control_plane/effect_program.ts | 4 + .../goals/first_party_host_admission.py | 12 +-- .../goals/source_session_lifetime.ts | 32 +++--- .../goals/source_session_recreation.py | 13 ++- .../goals/source_session_turn_effects.py | 27 +++-- loopx/control_plane/turn_driver/settlement.py | 39 ++++--- loopx/control_plane/turn_driver/settlement.ts | 41 ++++--- .../turn_journal_attempt_contract.ts | 4 +- .../project_registry_io_manifest_v1.json | 8 +- .../test_source_session_lifetime.py | 100 ++++++++++++++++++ .../source_session_lifetime.test.ts | 14 +++ tests/test_loopx_turn_executor.py | 24 +++++ 14 files changed, 287 insertions(+), 77 deletions(-) diff --git a/loopx/cli_commands/project.py b/loopx/cli_commands/project.py index 503b9c4998..e9813e167f 100644 --- a/loopx/cli_commands/project.py +++ b/loopx/cli_commands/project.py @@ -4,7 +4,7 @@ from ..control_plane.coordination.shadow_management import ShadowManagementError import argparse -from collections.abc import Callable +from collections.abc import Callable, Mapping from pathlib import Path from ..control_plane.projects.registry import ( @@ -102,7 +102,10 @@ def render_project_command_markdown(payload: dict[str, object]) -> str: f"- registry: `{payload.get('registry')}`", ] for field in ( + "status", "changed", + "replayed", + "gate_state", "resolution", "source", "project_id", @@ -110,6 +113,21 @@ def render_project_command_markdown(payload: dict[str, object]) -> str: ): if field in payload: lines.append(f"- {field}: `{payload.get(field)}`") + if payload.get("recovery_action"): + lines.append(f"- recovery_action: {payload.get('recovery_action')}") + pending_effects = payload.get("pending_effects") + if isinstance(pending_effects, list) and pending_effects: + lines.extend(["", "## Pending Turn effects", ""]) + for pending in pending_effects: + if not isinstance(pending, Mapping): + continue + lines.append( + "- " + f"turn_key=`{pending.get('turn_key')}`; " + f"step_kind=`{pending.get('step_kind')}`; " + f"reason=`{pending.get('reason')}`; " + f"recovery_action={pending.get('recovery_action')}" + ) if payload.get("error"): lines.append(f"- error: {payload.get('error')}") return "\n".join(lines) diff --git a/loopx/control_plane/effect_program.py b/loopx/control_plane/effect_program.py index 7148d03b15..c7877c76dd 100644 --- a/loopx/control_plane/effect_program.py +++ b/loopx/control_plane/effect_program.py @@ -6,7 +6,7 @@ from enum import StrEnum from functools import lru_cache from pathlib import Path -from typing import Any, Generic, TypeVar +from typing import Any, Generic, Literal, TypeAlias, TypeVar from .effect_runtime import EffectRuntimeRejected, effect_runtime_result @@ -176,6 +176,30 @@ class SettlementStepKind(StrEnum): TERMINAL_CLOSEOUT = "terminal_closeout" +TurnProviderStepKind: TypeAlias = Literal[ + SettlementStepKind.DURABLE_WRITEBACK, + SettlementStepKind.QUOTA_SPEND, + SettlementStepKind.TERMINAL_CLOSEOUT, +] +TURN_PROVIDER_STEP_KINDS: tuple[TurnProviderStepKind, ...] = ( + SettlementStepKind.DURABLE_WRITEBACK, + SettlementStepKind.QUOTA_SPEND, + SettlementStepKind.TERMINAL_CLOSEOUT, +) + + +def require_turn_provider_step_kind( + step_kind: SettlementStepKind, +) -> TurnProviderStepKind: + if step_kind is SettlementStepKind.DURABLE_WRITEBACK: + return step_kind + if step_kind is SettlementStepKind.QUOTA_SPEND: + return step_kind + if step_kind is SettlementStepKind.TERMINAL_CLOSEOUT: + return step_kind + raise ValueError("validation is not a provider effect step") + + class SettlementBindingKind(StrEnum): TODO = "todo" AUTONOMOUS_REPLAN = "autonomous_replan" diff --git a/loopx/control_plane/effect_program.ts b/loopx/control_plane/effect_program.ts index 181ae4c472..26418caaf7 100644 --- a/loopx/control_plane/effect_program.ts +++ b/loopx/control_plane/effect_program.ts @@ -97,6 +97,10 @@ export const SETTLEMENT_STEP_KINDS = [ "terminal_closeout", ] as const; export type SettlementStepKind = (typeof SETTLEMENT_STEP_KINDS)[number]; +export type TurnProviderStepKind = Exclude; +export const TURN_PROVIDER_STEP_KINDS = SETTLEMENT_STEP_KINDS.filter( + (kind): kind is TurnProviderStepKind => kind !== "validation", +); export const SETTLEMENT_BINDING_KINDS = [ "todo", diff --git a/loopx/control_plane/goals/first_party_host_admission.py b/loopx/control_plane/goals/first_party_host_admission.py index c353230fee..3b4fc29ecb 100644 --- a/loopx/control_plane/goals/first_party_host_admission.py +++ b/loopx/control_plane/goals/first_party_host_admission.py @@ -15,7 +15,7 @@ SOURCE_SESSION_PROFILE_ID, load_project_registry, ) -from ..effect_program import SettlementStepKind +from ..effect_program import TurnProviderStepKind from .source_session_registry_state import ( exact_goal_ref, guard_path, @@ -50,7 +50,7 @@ class FirstPartyHostTurnEffectAdmission: def _effect( self, - step_kind: SettlementStepKind, + step_kind: TurnProviderStepKind, effect_ref: str, ) -> SourceTurnEffect: goal_ref = self.goal_admission.planned_goal_ref @@ -70,7 +70,7 @@ def _effect( def prepare( self, - step_kind: SettlementStepKind, + step_kind: TurnProviderStepKind, effect_ref: str, persist_journal: JournalPersist, ) -> None: @@ -87,7 +87,7 @@ def prepare( def hold( self, - step_kind: SettlementStepKind, + step_kind: TurnProviderStepKind, effect_ref: str, persist_journal: JournalPersist, ) -> None: @@ -95,7 +95,7 @@ def hold( def release( self, - step_kind: SettlementStepKind, + step_kind: TurnProviderStepKind, effect_ref: str, persist_journal: JournalPersist, ) -> None: @@ -112,7 +112,7 @@ def release( def allows_absent_reexecute( self, - step_kind: SettlementStepKind, + step_kind: TurnProviderStepKind, effect_ref: str, ) -> bool: try: diff --git a/loopx/control_plane/goals/source_session_lifetime.ts b/loopx/control_plane/goals/source_session_lifetime.ts index f52640cf26..ca0e174a0a 100644 --- a/loopx/control_plane/goals/source_session_lifetime.ts +++ b/loopx/control_plane/goals/source_session_lifetime.ts @@ -1,6 +1,10 @@ -import type { JsonObject } from "../effect_program.ts"; +import { + TURN_PROVIDER_STEP_KINDS, + type JsonObject, + type TurnProviderStepKind, +} from "../effect_program.ts"; import { EffectRuntimeRequestError } from "../effect_runtime_errors.ts"; -import { jsonObject } from "../runtime_decode.ts"; +import { jsonObject, requireStringLiteral } from "../runtime_decode.ts"; import { parseExactGoalRef, type ExactGoalRef, @@ -66,16 +70,11 @@ export type GoalRecreationDecision = code: GoalRecreationRejection; }>; -type SourceTurnEffectStep = - | "durable_writeback" - | "quota_spend" - | "terminal_closeout"; - type SourceTurnEffectAdmission = Readonly<{ schema_version: typeof SOURCE_TURN_EFFECT_ADMISSION_SCHEMA_VERSION; goal_ref: WireGoalRef; turn_key: string; - step_kind: SourceTurnEffectStep; + step_kind: TurnProviderStepKind; effect_ref: string; }>; @@ -229,17 +228,12 @@ function goalRefsEqual(left: ExactGoalRef, right: ExactGoalRef): boolean { && left.goalInstanceId.value === right.goalInstanceId.value; } -function sourceTurnEffectStep(value: unknown): SourceTurnEffectStep { - if ( - value !== "durable_writeback" - && value !== "quota_spend" - && value !== "terminal_closeout" - ) { - throw new EffectRuntimeRequestError( - "step_kind must name a supported Turn settlement effect", - ); - } - return value; +function sourceTurnEffectStep(value: unknown): TurnProviderStepKind { + return requireStringLiteral( + value, + TURN_PROVIDER_STEP_KINDS, + "step_kind", + ); } function sourceTurnEffectAdmission( diff --git a/loopx/control_plane/goals/source_session_recreation.py b/loopx/control_plane/goals/source_session_recreation.py index 36a27560cc..24c7496365 100644 --- a/loopx/control_plane/goals/source_session_recreation.py +++ b/loopx/control_plane/goals/source_session_recreation.py @@ -243,16 +243,22 @@ def _drain_required_result( *, requested_goal_ref: dict[str, str], pending_effects: list[dict[str, str]], + gate_changed: bool, ) -> dict[str, Any]: return { "ok": False, "schema_version": "loopx_goal_recreation_v1", "status": "drain_required", - "changed": False, - "replayed": False, + "changed": gate_changed, + "replayed": not gate_changed, + "gate_state": "closing", "registry": str(request.registry_path), "retired_goal_ref": copy.deepcopy(requested_goal_ref), "pending_effects": pending_effects, + "recovery_action": ( + "Resolve every pending Turn effect, then retry recreate-goal " + "with the same operation_id." + ), "execution_authority": False, } @@ -329,7 +335,7 @@ def recreate_goal_instance(request: RecreateGoalRequest) -> dict[str, Any]: receipt=replay_receipt, replayed=True, ) - closing_gate = decide_source_turn_effect_close_locked( + closing_gate, gate_changed = decide_source_turn_effect_close_locked( registry_path=request.registry_path, goal_id=request.goal_id, requested_goal_ref=requested_goal_ref, @@ -362,6 +368,7 @@ def recreate_goal_instance(request: RecreateGoalRequest) -> dict[str, Any]: request, requested_goal_ref=requested_goal_ref, pending_effects=pending_effects, + gate_changed=gate_changed, ) with exclusive_cross_runtime_file_lock( diff --git a/loopx/control_plane/goals/source_session_turn_effects.py b/loopx/control_plane/goals/source_session_turn_effects.py index 48f746a920..4ed209ec28 100644 --- a/loopx/control_plane/goals/source_session_turn_effects.py +++ b/loopx/control_plane/goals/source_session_turn_effects.py @@ -12,7 +12,7 @@ exclusive_cross_runtime_file_lock, exclusive_file_lock, ) -from ..effect_program import SettlementStepKind +from ..effect_program import TurnProviderStepKind from ..effect_runtime import effect_runtime_result from ..projects.registry_codec import ( SOURCE_SESSION_PROFILE_ID, @@ -31,7 +31,21 @@ _GATE_SCHEMA = "loopx_source_turn_effect_gate_v1" _ADMISSION_SCHEMA = "loopx_source_turn_effect_admission_v1" -_HOLD_SCHEMA = "loopx_source_turn_effect_hold_v1" +SOURCE_TURN_EFFECT_HOLD_SCHEMA_VERSION = "loopx_source_turn_effect_hold_v1" +_PENDING_RECOVERY_ACTIONS = { + "executor_active": "Wait for this Turn to finish, then retry recreate-goal.", + "goal_ref_mismatch": "Repair the admission GoalRef before retrying recreate-goal.", + "journal_effect_conflict": "Repair this Turn journal before retrying recreate-goal.", + "journal_identity_invalid": "Repair this Turn identity before retrying recreate-goal.", + "journal_unreadable": "Repair this Turn journal before retrying recreate-goal.", + "provider_readback_required": ( + "Resume this Turn with provider readback, then retry recreate-goal." + ), + "turn_tail_conflict": "Repair this Turn tail before retrying recreate-goal.", + "turn_tail_recovery_required": ( + "Resume this Turn to finish its settlement tail, then retry recreate-goal." + ), +} SourceAdmissionFactory = Callable[[], Mapping[str, Any]] JournalPersist = Callable[[Mapping[str, Any]], None] @@ -47,7 +61,7 @@ def __init__(self, code: str) -> None: class SourceTurnEffect: goal_ref: Mapping[str, str] turn_key: str - step_kind: SettlementStepKind + step_kind: TurnProviderStepKind effect_ref: str journal_path: Path @@ -299,7 +313,7 @@ def decide_source_turn_effect_close_locked( reserved_goal_ref: Mapping[str, str], operation_id: str, request_digest: str, -) -> dict[str, Any]: +) -> tuple[dict[str, Any], bool]: decision = _decision( "goal.source_session.turn_effect.gate", { @@ -316,7 +330,7 @@ def decide_source_turn_effect_close_locked( gate = decision.get("gate") if not isinstance(gate, dict): raise RuntimeError("source Turn effect decision omitted gate") - return gate + return gate, decision["kind"] == "commit" def decide_source_turn_effect_publish_locked( @@ -414,6 +428,7 @@ def _pending_projection( "turn_key": str(admission.get("turn_key") or ""), "step_kind": str(admission.get("step_kind") or ""), "reason": reason, + "recovery_action": _PENDING_RECOVERY_ACTIONS[reason], } @@ -475,7 +490,7 @@ def drain_releasable_source_turn_effects( ) if hold is not None: expected_hold = { - "schema_version": _HOLD_SCHEMA, + "schema_version": SOURCE_TURN_EFFECT_HOLD_SCHEMA_VERSION, "status": "held", "step_kind": step_kind, "effect_ref": effect_ref, diff --git a/loopx/control_plane/turn_driver/settlement.py b/loopx/control_plane/turn_driver/settlement.py index 560347bff8..15984768ff 100644 --- a/loopx/control_plane/turn_driver/settlement.py +++ b/loopx/control_plane/turn_driver/settlement.py @@ -10,9 +10,14 @@ from ..effect_program import ( SettlementResult, SettlementStepKind, + TurnProviderStepKind, + require_turn_provider_step_kind, settlement_result_payload, ) from ..effect_runtime import effect_runtime_result +from ..goals.source_session_turn_effects import ( + SOURCE_TURN_EFFECT_HOLD_SCHEMA_VERSION, +) from ..settlement_driver import decode_settlement_result from .driver import selected_turn_todo from .transaction import ( @@ -39,28 +44,28 @@ class TurnSettlementEffectAdmission(Protocol): def prepare( self, - step_kind: SettlementStepKind, + step_kind: TurnProviderStepKind, effect_ref: str, persist_journal: SourceJournalPersist, ) -> None: ... def hold( self, - step_kind: SettlementStepKind, + step_kind: TurnProviderStepKind, effect_ref: str, persist_journal: SourceJournalPersist, ) -> None: ... def release( self, - step_kind: SettlementStepKind, + step_kind: TurnProviderStepKind, effect_ref: str, persist_journal: SourceJournalPersist, ) -> None: ... def allows_absent_reexecute( self, - step_kind: SettlementStepKind, + step_kind: TurnProviderStepKind, effect_ref: str, ) -> bool: ... @@ -74,7 +79,6 @@ class TurnSettlementState: TURN_SETTLEMENT_TRANSACTION_SCHEMA_VERSION = "loopx_turn_settlement_transaction_v0" TURN_SETTLEMENT_REDUCTION_SCHEMA_VERSION = "loopx_turn_settlement_reduction_v0" -SOURCE_TURN_EFFECT_HOLD_SCHEMA_VERSION = "loopx_source_turn_effect_hold_v1" def _invoke_turn_effect(effect: TurnEffect, effect_ref: str) -> Mapping[str, Any]: @@ -171,7 +175,7 @@ def prepare(self, step_kind: SettlementStepKind, effect_ref: str) -> None: self.persist() return self.source_effects.prepare( - step_kind, + require_turn_provider_step_kind(step_kind), effect_ref, self._require_source_persist(), ) @@ -182,7 +186,7 @@ def abort(self, step_kind: SettlementStepKind, effect_ref: str) -> None: self.persist() return self.source_effects.release( - step_kind, + require_turn_provider_step_kind(step_kind), effect_ref, self._require_source_persist(), ) @@ -230,7 +234,10 @@ def allows_absent_reexecute( ) -> bool: if self.source_effects is None: return True - return self.source_effects.allows_absent_reexecute(step_kind, effect_ref) + return self.source_effects.allows_absent_reexecute( + require_turn_provider_step_kind(step_kind), + effect_ref, + ) def hold_tail( self, @@ -241,7 +248,7 @@ def hold_tail( raise RuntimeError("source Turn effect admission is unavailable") self._set_tail_hold(step_kind, effect_ref) self.source_effects.hold( - step_kind, + require_turn_provider_step_kind(step_kind), effect_ref, self._require_source_persist(), ) @@ -258,7 +265,7 @@ def release_tail( if self.source_effects is None: raise RuntimeError("source Turn effect admission is unavailable") self.source_effects.release( - step_kind, + require_turn_provider_step_kind(step_kind), effect_ref, self._require_source_persist(), ) @@ -273,9 +280,17 @@ def _checkpoint_source_effect( persist = self._require_source_persist() if step_kind is self.deferred_release_step: self._set_tail_hold(step_kind, effect_ref) - self.source_effects.hold(step_kind, effect_ref, persist) + self.source_effects.hold( + require_turn_provider_step_kind(step_kind), + effect_ref, + persist, + ) return - self.source_effects.release(step_kind, effect_ref, persist) + self.source_effects.release( + require_turn_provider_step_kind(step_kind), + effect_ref, + persist, + ) @staticmethod def _tail_hold( diff --git a/loopx/control_plane/turn_driver/settlement.ts b/loopx/control_plane/turn_driver/settlement.ts index 33c49bf626..890d97a740 100644 --- a/loopx/control_plane/turn_driver/settlement.ts +++ b/loopx/control_plane/turn_driver/settlement.ts @@ -13,6 +13,8 @@ import { type SettlementIdentity, type SettlementResult, type SettlementStepKind, + TURN_PROVIDER_STEP_KINDS, + type TurnProviderStepKind, } from "../effect_program.ts"; import { optionalNonEmptyString, @@ -36,13 +38,6 @@ const BASE_SETTLEMENT_STEPS = [ "quota_spend", ] as const satisfies readonly SettlementStepKind[]; -const PROVIDER_STEP_KINDS = [ - "durable_writeback", - "quota_spend", - "terminal_closeout", -] as const; -type ProviderStepKind = (typeof PROVIDER_STEP_KINDS)[number]; - const PROVIDER_RESOLUTION_KINDS = ["committed", "absent", "unknown"] as const; type ProviderResolutionKind = (typeof PROVIDER_RESOLUTION_KINDS)[number]; @@ -70,19 +65,19 @@ interface ProviderObservation { } interface FailedProviderAttempt { - step_kind: ProviderStepKind; + step_kind: TurnProviderStepKind; payload: JsonObject; } interface ProviderExecutionEffect { - step_kind: ProviderStepKind; + step_kind: TurnProviderStepKind; action: "prepare_and_execute"; effect_ref: string; completed_phases: readonly string[]; } interface ProviderResolutionEffect { - step_kind: ProviderStepKind; + step_kind: TurnProviderStepKind; action: "resolve_prepared"; effect_ref: string; completed_phases: readonly string[]; @@ -106,8 +101,8 @@ interface TurnSettlementRequest { terminal_closeout_required: boolean; terminal_closeout_payload: JsonObject | null; failed_provider_attempt: FailedProviderAttempt | null; - effect_attempts: Partial>; - provider_observations: Partial>; + effect_attempts: Partial>; + provider_observations: Partial>; turn_result_kind: TurnResultKind | null; } @@ -150,7 +145,7 @@ function decodeFailedProviderAttempt( return { step_kind: requireStringLiteral( attempt.step_kind, - PROVIDER_STEP_KINDS, + TURN_PROVIDER_STEP_KINDS, "failed_provider_attempt.step_kind", ), payload: requireJsonObject( @@ -164,14 +159,14 @@ function decodeProviderRecord( value: unknown, label: string, decode: (value: unknown, label: string) => Value, -): Partial> { +): Partial> { if (value === null || value === undefined) return {}; const record = requireJsonObject(value, label); - const decoded: Partial> = {}; + const decoded: Partial> = {}; for (const [rawStep, rawValue] of Object.entries(record)) { const step = requireStringLiteral( rawStep, - PROVIDER_STEP_KINDS, + TURN_PROVIDER_STEP_KINDS, `${label} step`, ); decoded[step] = decode(rawValue, `${label}.${step}`); @@ -466,7 +461,7 @@ function failedState( function providerFailure( identity: SettlementIdentity, request: TurnSettlementRequest, - stepKind: ProviderStepKind, + stepKind: TurnProviderStepKind, receipts: SettlementResult["receipts"], ): TurnSettlementOutcome { const attempt = request.failed_provider_attempt; @@ -504,7 +499,7 @@ function providerFailure( function pendingProviderEffects( request: TurnSettlementRequest, identity: SettlementIdentity, - firstStep: ProviderStepKind, + firstStep: TurnProviderStepKind, ): readonly ProviderEffect[] { const completed = new Set(request.completed_phases); const effects: ProviderEffect[] = []; @@ -556,7 +551,7 @@ function terminalCloseoutRequestFailure( function reduceBaseProviderAction( request: TurnSettlementRequest, identity: SettlementIdentity, - stepKind: ProviderStepKind, + stepKind: TurnProviderStepKind, receipts: SettlementResult["receipts"], ): TurnSettlementReduction { const effects = pendingProviderEffects(request, identity, stepKind); @@ -646,15 +641,15 @@ function reduceTerminalCloseout( function providerExecution( request: TurnSettlementRequest, identity: SettlementIdentity, - firstStep: ProviderStepKind, + firstStep: TurnProviderStepKind, effects: readonly ProviderEffect[], receipts: SettlementResult["receipts"], ): TurnSettlementReduction { const attempts = Object.entries(request.effect_attempts) as Array< - [ProviderStepKind, PreparedEffectAttempt] + [TurnProviderStepKind, PreparedEffectAttempt] >; const observations = Object.entries(request.provider_observations) as Array< - [ProviderStepKind, ProviderObservation] + [TurnProviderStepKind, ProviderObservation] >; if (attempts.length === 0) { if (observations.length > 0) { @@ -865,7 +860,7 @@ function reduceTurnSettlementRequest( receipts = terminal.receipts; const danglingAttempt = Object.keys(request.effect_attempts)[0] as - | ProviderStepKind + | TurnProviderStepKind | undefined; if (danglingAttempt) { return reduction( diff --git a/loopx/control_plane/turn_driver/turn_journal_attempt_contract.ts b/loopx/control_plane/turn_driver/turn_journal_attempt_contract.ts index 90e90a1b84..d1b27644f7 100644 --- a/loopx/control_plane/turn_driver/turn_journal_attempt_contract.ts +++ b/loopx/control_plane/turn_driver/turn_journal_attempt_contract.ts @@ -1,9 +1,9 @@ /** The journal owner's prepared-intent contract, shared by writes and reads. */ import transactionContract from "../turn_transaction_contract.json" with { type: "json" }; -import { SETTLEMENT_STEP_KINDS, type JsonObject } from "../effect_program.ts"; +import { TURN_PROVIDER_STEP_KINDS, type JsonObject } from "../effect_program.ts"; const preparedStepKinds: ReadonlySet = new Set( - SETTLEMENT_STEP_KINDS.filter((kind) => kind !== "validation"), + TURN_PROVIDER_STEP_KINDS, ); interface AttemptViolation { diff --git a/loopx/semantics/project_registry_io_manifest_v1.json b/loopx/semantics/project_registry_io_manifest_v1.json index 3f4567a78a..61f07f0f1f 100644 --- a/loopx/semantics/project_registry_io_manifest_v1.json +++ b/loopx/semantics/project_registry_io_manifest_v1.json @@ -1351,7 +1351,7 @@ }, { "site": "loopx/control_plane/goals/source_session_recreation.py::.recreate_goal_instance::codec_transaction:source_session_registry_transaction#1", - "line": 286, + "line": 292, "column": 14, "kind": "codec_transaction", "api": "source_session_registry_transaction", @@ -1359,7 +1359,7 @@ }, { "site": "loopx/control_plane/goals/source_session_recreation.py::.recreate_goal_instance::codec_transaction:source_session_registry_transaction#2", - "line": 374, + "line": 381, "column": 14, "kind": "codec_transaction", "api": "source_session_registry_transaction", @@ -1375,7 +1375,7 @@ }, { "site": "loopx/control_plane/goals/source_session_turn_effects.py::._source_state::codec_read:load_project_registry#1", - "line": 121, + "line": 135, "column": 16, "kind": "codec_read", "api": "load_project_registry", @@ -1383,7 +1383,7 @@ }, { "site": "loopx/control_plane/goals/source_session_turn_effects.py::.drain_releasable_source_turn_effects::codec_read:load_project_registry#1", - "line": 429, + "line": 444, "column": 16, "kind": "codec_read", "api": "load_project_registry", diff --git a/tests/cli_commands/test_source_session_lifetime.py b/tests/cli_commands/test_source_session_lifetime.py index 7fdaf593d8..1d5fe64312 100644 --- a/tests/cli_commands/test_source_session_lifetime.py +++ b/tests/cli_commands/test_source_session_lifetime.py @@ -15,8 +15,20 @@ from loopx.control_plane.goals import ( source_session_recreation, source_session_registration, + source_session_turn_effects, +) +from loopx.control_plane.effect_program import ( + SettlementStepKind, + require_turn_provider_step_kind, +) +from loopx.control_plane.goals.first_party_host_admission import ( + FirstPartyHostGoalAdmission, ) from loopx.control_plane.projects import registry_codec +from loopx.control_plane.turn_driver.journal_store import ( + LOOPX_TURN_JOURNAL_SCHEMA_VERSION, + turn_journal_path, +) def _registration_arguments(registry_path: Path, knowledge_root: Path) -> list[str]: @@ -113,6 +125,94 @@ def _registry_payload(registry_path: Path) -> dict[str, object]: return json.loads(registry_path.read_text(encoding="utf-8"))[1] +def test_recreation_default_output_reports_durable_drain_transition( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + _knowledge_root, registry_path, registration = _register(tmp_path, capsys) + goal_ref = registration["goal_ref"] + assert isinstance(goal_ref, dict) + instance_a = str(goal_ref["goal_instance_id"]) + turn_key = "sha256:" + ("a" * 64) + effect_ref = f"{turn_key}#durable_writeback" + runtime_root = Path(str(_registry_payload(registry_path)["common_runtime_root"])) + journal_path = turn_journal_path( + runtime_root, + goal_id="atlas-import", + turn_key=turn_key, + ) + journal = { + "schema_version": LOOPX_TURN_JOURNAL_SCHEMA_VERSION, + "goal_id": "atlas-import", + "turn_key": turn_key, + "status": "in_progress", + "completed_phases": ["host_execute", "typed_result", "validation"], + "effect_attempts": { + "durable_writeback": { + "status": "prepared", + "effect_ref": effect_ref, + } + }, + } + goal_admission = FirstPartyHostGoalAdmission.for_plan( + registry_path=registry_path, + goal_id="atlas-import", + planned_goal_ref=goal_ref, + ) + effect_admission = goal_admission.turn_effect_admission( + turn_key=turn_key, + journal_path=journal_path, + ) + assert effect_admission is not None + + def persist_journal(_source_admission: dict[str, object]) -> None: + journal_path.parent.mkdir(parents=True, exist_ok=True) + journal_path.write_text(json.dumps(journal), encoding="utf-8") + + effect_admission.prepare( + require_turn_provider_step_kind(SettlementStepKind.DURABLE_WRITEBACK), + effect_ref, + persist_journal, + ) + + arguments = _recreation_arguments( + registry_path, + goal_instance_id=instance_a, + ) + del arguments[:2] + assert main(arguments) == 1 + rendered = capsys.readouterr().out + gate = json.loads( + source_session_turn_effects.source_turn_effect_gate_path( + registry_path, + "atlas-import", + ).read_text(encoding="utf-8") + ) + + assert gate["state"] == "closing" + assert "- status: `drain_required`" in rendered + assert "- changed: `True`" in rendered + assert "- gate_state: `closing`" in rendered + assert f"turn_key=`{turn_key}`" in rendered + assert "step_kind=`durable_writeback`" in rendered + assert "reason=`provider_readback_required`" in rendered + assert "Resume this Turn with provider readback" in rendered + assert "retry recreate-goal with the same operation_id" in rendered + + assert ( + main( + _recreation_arguments( + registry_path, + goal_instance_id=instance_a, + ) + ) + == 1 + ) + replay = json.loads(capsys.readouterr().out) + assert replay["changed"] is False + assert replay["replayed"] is True + + def test_registration_publishes_fresh_v2_without_global_sync( tmp_path: Path, capsys: pytest.CaptureFixture[str], diff --git a/tests/control_plane_ts/source_session_lifetime.test.ts b/tests/control_plane_ts/source_session_lifetime.test.ts index 51f41fb692..9501b6ae1a 100644 --- a/tests/control_plane_ts/source_session_lifetime.test.ts +++ b/tests/control_plane_ts/source_session_lifetime.test.ts @@ -211,6 +211,20 @@ test("Turn effects admit once and release while retirement is closing", () => { assert.equal(admitted.kind, "commit"); if (admitted.kind !== "commit") return; assert.equal(admitted.gate.state, "open"); + assert.throws( + () => decideSourceTurnEffectAdmission({ + profile_id: "source_session_v1", + requested_goal_ref: goalRef, + current_goal_ref: goalRef, + gate: admitted.gate, + admission: { + ...admission, + step_kind: "validation", + }, + existing_admission: null, + }), + /step_kind/, + ); assert.deepEqual( decideSourceTurnEffectAdmission({ profile_id: "source_session_v1", diff --git a/tests/test_loopx_turn_executor.py b/tests/test_loopx_turn_executor.py index 053297f25e..88a6713a8d 100644 --- a/tests/test_loopx_turn_executor.py +++ b/tests/test_loopx_turn_executor.py @@ -2297,10 +2297,34 @@ def crash_before_tail_confirmation( "turn_key": str(plan["transaction"]["turn_key"]), "step_kind": tail_step, "reason": "turn_tail_recovery_required", + "recovery_action": ( + "Resume this Turn to finish its settlement tail, " + "then retry recreate-goal." + ), } ] assert load_project_registry(registry)["goals"][0]["goal_instance_id"] == INSTANCE_A + journal_path = turn_journal_path( + runtime_root, + goal_id="fixture-goal", + turn_key=str(plan["transaction"]["turn_key"]), + ) + hold["schema_version"] = "loopx_source_turn_effect_hold_invalid" + journal_path.write_text(json.dumps(journal), encoding="utf-8") + + conflicted = recreate_goal_instance( + RecreateGoalRequest( + registry_path=registry, + goal_id="fixture-goal", + goal_instance_id=INSTANCE_A, + operation_id=f"recreate-after-{tail_step}-checkpoint", + ) + ) + assert conflicted["changed"] is False + assert conflicted["replayed"] is True + assert conflicted["pending_effects"][0]["reason"] == "turn_tail_conflict" + @pytest.mark.parametrize("first_scheduler_result", ["exception", "incomplete"]) def test_recreation_waits_for_tail_recovery( From b2ee0eb4a52f28f25cc3f72b47cbe055c925526c Mon Sep 17 00:00:00 2001 From: "duanjialing.777" Date: Wed, 30 Sep 2026 23:18:10 +0800 Subject: [PATCH 3/3] fix(goals): report partial recreation drain progress Signed-off-by: duanjialing.777 --- .../goals/source_session_recreation.py | 14 +-- .../goals/source_session_turn_effects.py | 22 +++- .../project_registry_io_manifest_v1.json | 4 +- .../test_source_session_lifetime.py | 102 ++++++++++++++++++ 4 files changed, 130 insertions(+), 12 deletions(-) diff --git a/loopx/control_plane/goals/source_session_recreation.py b/loopx/control_plane/goals/source_session_recreation.py index 24c7496365..849cb0c2a4 100644 --- a/loopx/control_plane/goals/source_session_recreation.py +++ b/loopx/control_plane/goals/source_session_recreation.py @@ -243,14 +243,14 @@ def _drain_required_result( *, requested_goal_ref: dict[str, str], pending_effects: list[dict[str, str]], - gate_changed: bool, + changed: bool, ) -> dict[str, Any]: return { "ok": False, "schema_version": "loopx_goal_recreation_v1", "status": "drain_required", - "changed": gate_changed, - "replayed": not gate_changed, + "changed": changed, + "replayed": not changed, "gate_state": "closing", "registry": str(request.registry_path), "retired_goal_ref": copy.deepcopy(requested_goal_ref), @@ -358,17 +358,17 @@ def recreate_goal_instance(request: RecreateGoalRequest) -> dict[str, Any]: gate=closing_gate, ) - pending_effects = drain_releasable_source_turn_effects( + drain_result = drain_releasable_source_turn_effects( registry_path=request.registry_path, goal_id=request.goal_id, requested_goal_ref=requested_goal_ref, ) - if pending_effects: + if drain_result.pending_effects: return _drain_required_result( request, requested_goal_ref=requested_goal_ref, - pending_effects=pending_effects, - gate_changed=gate_changed, + pending_effects=drain_result.pending_effects, + changed=gate_changed or drain_result.changed, ) with exclusive_cross_runtime_file_lock( diff --git a/loopx/control_plane/goals/source_session_turn_effects.py b/loopx/control_plane/goals/source_session_turn_effects.py index 4ed209ec28..e72e2556d7 100644 --- a/loopx/control_plane/goals/source_session_turn_effects.py +++ b/loopx/control_plane/goals/source_session_turn_effects.py @@ -75,6 +75,16 @@ def payload(self) -> dict[str, Any]: } +@dataclass(frozen=True, slots=True) +class SourceTurnEffectDrainResult: + pending_effects: list[dict[str, str]] + released_count: int + + @property + def changed(self) -> bool: + return self.released_count > 0 + + def _gate_root(registry_path: Path, goal_id: str) -> Path: return lifetime_root(registry_path) / "turn-settlement" / alias_digest(goal_id) @@ -437,7 +447,7 @@ def drain_releasable_source_turn_effects( registry_path: Path, goal_id: str, requested_goal_ref: Mapping[str, str], -) -> list[dict[str, str]]: +) -> SourceTurnEffectDrainResult: # turn_driver imports the Host admission owner, so defer this reverse edge. from ..turn_driver.journal_store import load_turn_journal, turn_journal_path @@ -446,6 +456,7 @@ def drain_releasable_source_turn_effects( if not runtime_root.is_absolute() or runtime_root.resolve() != runtime_root: raise ValueError("source-session common_runtime_root must be absolute") pending: list[dict[str, str]] = [] + released_count = 0 for admission_path in _admission_paths(registry_path, goal_id): admission = _read_object( admission_path, @@ -543,7 +554,7 @@ def drain_releasable_source_turn_effects( admission_path, label="source Turn effect admission", ) - _decision( + release_decision = _decision( "goal.source_session.turn_effect.release", { "profile_id": SOURCE_SESSION_PROFILE_ID, @@ -554,6 +565,11 @@ def drain_releasable_source_turn_effects( }, ) _remove_admission(admission_path) + if release_decision["kind"] == "commit": + released_count += 1 except LockAcquireTimeoutError: pending.append(_pending_projection(admission, reason="executor_active")) - return pending + return SourceTurnEffectDrainResult( + pending_effects=pending, + released_count=released_count, + ) diff --git a/loopx/semantics/project_registry_io_manifest_v1.json b/loopx/semantics/project_registry_io_manifest_v1.json index 61f07f0f1f..5636bb9a0e 100644 --- a/loopx/semantics/project_registry_io_manifest_v1.json +++ b/loopx/semantics/project_registry_io_manifest_v1.json @@ -1375,7 +1375,7 @@ }, { "site": "loopx/control_plane/goals/source_session_turn_effects.py::._source_state::codec_read:load_project_registry#1", - "line": 135, + "line": 145, "column": 16, "kind": "codec_read", "api": "load_project_registry", @@ -1383,7 +1383,7 @@ }, { "site": "loopx/control_plane/goals/source_session_turn_effects.py::.drain_releasable_source_turn_effects::codec_read:load_project_registry#1", - "line": 444, + "line": 454, "column": 16, "kind": "codec_read", "api": "load_project_registry", diff --git a/tests/cli_commands/test_source_session_lifetime.py b/tests/cli_commands/test_source_session_lifetime.py index 1d5fe64312..eaa7ebaaef 100644 --- a/tests/cli_commands/test_source_session_lifetime.py +++ b/tests/cli_commands/test_source_session_lifetime.py @@ -213,6 +213,108 @@ def persist_journal(_source_admission: dict[str, object]) -> None: assert replay["replayed"] is True +def test_recreation_retry_reports_partial_drain_as_change( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + _knowledge_root, registry_path, registration = _register(tmp_path, capsys) + goal_ref = registration["goal_ref"] + assert isinstance(goal_ref, dict) + instance_a = str(goal_ref["goal_instance_id"]) + runtime_root = Path(str(_registry_payload(registry_path)["common_runtime_root"])) + journals: dict[str, Path] = {} + + for marker in ("a", "b"): + turn_key = "sha256:" + (marker * 64) + effect_ref = f"{turn_key}#durable_writeback" + journal_path = turn_journal_path( + runtime_root, + goal_id="atlas-import", + turn_key=turn_key, + ) + journal = { + "schema_version": LOOPX_TURN_JOURNAL_SCHEMA_VERSION, + "goal_id": "atlas-import", + "turn_key": turn_key, + "status": "in_progress", + "completed_phases": ["host_execute", "typed_result", "validation"], + "effect_attempts": { + "durable_writeback": { + "status": "prepared", + "effect_ref": effect_ref, + } + }, + } + goal_admission = FirstPartyHostGoalAdmission.for_plan( + registry_path=registry_path, + goal_id="atlas-import", + planned_goal_ref=goal_ref, + ) + effect_admission = goal_admission.turn_effect_admission( + turn_key=turn_key, + journal_path=journal_path, + ) + assert effect_admission is not None + + def persist_journal( + _source_admission: dict[str, object], + *, + path: Path = journal_path, + payload: dict[str, object] = journal, + ) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload), encoding="utf-8") + + effect_admission.prepare( + require_turn_provider_step_kind(SettlementStepKind.DURABLE_WRITEBACK), + effect_ref, + persist_journal, + ) + journals[turn_key] = journal_path + + arguments = _recreation_arguments( + registry_path, + goal_instance_id=instance_a, + ) + admissions = ( + source_session_turn_effects.source_turn_effect_gate_path( + registry_path, + "atlas-import", + ).parent + / "admissions" + ) + + assert main(arguments) == 1 + first = json.loads(capsys.readouterr().out) + assert first["changed"] is True + assert first["replayed"] is False + assert len(first["pending_effects"]) == 2 + assert len(list(admissions.glob("*.json"))) == 2 + + released_turn_key = "sha256:" + ("a" * 64) + releasable = json.loads(journals[released_turn_key].read_text(encoding="utf-8")) + releasable.pop("effect_attempts") + journals[released_turn_key].write_text( + json.dumps(releasable), + encoding="utf-8", + ) + + assert main(arguments) == 1 + partial = json.loads(capsys.readouterr().out) + assert partial["changed"] is True + assert partial["replayed"] is False + assert [effect["turn_key"] for effect in partial["pending_effects"]] == [ + "sha256:" + ("b" * 64) + ] + assert len(list(admissions.glob("*.json"))) == 1 + + assert main(arguments) == 1 + replay = json.loads(capsys.readouterr().out) + assert replay["changed"] is False + assert replay["replayed"] is True + assert len(list(admissions.glob("*.json"))) == 1 + + def test_registration_publishes_fresh_v2_without_global_sync( tmp_path: Path, capsys: pytest.CaptureFixture[str],