From 02c9696f3105a4fc5c53390ab32a4201f8218802 Mon Sep 17 00:00:00 2001 From: Jiri Puc Date: Thu, 3 Sep 2026 11:05:12 +0200 Subject: [PATCH 1/3] feat(observations)!: dated events never collapse across dates (D106) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The observation adjudicator judged whether a new statement re-asserts an existing one from two bare strings, never reading the D41 event window every claim already carries. On LoCoMo conv-42 that folded three of seven dated tournament wins into earlier facts — October's "won a really big tournament last week" became evidence for January's "won his first tournament last week" because both say "last week" — and ten participation lineages collapsed into header boilerplate, swallowing two real entries. Add the deterministic temporal-compatibility rung D43's design named but never implemented: two dated events with disjoint resolved windows never interact and buy no verdict (also overriding the exact-statement shortcut); a dated event paired with an undated statement may still be judged for supersede/contradict, but an `evidence` verdict is coerced to `new` and recorded; undated pairs and overlapping-window events behave as before. The verdict prompt now shows both timelines. The candidate block derives each observation's event window from its supporting event-time claims. Roll the adjudicator generation, the adjudicate_observations flush component version, and the LoCoMo protocol (Full-v20 -> Full-v21). Add seven D106 fixture tests driven by a collapse-happy fake model, the D106 decision, the observations design amendment and worked example, the conv-42 evidence analysis, and the docs/status updates. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016p9xB9T7VYREuVmohDPKjR --- benchmarks/locomo/README.md | 15 +- benchmarks/locomo/__init__.py | 2 +- benchmarks/locomo/cli.py | 2 +- benchmarks/locomo/dataset.py | 4 +- benchmarks/locomo/model.py | 10 +- benchmarks/locomo/protocol.py | 14 +- benchmarks/locomo/runner.py | 10 +- benchmarks/locomo/sharding/README.md | 2 +- benchmarks/locomo/sharding/run_shard.sh | 4 +- decisions.md | 76 ++++ ...omo_conv42_recurring_event_adjudication.md | 129 +++++++ plan/designs/locomo_benchmark_design.md | 23 +- plan/designs/observations_design.md | 41 ++- plan/plans/phase-2-truth-machinery.md | 2 +- .../spine/observation_adjudication.py | 256 ++++++++++++-- src/rememberstack/workers/e3.py | 8 +- src/tests/benchmarks/test_locomo_protocol.py | 12 +- src/tests/benchmarks/test_locomo_runner.py | 10 +- .../benchmarks/test_locomo_store_backup.py | 6 +- .../spine/test_observation_adjudication.py | 333 +++++++++++++++++- website/src/app/docs/project-status/page.mdx | 8 +- 21 files changed, 887 insertions(+), 80 deletions(-) create mode 100644 plan/analysis/locomo_conv42_recurring_event_adjudication.md diff --git a/benchmarks/locomo/README.md b/benchmarks/locomo/README.md index ac85a979b..d5ca8f3a3 100644 --- a/benchmarks/locomo/README.md +++ b/benchmarks/locomo/README.md @@ -1,4 +1,4 @@ -# RS-LoCoMo-Full-v20 setup +# RS-LoCoMo-Full-v21 setup This directory contains the unshipped full-system LoCoMo adapter. It does not vendor or auto-download LoCoMo. Supply the exact pinned `locomo10.json` only after confirming its @@ -16,12 +16,12 @@ The safe first command is local and makes no API or model call: uv run --extra benchmark python -m benchmarks.locomo prepare \ --dataset /absolute/path/locomo10.json \ --tier smoke \ - --protocol full-v20 \ + --protocol full-v21 \ --output .benchmark-runs/locomo-smoke ``` The harness validates the pinned bytes, renders session documents, and fingerprints the -eight-question smoke plan. `--protocol` is prepare-only; `full-v20` is the one +eight-question smoke plan. `--protocol` is prepare-only; `full-v21` is the one current-system protocol, and every later stage reads that immutable choice from `run.json`. Do not run remote stages until reviewing [`locomo_benchmark_design.md`](../../plan/designs/locomo_benchmark_design.md). @@ -64,6 +64,15 @@ call sees the complete bounded candidate snapshot and returns a supplied candidate id or `new`. There is no insufficient-evidence result or confidence-routed frontier call. +V21 fingerprints the D106 observation adjudicator: dated events with disjoint +resolved windows never collapse onto each other, a dated event is never +`evidence` for an undated statement (nor the reverse), and the verdict prompt +shows both timelines. The `adjudicate_observations` component version pins +that generation; the dataset, rendered documents, retrieval surface, answer +and judge prompts, budgets, and scoring are those of v20. V20 and v21 scores +are directional, not a one-variable comparison — the fact layer a v21 store +serves differs from a v20 store's. + Build the image from the revision under test — Compose otherwise serves the published release image, and the harness refuses to run against an engine whose stamped revision does not match the prepared run: diff --git a/benchmarks/locomo/__init__.py b/benchmarks/locomo/__init__.py index 1a46ed3e8..d7dc5aa03 100644 --- a/benchmarks/locomo/__init__.py +++ b/benchmarks/locomo/__init__.py @@ -1 +1 @@ -"""The pinned RS-LoCoMo-Full-v20 benchmark adapter.""" +"""The pinned RS-LoCoMo-Full-v21 benchmark adapter.""" diff --git a/benchmarks/locomo/cli.py b/benchmarks/locomo/cli.py index f01cc26be..ece293744 100644 --- a/benchmarks/locomo/cli.py +++ b/benchmarks/locomo/cli.py @@ -118,7 +118,7 @@ def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="python -m benchmarks.locomo", description=( - "RS-LoCoMo-Full-v20: prepare is local; ingest/answer/judge require " + "RS-LoCoMo-Full-v21: prepare is local; ingest/answer/judge require " "explicit execution acknowledgements" ), ) diff --git a/benchmarks/locomo/dataset.py b/benchmarks/locomo/dataset.py index d29656056..cbca706ed 100644 --- a/benchmarks/locomo/dataset.py +++ b/benchmarks/locomo/dataset.py @@ -140,10 +140,10 @@ def load_manifest(tier: str) -> QuestionManifest: ) if manifest.dataset_commit != DATASET_COMMIT: raise DatasetValidationError( - "manifest dataset commit is not RS-LoCoMo-Full-v20" + "manifest dataset commit is not RS-LoCoMo-Full-v21" ) if manifest.dataset_sha256 != DATASET_SHA256: - raise DatasetValidationError("manifest dataset hash is not RS-LoCoMo-Full-v20") + raise DatasetValidationError("manifest dataset hash is not RS-LoCoMo-Full-v21") actual = item_ids_hash(item_ids=manifest.item_ids) if actual != manifest.item_ids_sha256: raise DatasetValidationError( diff --git a/benchmarks/locomo/model.py b/benchmarks/locomo/model.py index 33d5c9fb0..890cc1cf6 100644 --- a/benchmarks/locomo/model.py +++ b/benchmarks/locomo/model.py @@ -1,4 +1,4 @@ -"""Typed values for the full-system RS-LoCoMo-Full-v20 protocol.""" +"""Typed values for the full-system RS-LoCoMo-Full-v21 protocol.""" from __future__ import annotations @@ -28,8 +28,8 @@ Category = Literal[1, 2, 3, 4, 5] RetainedCategory = Literal[1, 2, 3, 4] Tier = Literal["smoke", "development", "publication"] -ProtocolKey = Literal["full-v20"] -ProtocolName = Literal["RS-LoCoMo-Full-v20"] +ProtocolKey = Literal["full-v21"] +ProtocolName = Literal["RS-LoCoMo-Full-v21"] SourceTimezoneBasis = Literal["assumed_utc"] AnswerAgentModel = Literal["openai/gpt-5.6-luna"] JudgeModel = Literal["openai/gpt-5.6-luna"] @@ -120,7 +120,7 @@ class QuestionManifest(FrozenModel): class RunConfiguration(FrozenModel): """Immutable identity of one prepared benchmark run.""" - protocol_name: ProtocolName = "RS-LoCoMo-Full-v20" + protocol_name: ProtocolName = "RS-LoCoMo-Full-v21" adapter_version: NonEmpty prepared_at: datetime repository_revision: NonEmpty @@ -393,7 +393,7 @@ class SessionDiagnosticSummary(FrozenModel): class RunSummary(FrozenModel): """Publication-ready local aggregate with no hidden denominator.""" - protocol_name: ProtocolName = "RS-LoCoMo-Full-v20" + protocol_name: ProtocolName = "RS-LoCoMo-Full-v21" protocol_fingerprint: NonEmpty tier: Tier questions: int = Field(ge=1) diff --git a/benchmarks/locomo/protocol.py b/benchmarks/locomo/protocol.py index 8859a1f3c..33d683e10 100644 --- a/benchmarks/locomo/protocol.py +++ b/benchmarks/locomo/protocol.py @@ -29,9 +29,9 @@ from rememberstack.model import Envelope from rememberstack.model import ToolDescriptor -PROTOCOL_NAME: Final = "RS-LoCoMo-Full-v20" -DEFAULT_PROTOCOL_KEY: Final = "full-v20" -ADAPTER_VERSION: Final = "locomo-full-adapter-2026.09-complete-values-v20" +PROTOCOL_NAME: Final = "RS-LoCoMo-Full-v21" +DEFAULT_PROTOCOL_KEY: Final = "full-v21" +ADAPTER_VERSION: Final = "locomo-full-adapter-2026.09-temporal-gate-v21" MAX_TOOL_CALLS: Final = 8 MAX_AGENT_CALLS: Final = 9 ANSWER_READER_RETRY_BUDGET: Final = 2 @@ -74,7 +74,7 @@ "no-types-1:binary-t4-1:document-t0-1" ), "adjudicate_observations": ( - "e3-obs-flush-2026.08a:claim-fanout-1:entity-fanout-1" + "e3-obs-flush-2026.09a:temporal-gate-1:claim-fanout-1:entity-fanout-1" ), "adjudicate_supersession": "adjudicator-2026.07b:temp0-1", "embed_claim": "p1-embed-claims-2026.07", @@ -211,8 +211,8 @@ class LoCoMoProtocol: answer_word_cap: int | None = None -_FULL_V20 = LoCoMoProtocol( - key="full-v20", +_FULL_V21 = LoCoMoProtocol( + key="full-v21", name=PROTOCOL_NAME, answer_agent_model=ANSWER_AGENT_MODEL, judge_model=JUDGE_MODEL, @@ -234,7 +234,7 @@ class LoCoMoProtocol: ) PROTOCOL_REGISTRY: Final[Mapping[ProtocolKey, LoCoMoProtocol]] = MappingProxyType( - {_FULL_V20.key: _FULL_V20} + {_FULL_V21.key: _FULL_V21} ) diff --git a/benchmarks/locomo/runner.py b/benchmarks/locomo/runner.py index e8eb67abd..60e037e33 100644 --- a/benchmarks/locomo/runner.py +++ b/benchmarks/locomo/runner.py @@ -411,7 +411,7 @@ def ingest_sample( _require_current_ingest_bindings(model_bindings=build.model_bindings) if build.document_binding_generation != EXPECTED_DOCUMENT_BINDING_GENERATION: raise ExecutionGuardError( - "deployment document binding generation differs from RS-LoCoMo-Full-v20" + "deployment document binding generation differs from RS-LoCoMo-Full-v21" ) _require_current_query_surface(context=context, client=client) _require_exact_live_ingests( @@ -570,7 +570,7 @@ def answer_sample( ): raise ExecutionGuardError( "the deployment did not report the exact completed" - " RS-LoCoMo-Full-v20 pipeline, live graph, and fresh P3 projection" + " RS-LoCoMo-Full-v21 pipeline, live graph, and fresh P3 projection" ) _require_serving_revision(context=context, readiness=readiness) prior_readiness = context.state.readiness.get(sample_id) @@ -1177,7 +1177,7 @@ def _validate_run( """Recompute immutable run identity before any local or remote stage.""" selected_protocol = protocol_for_name(configuration.protocol_name) if configuration.dataset_sha256 != DATASET_SHA256: - raise BenchmarkRunError("run dataset hash is not RS-LoCoMo-Full-v20") + raise BenchmarkRunError("run dataset hash is not RS-LoCoMo-Full-v21") if item_ids_hash(item_ids=manifest.item_ids) != manifest.item_ids_sha256: raise BenchmarkRunError("run manifest item hash changed") if manifest_bytes_hash(manifest=manifest) != configuration.manifest_sha256: @@ -1187,7 +1187,7 @@ def _validate_run( if manifest.tier != configuration.tier: raise BenchmarkRunError("run manifest tier changed") if configuration.dataset_commit != DATASET_COMMIT: - raise BenchmarkRunError("run dataset commit is not RS-LoCoMo-Full-v20") + raise BenchmarkRunError("run dataset commit is not RS-LoCoMo-Full-v21") if configuration.adapter_version != ADAPTER_VERSION: raise BenchmarkRunError("run adapter version differs from current code") if _models_hash(values=documents) != configuration.documents_sha256: @@ -1445,7 +1445,7 @@ def _require_current_ingest_bindings(*, model_bindings: dict[str, str]) -> None: if model_bindings.get(name) != expected.get(name) ) raise ExecutionGuardError( - "deployment ingest model bindings differ from RS-LoCoMo-Full-v20: " + "deployment ingest model bindings differ from RS-LoCoMo-Full-v21: " + ", ".join(mismatches) ) diff --git a/benchmarks/locomo/sharding/README.md b/benchmarks/locomo/sharding/README.md index 68c968079..1fe312992 100644 --- a/benchmarks/locomo/sharding/README.md +++ b/benchmarks/locomo/sharding/README.md @@ -178,7 +178,7 @@ as source-derived data. | --- | ---: | --- | | `LOCOMO_PYTHON` | `.venv/bin/python` | repository virtual-environment Python | | `LOCOMO_TIER` | `publication` | prepared manifest tier | -| `LOCOMO_PROTOCOL` | `full-v20` | prepare-time protocol key | +| `LOCOMO_PROTOCOL` | `full-v21` | prepare-time protocol key | | `LOCOMO_MOUNT_ROOT` | `$RUN_DIR/.mounts` | host/container-identical P3 mount root | | `LOCOMO_MAX_DOCUMENTS` | `100` | per-sample ingest authorization | | `LOCOMO_MAX_QUESTIONS` | `1540` | run-absolute answer item authorization | diff --git a/benchmarks/locomo/sharding/run_shard.sh b/benchmarks/locomo/sharding/run_shard.sh index 12086c58e..934a83ee2 100755 --- a/benchmarks/locomo/sharding/run_shard.sh +++ b/benchmarks/locomo/sharding/run_shard.sh @@ -25,7 +25,7 @@ run_dir=$2 dataset_path=$3 python_bin=${LOCOMO_PYTHON:-.venv/bin/python} tier=${LOCOMO_TIER:-publication} -protocol=${LOCOMO_PROTOCOL:-full-v20} +protocol=${LOCOMO_PROTOCOL:-full-v21} mount_root=${LOCOMO_MOUNT_ROOT:-$run_dir/.mounts} max_documents=${LOCOMO_MAX_DOCUMENTS:-100} max_questions=${LOCOMO_MAX_QUESTIONS:-1540} @@ -50,7 +50,7 @@ export GOOGLE_APPLICATION_CREDENTIALS=${LOCOMO_GCP_CREDENTIALS_FILE:-/etc/rememb export GOOGLE_API_CERTIFICATE_CONFIG=${LOCOMO_GCP_CERTIFICATE_CONFIG_FILE:-/etc/rememberstack/locomo-gcs/certificate-config.json} export GOOGLE_API_USE_CLIENT_CERTIFICATE=true -# RS-LoCoMo-Full-v20's non-secret ingest identity. Override ambient self-host +# RS-LoCoMo-Full-v21's non-secret ingest identity. Override ambient self-host # defaults so every shard runs the exact Luna/Qwen pipeline the protocol checks. export REMEMBERSTACK_STRUCTURER_MODEL=openai/gpt-5.6-luna export REMEMBERSTACK_SKELETON_CHECK_MODEL=openai/gpt-5.6-luna diff --git a/decisions.md b/decisions.md index baf971ede..7b350e1b4 100644 --- a/decisions.md +++ b/decisions.md @@ -4889,3 +4889,79 @@ retry; mention research or adoption in the prompt. **Amends.** Advances D104's Full-v19 benchmark identity only. It preserves D98 retrieval, D99's objective content-before-`Unknown` guard, and D104's bounded counterfactual instruction. + +## D106. Dated events never collapse across dates; evidence requires temporal compatibility + +**Decision (2026-09-03).** The observation adjudicator (D43) gains a +deterministic temporal-compatibility rung that runs before any model call, +reading the D41 valid-time each claim already carries: + +1. Two statements that both describe **datable events** + (`claim_valid_kind = 'event_time'`) whose resolved windows are **disjoint** + describe different events. They never interact — no `evidence`, no + `supersede`, no `contradict` — and no verdict is bought for the pair. This + holds even when the wording is byte-identical: "won a tournament last week" + said in January and again in October is two wins. +2. A dated event paired with an **undated** statement (a state, a period + figure, a summary such as "has been winning a few tournaments", or + testimony the extractor could not anchor) may still be judged for + `supersede`/`contradict`, but an `evidence` verdict for such a pair is + coerced to `new` and recorded. A specific dated event is never a + re-assertion of a vaguer statement, and a summary never re-asserts a + specific event. +3. Two undated statements are adjudicated exactly as before; two dated + events with overlapping windows go to the ladder as before (a re-mention + of the same event still collapses). + +The verdict prompt now also shows both statements' asserted dates and +resolved event windows, and states the rule, so the small model that does +run judges with the timeline in front of it. An observation's event window +is the span of its supporting current-testimony event claims; it is derived +at block time, not stored. The adjudicator generation, the +`adjudicate_observations` flush component version, and the LoCoMo protocol +(Full-v20 → Full-v21) roll; nothing else about D43 — the entity block, the +novelty gate, the no-cap rule, the fail-safe-to-coexist contract — changes. + +**Context.** The v0.11.0 LoCoMo `conv-42` run (protocol Full-v18, +`plan/analysis/locomo_conv42_recurring_event_adjudication.md`) extracted all +seven of Nate's tournament wins as clean, dated claims and resolved one Nate +entity, yet the fact layer kept only four distinct win facts. The +`observation_adjudications` transcript shows why: the small model saw two +bare strings, judged "won a really big video game tournament last week" +(October) to be `evidence` for "won his first video game tournament last +week" (January) because both say "last week", and folded the international +and Valorant wins into "has been winning a few gaming tournaments". Ten +participation lineages collapsed into the header boilerplate "Nate is a +participant.", swallowing two real tournament entries. The claims carried +resolved event dates the whole time; the adjudicator never read them. The +reader answered the counting question "At least five" against a gold of +seven with all seven wins in its testimony context — the fact envelope's +undercount is the plausible anchor. + +**Consequences.** Recurring same-shaped events survive as distinct facts, so +entity-anchored counting and enumeration have a complete fact layer to read. +Fewer verdict calls are bought (disjoint-event pairs cost zero). The +fail-safe direction is preserved and slightly widened: an undated re-mention +of a dated event now stays a separate observation rather than collapsing — +a duplicate, never a loss. Observation `valid_from` continues to be the +claim's `asserted_at`; using the resolved event window as the observation's +own validity is a separate change this decision does not make. Existing +stores re-adjudicate observations under the new generation on rebuild (D7). + +**Rejected.** Fixing only the verdict prompt (a model may still merge across +dates, and disjoint-date pairs should cost nothing); treating every +mixed-dating pair as non-interacting (would stop a dated event from +superseding a state it ends); dropping vague summaries at extraction +(source-faithful testimony is kept — D32 — the defect was absorption, not +extraction); a typed period/value column (D43 §4 rejects the typed schema; +the event window is already carried by the claim). + +**Design.** `plan/designs/observations_design.md` §3 (temporal compatibility +rung) and `plan/designs/locomo_benchmark_design.md` Full-v21 protocol. + +**Analysis.** `plan/analysis/locomo_conv42_recurring_event_adjudication.md`. + +**Amends.** D43's adjudication cascade (adds the deterministic rung its +design named but never implemented) and D105's Full-v20 benchmark identity. +Preserves D41 (claims stay the one home of asserted validity), D98 +retrieval, D100/D102 identity, and D104/D105 answer-prompt contracts. diff --git a/plan/analysis/locomo_conv42_recurring_event_adjudication.md b/plan/analysis/locomo_conv42_recurring_event_adjudication.md new file mode 100644 index 000000000..cd4442595 --- /dev/null +++ b/plan/analysis/locomo_conv42_recurring_event_adjudication.md @@ -0,0 +1,129 @@ +# LoCoMo `conv-42`: recurring events collapse in the observation layer + +**Date:** 2026-09-03 + +**Status:** analysis (non-binding); the decision it motivated is D106 + +**Scope:** why a store that extracted all seven of a speaker's tournament wins +as clean, dated claims and resolved one entity for him ended with four win +facts — and what the counting question's reader then did with it. Evidence is +read-only SQL against a retained v0.11.0 store; nothing here changes retrieval, +identity, or the answer prompt. + +## 1. Run coordinates + +| Field | Value | +| --- | --- | +| Engine | RememberStack v0.11.0, revision `e7b173a19e8a992ec57bf75ce6593373ab2fc2c5` | +| Protocol | `RS-LoCoMo-Full-v18`, publication tier, fingerprint `2c2d3070d7620b176c6396e1245f07e8e59f44bec01b09d837ac2e144b064df8` | +| Sample | `conv-42` (29 sessions, 629 turns, 199 retained questions) | +| Deployment | `3f134f36-6b59-4060-9318-bf77f2a3463b` on the managed-cloud benchmark host | +| Dataset SHA-256 | `79fa87e90f04081343b8c8debecb80a9a6842b76a7aa537dc9fdf651ea698ff4` | +| Scoring-base backup | `gs://remember-stack-locomo-backups/2c2d3070…/e7b173a1…/2026-09-01T12-19-28.807846Z/conv-42/20260902T072007Z-75e029f2` (verified) | +| Ingest outcome | 1,772 stage units succeeded, one `normalize_relations` dead letter replayed; 1,298 claims, 716 observations, 22 active entities, zero duplicate-name groups, one Nate and one Joanna | + +The answer stage stopped at 182 of 199 questions when the provider account +ran out of credits; the judge did not run. The two counting questions this +document is about were answered before the stop. + +## 2. The seven wins, layer by layer + +The gold answer to `conv-42/qa/0080` ("How many tournaments has Nate won?") +is **seven**, with evidence in seven sessions spread over ten months. + +**Extraction (E2) — 7 of 7.** Every win became a decontextualized, attributed +claim with a resolved D41 event window (`claim_valid_kind = 'event_time'`, +day precision): + +| Session | Claim text | Resolved event date | +| --- | --- | --- | +| D1 | Nate said he won his first video game tournament last week. | 2022-01-14 | +| D10 | Nate said that Nate won Nate's second tournament last week. | 2022-04-25 | +| D14 | Nate just won another regional video game tournament last week. | 2022-05-27 | +| D17 | Nate won Nate's fourth video game tournament on Friday. | 2022-07-08 | +| D19 | Nate won an international tournament yesterday. | 2022-08-21 | +| D22 | Nate said that Nate won a really big video game tournament last week. | 2022-09-29 | +| D27 | Nate won the final of a big Valorant tournament last Saturday. | 2022-11-05 | + +**Identity (D100/D102) — clean.** One active Nate entity (568 resolved +mentions), one Joanna (655). Every claim above is anchored on that Nate. + +**Fact layer (E3 + D43 adjudication) — 4 of 7.** The `observation_adjudications` +transcript for the same claims: + +| Incoming claim | Outcome | Absorbed into | +| --- | --- | --- | +| won his first video game tournament | `add` | — (observation, Jan) | +| won Nate's second tournament | `add` | — (observation, May) | +| won another regional tournament | `add` | — ("Winning the tournament was a huge confidence boost", Jun) | +| won Nate's fourth tournament | relation | `Nate —other:won→ "Nate's fourth video game tournament"` (the tournament minted as an entity) | +| won an international tournament yesterday | `noop` (evidence) | "Nate has been winning a few gaming tournaments" (May, undated summary) | +| won a really big tournament last week | `noop` (evidence) | **"won his first video game tournament last week" (Jan)** | +| won the final of a big Valorant tournament | `noop` (evidence) | "Nate has been winning a few gaming tournaments" | + +Three wins were folded into earlier facts as evidence rows. The October → +January merge is the decisive specimen: two claims whose resolved event +dates were nine months apart, judged the same fact because both say "last +week". The adjudicator's verdict prompt at that generation rendered exactly +two strings — `EXISTING` and `NEW` — and nothing about time, although both +claims carried resolved windows in the same table row the adjudicator read +`asserted_at` from. + +**Reader — "At least five".** The answer agent made one `answer_context` +call with the raw question and received all seven win claims in the +testimony envelope (sessions D1, D10, D14, D17, D19, D22, D27 — every gold +turn) and, in the fact envelope, the four surviving win facts plus the +summary "has been winning a few". It answered "At least five" against seven. +The fact envelope's undercount is the plausible anchor; the run cannot prove +the reader's arithmetic either way, and it ran at the protocol's +reasoning-effort `none`. + +## 3. The participation question: ten lineages into boilerplate + +`conv-42/qa/0078` ("How many video game tournaments has Nate participated +in?", gold nine) was answered "Four". Its fact-layer counterpart is one +observation, "Nate is a participant.", with `evidence_count = 10`. Its +supporting claims: + +- nine are the rendered session header — "Participants: Joanna and Nate", + "Nate is a participant.", "Joanna and Nate are participants." — one per + ingested session, source-faithful document framing; +- two are real, dated tournament entries: "currently participating in the + video game tournament again" (2022-03-24) and "tried playing in the local + Street Fighter tournament this time" (2022-04-25). + +The boilerplate is harmless on its own. The defect is that two dated events +were absorbed into an undated statement that means "participant of this +conversation": semantic adjacency ("participating" ≈ "participant") +overrode content and time. The header claims themselves are outside this +analysis (they are what the source says; whether document framing should +seed observations at all is a separate question for E2 selection). + +## 4. Where the fix belongs + +Not extraction: the summary "has been winning a few" and the header lines are +source-faithful claims (D32), and every win reached the store dated. Not +identity: one entity. Not retrieval: every win reached the reader. The defect +is in the adjudicator's *inputs* — it decided sameness from two strings while +the temporal discriminator sat unread — and in a missing rule: a dated event +is not a re-assertion of anything undated, and two dated events on different +days are two events regardless of wording. + +D106 adds that rule as the deterministic rung D43's design always named +before the model call, and shows both timelines in the prompt for the pairs +that still need judging. Replaying the seven wins through the new rung in the +test suite yields seven observations and buys no verdict for any +disjoint-window pair; the vague summary and the boilerplate state each keep +their own row. + +## 5. What this does not establish + +- No score. The run's judge did not execute, and 17 questions were never + answered; the per-question reads above are the answer agent's generated + answers against gold, not judged results. +- No claim that the temporal rung alone fixes the counting question. The + reader saw seven wins in testimony and still undercounted; whether a + complete fact layer changes its arithmetic is the next run's question. +- No change to observation `valid_from` (still the claim's `asserted_at`), + to the E2 selection of document-framing claims, or to the reader's + reasoning effort. Each is a separate decision. diff --git a/plan/designs/locomo_benchmark_design.md b/plan/designs/locomo_benchmark_design.md index 981ee657b..68fe146f0 100644 --- a/plan/designs/locomo_benchmark_design.md +++ b/plan/designs/locomo_benchmark_design.md @@ -1,7 +1,18 @@ # LoCoMo full-system benchmark design -> **Binding D105 amendment (2026-09-01).** The current protocol is -> `RS-LoCoMo-Full-v20`. It retains v19's dataset, ingestion, models, tools, +> **Binding D106 amendment (2026-09-03).** The current protocol is +> `RS-LoCoMo-Full-v21`. It retains v20's dataset, rendered documents, models, +> tools, budgets, answer and judge prompts, counterfactual and complete-answer +> instructions, content-before-`Unknown` harness guard, and no-review scoring +> rule. Its pinned `adjudicate_observations` component version now carries the +> D106 temporal-compatibility rung: dated events with disjoint resolved windows +> never collapse onto each other, and a dated event is never `evidence` for an +> undated statement. Ingestion provenance, protocol identity, and fingerprint +> roll; no retrieval, retry, model-effort, or call-budget behavior changes. +> V20 and v21 scores are directional because the fact layer differs. + +> **Historical D105 amendment (2026-09-01; superseded by D106).** The D105 +> protocol was `RS-LoCoMo-Full-v20`. It retained v19's dataset, ingestion, models, tools, > budgets, counterfactual instruction, content-before-`Unknown` harness guard, > and no-review scoring rule. Its answer prompt now also requires every > distinct retrieved value that directly satisfies the question, rather than @@ -79,7 +90,7 @@ and spend ceiling. ## 2. Fixed protocol ```text -protocol RS-LoCoMo-Full-v20 +protocol RS-LoCoMo-Full-v21 dataset commit 3eb6f2c585f5e1699204e3c3bdf7adc5c28cb376 dataset SHA-256 79fa87e90f04081343b8c8debecb80a9a6842b76a7aa537dc9fdf651ea698ff4 categories 1, 2, 3, 4 @@ -651,7 +662,7 @@ compatibility form. The response contains: same-snapshot proven-absent-anchor execution checks when live graph is required; - an overall `ready` that is the conjunction of the requested capabilities; - every non-secret ingestion/query model binding; and -- the non-secret `document_binding_generation`, which Full-v20 requires to be +- the non-secret `document_binding_generation`, which Full-v21 requires to be exactly `document-t0-v1` and stores in `run.json` plus the protocol fingerprint. @@ -797,11 +808,11 @@ Local preparation: uv run --extra benchmark python -m benchmarks.locomo prepare \ --dataset /absolute/path/locomo10.json \ --tier smoke \ - --protocol full-v20 \ + --protocol full-v21 \ --output .benchmark-runs/locomo-smoke ``` -`--protocol` exists only on `prepare`. The sole choice is `full-v20`; ingest, +`--protocol` exists only on `prepare`. The sole choice is `full-v21`; ingest, answer, judge, and summarize read it from the prepared run and expose no protocol override. diff --git a/plan/designs/observations_design.md b/plan/designs/observations_design.md index a5511e743..bc2d89825 100644 --- a/plan/designs/observations_design.md +++ b/plan/designs/observations_design.md @@ -142,6 +142,30 @@ When a claim asserts a value/property about entity *E*: state-vs-measurement from the `statement` (semantic), not from a typed column. This is the rule that replaces the dropped `about_period` columns: rather than recording the period structurally, the system simply never caps a period figure and lets same-period conflicts coexist. + + **Temporal compatibility — the deterministic rung (D106).** Before any model call, the + adjudicator reads what the claim already knows about *when*: its D41 `claim_valid_kind` and + resolved window. A claim that describes a **datable event** (`event_time` — a win, a visit, a + purchase; the extractor resolved "last Saturday" to a calendar day) is compared to the + candidate's own event window, which is the span of the event-time claims supporting it. + Then: + - **both dated events, disjoint windows → no interaction.** They are different things that + happened; the pair is skipped without buying a verdict. This holds even when the wording is + byte-identical — "won a tournament last week" said in January and again in October is two + wins, not one re-asserted. (The exact-statement shortcut above obeys the same rule.) + - **one dated event, one undated statement → `evidence` is forbidden.** The pair may still be + judged for supersede/contradict (a dated resignation can end a "is CEO" state), but a verdict + of `evidence` is coerced to `new` and recorded with its reason. A specific dated event is never + a re-assertion of a vaguer statement ("has been winning a few tournaments"), and a summary never + re-asserts a specific event — both stand. + - **both undated, or both dated with overlapping windows → the ladder as before.** A re-mention + of the *same* event (same resolved date) still collapses as evidence; headcounts, revenues and + stances are untouched. + The verdict prompt also shows both statements' asserted dates and resolved event windows, so the + model that does run judges with the timeline in front of it rather than two bare strings. The + rung is exactly the "deterministic value/period compare" step the cascade below always named; + it is what keeps recurring same-shaped events countable — the failure it closes is recorded in + `plan/analysis/locomo_conv42_recurring_event_adjudication.md`. 4. **Fail safe — a binding adjudicator contract (not just a hope).** This is the honest core of the untyped design: "never silently resolve" is **policy enforced in E3 + eval**, not a schema invariant. The binding rules: @@ -187,6 +211,20 @@ worst the adjudicator spends a bit more. Contrast a pure semantic-cluster approa prior headcount would be **invisible** and silently duplicated. Anchoring to the resolved entity is what buys this. +**Distinct events — seven tournament wins (D106).** A conversation reports, months apart, "I won +my first tournament last week", "won my second tournament last week", "won another regional +tournament", "won my fourth tournament on Friday", "won an international tournament yesterday", +"won a really big tournament last week", "won the Valorant final last Saturday". Every claim is a +dated event (`event_time`, day precision, all seven windows disjoint). Without the temporal rung a +small model reading only the two strings judged October's "won a really big tournament last week" +to be `evidence` for January's "won his first tournament last week" — both say "last week" — and +folded the international and Valorant wins into "has been winning a few tournaments"; the entity +ended with four win facts and a count question answered "at least five" against seven. With the +rung: each new win is compared to seven-minus-one disjoint-window events (skipped, zero verdicts) +and one undated summary (judged; an `evidence` verdict is coerced to `new`) → **seven observations, +one summary, no LLM spend on the disjoint pairs**. A second mention of the *same* win (same resolved +day) still collapses as evidence onto it. + ### Supersession appends — an observation is a time-slice, never an in-place edit An observation is **a time-slice of belief, not a mutable current-state record.** Supersession does two @@ -234,7 +272,8 @@ value → observation), and often the claim's embedding (E2 embeds claims for P1 - For a **hub entity**, the same vector step top-k ranks *which* candidates to compare (cheap math); a skipped far candidate costs at most a duplicate row, never a wrong supersede. 3. **Adjudicate the residue only (cheap → frontier).** Only similar-but-not-identical candidates escalate - the D4 cascade: deterministic value/period compare → small model → frontier LLM for the survivors. The + the D4 cascade: the deterministic temporal-compatibility compare (§3 step 3, D106 — disjoint dated + events skip the pair outright) → small model → frontier LLM for the survivors. The adjudicator decides same-property (+ same-period for a figure) and the outcome under the no-cap rule (state → supersede; measurement → contradict/coexist; same value → evidence; else new), and **fails safe to coexist** below the supersede margin. diff --git a/plan/plans/phase-2-truth-machinery.md b/plan/plans/phase-2-truth-machinery.md index 0ede35bba..e082d3623 100644 --- a/plan/plans/phase-2-truth-machinery.md +++ b/plan/plans/phase-2-truth-machinery.md @@ -16,7 +16,7 @@ review queue operable end-to-end; observation adjudicator passes its D43 accepta | WP-2.2 | Clustering + reversibility: gather/decide, black-hole guard, nDR, merge_events, redirects | registries §6; D21 | WP-2.1 | cluster worker | order-independence test; un-merge replay | done — PR #89: EntityClusterer (gather via blocking reach, decide via HAC centroid-linkage distance cut — never transitive closure), joint 1-hop neighborhood re-decision (order-independent, Klein test), black-hole guard (tightened cut on oversized blobs), blast-radius routing to review_queue (expected-impact ranked, hubs never auto), reversible merges (redirect + pre-merge snapshot + linked reversal events; double-unmerge refused) | | WP-2.3 | Ontology + predicate governance: seed core, packs, domain/range enforcement, `other:` funnel | registries §4, §7; D15, D18 | Phase 1 | registry content + normalizer checks | domain/range rejects hallucination sample | done — PR #90: system-shipped Work pack (Task/Decision/Goal + 6 predicates with signatures) installed as a unit with extend-never-fork anchor refusal; the D5 other: funnel (grammar-gated, tier=other registration, usage-counted, promotion-ranked, never leaks into the governed prompt vocabulary); registry-rendered normalizer prompts (descriptions + synonyms); hallucination sample rejected at ancestor-level signature matching | | WP-2.4 | Relation supersession cascade (blocking → novelty → cheap-first → outcomes) + adjudication transcripts | e2_e3 §5; D3–D4; schema §9 | WP-2.1 | adjudicator | S9-class green; zombie-fact test | done — PR #91: SupersessionAdjudicator (change-prone-only (subject, predicate) blocking → novelty gate past the LLM → small→frontier ladder → supersede/coexist/contradict outcomes applied atomically with append-only transcripts); normalize chains adjudicate_supersession for created relations; query surface gains valid_at as-of (S9-class) and the relation transcript (S8); zombie-fact and both-stand contradiction proofs | -| WP-2.5 | Observation adjudication (entity block, no-cap rule, contradiction groups, fail-safe coexist) | observations §3–4; D43 | WP-2.4 | observation adjudicator | **D43 eval gate green** (shipping criterion) | done — PR #92: the add-observation worker (entity block exact+exhaustive, novelty gate with zero-LLM exits, hub top-k ranking as ordering-only, small→frontier ladder), the binding fail-safe contract (supersede only above an explicit margin with a reason row — below it coexists; every cap audited), the no-cap rule for fixed-period measurements (contradictions both stand, grouped), corpus-redundancy evidence-collapse; contradiction P/R gate green over the golden set and recorded in eval_runs; S9 as-of over capped time-slices | +| WP-2.5 | Observation adjudication (entity block, no-cap rule, contradiction groups, fail-safe coexist) | observations §3–4; D43 | WP-2.4 | observation adjudicator | **D43 eval gate green** (shipping criterion) | done — PR #92: the add-observation worker (entity block exact+exhaustive, novelty gate with zero-LLM exits, hub top-k ranking as ordering-only, small→frontier ladder), the binding fail-safe contract (supersede only above an explicit margin with a reason row — below it coexists; every cap audited), the no-cap rule for fixed-period measurements (contradictions both stand, grouped), corpus-redundancy evidence-collapse; contradiction P/R gate green over the golden set and recorded in eval_runs; S9 as-of over capped time-slices. Amended 2026-09-03 by D106: the deterministic temporal-compatibility rung (disjoint dated events never collapse; a dated event is never evidence for an undated statement) after LoCoMo `conv-42` folded three of seven dated wins into earlier facts | | WP-2.6 | Review queue + CLI (cluster review, blast-radius bands, support_withdrawn triage verdicts) | registries §8; D24; lifecycle §4 triage | WP-2.2 | review tooling | verdicts write the designed rows (restore/invalidate) | done — PR #93: ReviewQueue (expected-impact-ranked listing, kind-checked locked verdicts), merge verdicts through the same reversible apply_merge mechanism as the clusterer (decided_by=human, snapshot kept), support_withdrawn triage (restore_support → review_restored currency event + claim currency + D54 recount; invalidate_fact → invalidated_at + recorded human adjudication; uncertain → deferred, marker stands), the remember CLI entry point (review list/decide) with the /docs/reference/cli page (D66) | | WP-2.7 | Un-merge ↔ supersession ripple spike + stance-holder resolution eval (D59) | registries §11.3; D59 | WP-2.4 | spike reports | recorded; thresholds adjusted | done — PR #94: executable spike (test_unmerge_ripple.py) + report (plan/analysis/unmerge_supersession_ripple_spike.md); findings: identity-set supersession blocking added (the merged identity's history is one person's history), un-merge flags cross-identity closures into review (split_cluster, never silently reopened or kept), same-identity closures don't ripple; stance-content guard locked (D59); ripple expected-impact starting point recorded | diff --git a/src/rememberstack/spine/observation_adjudication.py b/src/rememberstack/spine/observation_adjudication.py index 88cc02d99..6ef6cdb6a 100644 --- a/src/rememberstack/spine/observation_adjudication.py +++ b/src/rememberstack/spine/observation_adjudication.py @@ -13,7 +13,12 @@ figures contradict and both stand. """ +from __future__ import annotations + +from collections.abc import Mapping from collections.abc import Sequence +from dataclasses import dataclass +from typing import Any from typing import Final from uuid import UUID from uuid import uuid4 @@ -35,9 +40,15 @@ from rememberstack.ports.model_provider import ModelProviderPort from rememberstack.spine.rank_embed_cache import RankEmbedCache -OBSERVATION_ADJUDICATOR_VERSION: Final = "obs-adjudicator-2026.07b:temp0-1" +OBSERVATION_ADJUDICATOR_VERSION: Final = ( + "obs-adjudicator-2026.09a:temp0-1:temporal-gate-1" +) """The observation adjudicator generation (D12; replayed on rebuild, D7). -07b pins temperature=0.0 — generation parameters are part of provenance.""" +07b pins temperature=0.0 — generation parameters are part of provenance. +09a (D106) adds the temporal-compatibility rung: two dated events with +disjoint resolved windows never interact, a dated event never collapses as +evidence onto an undated statement (nor the reverse), and the verdict prompt +sees both statements' asserted dates and resolved event windows.""" _VERDICT_PROMPT: Final = """You adjudicate observations for a memory system. Both statements are believed facts about the SAME entity: @@ -45,6 +56,9 @@ EXISTING: {existing!r} NEW: {new!r} +EXISTING TIME: {existing_time} +NEW TIME: {new_time} + Judge semantically (there are no typed columns — "FY2023" vs "fiscal 2023" and "headcount" vs "staff count" are your equivalence calls): - evidence: same property, same value, overlapping validity — the new @@ -56,7 +70,16 @@ - contradict: same property AND same reporting period, incompatible value — both must stand, surfaced together. (Different property, or different period, is NOT a contradiction.) -- new: a different property, period, or thing — no interaction.""" +- new: a different property, period, or thing — no interaction. + +Time is decisive for EVENTS (a win, a visit, a purchase, a meeting). Two +statements about datable events with DIFFERENT resolved event dates describe +DIFFERENT events: always `new`, never `evidence`, even when the wording is +identical ("won a tournament last week" said in January and again in October +are two wins, not one re-asserted). A specific dated event is never +`evidence` for a vaguer summary ("has won a few tournaments"), and a summary +never re-asserts a specific event — keep both. Only a re-mention of the SAME +event (same resolved date, same thing) is `evidence`.""" class ObservationSettings(BaseSettings): @@ -238,8 +261,8 @@ def _apply_assertions_locked( if not assertions: return () claim_ids = list(dict.fromkeys(item.claim_id for item in assertions)) - asserted_by_claim = { - row["claim_id"]: row["asserted_at"] + timing_by_claim = { + row["claim_id"]: _timing_from_row(row) for row in connection.execute( _CLAIMS_ASSERTED, {"claim_ids": claim_ids} ).mappings() @@ -262,7 +285,7 @@ def _apply_assertions_locked( deployment_id=deployment_id, subject_entity_id=subject_entity_id, assertion=assertion, - asserted_at=asserted_by_claim.get(assertion.claim_id), + timing=timing_by_claim.get(assertion.claim_id, _UNDATED), candidates=candidates, meter=meter, call_key=f"{call_key}:{assertion_index}", @@ -277,18 +300,24 @@ def _add_with_block( deployment_id: UUID, subject_entity_id: UUID, assertion: ObservationAssertion, - asserted_at: object, + timing: _ClaimTiming, candidates: list[dict[str, object]], meter: CostMeterPort | None, call_key: str, ) -> UUID: """Apply one assertion while keeping the front-loaded block current.""" + asserted_at = timing.asserted_at + # An identical statement is the strongest same-fact signal, with one + # exception (D106): the same words about two DIFFERENT dated events + # ("won a tournament last week" in January and in October) are two + # facts, not one re-asserted. exact = next( ( candidate for candidate in candidates if candidate["statement"] == assertion.statement and bool(candidate["is_open"]) + and not _disjoint_dated_events(timing=timing, candidate=candidate) ), None, ) @@ -334,6 +363,7 @@ def _add_with_block( observation_id=observation_id, statement=assertion.statement, valid_from=asserted_at, + timing=timing, ) return observation_id # Capped state slices are history, not competitors for the next @@ -363,6 +393,7 @@ def _add_with_block( observation_id=observation_id, statement=assertion.statement, valid_from=asserted_at, + timing=timing, ) return observation_id ranked = self._rank( @@ -392,6 +423,7 @@ def _add_with_block( observation_id=observation_id, statement=assertion.statement, valid_from=asserted_at, + timing=timing, ) return observation_id return self._adjudicate_residue( @@ -401,7 +433,7 @@ def _add_with_block( statement=assertion.statement, claim_id=assertion.claim_id, doc_id=assertion.doc_id, - asserted_at=asserted_at, + timing=timing, ranked=ranked[: self._settings.hub_top_k], candidates=candidates, meter=meter, @@ -412,7 +444,12 @@ def judge_statements( self, *, existing: str, new: str ) -> tuple[ObservationOutcome, float]: """The bare pair-decision function — the D43 eval gate's surface.""" - verdict, method = self._ladder(existing=existing, new=new) + verdict, method = self._ladder( + existing=existing, + new=new, + existing_time=_render_timing(_UNDATED), + new_time=_render_timing(_UNDATED), + ) del method # the gate grades outcomes; rungs are graded per-run cost return verdict.outcome, verdict.confidence @@ -425,17 +462,42 @@ def _adjudicate_residue( statement: str, claim_id: UUID, doc_id: UUID, - asserted_at: object, + timing: _ClaimTiming, ranked: list[tuple[dict[str, object], float]], candidates: list[dict[str, object]], meter: CostMeterPort | None, call_key: str, ) -> UUID: - """Ladder the similar candidates; apply the first decisive outcome.""" + """Ladder the similar candidates; apply the first decisive outcome. + + The temporal-compatibility rung (D106) runs before every model call. + Two dated events whose resolved windows are disjoint are distinct + events and never interact, so no verdict is bought for them. A dated + event paired with an undated statement (or the reverse) may still be + judged for supersede/contradict, but an `evidence` verdict for such a + pair is coerced to `new`: a specific dated event is never a + re-assertion of a vaguer statement, and a summary never re-asserts a + specific event. + """ + asserted_at = timing.asserted_at + temporal_skips: list[dict[str, object]] = [] for candidate, similarity in ranked: + candidate_id = UUID(str(candidate["observation_id"])) + if _disjoint_dated_events(timing=timing, candidate=candidate): + temporal_skips.append( + { + "observation_id": str(candidate_id), + "reason": "disjoint dated events", + "similarity": similarity, + } + ) + continue + mixed_dating = _mixed_dating(timing=timing, candidate=candidate) verdict, method = self._ladder( existing=str(candidate["statement"]), new=statement, + existing_time=_render_timing(_candidate_timing(candidate)), + new_time=_render_timing(timing), meter=meter, call_key=f"{call_key}:verdict:{candidate['observation_id']}", ) @@ -443,7 +505,16 @@ def _adjudicate_residue( "similarity": similarity, "rationale": verdict.rationale, } - candidate_id = UUID(str(candidate["observation_id"])) + if mixed_dating and verdict.outcome is ObservationOutcome.EVIDENCE: + temporal_skips.append( + { + "observation_id": str(candidate_id), + "reason": "evidence coerced to new: dated event vs undated statement", + "similarity": similarity, + "rationale": verdict.rationale, + } + ) + continue if verdict.outcome is ObservationOutcome.EVIDENCE: self._evidence( connection=connection, @@ -499,6 +570,7 @@ def _adjudicate_residue( observation_id=new_id, statement=statement, valid_from=asserted_at, + timing=timing, ) return new_id if verdict.confidence < self._settings.supersede_margin: @@ -527,6 +599,7 @@ def _adjudicate_residue( observation_id=new_id, statement=statement, valid_from=asserted_at, + timing=timing, ) return new_id # D88 continuous ingest: direction follows source time, not @@ -581,6 +654,7 @@ def _adjudicate_residue( observation_id=new_id, statement=statement, valid_from=asserted_at, + timing=timing, is_open=False, ) return new_id @@ -689,7 +763,10 @@ def _adjudicate_residue( outcome="add", method="small_model", confidence=1.0, - features={"reason": "no candidate interacted"}, + features={ + "reason": "no candidate interacted", + "temporal_gate": temporal_skips, + }, related=None, contradiction_group=None, ) @@ -698,6 +775,7 @@ def _adjudicate_residue( observation_id=new_id, statement=statement, valid_from=asserted_at, + timing=timing, ) return new_id @@ -706,11 +784,15 @@ def _ladder( *, existing: str, new: str, + existing_time: str, + new_time: str, meter: CostMeterPort | None = None, call_key: str = "observation:verdict", ) -> tuple[ObservationVerdict, str]: """Small-model verdict, escalating to frontier below the floor.""" - prompt = _VERDICT_PROMPT.format(existing=existing, new=new) + prompt = _VERDICT_PROMPT.format( + existing=existing, new=new, existing_time=existing_time, new_time=new_time + ) verdict_call = self._model_provider.generate( request=ModelRequest( model=self._settings.small_model, prompt=prompt, temperature=0.0 @@ -950,7 +1032,7 @@ def _resplit_later_evidence( assertion=ObservationAssertion( statement=capped_statement, claim_id=claim_id, doc_id=doc_id ), - asserted_at=claim_asserted, + timing=_timing_from_row(row), candidates=candidates, meter=meter, call_key="d90_late_arrival_resplit", @@ -1019,6 +1101,7 @@ def _remember_candidate( contradiction_group: UUID | None = None, valid_from: object = None, is_open: bool = True, + timing: _ClaimTiming | None = None, ) -> None: """Expose one in-transaction insert to later assertions in the batch.""" candidates.append( @@ -1028,10 +1111,115 @@ def _remember_candidate( "contradiction_group": contradiction_group, "valid_from": valid_from, "is_open": is_open, + "event_from": None if timing is None else timing.event_from, + "event_until": None if timing is None else timing.event_until, } ) +@dataclass(frozen=True) +class _ClaimTiming: + """What the D41 record says about WHEN one piece of testimony applies. + + ``asserted_at`` is the source's own timestamp (the supersession boundary + the layer already used). ``event_from``/``event_until`` are the resolved + world-time window when the claim describes a datable EVENT + (``claim_valid_kind = 'event_time'``); both are ``None`` for undated + testimony, states, and period figures, so ``is_event`` is False there. + """ + + asserted_at: object = None + event_from: object = None + event_until: object = None + + @property + def is_event(self) -> bool: + """True when the testimony carries a resolved event window.""" + return self.event_from is not None + + +_UNDATED: Final = _ClaimTiming() + + +def _timing_from_row(row: Mapping[Any, Any]) -> _ClaimTiming: + """Read one claim row's asserted time and resolved event window.""" + is_event = row.get("valid_kind") == "event_time" + event_from = row.get("claim_valid_from") if is_event else None + if event_from is None: + return _ClaimTiming(asserted_at=row.get("asserted_at")) + event_until = row.get("claim_valid_until") + return _ClaimTiming( + asserted_at=row.get("asserted_at"), + event_from=event_from, + event_until=event_from if event_until is None else event_until, + ) + + +def _candidate_timing(candidate: Mapping[str, object]) -> _ClaimTiming: + """The block row's event window (aggregated over its supporting claims).""" + event_from = candidate.get("event_from") + if event_from is None: + return _ClaimTiming(asserted_at=candidate.get("valid_from")) + event_until = candidate.get("event_until") + return _ClaimTiming( + asserted_at=candidate.get("valid_from"), + event_from=event_from, + event_until=event_from if event_until is None else event_until, + ) + + +def _render_timing(timing: _ClaimTiming) -> str: + """One prompt line describing when testimony applies, honestly.""" + asserted = ( + "undated" + if timing.asserted_at is None + else f"asserted {_date_text(timing.asserted_at)}" + ) + if not timing.is_event: + return f"{asserted}; no resolved event date (a state, period figure, or undated statement)" + start = _date_text(timing.event_from) + end = _date_text(timing.event_until) + window = start if start == end else f"{start} to {end}" + return f"{asserted}; datable event resolved to {window}" + + +def _date_text(value: object) -> str: + """Render a timestamp as its calendar date; anything else verbatim.""" + date = getattr(value, "date", None) + return str(date()) if callable(date) else str(value) + + +def _disjoint_dated_events( + *, timing: _ClaimTiming, candidate: Mapping[str, object] +) -> bool: + """True when both sides are dated events whose windows never overlap. + + Two such statements describe different things that happened; they can + neither re-assert nor supersede nor contradict each other (D106). + """ + other = _candidate_timing(candidate) + if not (timing.is_event and other.is_event): + return False + return _windows_disjoint( + timing.event_from, timing.event_until, other.event_from, other.event_until + ) + + +def _mixed_dating(*, timing: _ClaimTiming, candidate: Mapping[str, object]) -> bool: + """True when exactly one side is a dated event (the other undated).""" + return timing.is_event != _candidate_timing(candidate).is_event + + +def _windows_disjoint( + left_from: object, left_until: object, right_from: object, right_until: object +) -> bool: + """Closed-interval disjointness; any incomparable value counts as overlap.""" + try: + return bool(left_until < right_from) or bool(right_until < left_from) # type: ignore[operator] + except TypeError: + return False + + def _cosine(a: tuple[float, ...], b: tuple[float, ...]) -> float: """Cosine similarity of two same-dimension vectors.""" if len(a) != len(b): @@ -1093,13 +1281,27 @@ def _is_later_in_total_order( _BLOCK_ENTITY = text( """ - SELECT observation_id, statement, contradiction_group, valid_from, - (valid_until IS NULL OR valid_until > now()) AS is_open - FROM observations - WHERE deployment_id = :deployment_id - AND subject_entity_id = :subject_entity_id - AND invalidated_at IS NULL - ORDER BY created_at + SELECT o.observation_id, o.statement, o.contradiction_group, o.valid_from, + (o.valid_until IS NULL OR o.valid_until > now()) AS is_open, + dated.event_from, dated.event_until + FROM observations o + -- D106: the resolved event window an observation stands for is the span + -- of its supporting dated-event testimony (D41 event_time claims only). + LEFT JOIN LATERAL ( + SELECT min(c.claim_valid_from) AS event_from, + max(coalesce(c.claim_valid_until, c.claim_valid_from)) AS event_until + FROM observation_evidence e + JOIN claims c ON c.claim_id = e.claim_id + WHERE e.observation_id = o.observation_id + AND e.stance = 'supports' + AND c.is_current_testimony + AND c.claim_valid_kind = 'event_time' + AND c.claim_valid_from IS NOT NULL + ) dated ON true + WHERE o.deployment_id = :deployment_id + AND o.subject_entity_id = :subject_entity_id + AND o.invalidated_at IS NULL + ORDER BY o.created_at """ ) @@ -1196,7 +1398,9 @@ def _is_later_in_total_order( _SELECT_EVIDENCE_FOR_OBS = text( """ - SELECT e.claim_id, e.doc_id, c.asserted_at + SELECT e.claim_id, e.doc_id, c.asserted_at, + c.claim_valid_kind::text AS valid_kind, + c.claim_valid_from, c.claim_valid_until FROM observation_evidence e JOIN claims c ON c.claim_id = e.claim_id WHERE e.deployment_id = :deployment_id @@ -1266,5 +1470,9 @@ def _is_later_in_total_order( ).bindparams(bindparam("features", type_=JSON)) _CLAIMS_ASSERTED = text( - "SELECT claim_id, asserted_at FROM claims WHERE claim_id = ANY(:claim_ids)" + """ + SELECT claim_id, asserted_at, claim_valid_kind::text AS valid_kind, + claim_valid_from, claim_valid_until + FROM claims WHERE claim_id = ANY(:claim_ids) + """ ) diff --git a/src/rememberstack/workers/e3.py b/src/rememberstack/workers/e3.py index 895900997..97b918960 100644 --- a/src/rememberstack/workers/e3.py +++ b/src/rememberstack/workers/e3.py @@ -85,8 +85,12 @@ def _run_profile_refresh(*, action: Callable[[], object], call_key: str) -> None Temperature=0.0 is part of provenance. """ -OBS_FLUSH_VERSION: Final = "e3-obs-flush-2026.08a:claim-fanout-1:entity-fanout-1" -"""Post-barrier observation flush generation (D88 §5.6; D90 entity fan-out).""" +OBS_FLUSH_VERSION: Final = ( + "e3-obs-flush-2026.09a:temporal-gate-1:claim-fanout-1:entity-fanout-1" +) +"""Post-barrier observation flush generation (D88 §5.6; D90 entity fan-out). +09a/temporal-gate-1: D106 — dated events with disjoint resolved windows never +collapse, and a dated event never becomes evidence for an undated statement.""" OBS_FLUSH_LEGACY_VERSION: Final = "e3-obs-flush-2026.08a:claim-fanout-1" """Pre-D90 version-serial obs flush component version (cutover only).""" diff --git a/src/tests/benchmarks/test_locomo_protocol.py b/src/tests/benchmarks/test_locomo_protocol.py index 020addddc..3003ac407 100644 --- a/src/tests/benchmarks/test_locomo_protocol.py +++ b/src/tests/benchmarks/test_locomo_protocol.py @@ -241,8 +241,8 @@ def test_current_protocol_pins_manifest_and_complete_read_plane() -> None: def test_protocol_is_v20_and_answer_prompt_has_reasoning_and_loop_guards() -> None: """The current identity, bounded inference, and loop discipline are locked.""" - assert PROTOCOL_NAME == "RS-LoCoMo-Full-v20" - assert DEFAULT_PROTOCOL_KEY == "full-v20" + assert PROTOCOL_NAME == "RS-LoCoMo-Full-v21" + assert DEFAULT_PROTOCOL_KEY == "full-v21" prompt = ANSWER_AGENT_PROMPT_TEMPLATE normalized_prompt = " ".join(prompt.split()) assert ( @@ -274,10 +274,10 @@ def test_protocol_is_v20_and_answer_prompt_has_reasoning_and_loop_guards() -> No def test_typed_protocol_registry_pins_answer_agent_identity_and_effort() -> None: - assert tuple(PROTOCOL_REGISTRY) == ("full-v20",) - protocol = PROTOCOL_REGISTRY["full-v20"] + assert tuple(PROTOCOL_REGISTRY) == ("full-v21",) + protocol = PROTOCOL_REGISTRY["full-v21"] - assert protocol.name == "RS-LoCoMo-Full-v20" + assert protocol.name == "RS-LoCoMo-Full-v21" assert protocol.answer_agent_model == "openai/gpt-5.6-luna" assert protocol.answer_agent_reasoning_effort == "none" assert protocol.judge_reasoning_effort == "none" @@ -315,7 +315,7 @@ def fake_prepare_run(**values: object) -> _Prepared: ) assert exit_code == 0 - assert selected == ["full-v20"] + assert selected == ["full-v21"] def test_summarize_cli_accepts_multiple_run_flags( diff --git a/src/tests/benchmarks/test_locomo_runner.py b/src/tests/benchmarks/test_locomo_runner.py index 4c34a2c7c..e4b162cc3 100644 --- a/src/tests/benchmarks/test_locomo_runner.py +++ b/src/tests/benchmarks/test_locomo_runner.py @@ -1047,7 +1047,7 @@ def test_answer_persists_usage_when_provider_drifts_after_tool_call() -> None: "invalid_first_step_completions", "invalid_reader_completions", ), - (("full-v20", "openai/gpt-5.6-luna", "none", 0, 2),), + (("full-v21", "openai/gpt-5.6-luna", "none", 0, 2),), ) def test_staged_mock_run_uses_prepared_protocol_and_resumes( protocol: ProtocolKey, @@ -1680,7 +1680,7 @@ def drifted_binding(request: httpx.Request) -> httpx.Response: def test_ingest_refuses_document_binding_generation_drift_before_upload( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - """Full-v20 cannot silently process with document-local T0 disabled.""" + """Full-v21 cannot silently process with document-local T0 disabled.""" _patch_prepared_inputs(monkeypatch=monkeypatch) run_dir = tmp_path / "run" prepare_run(dataset_path=tmp_path / "synthetic.json", tier="smoke", output=run_dir) @@ -1906,8 +1906,8 @@ def test_single_run_summary_json_is_unchanged( serialized = summarize_run(run_dir=run_dir).model_dump_json() assert serialized == ( - '{"protocol_name":"RS-LoCoMo-Full-v20","protocol_fingerprint":' - '"073711294b4c535ebdb51a0f7b1ddf74e314f9ca2cf110335b898c489f5ee916",' + '{"protocol_name":"RS-LoCoMo-Full-v21","protocol_fingerprint":' + '"0391b709fc76eb2cfc5748f62c6e3a294885bf2b329f1537715f6148ef455fef",' '"tier":"smoke","questions":1,"judge_correct":0,"judge_percent":0.0,' '"official_f1":0.0,"categories":[{"category":1,"questions":0,' '"judge_correct":0,"judge_percent":0.0,"official_f1":0.0},{"category":2,' @@ -2126,7 +2126,7 @@ def test_prepared_protocol_pins_current_surface_and_luna( dataset_path=tmp_path / "synthetic.json", tier="smoke", output=run_dir ) - assert prepared.protocol_name == "RS-LoCoMo-Full-v20" + assert prepared.protocol_name == "RS-LoCoMo-Full-v21" assert prepared.answer_agent_model == "openai/gpt-5.6-luna" assert prepared.answer_agent_reasoning_effort == "none" assert prepared.answer_reader_retry_budget == 2 diff --git a/src/tests/benchmarks/test_locomo_store_backup.py b/src/tests/benchmarks/test_locomo_store_backup.py index a31767548..06be91ac5 100644 --- a/src/tests/benchmarks/test_locomo_store_backup.py +++ b/src/tests/benchmarks/test_locomo_store_backup.py @@ -24,7 +24,7 @@ def _run_json(path: Path) -> None: (path / "run.json").write_text( json.dumps( { - "protocol_name": "RS-LoCoMo-Full-v20", + "protocol_name": "RS-LoCoMo-Full-v21", "protocol_fingerprint": "p" * 64, "repository_revision": "r" * 40, "prepared_at": "2026-08-11T00:00:00Z", @@ -660,7 +660,7 @@ def test_restore_validates_every_archive_before_running_docker( deployment_id="57000000-0000-0000-0000-000000000001", compose_project="rememberstack", run=store_backup.RunIdentity( - protocol_name="RS-LoCoMo-Full-v20", + protocol_name="RS-LoCoMo-Full-v21", protocol_fingerprint="p" * 64, repository_revision="r" * 40, prepared_at="2026-08-11T00:00:00Z", @@ -922,7 +922,7 @@ def test_runtime_validation_uses_the_image_revision_stamp( sample_id="conv-1", deployment_id=deployment_id, run=store_backup.RunIdentity( - protocol_name="RS-LoCoMo-Full-v20", + protocol_name="RS-LoCoMo-Full-v21", protocol_fingerprint="p" * 64, repository_revision=revision, prepared_at="2026-08-11T00:00:00Z", diff --git a/src/tests/spine/test_observation_adjudication.py b/src/tests/spine/test_observation_adjudication.py index 9ac3756ce..25bb2a187 100644 --- a/src/tests/spine/test_observation_adjudication.py +++ b/src/tests/spine/test_observation_adjudication.py @@ -153,13 +153,19 @@ def _add( statement: str, engine: Engine | None = None, asserted_at: str | None = None, + event_window: tuple[str, str] | None = None, + doc_id: UUID | None = None, ) -> UUID: """One observation through the cascade with a fresh claim. With `asserted_at`, a real dated claim row backs the testimony (the D41 seed the boundary math reads); without it, the testimony is undated. + With `event_window`, the claim is a datable EVENT resolved to that + day-precision `[from, until]` window (`claim_valid_kind = 'event_time'`), + the input the D106 temporal-compatibility rung reads. """ claim_id = uuid4() + doc_id = doc_id or uuid4() if asserted_at is not None and engine is not None: with engine.begin() as connection: connection.execute( @@ -167,17 +173,25 @@ def _add( "INSERT INTO claims (claim_id, deployment_id, doc_id," " chunk_id, claim_text, source_span, char_start, char_end," " anchor_ok, window_membership_ok, extractor_version," - " asserted_at)" + " asserted_at, claim_valid_kind, claim_valid_precision," + " claim_valid_from, claim_valid_until)" " VALUES (:c, :d, :doc, :ch, :s, :s, 0, 1, true, true," - " 'test', CAST(:a AS timestamptz))" + " 'test', CAST(:a AS timestamptz)," + " CAST(:kind AS claim_valid_kind)," + " CAST(:precision AS claim_valid_precision)," + " CAST(:vf AS timestamptz), CAST(:vu AS timestamptz))" ), { "c": claim_id, "d": _DEPLOYMENT_ID, - "doc": uuid4(), + "doc": doc_id, "ch": uuid4(), "s": statement, "a": asserted_at, + "kind": None if event_window is None else "event_time", + "precision": "unknown" if event_window is None else "day", + "vf": None if event_window is None else event_window[0], + "vu": None if event_window is None else event_window[1], }, ) return adjudicator.add_observation( @@ -185,10 +199,88 @@ def _add( subject_entity_id=entity, statement=statement, claim_id=claim_id, - doc_id=uuid4(), + doc_id=doc_id, ) +def _collapse_happy_router(prompt: str, type_name: str) -> dict[str, object]: + """The failure the D106 rung exists to stop: a small model that reads + "won a tournament last week" twice and calls the second a re-assertion. + + It answers `evidence` for any pair about the same recurring activity — + exactly what the conv-42 run's adjudicator did to five of seven wins — + so a test passes only when the deterministic rung keeps it from mattering. + """ + if type_name != "ObservationVerdict": + raise AssertionError(f"unexpected generate call: {type_name}") + lower = prompt.lower() + if "tournament" in lower or "participa" in lower or "competing" in lower: + return {"outcome": "evidence", "confidence": 0.95} + return {"outcome": "new", "confidence": 0.9} + + +def _observations(*, engine: Engine, entity: UUID) -> list[dict[str, object]]: + """The entity's live observations, oldest window first.""" + with engine.connect() as connection: + return [ + dict(row) + for row in connection.execute( + text( + "SELECT observation_id, statement, valid_from, evidence_count" + " FROM observations" + " WHERE subject_entity_id = :e AND invalidated_at IS NULL" + " ORDER BY valid_from NULLS LAST, statement" + ), + {"e": entity}, + ).mappings() + ] + + +def _verdict_calls(provider: FakeModelProvider) -> int: + """How many verdict prompts the ladder actually bought.""" + return sum(1 for prompt in provider.generated_prompts if "EXISTING:" in prompt) + + +# Seven distinct wins as conv-42 reported them: same shape, months apart. +_SEVEN_WINS: tuple[tuple[str, str, str], ...] = ( + ( + "Nate said he won his first video game tournament last week.", + "2022-01-21T19:31:00Z", + "2022-01-14", + ), + ( + "Nate said that Nate won Nate's second tournament last week.", + "2022-05-02T11:54:00Z", + "2022-04-25", + ), + ( + "Nate just won another regional video game tournament last week.", + "2022-06-03T17:44:00Z", + "2022-05-27", + ), + ( + "Nate won Nate's fourth video game tournament on Friday.", + "2022-07-10T14:34:00Z", + "2022-07-08", + ), + ( + "Nate won an international tournament yesterday.", + "2022-08-22T10:57:00Z", + "2022-08-21", + ), + ( + "Nate said that Nate won a really big video game tournament last week.", + "2022-10-06T11:15:00Z", + "2022-09-29", + ), + ( + "Nate won the final of a big Valorant tournament last Saturday.", + "2022-11-07T20:10:00Z", + "2022-11-05", + ), +) + + def test_d90_staggered_late_arrival_resplit_shapes(database_engine: Engine) -> None: """D90 §5.5.3: A{t1,t3} first then B{t2} yields A[t1,t2), B[t2,t3), A[t3,∞).""" adjudicator, _provider = _adjudicator(engine=database_engine) @@ -674,3 +766,236 @@ def test_stance_content_never_becomes_a_fact(database_engine: Engine) -> None: ).scalar_one() assert subjects == [team] # anchored on the holder, nowhere else assert relations == 0 # no fact about Atlas was derived + + +def test_d106_seven_dated_wins_survive_as_seven_facts(database_engine: Engine) -> None: + """The conv-42 counting case: seven same-shaped wins on seven dates must + stay seven observations, and the rung must not even buy a verdict for + pairs whose event windows are disjoint (they are different events).""" + adjudicator, provider = _adjudicator( + engine=database_engine, router=_collapse_happy_router + ) + nate = _entity(engine=database_engine) + for statement, asserted_at, day in _SEVEN_WINS: + _add( + adjudicator=adjudicator, + entity=nate, + statement=statement, + engine=database_engine, + asserted_at=asserted_at, + event_window=(day, day), + ) + rows = _observations(engine=database_engine, entity=nate) + assert [row["statement"] for row in rows] == [win[0] for win in _SEVEN_WINS] + assert all(row["evidence_count"] == 1 for row in rows) + assert _verdict_calls(provider) == 0 # every pair was gated: no LLM spend + with database_engine.connect() as connection: + noops = connection.execute( + text( + "SELECT count(*) FROM observation_adjudications a" + " JOIN observations o ON o.observation_id = a.observation_id" + " WHERE o.subject_entity_id = :e AND a.outcome = 'noop'" + ), + {"e": nate}, + ).scalar_one() + assert noops == 0 + + +def test_d106_arrival_order_does_not_change_the_count(database_engine: Engine) -> None: + """Reverse (and shuffled) arrival yields the same seven facts.""" + adjudicator, _provider = _adjudicator( + engine=database_engine, router=_collapse_happy_router + ) + nate = _entity(engine=database_engine) + shuffled = ( + _SEVEN_WINS[6], + _SEVEN_WINS[2], + _SEVEN_WINS[0], + _SEVEN_WINS[4], + _SEVEN_WINS[1], + _SEVEN_WINS[5], + _SEVEN_WINS[3], + ) + for statement, asserted_at, day in shuffled: + _add( + adjudicator=adjudicator, + entity=nate, + statement=statement, + engine=database_engine, + asserted_at=asserted_at, + event_window=(day, day), + ) + rows = _observations(engine=database_engine, entity=nate) + assert len(rows) == 7 + assert [row["statement"] for row in rows] == [win[0] for win in _SEVEN_WINS] + + +def test_d106_same_event_mentioned_twice_still_collapses( + database_engine: Engine, +) -> None: + """The rung is about DIFFERENT events: a re-mention of one event (same + resolved date) still collapses to one observation with two lineages.""" + adjudicator, provider = _adjudicator( + engine=database_engine, router=_collapse_happy_router + ) + nate = _entity(engine=database_engine) + statement = "Nate won an international tournament yesterday." + for asserted_at in ("2022-08-22T10:57:00Z", "2022-08-22T18:00:00Z"): + _add( + adjudicator=adjudicator, + entity=nate, + statement=statement, + engine=database_engine, + asserted_at=asserted_at, + event_window=("2022-08-21", "2022-08-21"), + ) + rows = _observations(engine=database_engine, entity=nate) + assert len(rows) == 1 and rows[0]["evidence_count"] == 2 + assert _verdict_calls(provider) == 0 # exact re-assertion: zero LLM + + +def test_d106_identical_words_about_two_dates_are_two_events( + database_engine: Engine, +) -> None: + """ "won a tournament last week" in January and again in October is two + wins even though the strings are byte-identical.""" + adjudicator, provider = _adjudicator( + engine=database_engine, router=_collapse_happy_router + ) + nate = _entity(engine=database_engine) + statement = "Nate said that Nate won a video game tournament last week." + for asserted_at, day in ( + ("2022-01-21T19:31:00Z", "2022-01-14"), + ("2022-10-06T11:15:00Z", "2022-09-29"), + ): + _add( + adjudicator=adjudicator, + entity=nate, + statement=statement, + engine=database_engine, + asserted_at=asserted_at, + event_window=(day, day), + ) + rows = _observations(engine=database_engine, entity=nate) + assert len(rows) == 2 and all(row["evidence_count"] == 1 for row in rows) + assert _verdict_calls(provider) == 0 + + +def test_d106_a_vague_summary_never_absorbs_a_dated_event( + database_engine: Engine, +) -> None: + """ "has been winning a few tournaments" (undated) must not swallow the + later dated international win as evidence, whatever the model says — + and the verdict prompt must have shown the model both timelines.""" + adjudicator, provider = _adjudicator( + engine=database_engine, router=_collapse_happy_router + ) + nate = _entity(engine=database_engine) + _add( + adjudicator=adjudicator, + entity=nate, + statement="Nate said that Nate has been winning a few gaming tournaments.", + engine=database_engine, + asserted_at="2022-05-02T11:54:00Z", + ) + dated = _add( + adjudicator=adjudicator, + entity=nate, + statement="Nate won an international tournament yesterday.", + engine=database_engine, + asserted_at="2022-08-22T10:57:00Z", + event_window=("2022-08-21", "2022-08-21"), + ) + rows = _observations(engine=database_engine, entity=nate) + assert len(rows) == 2 and all(row["evidence_count"] == 1 for row in rows) + assert _verdict_calls(provider) == 1 # mixed pair: judged, then coerced + last_prompt = provider.generated_prompts[-1] + assert "EXISTING TIME: asserted 2022-05-02; no resolved event date" in last_prompt + assert "NEW TIME: asserted 2022-08-22; datable event resolved to 2022-08-21" in ( + last_prompt + ) + with database_engine.connect() as connection: + features = connection.execute( + text( + "SELECT features FROM observation_adjudications" + " WHERE observation_id = :o AND outcome = 'add'" + ), + {"o": dated}, + ).scalar_one() + assert features["temporal_gate"][0]["reason"].startswith("evidence coerced to new") + + +def test_d106_a_dated_event_never_absorbs_a_later_summary( + database_engine: Engine, +) -> None: + """The mirror image: a summary arriving after a dated event coexists.""" + adjudicator, _provider = _adjudicator( + engine=database_engine, router=_collapse_happy_router + ) + nate = _entity(engine=database_engine) + _add( + adjudicator=adjudicator, + entity=nate, + statement="Nate won an international tournament yesterday.", + engine=database_engine, + asserted_at="2022-08-22T10:57:00Z", + event_window=("2022-08-21", "2022-08-21"), + ) + _add( + adjudicator=adjudicator, + entity=nate, + statement="Nate said that Nate has been winning a few gaming tournaments.", + engine=database_engine, + asserted_at="2022-10-06T11:15:00Z", + ) + assert len(_observations(engine=database_engine, entity=nate)) == 2 + + +def test_d106_boilerplate_state_never_absorbs_dated_participation( + database_engine: Engine, +) -> None: + """conv-42's "Nate is a participant." (document-header boilerplate, nine + lineages) swallowed two real tournament entries. The boilerplate may + still collapse onto itself; the dated events stay distinct facts.""" + adjudicator, _provider = _adjudicator( + engine=database_engine, router=_collapse_happy_router + ) + nate = _entity(engine=database_engine) + for asserted_at in ( + "2022-01-21T19:31:00Z", + "2022-02-07T09:00:00Z", + "2022-03-01T09:00:00Z", + ): + _add( + adjudicator=adjudicator, + entity=nate, + statement="Nate is a participant.", + engine=database_engine, + asserted_at=asserted_at, + ) + for statement, asserted_at, day in ( + ( + "Nate said that Nate is currently participating in the video game tournament again.", + "2022-03-24T15:00:00Z", + "2022-03-24", + ), + ( + "Nate said that Nate tried playing in the local Street Fighter tournament this time.", + "2022-04-25T15:00:00Z", + "2022-04-25", + ), + ): + _add( + adjudicator=adjudicator, + entity=nate, + statement=statement, + engine=database_engine, + asserted_at=asserted_at, + event_window=(day, day), + ) + rows = _observations(engine=database_engine, entity=nate) + assert len(rows) == 3 + boilerplate = next( + row for row in rows if row["statement"] == "Nate is a participant." + ) + assert boilerplate["evidence_count"] == 3 diff --git a/website/src/app/docs/project-status/page.mdx b/website/src/app/docs/project-status/page.mdx index 93f3ebb89..c554a4977 100644 --- a/website/src/app/docs/project-status/page.mdx +++ b/website/src/app/docs/project-status/page.mdx @@ -250,8 +250,14 @@ examined, and the failure looks like the deployment being down. hypothetical may follow causal or motivational relationships in retrieved evidence even when no source states the hypothetical verbatim. It adds no retrieval, retry, reasoning-effort, or call-budget work. -- The current `RS-LoCoMo-Full-v20` also requires every distinct retrieved value +- Historical `RS-LoCoMo-Full-v20` also requires every distinct retrieved value that directly satisfies the question. It does not add a retrieval or retry. +- The current `RS-LoCoMo-Full-v21` pins the D106 observation adjudicator: + dated events with disjoint resolved windows never collapse onto each other, + and a dated event is never `evidence` for an undated statement. Recurring + same-shaped events (seven tournament wins months apart) now survive as + distinct facts. It changes ingestion provenance only; retrieval, prompts, + budgets, and scoring are those of v20. ## What landed after v0.6.0 (in v0.7.0; historical resolver generation) From 80159af1dbb589b0620120a1ecb33f12aedbe166 Mon Sep 17 00:00:00 2001 From: Jiri Puc Date: Thu, 3 Sep 2026 12:26:54 +0200 Subject: [PATCH 2/3] fix(observations): bound verdicts by temporal relation instead of skipping pairs (D106 review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex review of #360 found the first cut over-broad: disjoint dated events were declared unrelated without a verdict, which would double-count one occurrence whose date two sources dispute and suppress its contradiction group. Disjoint pairs now reach the date-aware model and may only contradict or stay new; evidence and supersede are coerced to new and recorded. The exact-statement shortcut requires temporal compatibility (identical text beside a dated/undated or disjoint-window twin coexists, decided without a model). Open-ended D41 windows stay unbounded instead of collapsing to a point. In-batch candidates carry timing on every insert and widen their windows as they absorb evidence. Every verdict record carries the coercions before it. The entity flush handler reports the claimed unit's own component generation to its barrier so units enqueued before the OBS_FLUSH_VERSION roll drain under the generation their barrier counts; D106 states the stop-drain-rebuild rollout. The verdict prompt labels each side's two clocks ("said on" — the source's date; "is about" — the resolved world-time of any D41 kind) and defines them. Six more proofs; stale v20-as-current references corrected. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016p9xB9T7VYREuVmohDPKjR --- benchmarks/locomo/README.md | 8 +- decisions.md | 76 ++- plan/designs/locomo_benchmark_design.md | 7 +- plan/designs/observations_design.md | 52 +- .../spine/observation_adjudication.py | 445 +++++++++++++----- src/rememberstack/workers/e3.py | 6 +- src/tests/benchmarks/test_locomo_protocol.py | 2 +- .../spine/test_observation_adjudication.py | 333 +++++++++++-- .../test_e3_entity_obs_flush_fanout.py | 12 + website/src/app/docs/project-status/page.mdx | 5 +- 10 files changed, 738 insertions(+), 208 deletions(-) diff --git a/benchmarks/locomo/README.md b/benchmarks/locomo/README.md index d5ca8f3a3..c9e997b0b 100644 --- a/benchmarks/locomo/README.md +++ b/benchmarks/locomo/README.md @@ -65,9 +65,11 @@ candidate id or `new`. There is no insufficient-evidence result or confidence-routed frontier call. V21 fingerprints the D106 observation adjudicator: dated events with disjoint -resolved windows never collapse onto each other, a dated event is never -`evidence` for an undated statement (nor the reverse), and the verdict prompt -shows both timelines. The `adjudicate_observations` component version pins +resolved windows never collapse onto or supersede each other (they may only +contradict or stay distinct), a dated event is never `evidence` for an undated +statement (nor the reverse), open-ended windows stay unbounded, and the verdict +prompt shows when each statement was said and what time it is about. The +`adjudicate_observations` component version pins that generation; the dataset, rendered documents, retrieval surface, answer and judge prompts, budgets, and scoring are those of v20. V20 and v21 scores are directional, not a one-variable comparison — the fact layer a v21 store diff --git a/decisions.md b/decisions.md index 7b350e1b4..0d0dead08 100644 --- a/decisions.md +++ b/decisions.md @@ -2910,7 +2910,11 @@ may proceed to its first tagged artifact proof after CLA activation. ## D78. LoCoMo measures the ordinary OSS query system, not a claims-only shortcut -> **D105 amendment.** The current protocol is `RS-LoCoMo-Full-v20`. It retains +> **D106 amendment.** The current protocol is `RS-LoCoMo-Full-v21`. It retains +> v20's dataset, rendered documents, models, tools, budgets, prompts, and +> scoring and pins the D106 observation adjudicator generation. + +> **Historical D105 amendment.** The D105 protocol was `RS-LoCoMo-Full-v20`. It retains > v19's surface, dataset, ingestion, models, tools, and budgets while adding the > D105 complete-direct-values answer instruction. > @@ -4898,26 +4902,35 @@ reading the D41 valid-time each claim already carries: 1. Two statements that both describe **datable events** (`claim_valid_kind = 'event_time'`) whose resolved windows are **disjoint** - describe different events. They never interact — no `evidence`, no - `supersede`, no `contradict` — and no verdict is bought for the pair. This - holds even when the wording is byte-identical: "won a tournament last week" - said in January and again in October is two wins. + are different occurrences unless the date-aware model finds they name the + *same* occurrence with disputed dates. Such a pair may therefore only + `contradict` (both stand, grouped) or stay `new`; an `evidence` or + `supersede` verdict is coerced to `new` and recorded. This holds even when + the wording is byte-identical — "won a tournament last week" said in + January and again in October is two wins — and identical text is then + kept apart without any model call. 2. A dated event paired with an **undated** statement (a state, a period figure, a summary such as "has been winning a few tournaments", or testimony the extractor could not anchor) may still be judged for - `supersede`/`contradict`, but an `evidence` verdict for such a pair is - coerced to `new` and recorded. A specific dated event is never a - re-assertion of a vaguer statement, and a summary never re-asserts a - specific event. + `supersede`/`contradict` — a dated resignation can end an undated "is + CEO" state — but an `evidence` verdict for such a pair is coerced to `new` + and recorded, and identical text is kept apart without a model call. A + specific dated event is never a re-assertion of a vaguer statement, and a + summary never re-asserts a specific event. 3. Two undated statements are adjudicated exactly as before; two dated - events with overlapping windows go to the ladder as before (a re-mention - of the same event still collapses). - -The verdict prompt now also shows both statements' asserted dates and -resolved event windows, and states the rule, so the small model that does -run judges with the timeline in front of it. An observation's event window -is the span of its supporting current-testimony event claims; it is derived -at block time, not stored. The adjudicator generation, the + events with overlapping windows — including a year-level and a day-level + window for one occurrence — go to the ladder as before, and a re-mention + of the same event still collapses. +4. A D41 open interval (a resolved start with no end) is unbounded, never a + point: it overlaps every later window. + +The verdict prompt now shows, for each statement, when the source *said it* +and what world-time it *is about* (the resolved D41 window of any kind), and +states the rule, so the model judges with both timelines in front of it. An +observation's windows are the span of its supporting current-testimony +claims, derived at block time and kept current within a batch as the row +absorbs evidence; nothing is stored. Every adjudication record carries the +coercions that preceded it. The adjudicator generation, the `adjudicate_observations` flush component version, and the LoCoMo protocol (Full-v20 → Full-v21) roll; nothing else about D43 — the entity block, the novelty gate, the no-cap rule, the fail-safe-to-coexist contract — changes. @@ -4939,17 +4952,28 @@ seven with all seven wins in its testimony context — the fact envelope's undercount is the plausible anchor. **Consequences.** Recurring same-shaped events survive as distinct facts, so -entity-anchored counting and enumeration have a complete fact layer to read. -Fewer verdict calls are bought (disjoint-event pairs cost zero). The -fail-safe direction is preserved and slightly widened: an undated re-mention -of a dated event now stays a separate observation rather than collapsing — -a duplicate, never a loss. Observation `valid_from` continues to be the -claim's `asserted_at`; using the resolved event window as the observation's -own validity is a separate change this decision does not make. Existing -stores re-adjudicate observations under the new generation on rebuild (D7). +entity-anchored counting and enumeration have a complete fact layer to read, +while a genuine same-occurrence date dispute can still surface as a +contradiction. Verdict spend is unchanged in shape (similar pairs are still +judged; only identical-text decisions become model-free). The fail-safe +direction is preserved and slightly widened: an undated re-mention of a +dated event now stays a separate observation rather than collapsing — a +duplicate, never a loss. Observation `valid_from` continues to be the +claim's `asserted_at`; using the resolved window as the observation's own +validity is a separate change this decision does not make. + +Rollout is stop-drain-rebuild. The `adjudicate_observations` flush handler +reports the claimed unit's *own* component generation to its barrier, so +entity units enqueued before the roll complete under the generation their +barrier counts and drain cleanly; readiness then reports the stage at the old +generation until the deployment is rebuilt under the new one (D7/D12). +Existing stores re-adjudicate observations under the new generation on that +rebuild. **Rejected.** Fixing only the verdict prompt (a model may still merge across -dates, and disjoint-date pairs should cost nothing); treating every +dates; the coercions must be deterministic); skipping disjoint-date pairs +without any verdict (would silently double-count one occurrence whose date +two sources dispute, and suppress the contradiction group); treating every mixed-dating pair as non-interacting (would stop a dated event from superseding a state it ends); dropping vague summaries at extraction (source-faithful testimony is kept — D32 — the defect was absorption, not diff --git a/plan/designs/locomo_benchmark_design.md b/plan/designs/locomo_benchmark_design.md index 68fe146f0..d528728a1 100644 --- a/plan/designs/locomo_benchmark_design.md +++ b/plan/designs/locomo_benchmark_design.md @@ -6,8 +6,9 @@ > instructions, content-before-`Unknown` harness guard, and no-review scoring > rule. Its pinned `adjudicate_observations` component version now carries the > D106 temporal-compatibility rung: dated events with disjoint resolved windows -> never collapse onto each other, and a dated event is never `evidence` for an -> undated statement. Ingestion provenance, protocol identity, and fingerprint +> never collapse onto or supersede each other (a same-occurrence date dispute +> may contradict), and a dated event is never `evidence` for an undated +> statement. Ingestion provenance, protocol identity, and fingerprint > roll; no retrieval, retry, model-effort, or call-budget behavior changes. > V20 and v21 scores are directional because the fact layer differs. @@ -64,7 +65,7 @@ > **Status:** binding current-system protocol contract. Real provider execution > remains operator-invoked. Accepting this design does not itself authorize a -> paid v20 run. +> paid run of the current protocol. ## 1. Acceptance boundary diff --git a/plan/designs/observations_design.md b/plan/designs/observations_design.md index bc2d89825..6830d4c67 100644 --- a/plan/designs/observations_design.md +++ b/plan/designs/observations_design.md @@ -148,23 +148,32 @@ When a claim asserts a value/property about entity *E*: resolved window. A claim that describes a **datable event** (`event_time` — a win, a visit, a purchase; the extractor resolved "last Saturday" to a calendar day) is compared to the candidate's own event window, which is the span of the event-time claims supporting it. - Then: - - **both dated events, disjoint windows → no interaction.** They are different things that - happened; the pair is skipped without buying a verdict. This holds even when the wording is - byte-identical — "won a tournament last week" said in January and again in October is two - wins, not one re-asserted. (The exact-statement shortcut above obeys the same rule.) + A resolved start with no end is a D41 *open* interval and stays unbounded (it overlaps every later + window). Then: + - **both dated events, disjoint windows → `evidence` and `supersede` are forbidden.** Two things + that happened on different days are different occurrences — *unless* two sources are disagreeing + about the date of one named occurrence ("the Valorant final on Friday" vs "…on Saturday"), which + only the date-aware model can tell. So the pair is still judged, but may only `contradict` (both + stand, grouped) or stay `new`; an `evidence` or `supersede` verdict is coerced to `new` and + recorded with its reason. This holds even when the wording is byte-identical — "won a tournament + last week" said in January and again in October is two wins, not one re-asserted — and identical + text is then kept apart *without* a verdict (the exact-statement shortcut obeys the same rule). - **one dated event, one undated statement → `evidence` is forbidden.** The pair may still be judged for supersede/contradict (a dated resignation can end a "is CEO" state), but a verdict - of `evidence` is coerced to `new` and recorded with its reason. A specific dated event is never - a re-assertion of a vaguer statement ("has been winning a few tournaments"), and a summary never - re-asserts a specific event — both stand. + of `evidence` is coerced to `new` and recorded with its reason; identical text is again kept + apart without a verdict. A specific dated event is never a re-assertion of a vaguer statement + ("has been winning a few tournaments"), and a summary never re-asserts a specific event — both + stand. - **both undated, or both dated with overlapping windows → the ladder as before.** A re-mention - of the *same* event (same resolved date) still collapses as evidence; headcounts, revenues and - stances are untouched. - The verdict prompt also shows both statements' asserted dates and resolved event windows, so the - model that does run judges with the timeline in front of it rather than two bare strings. The - rung is exactly the "deterministic value/period compare" step the cascade below always named; - it is what keeps recurring same-shaped events countable — the failure it closes is recorded in + of the *same* event (same resolved date, or a year-level and a day-level window for one + occurrence) still collapses as evidence; headcounts, revenues and stances are untouched. + The verdict prompt shows, for each statement, when the source *said it* and what world-time it *is + about* (the resolved window of any D41 kind, not only events), and defines both, so the model + judges with the timelines in front of it rather than two bare strings. Within one batch a block row + that absorbs evidence widens its in-memory windows, so a later overlapping claim is not split off + as a different occurrence. Every adjudication record carries the coercions that preceded it. The + rung is the "deterministic value/period compare" step the cascade below always named; it is what + keeps recurring same-shaped events countable — the failure it closes is recorded in `plan/analysis/locomo_conv42_recurring_event_adjudication.md`. 4. **Fail safe — a binding adjudicator contract (not just a hope).** This is the honest core of the untyped design: "never silently resolve" is **policy enforced in E3 + eval**, not a schema invariant. @@ -220,10 +229,12 @@ small model reading only the two strings judged October's "won a really big tour to be `evidence` for January's "won his first tournament last week" — both say "last week" — and folded the international and Valorant wins into "has been winning a few tournaments"; the entity ended with four win facts and a count question answered "at least five" against seven. With the -rung: each new win is compared to seven-minus-one disjoint-window events (skipped, zero verdicts) -and one undated summary (judged; an `evidence` verdict is coerced to `new`) → **seven observations, -one summary, no LLM spend on the disjoint pairs**. A second mention of the *same* win (same resolved -day) still collapses as evidence onto it. +rung: each new win is still judged against the similar earlier wins and the undated summary, but +with both timelines in the prompt, and whatever the model answers, an `evidence` or `supersede` +verdict against a disjoint-window win — or an `evidence` verdict against the summary — is coerced to +`new` and recorded → **seven observations, one summary**. A second mention of the *same* win (same +resolved day) still collapses as evidence onto it, and two sources disagreeing about *which* day one +named final was won may `contradict`, so both stand grouped. ### Supersession appends — an observation is a time-slice, never an in-place edit @@ -272,8 +283,9 @@ value → observation), and often the claim's embedding (E2 embeds claims for P1 - For a **hub entity**, the same vector step top-k ranks *which* candidates to compare (cheap math); a skipped far candidate costs at most a duplicate row, never a wrong supersede. 3. **Adjudicate the residue only (cheap → frontier).** Only similar-but-not-identical candidates escalate - the D4 cascade: the deterministic temporal-compatibility compare (§3 step 3, D106 — disjoint dated - events skip the pair outright) → small model → frontier LLM for the survivors. The + the D4 cascade: the deterministic temporal-compatibility compare (§3 step 3, D106 — it bounds which + verdicts a pair may take, and decides identical text alone) → small model → frontier LLM for the + survivors. The adjudicator decides same-property (+ same-period for a figure) and the outcome under the no-cap rule (state → supersede; measurement → contradict/coexist; same value → evidence; else new), and **fails safe to coexist** below the supersede margin. diff --git a/src/rememberstack/spine/observation_adjudication.py b/src/rememberstack/spine/observation_adjudication.py index 6ef6cdb6a..3d9715684 100644 --- a/src/rememberstack/spine/observation_adjudication.py +++ b/src/rememberstack/spine/observation_adjudication.py @@ -46,18 +46,30 @@ """The observation adjudicator generation (D12; replayed on rebuild, D7). 07b pins temperature=0.0 — generation parameters are part of provenance. 09a (D106) adds the temporal-compatibility rung: two dated events with -disjoint resolved windows never interact, a dated event never collapses as -evidence onto an undated statement (nor the reverse), and the verdict prompt -sees both statements' asserted dates and resolved event windows.""" +disjoint resolved windows never collapse or supersede (they may only +contradict or stay distinct), a dated event never collapses as evidence onto +an undated statement (nor the reverse), identical text is collapsed only when +temporally compatible, open-ended windows stay unbounded, and the verdict +prompt shows both statements' said-on dates and is-about windows.""" _VERDICT_PROMPT: Final = """You adjudicate observations for a memory system. Both statements are believed facts about the SAME entity: EXISTING: {existing!r} + said on: {existing_said_on} + is about: {existing_about} NEW: {new!r} - -EXISTING TIME: {existing_time} -NEW TIME: {new_time} + said on: {new_said_on} + is about: {new_about} + +Two clocks are shown for each statement. "said on" is the source's own date — +when the document was written or the conversation took place — and is NOT +when the described thing happened. "is about" is the world-time the statement +refers to, resolved from the source's wording against its said-on date: "last +week" said on 2022-10-06 is about the week before that date, not the week +before 2022-01-21, so two statements can both say "last week" and be about +days months apart. When the source tied nothing to a date, "is about" says so. +When either statement was ingested is irrelevant here and is not shown. Judge semantically (there are no typed columns — "FY2023" vs "fiscal 2023" and "headcount" vs "staff count" are your equivalence calls): @@ -73,13 +85,17 @@ - new: a different property, period, or thing — no interaction. Time is decisive for EVENTS (a win, a visit, a purchase, a meeting). Two -statements about datable events with DIFFERENT resolved event dates describe -DIFFERENT events: always `new`, never `evidence`, even when the wording is -identical ("won a tournament last week" said in January and again in October -are two wins, not one re-asserted). A specific dated event is never -`evidence` for a vaguer summary ("has won a few tournaments"), and a summary -never re-asserts a specific event — keep both. Only a re-mention of the SAME -event (same resolved date, same thing) is `evidence`.""" +statements about datable events whose "is about" windows do NOT overlap are +two different occurrences — `new` — even when the wording is identical ("won +a tournament last week" said in January and again in October are two wins, +not one re-asserted). The one exception: when they plainly name the SAME +single occurrence and merely disagree about its date ("the Valorant final on +Friday" vs "the Valorant final on Saturday"), answer `contradict` so both +stand. Overlapping windows of different precision (a year-level claim and a +day-level one) may well be the same occurrence — judge by the wording. A +specific dated event is never `evidence` for a vaguer summary ("has won a +few tournaments"), and a summary never re-asserts a specific event — keep +both.""" class ObservationSettings(BaseSettings): @@ -307,20 +323,61 @@ def _add_with_block( ) -> UUID: """Apply one assertion while keeping the front-loaded block current.""" asserted_at = timing.asserted_at - # An identical statement is the strongest same-fact signal, with one - # exception (D106): the same words about two DIFFERENT dated events - # ("won a tournament last week" in January and in October) are two - # facts, not one re-asserted. + # An identical statement is the strongest same-fact signal — unless + # the two are not temporally compatible (D106): the same words about + # two dated events on different days ("won a tournament last week" in + # January and in October), or a dated event beside an undated copy, + # are separate rows, not one re-asserted. That decision needs no + # model, so it is taken here and recorded. + identical = [ + candidate + for candidate in candidates + if candidate["statement"] == assertion.statement + and bool(candidate["is_open"]) + ] exact = next( ( candidate - for candidate in candidates - if candidate["statement"] == assertion.statement - and bool(candidate["is_open"]) - and not _disjoint_dated_events(timing=timing, candidate=candidate) + for candidate in identical + if _evidence_compatible(timing=timing, candidate=candidate) ), None, ) + if exact is None and identical: + observation_id = self._insert_new( + connection=connection, + deployment_id=deployment_id, + subject_entity_id=subject_entity_id, + statement=assertion.statement, + claim_id=assertion.claim_id, + doc_id=assertion.doc_id, + valid_from=asserted_at, + outcome="add", + method="exact", # the identical-text rung decided; no model ran + confidence=1.0, + features={ + "reason": "identical statement, temporally incompatible -> coexist", + "temporal_gate": [ + { + "observation_id": str(candidate["observation_id"]), + "relation": _temporal_relation( + timing=timing, candidate=candidate + ), + } + for candidate in identical + ], + }, + related=UUID(str(identical[0]["observation_id"])), + contradiction_group=None, + ) + _remember_candidate( + candidates=candidates, + observation_id=observation_id, + statement=assertion.statement, + valid_from=asserted_at, + timing=timing, + ) + return observation_id if exact is not None: observation_id = UUID(str(exact["observation_id"])) self._evidence( @@ -330,6 +387,7 @@ def _add_with_block( claim_id=assertion.claim_id, doc_id=assertion.doc_id, ) + _absorb_timing(candidate=exact, timing=timing) # D88 continuous ingest: equivalent evidence must not leave # valid_from dependent on which version flushed first. Pull the # open window back to the source-earliest assertion time. @@ -445,10 +503,7 @@ def judge_statements( ) -> tuple[ObservationOutcome, float]: """The bare pair-decision function — the D43 eval gate's surface.""" verdict, method = self._ladder( - existing=existing, - new=new, - existing_time=_render_timing(_UNDATED), - new_time=_render_timing(_UNDATED), + existing=existing, new=new, existing_timing=_UNDATED, new_timing=_UNDATED ) del method # the gate grades outcomes; rungs are graded per-run cost return verdict.outcome, verdict.confidence @@ -470,46 +525,45 @@ def _adjudicate_residue( ) -> UUID: """Ladder the similar candidates; apply the first decisive outcome. - The temporal-compatibility rung (D106) runs before every model call. - Two dated events whose resolved windows are disjoint are distinct - events and never interact, so no verdict is bought for them. A dated - event paired with an undated statement (or the reverse) may still be - judged for supersede/contradict, but an `evidence` verdict for such a - pair is coerced to `new`: a specific dated event is never a - re-assertion of a vaguer statement, and a summary never re-asserts a - specific event. + The temporal-compatibility rung (D106) bounds what a verdict may do, + using the D41 windows the claims already carry. Two dated events whose + resolved windows are disjoint are different occurrences unless the + model finds they name the SAME occurrence with disputed dates: they may + `contradict` (both stand, grouped) or stay `new`; `evidence` and + `supersede` are coerced to `new` and recorded. A dated event beside an + undated statement may still supersede or contradict it (a dated + resignation ends a "is CEO" state), but `evidence` is coerced to `new`: + a specific dated event is never a re-assertion of a vaguer statement, + and a summary never re-asserts a specific event. Undated pairs and + overlapping dated pairs are judged exactly as before. """ asserted_at = timing.asserted_at - temporal_skips: list[dict[str, object]] = [] + coercions: list[dict[str, object]] = [] for candidate, similarity in ranked: candidate_id = UUID(str(candidate["observation_id"])) - if _disjoint_dated_events(timing=timing, candidate=candidate): - temporal_skips.append( - { - "observation_id": str(candidate_id), - "reason": "disjoint dated events", - "similarity": similarity, - } - ) - continue - mixed_dating = _mixed_dating(timing=timing, candidate=candidate) + relation = _temporal_relation(timing=timing, candidate=candidate) verdict, method = self._ladder( existing=str(candidate["statement"]), new=statement, - existing_time=_render_timing(_candidate_timing(candidate)), - new_time=_render_timing(timing), + existing_timing=_candidate_timing(candidate), + new_timing=timing, meter=meter, call_key=f"{call_key}:verdict:{candidate['observation_id']}", ) features: dict[str, object] = { "similarity": similarity, "rationale": verdict.rationale, + "temporal_relation": relation, + "temporal_gate": list(coercions), } - if mixed_dating and verdict.outcome is ObservationOutcome.EVIDENCE: - temporal_skips.append( + coerced = _coerced_reason(relation=relation, outcome=verdict.outcome) + if coerced is not None: + coercions.append( { "observation_id": str(candidate_id), - "reason": "evidence coerced to new: dated event vs undated statement", + "verdict": verdict.outcome.value, + "relation": relation, + "reason": coerced, "similarity": similarity, "rationale": verdict.rationale, } @@ -523,6 +577,7 @@ def _adjudicate_residue( claim_id=claim_id, doc_id=doc_id, ) + _absorb_timing(candidate=candidate, timing=timing) self._pull_valid_from_earlier( connection=connection, deployment_id=deployment_id, @@ -700,6 +755,7 @@ def _adjudicate_residue( observation_id=new_id, statement=statement, valid_from=asserted_at, + timing=timing, ) # D90 §5.5.3: evidence already on O after the incoming order # key must re-enter the ladder (staggered multi-version). @@ -749,6 +805,7 @@ def _adjudicate_residue( statement=statement, contradiction_group=group, valid_from=asserted_at, + timing=timing, ) return new_id # ObservationOutcome.NEW: no interaction with this candidate @@ -763,10 +820,7 @@ def _adjudicate_residue( outcome="add", method="small_model", confidence=1.0, - features={ - "reason": "no candidate interacted", - "temporal_gate": temporal_skips, - }, + features={"reason": "no candidate interacted", "temporal_gate": coercions}, related=None, contradiction_group=None, ) @@ -784,14 +838,19 @@ def _ladder( *, existing: str, new: str, - existing_time: str, - new_time: str, + existing_timing: _ClaimTiming, + new_timing: _ClaimTiming, meter: CostMeterPort | None = None, call_key: str = "observation:verdict", ) -> tuple[ObservationVerdict, str]: """Small-model verdict, escalating to frontier below the floor.""" prompt = _VERDICT_PROMPT.format( - existing=existing, new=new, existing_time=existing_time, new_time=new_time + existing=existing, + new=new, + existing_said_on=_render_said_on(existing_timing), + existing_about=_render_about(existing_timing), + new_said_on=_render_said_on(new_timing), + new_about=_render_about(new_timing), ) verdict_call = self._model_provider.generate( request=ModelRequest( @@ -1113,6 +1172,8 @@ def _remember_candidate( "is_open": is_open, "event_from": None if timing is None else timing.event_from, "event_until": None if timing is None else timing.event_until, + "about_from": None if timing is None else timing.about_from, + "about_until": None if timing is None else timing.about_until, } ) @@ -1121,66 +1182,117 @@ def _remember_candidate( class _ClaimTiming: """What the D41 record says about WHEN one piece of testimony applies. - ``asserted_at`` is the source's own timestamp (the supersession boundary - the layer already used). ``event_from``/``event_until`` are the resolved - world-time window when the claim describes a datable EVENT - (``claim_valid_kind = 'event_time'``); both are ``None`` for undated - testimony, states, and period figures, so ``is_event`` is False there. + Two clocks. ``asserted_at`` is when the SOURCE said it — the document's + or conversation's own timestamp (the supersession boundary the layer + already used). ``about_from``/``about_until`` is the world-time the + statement is ABOUT, resolved by the extractor from the source's wording + against that date, and ``about_kind`` is the D41 ``claim_valid_kind`` that + says what sort of interval it is: ``event_time`` (a datable event — the + only kind the temporal-compatibility rung acts on), ``measurement_period`` + / ``effective_period`` / ``proposition_validity`` (a figure or state tied + to a span), or ``period`` for a block row whose supporting claims are + aggregated. All three are ``None`` when the source tied nothing to a date. + When the statement was ingested is deliberately not part of this record. """ asserted_at: object = None - event_from: object = None - event_until: object = None + about_kind: str | None = None + about_from: object = None + about_until: object = None @property def is_event(self) -> bool: - """True when the testimony carries a resolved event window.""" - return self.event_from is not None + """True when the testimony is a datable event with a resolved window.""" + return self.about_kind == "event_time" and self.about_from is not None + + @property + def event_from(self) -> object: + """The event window start, or ``None`` when this is not a dated event.""" + return self.about_from if self.is_event else None + + @property + def event_until(self) -> object: + """The event window end, or ``None`` when this is not a dated event.""" + return self.about_until if self.is_event else None _UNDATED: Final = _ClaimTiming() def _timing_from_row(row: Mapping[Any, Any]) -> _ClaimTiming: - """Read one claim row's asserted time and resolved event window.""" - is_event = row.get("valid_kind") == "event_time" - event_from = row.get("claim_valid_from") if is_event else None - if event_from is None: + """Read one claim row's said-on time and resolved about-window. + + A ``NULL`` ``claim_valid_until`` beside a resolved ``claim_valid_from`` is + a D41 open interval: the end is unbounded and stays ``None``. + """ + about_from = row.get("claim_valid_from") + if about_from is None: return _ClaimTiming(asserted_at=row.get("asserted_at")) - event_until = row.get("claim_valid_until") return _ClaimTiming( asserted_at=row.get("asserted_at"), - event_from=event_from, - event_until=event_from if event_until is None else event_until, + about_kind=row.get("valid_kind"), + about_from=about_from, + about_until=row.get("claim_valid_until"), ) def _candidate_timing(candidate: Mapping[str, object]) -> _ClaimTiming: - """The block row's event window (aggregated over its supporting claims).""" + """The block row's timing, aggregated over its supporting claims. + + ``valid_from`` is the earliest said-on date of its testimony. The event + window (dated-event claims only) drives the rung; the wider about-window + (any D41 kind) is shown to the model as ``period`` when no event exists. + A ``None`` end is an open (unbounded) window, as in the claim rows. + """ + said_on = candidate.get("valid_from") event_from = candidate.get("event_from") - if event_from is None: - return _ClaimTiming(asserted_at=candidate.get("valid_from")) - event_until = candidate.get("event_until") + if event_from is not None: + return _ClaimTiming( + asserted_at=said_on, + about_kind="event_time", + about_from=event_from, + about_until=candidate.get("event_until"), + ) + about_from = candidate.get("about_from") + if about_from is None: + return _ClaimTiming(asserted_at=said_on) return _ClaimTiming( - asserted_at=candidate.get("valid_from"), - event_from=event_from, - event_until=event_from if event_until is None else event_until, + asserted_at=said_on, + about_kind="period", + about_from=about_from, + about_until=candidate.get("about_until"), ) -def _render_timing(timing: _ClaimTiming) -> str: - """One prompt line describing when testimony applies, honestly.""" - asserted = ( - "undated" - if timing.asserted_at is None - else f"asserted {_date_text(timing.asserted_at)}" - ) - if not timing.is_event: - return f"{asserted}; no resolved event date (a state, period figure, or undated statement)" - start = _date_text(timing.event_from) - end = _date_text(timing.event_until) - window = start if start == end else f"{start} to {end}" - return f"{asserted}; datable event resolved to {window}" +def _render_said_on(timing: _ClaimTiming) -> str: + """The "said on" prompt value: the source's own date, or its absence.""" + if timing.asserted_at is None: + return "unknown (the source carries no date)" + return _date_text(timing.asserted_at) + + +def _render_about(timing: _ClaimTiming) -> str: + """The "is about" prompt value: the resolved world-time, or its absence.""" + if timing.about_from is None: + return ( + "no specific time given (a state, summary, or figure the source" + " did not tie to a date)" + ) + start = _date_text(timing.about_from) + if timing.about_until is None: + span = f"from {start} onward (no end given)" + else: + end = _date_text(timing.about_until) + span = start if start == end else f"{start} to {end}" + if timing.is_event: + return ( + f"a dated event on {span}" + if " to " not in span and "onward" not in span + else f"a dated event within {span}" + ) + if " to " in span or "onward" in span: + return f"the period {span} (a state or figure tied to that span, not a dated event)" + return f"the day {span} (a state or figure tied to that day, not a dated event)" def _date_text(value: object) -> str: @@ -1189,33 +1301,125 @@ def _date_text(value: object) -> str: return str(date()) if callable(date) else str(value) -def _disjoint_dated_events( +def _temporal_relation(*, timing: _ClaimTiming, candidate: Mapping[str, object]) -> str: + """How the incoming testimony's timing relates to a block row's (D106). + + ``undated`` — neither side is a dated event (states, figures, unanchored + testimony): judged as before. ``overlapping`` — both are dated events + whose windows touch or overlap: judged as before. ``disjoint`` — both + dated events, windows apart: different occurrences unless the model + finds one occurrence with disputed dates. ``mixed`` — exactly one side is + a dated event. + """ + other = _candidate_timing(candidate) + if timing.is_event and other.is_event: + apart = _windows_disjoint( + timing.event_from, timing.event_until, other.event_from, other.event_until + ) + return "disjoint" if apart else "overlapping" + if timing.is_event or other.is_event: + return "mixed" + return "undated" + + +def _coerced_reason(*, relation: str, outcome: ObservationOutcome) -> str | None: + """The reason a verdict is coerced to ``new`` under D106, or ``None``.""" + if relation == "disjoint" and outcome in ( + ObservationOutcome.EVIDENCE, + ObservationOutcome.SUPERSEDE, + ): + return ( + f"{outcome.value} coerced to new: dated events on different days are" + " different occurrences (a same-occurrence date dispute is contradict)" + ) + if relation == "mixed" and outcome is ObservationOutcome.EVIDENCE: + return ( + "evidence coerced to new: a dated event never re-asserts an undated" + " statement, nor the reverse" + ) + return None + + +def _evidence_compatible( *, timing: _ClaimTiming, candidate: Mapping[str, object] ) -> bool: - """True when both sides are dated events whose windows never overlap. + """True when identical text may collapse without a verdict (D106). - Two such statements describe different things that happened; they can - neither re-assert nor supersede nor contradict each other (D106). + Both undated, or both dated events whose windows overlap. Disjoint dated + events and mixed dating are separate rows even for identical text. """ - other = _candidate_timing(candidate) - if not (timing.is_event and other.is_event): - return False - return _windows_disjoint( - timing.event_from, timing.event_until, other.event_from, other.event_until + return _temporal_relation(timing=timing, candidate=candidate) in ( + "undated", + "overlapping", ) -def _mixed_dating(*, timing: _ClaimTiming, candidate: Mapping[str, object]) -> bool: - """True when exactly one side is a dated event (the other undated).""" - return timing.is_event != _candidate_timing(candidate).is_event +def _absorb_timing(*, candidate: dict[str, object], timing: _ClaimTiming) -> None: + """Widen a block row's in-memory windows after it absorbs new evidence. + + The database aggregate widens on the next block read; within one batch + the front-loaded row must not go stale, or a later overlapping claim + would be split off as a different occurrence (D106). + """ + if timing.is_event: + event_defined = candidate.get("event_from") is not None + candidate["event_from"] = _earliest( + candidate.get("event_from"), timing.event_from + ) + candidate["event_until"] = _latest_or_open( + candidate.get("event_until"), + timing.event_until, + existing_defined=event_defined, + ) + if timing.about_from is not None: + about_defined = candidate.get("about_from") is not None + candidate["about_from"] = _earliest( + candidate.get("about_from"), timing.about_from + ) + candidate["about_until"] = _latest_or_open( + candidate.get("about_until"), + timing.about_until, + existing_defined=about_defined, + ) + + +def _earliest(left: object, right: object) -> object: + """The earlier of two window starts; a missing start defers to the other.""" + if left is None: + return right + if right is None: + return left + try: + return left if left <= right else right # type: ignore[operator] + except TypeError: + return left + + +def _latest_or_open(left: object, right: object, *, existing_defined: bool) -> object: + """The later of two window ends, where ``None`` on a defined window is open.""" + if not existing_defined: + return right + if left is None or right is None: + return None + try: + return left if left >= right else right # type: ignore[operator] + except TypeError: + return left def _windows_disjoint( left_from: object, left_until: object, right_from: object, right_until: object ) -> bool: - """Closed-interval disjointness; any incomparable value counts as overlap.""" + """Closed-interval disjointness with ``None`` ends unbounded. + + Any incomparable value counts as overlap (the fail-safe direction). + """ try: - return bool(left_until < right_from) or bool(right_until < left_from) # type: ignore[operator] + if left_until is not None and bool(left_until < right_from): # type: ignore[operator] + return True + if right_until is not None and bool(right_until < left_from): # type: ignore[operator] + return True + return False except TypeError: return False @@ -1283,21 +1487,36 @@ def _is_later_in_total_order( """ SELECT o.observation_id, o.statement, o.contradiction_group, o.valid_from, (o.valid_until IS NULL OR o.valid_until > now()) AS is_open, - dated.event_from, dated.event_until + timing.event_from, timing.event_until, + timing.about_from, timing.about_until FROM observations o - -- D106: the resolved event window an observation stands for is the span - -- of its supporting dated-event testimony (D41 event_time claims only). + -- D106: what an observation is ABOUT in world-time is the span of its + -- supporting current testimony's D41 windows. The event window (dated + -- event claims only) drives the rung; the wider about-window is shown + -- to the model. LEFT JOIN LATERAL ( - SELECT min(c.claim_valid_from) AS event_from, - max(coalesce(c.claim_valid_until, c.claim_valid_from)) AS event_until + -- A NULL claim_valid_until beside a non-NULL claim_valid_from is a + -- D41 open interval (unbounded end); one open claim makes the + -- aggregate open, so the end stays NULL rather than a false maximum. + SELECT min(c.claim_valid_from) + FILTER (WHERE c.claim_valid_kind = 'event_time') AS event_from, + CASE WHEN bool_or(c.claim_valid_until IS NULL) + FILTER (WHERE c.claim_valid_kind = 'event_time') + THEN NULL + ELSE max(c.claim_valid_until) + FILTER (WHERE c.claim_valid_kind = 'event_time') + END AS event_until, + min(c.claim_valid_from) AS about_from, + CASE WHEN bool_or(c.claim_valid_until IS NULL) THEN NULL + ELSE max(c.claim_valid_until) + END AS about_until FROM observation_evidence e JOIN claims c ON c.claim_id = e.claim_id WHERE e.observation_id = o.observation_id AND e.stance = 'supports' AND c.is_current_testimony - AND c.claim_valid_kind = 'event_time' AND c.claim_valid_from IS NOT NULL - ) dated ON true + ) timing ON true WHERE o.deployment_id = :deployment_id AND o.subject_entity_id = :subject_entity_id AND o.invalidated_at IS NULL diff --git a/src/rememberstack/workers/e3.py b/src/rememberstack/workers/e3.py index 97b918960..acfcbde81 100644 --- a/src/rememberstack/workers/e3.py +++ b/src/rememberstack/workers/e3.py @@ -661,7 +661,11 @@ def _handle_entity_unit( extractor_version=extractor_version, content_hash=content_hash, lane=work.lane, - obs_flush_component_version=OBS_FLUSH_VERSION, + # Report the claimed unit's own generation, not the current + # constant: a unit enqueued before an OBS_FLUSH_VERSION roll + # must complete under the generation its barrier counts, or + # the barrier never closes (D106 rollout contract). + obs_flush_component_version=work.component_version, doc_id=doc_id, ), ) diff --git a/src/tests/benchmarks/test_locomo_protocol.py b/src/tests/benchmarks/test_locomo_protocol.py index 3003ac407..cda74dffc 100644 --- a/src/tests/benchmarks/test_locomo_protocol.py +++ b/src/tests/benchmarks/test_locomo_protocol.py @@ -239,7 +239,7 @@ def test_current_protocol_pins_manifest_and_complete_read_plane() -> None: assert len(tool_catalog_sha256()) == 64 -def test_protocol_is_v20_and_answer_prompt_has_reasoning_and_loop_guards() -> None: +def test_protocol_is_v21_and_answer_prompt_has_reasoning_and_loop_guards() -> None: """The current identity, bounded inference, and loop discipline are locked.""" assert PROTOCOL_NAME == "RS-LoCoMo-Full-v21" assert DEFAULT_PROTOCOL_KEY == "full-v21" diff --git a/src/tests/spine/test_observation_adjudication.py b/src/tests/spine/test_observation_adjudication.py index 25bb2a187..a14ffd553 100644 --- a/src/tests/spine/test_observation_adjudication.py +++ b/src/tests/spine/test_observation_adjudication.py @@ -153,7 +153,7 @@ def _add( statement: str, engine: Engine | None = None, asserted_at: str | None = None, - event_window: tuple[str, str] | None = None, + event_window: tuple[str, str | None] | None = None, doc_id: UUID | None = None, ) -> UUID: """One observation through the cascade with a fresh claim. @@ -161,39 +161,21 @@ def _add( With `asserted_at`, a real dated claim row backs the testimony (the D41 seed the boundary math reads); without it, the testimony is undated. With `event_window`, the claim is a datable EVENT resolved to that - day-precision `[from, until]` window (`claim_valid_kind = 'event_time'`), - the input the D106 temporal-compatibility rung reads. + `[from, until]` window (`claim_valid_kind = 'event_time'`, day precision; + an `until` of `None` is a D41 open interval), the input the D106 + temporal-compatibility rung reads. """ - claim_id = uuid4() doc_id = doc_id or uuid4() + claim_id = uuid4() if asserted_at is not None and engine is not None: - with engine.begin() as connection: - connection.execute( - text( - "INSERT INTO claims (claim_id, deployment_id, doc_id," - " chunk_id, claim_text, source_span, char_start, char_end," - " anchor_ok, window_membership_ok, extractor_version," - " asserted_at, claim_valid_kind, claim_valid_precision," - " claim_valid_from, claim_valid_until)" - " VALUES (:c, :d, :doc, :ch, :s, :s, 0, 1, true, true," - " 'test', CAST(:a AS timestamptz)," - " CAST(:kind AS claim_valid_kind)," - " CAST(:precision AS claim_valid_precision)," - " CAST(:vf AS timestamptz), CAST(:vu AS timestamptz))" - ), - { - "c": claim_id, - "d": _DEPLOYMENT_ID, - "doc": doc_id, - "ch": uuid4(), - "s": statement, - "a": asserted_at, - "kind": None if event_window is None else "event_time", - "precision": "unknown" if event_window is None else "day", - "vf": None if event_window is None else event_window[0], - "vu": None if event_window is None else event_window[1], - }, - ) + _claim( + engine=engine, + claim_id=claim_id, + doc_id=doc_id, + statement=statement, + asserted_at=asserted_at, + event_window=event_window, + ) return adjudicator.add_observation( deployment_id=_DEPLOYMENT_ID, subject_entity_id=entity, @@ -203,6 +185,48 @@ def _add( ) +def _claim( + *, + engine: Engine, + claim_id: UUID, + doc_id: UUID, + statement: str, + asserted_at: str, + event_window: tuple[str, str | None] | None = None, +) -> None: + """Insert one dated claim row, optionally a resolved D41 event window.""" + precision = "unknown" + if event_window is not None: + precision = "open" if event_window[1] is None else "day" + with engine.begin() as connection: + connection.execute( + text( + "INSERT INTO claims (claim_id, deployment_id, doc_id," + " chunk_id, claim_text, source_span, char_start, char_end," + " anchor_ok, window_membership_ok, extractor_version," + " asserted_at, claim_valid_kind, claim_valid_precision," + " claim_valid_from, claim_valid_until)" + " VALUES (:c, :d, :doc, :ch, :s, :s, 0, 1, true, true," + " 'test', CAST(:a AS timestamptz)," + " CAST(:kind AS claim_valid_kind)," + " CAST(:precision AS claim_valid_precision)," + " CAST(:vf AS timestamptz), CAST(:vu AS timestamptz))" + ), + { + "c": claim_id, + "d": _DEPLOYMENT_ID, + "doc": doc_id, + "ch": uuid4(), + "s": statement, + "a": asserted_at, + "kind": None if event_window is None else "event_time", + "precision": precision, + "vf": None if event_window is None else event_window[0], + "vu": None if event_window is None else event_window[1], + }, + ) + + def _collapse_happy_router(prompt: str, type_name: str) -> dict[str, object]: """The failure the D106 rung exists to stop: a small model that reads "won a tournament last week" twice and calls the second a re-assertion. @@ -770,8 +794,7 @@ def test_stance_content_never_becomes_a_fact(database_engine: Engine) -> None: def test_d106_seven_dated_wins_survive_as_seven_facts(database_engine: Engine) -> None: """The conv-42 counting case: seven same-shaped wins on seven dates must - stay seven observations, and the rung must not even buy a verdict for - pairs whose event windows are disjoint (they are different events).""" + stay seven observations whatever the model says about any pair.""" adjudicator, provider = _adjudicator( engine=database_engine, router=_collapse_happy_router ) @@ -788,7 +811,10 @@ def test_d106_seven_dated_wins_survive_as_seven_facts(database_engine: Engine) - rows = _observations(engine=database_engine, entity=nate) assert [row["statement"] for row in rows] == [win[0] for win in _SEVEN_WINS] assert all(row["evidence_count"] == 1 for row in rows) - assert _verdict_calls(provider) == 0 # every pair was gated: no LLM spend + # Disjoint dated pairs still reach the date-aware model (a same-occurrence + # date dispute must be able to surface as a contradiction); its + # collapse-happy `evidence` verdicts are coerced to `new` and audited. + assert _verdict_calls(provider) > 0 with database_engine.connect() as connection: noops = connection.execute( text( @@ -878,7 +904,7 @@ def test_d106_identical_words_about_two_dates_are_two_events( ) rows = _observations(engine=database_engine, entity=nate) assert len(rows) == 2 and all(row["evidence_count"] == 1 for row in rows) - assert _verdict_calls(provider) == 0 + assert _verdict_calls(provider) == 0 # identical text: decided without a model def test_d106_a_vague_summary_never_absorbs_a_dated_event( @@ -910,10 +936,8 @@ def test_d106_a_vague_summary_never_absorbs_a_dated_event( assert len(rows) == 2 and all(row["evidence_count"] == 1 for row in rows) assert _verdict_calls(provider) == 1 # mixed pair: judged, then coerced last_prompt = provider.generated_prompts[-1] - assert "EXISTING TIME: asserted 2022-05-02; no resolved event date" in last_prompt - assert "NEW TIME: asserted 2022-08-22; datable event resolved to 2022-08-21" in ( - last_prompt - ) + assert "said on: 2022-05-02\n is about: no specific time given" in last_prompt + assert "said on: 2022-08-22\n is about: a dated event on 2022-08-21" in last_prompt with database_engine.connect() as connection: features = connection.execute( text( @@ -999,3 +1023,234 @@ def test_d106_boilerplate_state_never_absorbs_dated_participation( row for row in rows if row["statement"] == "Nate is a participant." ) assert boilerplate["evidence_count"] == 3 + + +def test_d106_identical_text_dated_beside_undated_coexists( + database_engine: Engine, +) -> None: + """One anchored and one unanchored extraction of the same sentence are + kept apart (a duplicate, never a silent merge), with no model call.""" + adjudicator, provider = _adjudicator( + engine=database_engine, router=_collapse_happy_router + ) + alice = _entity(engine=database_engine) + statement = "Alice said she resigned from the tournament committee yesterday." + _add( + adjudicator=adjudicator, + entity=alice, + statement=statement, + engine=database_engine, + asserted_at="2026-03-02T09:00:00Z", + event_window=("2026-03-01", "2026-03-01"), + ) + _add( + adjudicator=adjudicator, + entity=alice, + statement=statement, + engine=database_engine, + asserted_at="2026-03-09T09:00:00Z", + ) + rows = _observations(engine=database_engine, entity=alice) + assert len(rows) == 2 and all(row["evidence_count"] == 1 for row in rows) + assert _verdict_calls(provider) == 0 + + +def test_d106_open_window_is_unbounded_not_a_point(database_engine: Engine) -> None: + """A D41 open interval ``[from, ∞)`` overlaps every later day, so a later + dated re-mention collapses onto it instead of splitting off.""" + adjudicator, _provider = _adjudicator( + engine=database_engine, router=_collapse_happy_router + ) + nate = _entity(engine=database_engine) + statement = "Nate has been competing in the regional tournament circuit." + _add( + adjudicator=adjudicator, + entity=nate, + statement=statement, + engine=database_engine, + asserted_at="2026-01-02T09:00:00Z", + event_window=("2026-01-01", None), + ) + _add( + adjudicator=adjudicator, + entity=nate, + statement=statement, + engine=database_engine, + asserted_at="2026-02-16T09:00:00Z", + event_window=("2026-02-15", "2026-02-15"), + ) + rows = _observations(engine=database_engine, entity=nate) + assert len(rows) == 1 and rows[0]["evidence_count"] == 2 + + +def test_d106_overlapping_windows_of_different_precision_may_collapse( + database_engine: Engine, +) -> None: + """A year-level claim and a day-level claim inside that year overlap, so + the model's `evidence` verdict is honoured (no coercion).""" + adjudicator, provider = _adjudicator( + engine=database_engine, router=_collapse_happy_router + ) + nate = _entity(engine=database_engine) + _add( + adjudicator=adjudicator, + entity=nate, + statement="Nate said he won a video game tournament last year.", + engine=database_engine, + asserted_at="2023-02-01T09:00:00Z", + event_window=("2022-01-01", "2022-12-31"), + ) + _add( + adjudicator=adjudicator, + entity=nate, + statement="Nate won Nate's fourth video game tournament on Friday.", + engine=database_engine, + asserted_at="2022-07-10T14:34:00Z", + event_window=("2022-07-08", "2022-07-08"), + ) + rows = _observations(engine=database_engine, entity=nate) + assert len(rows) == 1 and rows[0]["evidence_count"] == 2 + assert _verdict_calls(provider) == 1 + assert ( + "is about: a dated event within 2022-01-01 to 2022-12-31" + in (provider.generated_prompts[-1]) + ) + + +def test_d106_batch_widens_the_candidate_window_as_it_absorbs( + database_engine: Engine, +) -> None: + """Within one batch the front-loaded row must not go stale: after + ``[Jan 1–10]`` absorbs an overlapping ``[Jan 10–20]`` re-mention, a + ``[Jan 20]`` re-mention still overlaps and collapses too.""" + adjudicator, provider = _adjudicator( + engine=database_engine, router=_collapse_happy_router + ) + nate = _entity(engine=database_engine) + statement = "Nate said he was competing in the winter tournament." + doc_id = uuid4() + claims = [] + for asserted_at, window in ( + ("2026-01-11T09:00:00Z", ("2026-01-01", "2026-01-10")), + ("2026-01-21T09:00:00Z", ("2026-01-10", "2026-01-20")), + ("2026-01-21T18:00:00Z", ("2026-01-20", "2026-01-20")), + ): + claim_id = uuid4() + _claim( + engine=database_engine, + claim_id=claim_id, + doc_id=doc_id, + statement=statement, + asserted_at=asserted_at, + event_window=window, + ) + claims.append(claim_id) + adjudicator.add_observations( + deployment_id=_DEPLOYMENT_ID, + subject_entity_id=nate, + assertions=tuple( + ObservationAssertion(statement=statement, claim_id=claim_id, doc_id=doc_id) + for claim_id in claims + ), + ) + rows = _observations(engine=database_engine, entity=nate) + assert len(rows) == 1 + with database_engine.connect() as connection: + evidence = connection.execute( + text("SELECT count(*) FROM observation_evidence WHERE observation_id = :o"), + {"o": rows[0]["observation_id"]}, + ).scalar_one() + assert evidence == 3 + assert _verdict_calls(provider) == 0 + + +def test_d106_disjoint_dates_may_still_contradict_one_occurrence( + database_engine: Engine, +) -> None: + """Two sources disagreeing about the date of ONE named final are a + contradiction (both stand, grouped) — not two finals, not one merged.""" + + def router(prompt: str, type_name: str) -> dict[str, object]: + assert type_name == "ObservationVerdict" + return {"outcome": "contradict", "confidence": 0.9} + + adjudicator, provider = _adjudicator(engine=database_engine, router=router) + nate = _entity(engine=database_engine) + _add( + adjudicator=adjudicator, + entity=nate, + statement="Nate won the 2022 Valorant regional final on Friday.", + engine=database_engine, + asserted_at="2022-11-07T20:10:00Z", + event_window=("2022-11-04", "2022-11-04"), + ) + _add( + adjudicator=adjudicator, + entity=nate, + statement="Nate won the 2022 Valorant regional final on Saturday.", + engine=database_engine, + asserted_at="2022-11-08T20:10:00Z", + event_window=("2022-11-05", "2022-11-05"), + ) + with database_engine.connect() as connection: + groups = ( + connection.execute( + text( + "SELECT contradiction_group FROM observations" + " WHERE subject_entity_id = :e" + ), + {"e": nate}, + ) + .scalars() + .all() + ) + assert len(groups) == 2 and groups[0] is not None and groups[0] == groups[1] + assert _verdict_calls(provider) == 1 + + +def test_d106_a_dated_event_may_still_supersede_an_undated_state( + database_engine: Engine, +) -> None: + """Mixed dating forbids only `evidence`: a dated resignation can end an + undated "is CEO" state under the ordinary cap contract.""" + + def router(prompt: str, type_name: str) -> dict[str, object]: + assert type_name == "ObservationVerdict" + return { + "outcome": "supersede", + "confidence": 0.9, + "rationale": "the resignation ends the CEO state", + } + + adjudicator, _provider = _adjudicator(engine=database_engine, router=router) + alice = _entity(engine=database_engine) + _add( + adjudicator=adjudicator, + entity=alice, + statement="Alice is the CEO of Acme.", + engine=database_engine, + asserted_at="2025-06-01T09:00:00Z", + ) + _add( + adjudicator=adjudicator, + entity=alice, + statement="Alice resigned as CEO of Acme yesterday.", + engine=database_engine, + asserted_at="2026-03-02T09:00:00Z", + event_window=("2026-03-01", "2026-03-01"), + ) + with database_engine.connect() as connection: + rows = [ + dict(row) + for row in connection.execute( + text( + "SELECT statement, valid_until FROM observations" + " WHERE subject_entity_id = :e ORDER BY valid_from" + ), + {"e": alice}, + ).mappings() + ] + assert len(rows) == 2 + assert rows[0]["statement"] == "Alice is the CEO of Acme." + assert rows[0]["valid_until"] is not None # capped by the dated event + assert rows[1]["valid_until"] is None diff --git a/src/tests/workers/test_e3_entity_obs_flush_fanout.py b/src/tests/workers/test_e3_entity_obs_flush_fanout.py index cd9014aa9..58c0c7e0d 100644 --- a/src/tests/workers/test_e3_entity_obs_flush_fanout.py +++ b/src/tests/workers/test_e3_entity_obs_flush_fanout.py @@ -162,3 +162,15 @@ def test_entity_obs_flush_barrier_fields() -> None: "obs_flush_component_version", ): assert name in fields + + +def test_entity_handler_reports_the_claimed_units_own_generation() -> None: + """D106 rollout: a unit enqueued before an OBS_FLUSH_VERSION roll must + complete under the generation its barrier counts, or the barrier never + closes — so the entity handler reports ``work.component_version``, not + the current constant, to ``EntityObsFlushBarrier``.""" + import inspect + + source = inspect.getsource(e3.AdjudicateObservationsHandler._handle_entity_unit) + assert "obs_flush_component_version=work.component_version" in source + assert "obs_flush_component_version=OBS_FLUSH_VERSION" not in source diff --git a/website/src/app/docs/project-status/page.mdx b/website/src/app/docs/project-status/page.mdx index c554a4977..73a97746c 100644 --- a/website/src/app/docs/project-status/page.mdx +++ b/website/src/app/docs/project-status/page.mdx @@ -253,8 +253,9 @@ examined, and the failure looks like the deployment being down. - Historical `RS-LoCoMo-Full-v20` also requires every distinct retrieved value that directly satisfies the question. It does not add a retrieval or retry. - The current `RS-LoCoMo-Full-v21` pins the D106 observation adjudicator: - dated events with disjoint resolved windows never collapse onto each other, - and a dated event is never `evidence` for an undated statement. Recurring + dated events with disjoint resolved windows never collapse onto or supersede + each other (a same-occurrence date dispute may contradict), and a dated + event is never `evidence` for an undated statement. Recurring same-shaped events (seven tournament wins months apart) now survive as distinct facts. It changes ingestion provenance only; retrieval, prompts, budgets, and scoring are those of v20. From 7697011fad9929fa45f714cd2f871f6989a05a27 Mon Sep 17 00:00:00 2001 From: Jiri Puc Date: Thu, 3 Sep 2026 13:24:46 +0200 Subject: [PATCH 3/3] test(spikes): match the D106 block and claim-timing statements by stable fragments The operational-scale battery gates "one block read and one claim-timing read per entity batch" by matching literal SQL prefixes. D106 aliases the observations table and selects the D41 valid-time columns, so the old prefixes no longer occurred although both statements still run exactly once. Match stable fragments of the new statements instead; the counts and transaction gates are unchanged. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016p9xB9T7VYREuVmohDPKjR --- src/tests/spikes/test_operational_scale.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/tests/spikes/test_operational_scale.py b/src/tests/spikes/test_operational_scale.py index 4397e3e5d..dcaebe0ab 100644 --- a/src/tests/spikes/test_operational_scale.py +++ b/src/tests/spikes/test_operational_scale.py @@ -715,12 +715,18 @@ def _provider_neutral_batching( expected_hydration_statements = math.ceil( len(claim_ids) / INTERACTIVE_HYDRATION_BATCH_SIZE ) + # The front-loaded block and the batch's claim-timing read must each run + # once per entity batch (D43/D88). Match stable fragments of the two + # statements: the block reads `observations o` with its D106 timing + # lateral; the timing read selects `asserted_at` plus the D41 valid-time + # columns for the batch's claim ids. block_reads = sum( - "SELECT observation_id, statement, contradiction_group" in statement + "FROM observations o" in statement and "invalidated_at IS NULL" in statement for statement in observation_probe.statements ) timestamp_reads = sum( - "SELECT claim_id, asserted_at FROM claims" in statement + "SELECT claim_id, asserted_at" in statement + and "FROM claims WHERE claim_id = ANY(" in statement for statement in observation_probe.statements ) return OperationalScaleMeasurement(