diff --git a/benchmarks/locomo/README.md b/benchmarks/locomo/README.md index ac85a979..c9e997b0 100644 --- a/benchmarks/locomo/README.md +++ b/benchmarks/locomo/README.md @@ -1,4 +1,4 @@ -# RS-LoCoMo-Full-v20 setup +# RS-LoCoMo-Full-v21 setup This directory contains the unshipped full-system LoCoMo adapter. It does not vendor or auto-download LoCoMo. Supply the exact pinned `locomo10.json` only after confirming its @@ -16,12 +16,12 @@ The safe first command is local and makes no API or model call: uv run --extra benchmark python -m benchmarks.locomo prepare \ --dataset /absolute/path/locomo10.json \ --tier smoke \ - --protocol full-v20 \ + --protocol full-v21 \ --output .benchmark-runs/locomo-smoke ``` The harness validates the pinned bytes, renders session documents, and fingerprints the -eight-question smoke plan. `--protocol` is prepare-only; `full-v20` is the one +eight-question smoke plan. `--protocol` is prepare-only; `full-v21` is the one current-system protocol, and every later stage reads that immutable choice from `run.json`. Do not run remote stages until reviewing [`locomo_benchmark_design.md`](../../plan/designs/locomo_benchmark_design.md). @@ -64,6 +64,17 @@ call sees the complete bounded candidate snapshot and returns a supplied candidate id or `new`. There is no insufficient-evidence result or confidence-routed frontier call. +V21 fingerprints the D106 observation adjudicator: dated events with disjoint +resolved windows never collapse onto or supersede each other (they may only +contradict or stay distinct), a dated event is never `evidence` for an undated +statement (nor the reverse), open-ended windows stay unbounded, and the verdict +prompt shows when each statement was said and what time it is about. The +`adjudicate_observations` component version pins +that generation; the dataset, rendered documents, retrieval surface, answer +and judge prompts, budgets, and scoring are those of v20. V20 and v21 scores +are directional, not a one-variable comparison — the fact layer a v21 store +serves differs from a v20 store's. + Build the image from the revision under test — Compose otherwise serves the published release image, and the harness refuses to run against an engine whose stamped revision does not match the prepared run: diff --git a/benchmarks/locomo/__init__.py b/benchmarks/locomo/__init__.py index 1a46ed3e..d7dc5aa0 100644 --- a/benchmarks/locomo/__init__.py +++ b/benchmarks/locomo/__init__.py @@ -1 +1 @@ -"""The pinned RS-LoCoMo-Full-v20 benchmark adapter.""" +"""The pinned RS-LoCoMo-Full-v21 benchmark adapter.""" diff --git a/benchmarks/locomo/cli.py b/benchmarks/locomo/cli.py index f01cc26b..ece29374 100644 --- a/benchmarks/locomo/cli.py +++ b/benchmarks/locomo/cli.py @@ -118,7 +118,7 @@ def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="python -m benchmarks.locomo", description=( - "RS-LoCoMo-Full-v20: prepare is local; ingest/answer/judge require " + "RS-LoCoMo-Full-v21: prepare is local; ingest/answer/judge require " "explicit execution acknowledgements" ), ) diff --git a/benchmarks/locomo/dataset.py b/benchmarks/locomo/dataset.py index d2965605..cbca706e 100644 --- a/benchmarks/locomo/dataset.py +++ b/benchmarks/locomo/dataset.py @@ -140,10 +140,10 @@ def load_manifest(tier: str) -> QuestionManifest: ) if manifest.dataset_commit != DATASET_COMMIT: raise DatasetValidationError( - "manifest dataset commit is not RS-LoCoMo-Full-v20" + "manifest dataset commit is not RS-LoCoMo-Full-v21" ) if manifest.dataset_sha256 != DATASET_SHA256: - raise DatasetValidationError("manifest dataset hash is not RS-LoCoMo-Full-v20") + raise DatasetValidationError("manifest dataset hash is not RS-LoCoMo-Full-v21") actual = item_ids_hash(item_ids=manifest.item_ids) if actual != manifest.item_ids_sha256: raise DatasetValidationError( diff --git a/benchmarks/locomo/model.py b/benchmarks/locomo/model.py index 33d5c9fb..890cc1cf 100644 --- a/benchmarks/locomo/model.py +++ b/benchmarks/locomo/model.py @@ -1,4 +1,4 @@ -"""Typed values for the full-system RS-LoCoMo-Full-v20 protocol.""" +"""Typed values for the full-system RS-LoCoMo-Full-v21 protocol.""" from __future__ import annotations @@ -28,8 +28,8 @@ Category = Literal[1, 2, 3, 4, 5] RetainedCategory = Literal[1, 2, 3, 4] Tier = Literal["smoke", "development", "publication"] -ProtocolKey = Literal["full-v20"] -ProtocolName = Literal["RS-LoCoMo-Full-v20"] +ProtocolKey = Literal["full-v21"] +ProtocolName = Literal["RS-LoCoMo-Full-v21"] SourceTimezoneBasis = Literal["assumed_utc"] AnswerAgentModel = Literal["openai/gpt-5.6-luna"] JudgeModel = Literal["openai/gpt-5.6-luna"] @@ -120,7 +120,7 @@ class QuestionManifest(FrozenModel): class RunConfiguration(FrozenModel): """Immutable identity of one prepared benchmark run.""" - protocol_name: ProtocolName = "RS-LoCoMo-Full-v20" + protocol_name: ProtocolName = "RS-LoCoMo-Full-v21" adapter_version: NonEmpty prepared_at: datetime repository_revision: NonEmpty @@ -393,7 +393,7 @@ class SessionDiagnosticSummary(FrozenModel): class RunSummary(FrozenModel): """Publication-ready local aggregate with no hidden denominator.""" - protocol_name: ProtocolName = "RS-LoCoMo-Full-v20" + protocol_name: ProtocolName = "RS-LoCoMo-Full-v21" protocol_fingerprint: NonEmpty tier: Tier questions: int = Field(ge=1) diff --git a/benchmarks/locomo/protocol.py b/benchmarks/locomo/protocol.py index 8859a1f3..33d683e1 100644 --- a/benchmarks/locomo/protocol.py +++ b/benchmarks/locomo/protocol.py @@ -29,9 +29,9 @@ from rememberstack.model import Envelope from rememberstack.model import ToolDescriptor -PROTOCOL_NAME: Final = "RS-LoCoMo-Full-v20" -DEFAULT_PROTOCOL_KEY: Final = "full-v20" -ADAPTER_VERSION: Final = "locomo-full-adapter-2026.09-complete-values-v20" +PROTOCOL_NAME: Final = "RS-LoCoMo-Full-v21" +DEFAULT_PROTOCOL_KEY: Final = "full-v21" +ADAPTER_VERSION: Final = "locomo-full-adapter-2026.09-temporal-gate-v21" MAX_TOOL_CALLS: Final = 8 MAX_AGENT_CALLS: Final = 9 ANSWER_READER_RETRY_BUDGET: Final = 2 @@ -74,7 +74,7 @@ "no-types-1:binary-t4-1:document-t0-1" ), "adjudicate_observations": ( - "e3-obs-flush-2026.08a:claim-fanout-1:entity-fanout-1" + "e3-obs-flush-2026.09a:temporal-gate-1:claim-fanout-1:entity-fanout-1" ), "adjudicate_supersession": "adjudicator-2026.07b:temp0-1", "embed_claim": "p1-embed-claims-2026.07", @@ -211,8 +211,8 @@ class LoCoMoProtocol: answer_word_cap: int | None = None -_FULL_V20 = LoCoMoProtocol( - key="full-v20", +_FULL_V21 = LoCoMoProtocol( + key="full-v21", name=PROTOCOL_NAME, answer_agent_model=ANSWER_AGENT_MODEL, judge_model=JUDGE_MODEL, @@ -234,7 +234,7 @@ class LoCoMoProtocol: ) PROTOCOL_REGISTRY: Final[Mapping[ProtocolKey, LoCoMoProtocol]] = MappingProxyType( - {_FULL_V20.key: _FULL_V20} + {_FULL_V21.key: _FULL_V21} ) diff --git a/benchmarks/locomo/runner.py b/benchmarks/locomo/runner.py index e8eb67ab..60e037e3 100644 --- a/benchmarks/locomo/runner.py +++ b/benchmarks/locomo/runner.py @@ -411,7 +411,7 @@ def ingest_sample( _require_current_ingest_bindings(model_bindings=build.model_bindings) if build.document_binding_generation != EXPECTED_DOCUMENT_BINDING_GENERATION: raise ExecutionGuardError( - "deployment document binding generation differs from RS-LoCoMo-Full-v20" + "deployment document binding generation differs from RS-LoCoMo-Full-v21" ) _require_current_query_surface(context=context, client=client) _require_exact_live_ingests( @@ -570,7 +570,7 @@ def answer_sample( ): raise ExecutionGuardError( "the deployment did not report the exact completed" - " RS-LoCoMo-Full-v20 pipeline, live graph, and fresh P3 projection" + " RS-LoCoMo-Full-v21 pipeline, live graph, and fresh P3 projection" ) _require_serving_revision(context=context, readiness=readiness) prior_readiness = context.state.readiness.get(sample_id) @@ -1177,7 +1177,7 @@ def _validate_run( """Recompute immutable run identity before any local or remote stage.""" selected_protocol = protocol_for_name(configuration.protocol_name) if configuration.dataset_sha256 != DATASET_SHA256: - raise BenchmarkRunError("run dataset hash is not RS-LoCoMo-Full-v20") + raise BenchmarkRunError("run dataset hash is not RS-LoCoMo-Full-v21") if item_ids_hash(item_ids=manifest.item_ids) != manifest.item_ids_sha256: raise BenchmarkRunError("run manifest item hash changed") if manifest_bytes_hash(manifest=manifest) != configuration.manifest_sha256: @@ -1187,7 +1187,7 @@ def _validate_run( if manifest.tier != configuration.tier: raise BenchmarkRunError("run manifest tier changed") if configuration.dataset_commit != DATASET_COMMIT: - raise BenchmarkRunError("run dataset commit is not RS-LoCoMo-Full-v20") + raise BenchmarkRunError("run dataset commit is not RS-LoCoMo-Full-v21") if configuration.adapter_version != ADAPTER_VERSION: raise BenchmarkRunError("run adapter version differs from current code") if _models_hash(values=documents) != configuration.documents_sha256: @@ -1445,7 +1445,7 @@ def _require_current_ingest_bindings(*, model_bindings: dict[str, str]) -> None: if model_bindings.get(name) != expected.get(name) ) raise ExecutionGuardError( - "deployment ingest model bindings differ from RS-LoCoMo-Full-v20: " + "deployment ingest model bindings differ from RS-LoCoMo-Full-v21: " + ", ".join(mismatches) ) diff --git a/benchmarks/locomo/sharding/README.md b/benchmarks/locomo/sharding/README.md index 68c96807..1fe31299 100644 --- a/benchmarks/locomo/sharding/README.md +++ b/benchmarks/locomo/sharding/README.md @@ -178,7 +178,7 @@ as source-derived data. | --- | ---: | --- | | `LOCOMO_PYTHON` | `.venv/bin/python` | repository virtual-environment Python | | `LOCOMO_TIER` | `publication` | prepared manifest tier | -| `LOCOMO_PROTOCOL` | `full-v20` | prepare-time protocol key | +| `LOCOMO_PROTOCOL` | `full-v21` | prepare-time protocol key | | `LOCOMO_MOUNT_ROOT` | `$RUN_DIR/.mounts` | host/container-identical P3 mount root | | `LOCOMO_MAX_DOCUMENTS` | `100` | per-sample ingest authorization | | `LOCOMO_MAX_QUESTIONS` | `1540` | run-absolute answer item authorization | diff --git a/benchmarks/locomo/sharding/run_shard.sh b/benchmarks/locomo/sharding/run_shard.sh index 12086c58..934a83ee 100755 --- a/benchmarks/locomo/sharding/run_shard.sh +++ b/benchmarks/locomo/sharding/run_shard.sh @@ -25,7 +25,7 @@ run_dir=$2 dataset_path=$3 python_bin=${LOCOMO_PYTHON:-.venv/bin/python} tier=${LOCOMO_TIER:-publication} -protocol=${LOCOMO_PROTOCOL:-full-v20} +protocol=${LOCOMO_PROTOCOL:-full-v21} mount_root=${LOCOMO_MOUNT_ROOT:-$run_dir/.mounts} max_documents=${LOCOMO_MAX_DOCUMENTS:-100} max_questions=${LOCOMO_MAX_QUESTIONS:-1540} @@ -50,7 +50,7 @@ export GOOGLE_APPLICATION_CREDENTIALS=${LOCOMO_GCP_CREDENTIALS_FILE:-/etc/rememb export GOOGLE_API_CERTIFICATE_CONFIG=${LOCOMO_GCP_CERTIFICATE_CONFIG_FILE:-/etc/rememberstack/locomo-gcs/certificate-config.json} export GOOGLE_API_USE_CLIENT_CERTIFICATE=true -# RS-LoCoMo-Full-v20's non-secret ingest identity. Override ambient self-host +# RS-LoCoMo-Full-v21's non-secret ingest identity. Override ambient self-host # defaults so every shard runs the exact Luna/Qwen pipeline the protocol checks. export REMEMBERSTACK_STRUCTURER_MODEL=openai/gpt-5.6-luna export REMEMBERSTACK_SKELETON_CHECK_MODEL=openai/gpt-5.6-luna diff --git a/decisions.md b/decisions.md index baf971ed..0d0dead0 100644 --- a/decisions.md +++ b/decisions.md @@ -2910,7 +2910,11 @@ may proceed to its first tagged artifact proof after CLA activation. ## D78. LoCoMo measures the ordinary OSS query system, not a claims-only shortcut -> **D105 amendment.** The current protocol is `RS-LoCoMo-Full-v20`. It retains +> **D106 amendment.** The current protocol is `RS-LoCoMo-Full-v21`. It retains +> v20's dataset, rendered documents, models, tools, budgets, prompts, and +> scoring and pins the D106 observation adjudicator generation. + +> **Historical D105 amendment.** The D105 protocol was `RS-LoCoMo-Full-v20`. It retains > v19's surface, dataset, ingestion, models, tools, and budgets while adding the > D105 complete-direct-values answer instruction. > @@ -4889,3 +4893,99 @@ retry; mention research or adoption in the prompt. **Amends.** Advances D104's Full-v19 benchmark identity only. It preserves D98 retrieval, D99's objective content-before-`Unknown` guard, and D104's bounded counterfactual instruction. + +## D106. Dated events never collapse across dates; evidence requires temporal compatibility + +**Decision (2026-09-03).** The observation adjudicator (D43) gains a +deterministic temporal-compatibility rung that runs before any model call, +reading the D41 valid-time each claim already carries: + +1. Two statements that both describe **datable events** + (`claim_valid_kind = 'event_time'`) whose resolved windows are **disjoint** + are different occurrences unless the date-aware model finds they name the + *same* occurrence with disputed dates. Such a pair may therefore only + `contradict` (both stand, grouped) or stay `new`; an `evidence` or + `supersede` verdict is coerced to `new` and recorded. This holds even when + the wording is byte-identical — "won a tournament last week" said in + January and again in October is two wins — and identical text is then + kept apart without any model call. +2. A dated event paired with an **undated** statement (a state, a period + figure, a summary such as "has been winning a few tournaments", or + testimony the extractor could not anchor) may still be judged for + `supersede`/`contradict` — a dated resignation can end an undated "is + CEO" state — but an `evidence` verdict for such a pair is coerced to `new` + and recorded, and identical text is kept apart without a model call. A + specific dated event is never a re-assertion of a vaguer statement, and a + summary never re-asserts a specific event. +3. Two undated statements are adjudicated exactly as before; two dated + events with overlapping windows — including a year-level and a day-level + window for one occurrence — go to the ladder as before, and a re-mention + of the same event still collapses. +4. A D41 open interval (a resolved start with no end) is unbounded, never a + point: it overlaps every later window. + +The verdict prompt now shows, for each statement, when the source *said it* +and what world-time it *is about* (the resolved D41 window of any kind), and +states the rule, so the model judges with both timelines in front of it. An +observation's windows are the span of its supporting current-testimony +claims, derived at block time and kept current within a batch as the row +absorbs evidence; nothing is stored. Every adjudication record carries the +coercions that preceded it. The adjudicator generation, the +`adjudicate_observations` flush component version, and the LoCoMo protocol +(Full-v20 → Full-v21) roll; nothing else about D43 — the entity block, the +novelty gate, the no-cap rule, the fail-safe-to-coexist contract — changes. + +**Context.** The v0.11.0 LoCoMo `conv-42` run (protocol Full-v18, +`plan/analysis/locomo_conv42_recurring_event_adjudication.md`) extracted all +seven of Nate's tournament wins as clean, dated claims and resolved one Nate +entity, yet the fact layer kept only four distinct win facts. The +`observation_adjudications` transcript shows why: the small model saw two +bare strings, judged "won a really big video game tournament last week" +(October) to be `evidence` for "won his first video game tournament last +week" (January) because both say "last week", and folded the international +and Valorant wins into "has been winning a few gaming tournaments". Ten +participation lineages collapsed into the header boilerplate "Nate is a +participant.", swallowing two real tournament entries. The claims carried +resolved event dates the whole time; the adjudicator never read them. The +reader answered the counting question "At least five" against a gold of +seven with all seven wins in its testimony context — the fact envelope's +undercount is the plausible anchor. + +**Consequences.** Recurring same-shaped events survive as distinct facts, so +entity-anchored counting and enumeration have a complete fact layer to read, +while a genuine same-occurrence date dispute can still surface as a +contradiction. Verdict spend is unchanged in shape (similar pairs are still +judged; only identical-text decisions become model-free). The fail-safe +direction is preserved and slightly widened: an undated re-mention of a +dated event now stays a separate observation rather than collapsing — a +duplicate, never a loss. Observation `valid_from` continues to be the +claim's `asserted_at`; using the resolved window as the observation's own +validity is a separate change this decision does not make. + +Rollout is stop-drain-rebuild. The `adjudicate_observations` flush handler +reports the claimed unit's *own* component generation to its barrier, so +entity units enqueued before the roll complete under the generation their +barrier counts and drain cleanly; readiness then reports the stage at the old +generation until the deployment is rebuilt under the new one (D7/D12). +Existing stores re-adjudicate observations under the new generation on that +rebuild. + +**Rejected.** Fixing only the verdict prompt (a model may still merge across +dates; the coercions must be deterministic); skipping disjoint-date pairs +without any verdict (would silently double-count one occurrence whose date +two sources dispute, and suppress the contradiction group); treating every +mixed-dating pair as non-interacting (would stop a dated event from +superseding a state it ends); dropping vague summaries at extraction +(source-faithful testimony is kept — D32 — the defect was absorption, not +extraction); a typed period/value column (D43 §4 rejects the typed schema; +the event window is already carried by the claim). + +**Design.** `plan/designs/observations_design.md` §3 (temporal compatibility +rung) and `plan/designs/locomo_benchmark_design.md` Full-v21 protocol. + +**Analysis.** `plan/analysis/locomo_conv42_recurring_event_adjudication.md`. + +**Amends.** D43's adjudication cascade (adds the deterministic rung its +design named but never implemented) and D105's Full-v20 benchmark identity. +Preserves D41 (claims stay the one home of asserted validity), D98 +retrieval, D100/D102 identity, and D104/D105 answer-prompt contracts. diff --git a/plan/analysis/locomo_conv42_recurring_event_adjudication.md b/plan/analysis/locomo_conv42_recurring_event_adjudication.md new file mode 100644 index 00000000..cd444259 --- /dev/null +++ b/plan/analysis/locomo_conv42_recurring_event_adjudication.md @@ -0,0 +1,129 @@ +# LoCoMo `conv-42`: recurring events collapse in the observation layer + +**Date:** 2026-09-03 + +**Status:** analysis (non-binding); the decision it motivated is D106 + +**Scope:** why a store that extracted all seven of a speaker's tournament wins +as clean, dated claims and resolved one entity for him ended with four win +facts — and what the counting question's reader then did with it. Evidence is +read-only SQL against a retained v0.11.0 store; nothing here changes retrieval, +identity, or the answer prompt. + +## 1. Run coordinates + +| Field | Value | +| --- | --- | +| Engine | RememberStack v0.11.0, revision `e7b173a19e8a992ec57bf75ce6593373ab2fc2c5` | +| Protocol | `RS-LoCoMo-Full-v18`, publication tier, fingerprint `2c2d3070d7620b176c6396e1245f07e8e59f44bec01b09d837ac2e144b064df8` | +| Sample | `conv-42` (29 sessions, 629 turns, 199 retained questions) | +| Deployment | `3f134f36-6b59-4060-9318-bf77f2a3463b` on the managed-cloud benchmark host | +| Dataset SHA-256 | `79fa87e90f04081343b8c8debecb80a9a6842b76a7aa537dc9fdf651ea698ff4` | +| Scoring-base backup | `gs://remember-stack-locomo-backups/2c2d3070…/e7b173a1…/2026-09-01T12-19-28.807846Z/conv-42/20260902T072007Z-75e029f2` (verified) | +| Ingest outcome | 1,772 stage units succeeded, one `normalize_relations` dead letter replayed; 1,298 claims, 716 observations, 22 active entities, zero duplicate-name groups, one Nate and one Joanna | + +The answer stage stopped at 182 of 199 questions when the provider account +ran out of credits; the judge did not run. The two counting questions this +document is about were answered before the stop. + +## 2. The seven wins, layer by layer + +The gold answer to `conv-42/qa/0080` ("How many tournaments has Nate won?") +is **seven**, with evidence in seven sessions spread over ten months. + +**Extraction (E2) — 7 of 7.** Every win became a decontextualized, attributed +claim with a resolved D41 event window (`claim_valid_kind = 'event_time'`, +day precision): + +| Session | Claim text | Resolved event date | +| --- | --- | --- | +| D1 | Nate said he won his first video game tournament last week. | 2022-01-14 | +| D10 | Nate said that Nate won Nate's second tournament last week. | 2022-04-25 | +| D14 | Nate just won another regional video game tournament last week. | 2022-05-27 | +| D17 | Nate won Nate's fourth video game tournament on Friday. | 2022-07-08 | +| D19 | Nate won an international tournament yesterday. | 2022-08-21 | +| D22 | Nate said that Nate won a really big video game tournament last week. | 2022-09-29 | +| D27 | Nate won the final of a big Valorant tournament last Saturday. | 2022-11-05 | + +**Identity (D100/D102) — clean.** One active Nate entity (568 resolved +mentions), one Joanna (655). Every claim above is anchored on that Nate. + +**Fact layer (E3 + D43 adjudication) — 4 of 7.** The `observation_adjudications` +transcript for the same claims: + +| Incoming claim | Outcome | Absorbed into | +| --- | --- | --- | +| won his first video game tournament | `add` | — (observation, Jan) | +| won Nate's second tournament | `add` | — (observation, May) | +| won another regional tournament | `add` | — ("Winning the tournament was a huge confidence boost", Jun) | +| won Nate's fourth tournament | relation | `Nate —other:won→ "Nate's fourth video game tournament"` (the tournament minted as an entity) | +| won an international tournament yesterday | `noop` (evidence) | "Nate has been winning a few gaming tournaments" (May, undated summary) | +| won a really big tournament last week | `noop` (evidence) | **"won his first video game tournament last week" (Jan)** | +| won the final of a big Valorant tournament | `noop` (evidence) | "Nate has been winning a few gaming tournaments" | + +Three wins were folded into earlier facts as evidence rows. The October → +January merge is the decisive specimen: two claims whose resolved event +dates were nine months apart, judged the same fact because both say "last +week". The adjudicator's verdict prompt at that generation rendered exactly +two strings — `EXISTING` and `NEW` — and nothing about time, although both +claims carried resolved windows in the same table row the adjudicator read +`asserted_at` from. + +**Reader — "At least five".** The answer agent made one `answer_context` +call with the raw question and received all seven win claims in the +testimony envelope (sessions D1, D10, D14, D17, D19, D22, D27 — every gold +turn) and, in the fact envelope, the four surviving win facts plus the +summary "has been winning a few". It answered "At least five" against seven. +The fact envelope's undercount is the plausible anchor; the run cannot prove +the reader's arithmetic either way, and it ran at the protocol's +reasoning-effort `none`. + +## 3. The participation question: ten lineages into boilerplate + +`conv-42/qa/0078` ("How many video game tournaments has Nate participated +in?", gold nine) was answered "Four". Its fact-layer counterpart is one +observation, "Nate is a participant.", with `evidence_count = 10`. Its +supporting claims: + +- nine are the rendered session header — "Participants: Joanna and Nate", + "Nate is a participant.", "Joanna and Nate are participants." — one per + ingested session, source-faithful document framing; +- two are real, dated tournament entries: "currently participating in the + video game tournament again" (2022-03-24) and "tried playing in the local + Street Fighter tournament this time" (2022-04-25). + +The boilerplate is harmless on its own. The defect is that two dated events +were absorbed into an undated statement that means "participant of this +conversation": semantic adjacency ("participating" ≈ "participant") +overrode content and time. The header claims themselves are outside this +analysis (they are what the source says; whether document framing should +seed observations at all is a separate question for E2 selection). + +## 4. Where the fix belongs + +Not extraction: the summary "has been winning a few" and the header lines are +source-faithful claims (D32), and every win reached the store dated. Not +identity: one entity. Not retrieval: every win reached the reader. The defect +is in the adjudicator's *inputs* — it decided sameness from two strings while +the temporal discriminator sat unread — and in a missing rule: a dated event +is not a re-assertion of anything undated, and two dated events on different +days are two events regardless of wording. + +D106 adds that rule as the deterministic rung D43's design always named +before the model call, and shows both timelines in the prompt for the pairs +that still need judging. Replaying the seven wins through the new rung in the +test suite yields seven observations and buys no verdict for any +disjoint-window pair; the vague summary and the boilerplate state each keep +their own row. + +## 5. What this does not establish + +- No score. The run's judge did not execute, and 17 questions were never + answered; the per-question reads above are the answer agent's generated + answers against gold, not judged results. +- No claim that the temporal rung alone fixes the counting question. The + reader saw seven wins in testimony and still undercounted; whether a + complete fact layer changes its arithmetic is the next run's question. +- No change to observation `valid_from` (still the claim's `asserted_at`), + to the E2 selection of document-framing claims, or to the reader's + reasoning effort. Each is a separate decision. diff --git a/plan/designs/locomo_benchmark_design.md b/plan/designs/locomo_benchmark_design.md index 981ee657..d528728a 100644 --- a/plan/designs/locomo_benchmark_design.md +++ b/plan/designs/locomo_benchmark_design.md @@ -1,7 +1,19 @@ # LoCoMo full-system benchmark design -> **Binding D105 amendment (2026-09-01).** The current protocol is -> `RS-LoCoMo-Full-v20`. It retains v19's dataset, ingestion, models, tools, +> **Binding D106 amendment (2026-09-03).** The current protocol is +> `RS-LoCoMo-Full-v21`. It retains v20's dataset, rendered documents, models, +> tools, budgets, answer and judge prompts, counterfactual and complete-answer +> instructions, content-before-`Unknown` harness guard, and no-review scoring +> rule. Its pinned `adjudicate_observations` component version now carries the +> D106 temporal-compatibility rung: dated events with disjoint resolved windows +> never collapse onto or supersede each other (a same-occurrence date dispute +> may contradict), and a dated event is never `evidence` for an undated +> statement. Ingestion provenance, protocol identity, and fingerprint +> roll; no retrieval, retry, model-effort, or call-budget behavior changes. +> V20 and v21 scores are directional because the fact layer differs. + +> **Historical D105 amendment (2026-09-01; superseded by D106).** The D105 +> protocol was `RS-LoCoMo-Full-v20`. It retained v19's dataset, ingestion, models, tools, > budgets, counterfactual instruction, content-before-`Unknown` harness guard, > and no-review scoring rule. Its answer prompt now also requires every > distinct retrieved value that directly satisfies the question, rather than @@ -53,7 +65,7 @@ > **Status:** binding current-system protocol contract. Real provider execution > remains operator-invoked. Accepting this design does not itself authorize a -> paid v20 run. +> paid run of the current protocol. ## 1. Acceptance boundary @@ -79,7 +91,7 @@ and spend ceiling. ## 2. Fixed protocol ```text -protocol RS-LoCoMo-Full-v20 +protocol RS-LoCoMo-Full-v21 dataset commit 3eb6f2c585f5e1699204e3c3bdf7adc5c28cb376 dataset SHA-256 79fa87e90f04081343b8c8debecb80a9a6842b76a7aa537dc9fdf651ea698ff4 categories 1, 2, 3, 4 @@ -651,7 +663,7 @@ compatibility form. The response contains: same-snapshot proven-absent-anchor execution checks when live graph is required; - an overall `ready` that is the conjunction of the requested capabilities; - every non-secret ingestion/query model binding; and -- the non-secret `document_binding_generation`, which Full-v20 requires to be +- the non-secret `document_binding_generation`, which Full-v21 requires to be exactly `document-t0-v1` and stores in `run.json` plus the protocol fingerprint. @@ -797,11 +809,11 @@ Local preparation: uv run --extra benchmark python -m benchmarks.locomo prepare \ --dataset /absolute/path/locomo10.json \ --tier smoke \ - --protocol full-v20 \ + --protocol full-v21 \ --output .benchmark-runs/locomo-smoke ``` -`--protocol` exists only on `prepare`. The sole choice is `full-v20`; ingest, +`--protocol` exists only on `prepare`. The sole choice is `full-v21`; ingest, answer, judge, and summarize read it from the prepared run and expose no protocol override. diff --git a/plan/designs/observations_design.md b/plan/designs/observations_design.md index a5511e74..6830d4c6 100644 --- a/plan/designs/observations_design.md +++ b/plan/designs/observations_design.md @@ -142,6 +142,39 @@ When a claim asserts a value/property about entity *E*: state-vs-measurement from the `statement` (semantic), not from a typed column. This is the rule that replaces the dropped `about_period` columns: rather than recording the period structurally, the system simply never caps a period figure and lets same-period conflicts coexist. + + **Temporal compatibility — the deterministic rung (D106).** Before any model call, the + adjudicator reads what the claim already knows about *when*: its D41 `claim_valid_kind` and + resolved window. A claim that describes a **datable event** (`event_time` — a win, a visit, a + purchase; the extractor resolved "last Saturday" to a calendar day) is compared to the + candidate's own event window, which is the span of the event-time claims supporting it. + A resolved start with no end is a D41 *open* interval and stays unbounded (it overlaps every later + window). Then: + - **both dated events, disjoint windows → `evidence` and `supersede` are forbidden.** Two things + that happened on different days are different occurrences — *unless* two sources are disagreeing + about the date of one named occurrence ("the Valorant final on Friday" vs "…on Saturday"), which + only the date-aware model can tell. So the pair is still judged, but may only `contradict` (both + stand, grouped) or stay `new`; an `evidence` or `supersede` verdict is coerced to `new` and + recorded with its reason. This holds even when the wording is byte-identical — "won a tournament + last week" said in January and again in October is two wins, not one re-asserted — and identical + text is then kept apart *without* a verdict (the exact-statement shortcut obeys the same rule). + - **one dated event, one undated statement → `evidence` is forbidden.** The pair may still be + judged for supersede/contradict (a dated resignation can end a "is CEO" state), but a verdict + of `evidence` is coerced to `new` and recorded with its reason; identical text is again kept + apart without a verdict. A specific dated event is never a re-assertion of a vaguer statement + ("has been winning a few tournaments"), and a summary never re-asserts a specific event — both + stand. + - **both undated, or both dated with overlapping windows → the ladder as before.** A re-mention + of the *same* event (same resolved date, or a year-level and a day-level window for one + occurrence) still collapses as evidence; headcounts, revenues and stances are untouched. + The verdict prompt shows, for each statement, when the source *said it* and what world-time it *is + about* (the resolved window of any D41 kind, not only events), and defines both, so the model + judges with the timelines in front of it rather than two bare strings. Within one batch a block row + that absorbs evidence widens its in-memory windows, so a later overlapping claim is not split off + as a different occurrence. Every adjudication record carries the coercions that preceded it. The + rung is the "deterministic value/period compare" step the cascade below always named; it is what + keeps recurring same-shaped events countable — the failure it closes is recorded in + `plan/analysis/locomo_conv42_recurring_event_adjudication.md`. 4. **Fail safe — a binding adjudicator contract (not just a hope).** This is the honest core of the untyped design: "never silently resolve" is **policy enforced in E3 + eval**, not a schema invariant. The binding rules: @@ -187,6 +220,22 @@ worst the adjudicator spends a bit more. Contrast a pure semantic-cluster approa prior headcount would be **invisible** and silently duplicated. Anchoring to the resolved entity is what buys this. +**Distinct events — seven tournament wins (D106).** A conversation reports, months apart, "I won +my first tournament last week", "won my second tournament last week", "won another regional +tournament", "won my fourth tournament on Friday", "won an international tournament yesterday", +"won a really big tournament last week", "won the Valorant final last Saturday". Every claim is a +dated event (`event_time`, day precision, all seven windows disjoint). Without the temporal rung a +small model reading only the two strings judged October's "won a really big tournament last week" +to be `evidence` for January's "won his first tournament last week" — both say "last week" — and +folded the international and Valorant wins into "has been winning a few tournaments"; the entity +ended with four win facts and a count question answered "at least five" against seven. With the +rung: each new win is still judged against the similar earlier wins and the undated summary, but +with both timelines in the prompt, and whatever the model answers, an `evidence` or `supersede` +verdict against a disjoint-window win — or an `evidence` verdict against the summary — is coerced to +`new` and recorded → **seven observations, one summary**. A second mention of the *same* win (same +resolved day) still collapses as evidence onto it, and two sources disagreeing about *which* day one +named final was won may `contradict`, so both stand grouped. + ### Supersession appends — an observation is a time-slice, never an in-place edit An observation is **a time-slice of belief, not a mutable current-state record.** Supersession does two @@ -234,7 +283,9 @@ value → observation), and often the claim's embedding (E2 embeds claims for P1 - For a **hub entity**, the same vector step top-k ranks *which* candidates to compare (cheap math); a skipped far candidate costs at most a duplicate row, never a wrong supersede. 3. **Adjudicate the residue only (cheap → frontier).** Only similar-but-not-identical candidates escalate - the D4 cascade: deterministic value/period compare → small model → frontier LLM for the survivors. The + the D4 cascade: the deterministic temporal-compatibility compare (§3 step 3, D106 — it bounds which + verdicts a pair may take, and decides identical text alone) → small model → frontier LLM for the + survivors. The adjudicator decides same-property (+ same-period for a figure) and the outcome under the no-cap rule (state → supersede; measurement → contradict/coexist; same value → evidence; else new), and **fails safe to coexist** below the supersede margin. diff --git a/plan/plans/phase-2-truth-machinery.md b/plan/plans/phase-2-truth-machinery.md index 0ede35bb..e082d362 100644 --- a/plan/plans/phase-2-truth-machinery.md +++ b/plan/plans/phase-2-truth-machinery.md @@ -16,7 +16,7 @@ review queue operable end-to-end; observation adjudicator passes its D43 accepta | WP-2.2 | Clustering + reversibility: gather/decide, black-hole guard, nDR, merge_events, redirects | registries §6; D21 | WP-2.1 | cluster worker | order-independence test; un-merge replay | done — PR #89: EntityClusterer (gather via blocking reach, decide via HAC centroid-linkage distance cut — never transitive closure), joint 1-hop neighborhood re-decision (order-independent, Klein test), black-hole guard (tightened cut on oversized blobs), blast-radius routing to review_queue (expected-impact ranked, hubs never auto), reversible merges (redirect + pre-merge snapshot + linked reversal events; double-unmerge refused) | | WP-2.3 | Ontology + predicate governance: seed core, packs, domain/range enforcement, `other:` funnel | registries §4, §7; D15, D18 | Phase 1 | registry content + normalizer checks | domain/range rejects hallucination sample | done — PR #90: system-shipped Work pack (Task/Decision/Goal + 6 predicates with signatures) installed as a unit with extend-never-fork anchor refusal; the D5 other: funnel (grammar-gated, tier=other registration, usage-counted, promotion-ranked, never leaks into the governed prompt vocabulary); registry-rendered normalizer prompts (descriptions + synonyms); hallucination sample rejected at ancestor-level signature matching | | WP-2.4 | Relation supersession cascade (blocking → novelty → cheap-first → outcomes) + adjudication transcripts | e2_e3 §5; D3–D4; schema §9 | WP-2.1 | adjudicator | S9-class green; zombie-fact test | done — PR #91: SupersessionAdjudicator (change-prone-only (subject, predicate) blocking → novelty gate past the LLM → small→frontier ladder → supersede/coexist/contradict outcomes applied atomically with append-only transcripts); normalize chains adjudicate_supersession for created relations; query surface gains valid_at as-of (S9-class) and the relation transcript (S8); zombie-fact and both-stand contradiction proofs | -| WP-2.5 | Observation adjudication (entity block, no-cap rule, contradiction groups, fail-safe coexist) | observations §3–4; D43 | WP-2.4 | observation adjudicator | **D43 eval gate green** (shipping criterion) | done — PR #92: the add-observation worker (entity block exact+exhaustive, novelty gate with zero-LLM exits, hub top-k ranking as ordering-only, small→frontier ladder), the binding fail-safe contract (supersede only above an explicit margin with a reason row — below it coexists; every cap audited), the no-cap rule for fixed-period measurements (contradictions both stand, grouped), corpus-redundancy evidence-collapse; contradiction P/R gate green over the golden set and recorded in eval_runs; S9 as-of over capped time-slices | +| WP-2.5 | Observation adjudication (entity block, no-cap rule, contradiction groups, fail-safe coexist) | observations §3–4; D43 | WP-2.4 | observation adjudicator | **D43 eval gate green** (shipping criterion) | done — PR #92: the add-observation worker (entity block exact+exhaustive, novelty gate with zero-LLM exits, hub top-k ranking as ordering-only, small→frontier ladder), the binding fail-safe contract (supersede only above an explicit margin with a reason row — below it coexists; every cap audited), the no-cap rule for fixed-period measurements (contradictions both stand, grouped), corpus-redundancy evidence-collapse; contradiction P/R gate green over the golden set and recorded in eval_runs; S9 as-of over capped time-slices. Amended 2026-09-03 by D106: the deterministic temporal-compatibility rung (disjoint dated events never collapse; a dated event is never evidence for an undated statement) after LoCoMo `conv-42` folded three of seven dated wins into earlier facts | | WP-2.6 | Review queue + CLI (cluster review, blast-radius bands, support_withdrawn triage verdicts) | registries §8; D24; lifecycle §4 triage | WP-2.2 | review tooling | verdicts write the designed rows (restore/invalidate) | done — PR #93: ReviewQueue (expected-impact-ranked listing, kind-checked locked verdicts), merge verdicts through the same reversible apply_merge mechanism as the clusterer (decided_by=human, snapshot kept), support_withdrawn triage (restore_support → review_restored currency event + claim currency + D54 recount; invalidate_fact → invalidated_at + recorded human adjudication; uncertain → deferred, marker stands), the remember CLI entry point (review list/decide) with the /docs/reference/cli page (D66) | | WP-2.7 | Un-merge ↔ supersession ripple spike + stance-holder resolution eval (D59) | registries §11.3; D59 | WP-2.4 | spike reports | recorded; thresholds adjusted | done — PR #94: executable spike (test_unmerge_ripple.py) + report (plan/analysis/unmerge_supersession_ripple_spike.md); findings: identity-set supersession blocking added (the merged identity's history is one person's history), un-merge flags cross-identity closures into review (split_cluster, never silently reopened or kept), same-identity closures don't ripple; stance-content guard locked (D59); ripple expected-impact starting point recorded | diff --git a/src/rememberstack/spine/observation_adjudication.py b/src/rememberstack/spine/observation_adjudication.py index 88cc02d9..3d971568 100644 --- a/src/rememberstack/spine/observation_adjudication.py +++ b/src/rememberstack/spine/observation_adjudication.py @@ -13,7 +13,12 @@ figures contradict and both stand. """ +from __future__ import annotations + +from collections.abc import Mapping from collections.abc import Sequence +from dataclasses import dataclass +from typing import Any from typing import Final from uuid import UUID from uuid import uuid4 @@ -35,15 +40,36 @@ from rememberstack.ports.model_provider import ModelProviderPort from rememberstack.spine.rank_embed_cache import RankEmbedCache -OBSERVATION_ADJUDICATOR_VERSION: Final = "obs-adjudicator-2026.07b:temp0-1" +OBSERVATION_ADJUDICATOR_VERSION: Final = ( + "obs-adjudicator-2026.09a:temp0-1:temporal-gate-1" +) """The observation adjudicator generation (D12; replayed on rebuild, D7). -07b pins temperature=0.0 — generation parameters are part of provenance.""" +07b pins temperature=0.0 — generation parameters are part of provenance. +09a (D106) adds the temporal-compatibility rung: two dated events with +disjoint resolved windows never collapse or supersede (they may only +contradict or stay distinct), a dated event never collapses as evidence onto +an undated statement (nor the reverse), identical text is collapsed only when +temporally compatible, open-ended windows stay unbounded, and the verdict +prompt shows both statements' said-on dates and is-about windows.""" _VERDICT_PROMPT: Final = """You adjudicate observations for a memory system. Both statements are believed facts about the SAME entity: EXISTING: {existing!r} + said on: {existing_said_on} + is about: {existing_about} NEW: {new!r} + said on: {new_said_on} + is about: {new_about} + +Two clocks are shown for each statement. "said on" is the source's own date — +when the document was written or the conversation took place — and is NOT +when the described thing happened. "is about" is the world-time the statement +refers to, resolved from the source's wording against its said-on date: "last +week" said on 2022-10-06 is about the week before that date, not the week +before 2022-01-21, so two statements can both say "last week" and be about +days months apart. When the source tied nothing to a date, "is about" says so. +When either statement was ingested is irrelevant here and is not shown. Judge semantically (there are no typed columns — "FY2023" vs "fiscal 2023" and "headcount" vs "staff count" are your equivalence calls): @@ -56,7 +82,20 @@ - contradict: same property AND same reporting period, incompatible value — both must stand, surfaced together. (Different property, or different period, is NOT a contradiction.) -- new: a different property, period, or thing — no interaction.""" +- new: a different property, period, or thing — no interaction. + +Time is decisive for EVENTS (a win, a visit, a purchase, a meeting). Two +statements about datable events whose "is about" windows do NOT overlap are +two different occurrences — `new` — even when the wording is identical ("won +a tournament last week" said in January and again in October are two wins, +not one re-asserted). The one exception: when they plainly name the SAME +single occurrence and merely disagree about its date ("the Valorant final on +Friday" vs "the Valorant final on Saturday"), answer `contradict` so both +stand. Overlapping windows of different precision (a year-level claim and a +day-level one) may well be the same occurrence — judge by the wording. A +specific dated event is never `evidence` for a vaguer summary ("has won a +few tournaments"), and a summary never re-asserts a specific event — keep +both.""" class ObservationSettings(BaseSettings): @@ -238,8 +277,8 @@ def _apply_assertions_locked( if not assertions: return () claim_ids = list(dict.fromkeys(item.claim_id for item in assertions)) - asserted_by_claim = { - row["claim_id"]: row["asserted_at"] + timing_by_claim = { + row["claim_id"]: _timing_from_row(row) for row in connection.execute( _CLAIMS_ASSERTED, {"claim_ids": claim_ids} ).mappings() @@ -262,7 +301,7 @@ def _apply_assertions_locked( deployment_id=deployment_id, subject_entity_id=subject_entity_id, assertion=assertion, - asserted_at=asserted_by_claim.get(assertion.claim_id), + timing=timing_by_claim.get(assertion.claim_id, _UNDATED), candidates=candidates, meter=meter, call_key=f"{call_key}:{assertion_index}", @@ -277,21 +316,68 @@ def _add_with_block( deployment_id: UUID, subject_entity_id: UUID, assertion: ObservationAssertion, - asserted_at: object, + timing: _ClaimTiming, candidates: list[dict[str, object]], meter: CostMeterPort | None, call_key: str, ) -> UUID: """Apply one assertion while keeping the front-loaded block current.""" + asserted_at = timing.asserted_at + # An identical statement is the strongest same-fact signal — unless + # the two are not temporally compatible (D106): the same words about + # two dated events on different days ("won a tournament last week" in + # January and in October), or a dated event beside an undated copy, + # are separate rows, not one re-asserted. That decision needs no + # model, so it is taken here and recorded. + identical = [ + candidate + for candidate in candidates + if candidate["statement"] == assertion.statement + and bool(candidate["is_open"]) + ] exact = next( ( candidate - for candidate in candidates - if candidate["statement"] == assertion.statement - and bool(candidate["is_open"]) + for candidate in identical + if _evidence_compatible(timing=timing, candidate=candidate) ), None, ) + if exact is None and identical: + observation_id = self._insert_new( + connection=connection, + deployment_id=deployment_id, + subject_entity_id=subject_entity_id, + statement=assertion.statement, + claim_id=assertion.claim_id, + doc_id=assertion.doc_id, + valid_from=asserted_at, + outcome="add", + method="exact", # the identical-text rung decided; no model ran + confidence=1.0, + features={ + "reason": "identical statement, temporally incompatible -> coexist", + "temporal_gate": [ + { + "observation_id": str(candidate["observation_id"]), + "relation": _temporal_relation( + timing=timing, candidate=candidate + ), + } + for candidate in identical + ], + }, + related=UUID(str(identical[0]["observation_id"])), + contradiction_group=None, + ) + _remember_candidate( + candidates=candidates, + observation_id=observation_id, + statement=assertion.statement, + valid_from=asserted_at, + timing=timing, + ) + return observation_id if exact is not None: observation_id = UUID(str(exact["observation_id"])) self._evidence( @@ -301,6 +387,7 @@ def _add_with_block( claim_id=assertion.claim_id, doc_id=assertion.doc_id, ) + _absorb_timing(candidate=exact, timing=timing) # D88 continuous ingest: equivalent evidence must not leave # valid_from dependent on which version flushed first. Pull the # open window back to the source-earliest assertion time. @@ -334,6 +421,7 @@ def _add_with_block( observation_id=observation_id, statement=assertion.statement, valid_from=asserted_at, + timing=timing, ) return observation_id # Capped state slices are history, not competitors for the next @@ -363,6 +451,7 @@ def _add_with_block( observation_id=observation_id, statement=assertion.statement, valid_from=asserted_at, + timing=timing, ) return observation_id ranked = self._rank( @@ -392,6 +481,7 @@ def _add_with_block( observation_id=observation_id, statement=assertion.statement, valid_from=asserted_at, + timing=timing, ) return observation_id return self._adjudicate_residue( @@ -401,7 +491,7 @@ def _add_with_block( statement=assertion.statement, claim_id=assertion.claim_id, doc_id=assertion.doc_id, - asserted_at=asserted_at, + timing=timing, ranked=ranked[: self._settings.hub_top_k], candidates=candidates, meter=meter, @@ -412,7 +502,9 @@ def judge_statements( self, *, existing: str, new: str ) -> tuple[ObservationOutcome, float]: """The bare pair-decision function — the D43 eval gate's surface.""" - verdict, method = self._ladder(existing=existing, new=new) + verdict, method = self._ladder( + existing=existing, new=new, existing_timing=_UNDATED, new_timing=_UNDATED + ) del method # the gate grades outcomes; rungs are graded per-run cost return verdict.outcome, verdict.confidence @@ -425,25 +517,58 @@ def _adjudicate_residue( statement: str, claim_id: UUID, doc_id: UUID, - asserted_at: object, + timing: _ClaimTiming, ranked: list[tuple[dict[str, object], float]], candidates: list[dict[str, object]], meter: CostMeterPort | None, call_key: str, ) -> UUID: - """Ladder the similar candidates; apply the first decisive outcome.""" + """Ladder the similar candidates; apply the first decisive outcome. + + The temporal-compatibility rung (D106) bounds what a verdict may do, + using the D41 windows the claims already carry. Two dated events whose + resolved windows are disjoint are different occurrences unless the + model finds they name the SAME occurrence with disputed dates: they may + `contradict` (both stand, grouped) or stay `new`; `evidence` and + `supersede` are coerced to `new` and recorded. A dated event beside an + undated statement may still supersede or contradict it (a dated + resignation ends a "is CEO" state), but `evidence` is coerced to `new`: + a specific dated event is never a re-assertion of a vaguer statement, + and a summary never re-asserts a specific event. Undated pairs and + overlapping dated pairs are judged exactly as before. + """ + asserted_at = timing.asserted_at + coercions: list[dict[str, object]] = [] for candidate, similarity in ranked: + candidate_id = UUID(str(candidate["observation_id"])) + relation = _temporal_relation(timing=timing, candidate=candidate) verdict, method = self._ladder( existing=str(candidate["statement"]), new=statement, + existing_timing=_candidate_timing(candidate), + new_timing=timing, meter=meter, call_key=f"{call_key}:verdict:{candidate['observation_id']}", ) features: dict[str, object] = { "similarity": similarity, "rationale": verdict.rationale, + "temporal_relation": relation, + "temporal_gate": list(coercions), } - candidate_id = UUID(str(candidate["observation_id"])) + coerced = _coerced_reason(relation=relation, outcome=verdict.outcome) + if coerced is not None: + coercions.append( + { + "observation_id": str(candidate_id), + "verdict": verdict.outcome.value, + "relation": relation, + "reason": coerced, + "similarity": similarity, + "rationale": verdict.rationale, + } + ) + continue if verdict.outcome is ObservationOutcome.EVIDENCE: self._evidence( connection=connection, @@ -452,6 +577,7 @@ def _adjudicate_residue( claim_id=claim_id, doc_id=doc_id, ) + _absorb_timing(candidate=candidate, timing=timing) self._pull_valid_from_earlier( connection=connection, deployment_id=deployment_id, @@ -499,6 +625,7 @@ def _adjudicate_residue( observation_id=new_id, statement=statement, valid_from=asserted_at, + timing=timing, ) return new_id if verdict.confidence < self._settings.supersede_margin: @@ -527,6 +654,7 @@ def _adjudicate_residue( observation_id=new_id, statement=statement, valid_from=asserted_at, + timing=timing, ) return new_id # D88 continuous ingest: direction follows source time, not @@ -581,6 +709,7 @@ def _adjudicate_residue( observation_id=new_id, statement=statement, valid_from=asserted_at, + timing=timing, is_open=False, ) return new_id @@ -626,6 +755,7 @@ def _adjudicate_residue( observation_id=new_id, statement=statement, valid_from=asserted_at, + timing=timing, ) # D90 §5.5.3: evidence already on O after the incoming order # key must re-enter the ladder (staggered multi-version). @@ -675,6 +805,7 @@ def _adjudicate_residue( statement=statement, contradiction_group=group, valid_from=asserted_at, + timing=timing, ) return new_id # ObservationOutcome.NEW: no interaction with this candidate @@ -689,7 +820,7 @@ def _adjudicate_residue( outcome="add", method="small_model", confidence=1.0, - features={"reason": "no candidate interacted"}, + features={"reason": "no candidate interacted", "temporal_gate": coercions}, related=None, contradiction_group=None, ) @@ -698,6 +829,7 @@ def _adjudicate_residue( observation_id=new_id, statement=statement, valid_from=asserted_at, + timing=timing, ) return new_id @@ -706,11 +838,20 @@ def _ladder( *, existing: str, new: str, + existing_timing: _ClaimTiming, + new_timing: _ClaimTiming, meter: CostMeterPort | None = None, call_key: str = "observation:verdict", ) -> tuple[ObservationVerdict, str]: """Small-model verdict, escalating to frontier below the floor.""" - prompt = _VERDICT_PROMPT.format(existing=existing, new=new) + prompt = _VERDICT_PROMPT.format( + existing=existing, + new=new, + existing_said_on=_render_said_on(existing_timing), + existing_about=_render_about(existing_timing), + new_said_on=_render_said_on(new_timing), + new_about=_render_about(new_timing), + ) verdict_call = self._model_provider.generate( request=ModelRequest( model=self._settings.small_model, prompt=prompt, temperature=0.0 @@ -950,7 +1091,7 @@ def _resplit_later_evidence( assertion=ObservationAssertion( statement=capped_statement, claim_id=claim_id, doc_id=doc_id ), - asserted_at=claim_asserted, + timing=_timing_from_row(row), candidates=candidates, meter=meter, call_key="d90_late_arrival_resplit", @@ -1019,6 +1160,7 @@ def _remember_candidate( contradiction_group: UUID | None = None, valid_from: object = None, is_open: bool = True, + timing: _ClaimTiming | None = None, ) -> None: """Expose one in-transaction insert to later assertions in the batch.""" candidates.append( @@ -1028,10 +1170,260 @@ def _remember_candidate( "contradiction_group": contradiction_group, "valid_from": valid_from, "is_open": is_open, + "event_from": None if timing is None else timing.event_from, + "event_until": None if timing is None else timing.event_until, + "about_from": None if timing is None else timing.about_from, + "about_until": None if timing is None else timing.about_until, } ) +@dataclass(frozen=True) +class _ClaimTiming: + """What the D41 record says about WHEN one piece of testimony applies. + + Two clocks. ``asserted_at`` is when the SOURCE said it — the document's + or conversation's own timestamp (the supersession boundary the layer + already used). ``about_from``/``about_until`` is the world-time the + statement is ABOUT, resolved by the extractor from the source's wording + against that date, and ``about_kind`` is the D41 ``claim_valid_kind`` that + says what sort of interval it is: ``event_time`` (a datable event — the + only kind the temporal-compatibility rung acts on), ``measurement_period`` + / ``effective_period`` / ``proposition_validity`` (a figure or state tied + to a span), or ``period`` for a block row whose supporting claims are + aggregated. All three are ``None`` when the source tied nothing to a date. + When the statement was ingested is deliberately not part of this record. + """ + + asserted_at: object = None + about_kind: str | None = None + about_from: object = None + about_until: object = None + + @property + def is_event(self) -> bool: + """True when the testimony is a datable event with a resolved window.""" + return self.about_kind == "event_time" and self.about_from is not None + + @property + def event_from(self) -> object: + """The event window start, or ``None`` when this is not a dated event.""" + return self.about_from if self.is_event else None + + @property + def event_until(self) -> object: + """The event window end, or ``None`` when this is not a dated event.""" + return self.about_until if self.is_event else None + + +_UNDATED: Final = _ClaimTiming() + + +def _timing_from_row(row: Mapping[Any, Any]) -> _ClaimTiming: + """Read one claim row's said-on time and resolved about-window. + + A ``NULL`` ``claim_valid_until`` beside a resolved ``claim_valid_from`` is + a D41 open interval: the end is unbounded and stays ``None``. + """ + about_from = row.get("claim_valid_from") + if about_from is None: + return _ClaimTiming(asserted_at=row.get("asserted_at")) + return _ClaimTiming( + asserted_at=row.get("asserted_at"), + about_kind=row.get("valid_kind"), + about_from=about_from, + about_until=row.get("claim_valid_until"), + ) + + +def _candidate_timing(candidate: Mapping[str, object]) -> _ClaimTiming: + """The block row's timing, aggregated over its supporting claims. + + ``valid_from`` is the earliest said-on date of its testimony. The event + window (dated-event claims only) drives the rung; the wider about-window + (any D41 kind) is shown to the model as ``period`` when no event exists. + A ``None`` end is an open (unbounded) window, as in the claim rows. + """ + said_on = candidate.get("valid_from") + event_from = candidate.get("event_from") + if event_from is not None: + return _ClaimTiming( + asserted_at=said_on, + about_kind="event_time", + about_from=event_from, + about_until=candidate.get("event_until"), + ) + about_from = candidate.get("about_from") + if about_from is None: + return _ClaimTiming(asserted_at=said_on) + return _ClaimTiming( + asserted_at=said_on, + about_kind="period", + about_from=about_from, + about_until=candidate.get("about_until"), + ) + + +def _render_said_on(timing: _ClaimTiming) -> str: + """The "said on" prompt value: the source's own date, or its absence.""" + if timing.asserted_at is None: + return "unknown (the source carries no date)" + return _date_text(timing.asserted_at) + + +def _render_about(timing: _ClaimTiming) -> str: + """The "is about" prompt value: the resolved world-time, or its absence.""" + if timing.about_from is None: + return ( + "no specific time given (a state, summary, or figure the source" + " did not tie to a date)" + ) + start = _date_text(timing.about_from) + if timing.about_until is None: + span = f"from {start} onward (no end given)" + else: + end = _date_text(timing.about_until) + span = start if start == end else f"{start} to {end}" + if timing.is_event: + return ( + f"a dated event on {span}" + if " to " not in span and "onward" not in span + else f"a dated event within {span}" + ) + if " to " in span or "onward" in span: + return f"the period {span} (a state or figure tied to that span, not a dated event)" + return f"the day {span} (a state or figure tied to that day, not a dated event)" + + +def _date_text(value: object) -> str: + """Render a timestamp as its calendar date; anything else verbatim.""" + date = getattr(value, "date", None) + return str(date()) if callable(date) else str(value) + + +def _temporal_relation(*, timing: _ClaimTiming, candidate: Mapping[str, object]) -> str: + """How the incoming testimony's timing relates to a block row's (D106). + + ``undated`` — neither side is a dated event (states, figures, unanchored + testimony): judged as before. ``overlapping`` — both are dated events + whose windows touch or overlap: judged as before. ``disjoint`` — both + dated events, windows apart: different occurrences unless the model + finds one occurrence with disputed dates. ``mixed`` — exactly one side is + a dated event. + """ + other = _candidate_timing(candidate) + if timing.is_event and other.is_event: + apart = _windows_disjoint( + timing.event_from, timing.event_until, other.event_from, other.event_until + ) + return "disjoint" if apart else "overlapping" + if timing.is_event or other.is_event: + return "mixed" + return "undated" + + +def _coerced_reason(*, relation: str, outcome: ObservationOutcome) -> str | None: + """The reason a verdict is coerced to ``new`` under D106, or ``None``.""" + if relation == "disjoint" and outcome in ( + ObservationOutcome.EVIDENCE, + ObservationOutcome.SUPERSEDE, + ): + return ( + f"{outcome.value} coerced to new: dated events on different days are" + " different occurrences (a same-occurrence date dispute is contradict)" + ) + if relation == "mixed" and outcome is ObservationOutcome.EVIDENCE: + return ( + "evidence coerced to new: a dated event never re-asserts an undated" + " statement, nor the reverse" + ) + return None + + +def _evidence_compatible( + *, timing: _ClaimTiming, candidate: Mapping[str, object] +) -> bool: + """True when identical text may collapse without a verdict (D106). + + Both undated, or both dated events whose windows overlap. Disjoint dated + events and mixed dating are separate rows even for identical text. + """ + return _temporal_relation(timing=timing, candidate=candidate) in ( + "undated", + "overlapping", + ) + + +def _absorb_timing(*, candidate: dict[str, object], timing: _ClaimTiming) -> None: + """Widen a block row's in-memory windows after it absorbs new evidence. + + The database aggregate widens on the next block read; within one batch + the front-loaded row must not go stale, or a later overlapping claim + would be split off as a different occurrence (D106). + """ + if timing.is_event: + event_defined = candidate.get("event_from") is not None + candidate["event_from"] = _earliest( + candidate.get("event_from"), timing.event_from + ) + candidate["event_until"] = _latest_or_open( + candidate.get("event_until"), + timing.event_until, + existing_defined=event_defined, + ) + if timing.about_from is not None: + about_defined = candidate.get("about_from") is not None + candidate["about_from"] = _earliest( + candidate.get("about_from"), timing.about_from + ) + candidate["about_until"] = _latest_or_open( + candidate.get("about_until"), + timing.about_until, + existing_defined=about_defined, + ) + + +def _earliest(left: object, right: object) -> object: + """The earlier of two window starts; a missing start defers to the other.""" + if left is None: + return right + if right is None: + return left + try: + return left if left <= right else right # type: ignore[operator] + except TypeError: + return left + + +def _latest_or_open(left: object, right: object, *, existing_defined: bool) -> object: + """The later of two window ends, where ``None`` on a defined window is open.""" + if not existing_defined: + return right + if left is None or right is None: + return None + try: + return left if left >= right else right # type: ignore[operator] + except TypeError: + return left + + +def _windows_disjoint( + left_from: object, left_until: object, right_from: object, right_until: object +) -> bool: + """Closed-interval disjointness with ``None`` ends unbounded. + + Any incomparable value counts as overlap (the fail-safe direction). + """ + try: + if left_until is not None and bool(left_until < right_from): # type: ignore[operator] + return True + if right_until is not None and bool(right_until < left_from): # type: ignore[operator] + return True + return False + except TypeError: + return False + + def _cosine(a: tuple[float, ...], b: tuple[float, ...]) -> float: """Cosine similarity of two same-dimension vectors.""" if len(a) != len(b): @@ -1093,13 +1485,42 @@ def _is_later_in_total_order( _BLOCK_ENTITY = text( """ - SELECT observation_id, statement, contradiction_group, valid_from, - (valid_until IS NULL OR valid_until > now()) AS is_open - FROM observations - WHERE deployment_id = :deployment_id - AND subject_entity_id = :subject_entity_id - AND invalidated_at IS NULL - ORDER BY created_at + SELECT o.observation_id, o.statement, o.contradiction_group, o.valid_from, + (o.valid_until IS NULL OR o.valid_until > now()) AS is_open, + timing.event_from, timing.event_until, + timing.about_from, timing.about_until + FROM observations o + -- D106: what an observation is ABOUT in world-time is the span of its + -- supporting current testimony's D41 windows. The event window (dated + -- event claims only) drives the rung; the wider about-window is shown + -- to the model. + LEFT JOIN LATERAL ( + -- A NULL claim_valid_until beside a non-NULL claim_valid_from is a + -- D41 open interval (unbounded end); one open claim makes the + -- aggregate open, so the end stays NULL rather than a false maximum. + SELECT min(c.claim_valid_from) + FILTER (WHERE c.claim_valid_kind = 'event_time') AS event_from, + CASE WHEN bool_or(c.claim_valid_until IS NULL) + FILTER (WHERE c.claim_valid_kind = 'event_time') + THEN NULL + ELSE max(c.claim_valid_until) + FILTER (WHERE c.claim_valid_kind = 'event_time') + END AS event_until, + min(c.claim_valid_from) AS about_from, + CASE WHEN bool_or(c.claim_valid_until IS NULL) THEN NULL + ELSE max(c.claim_valid_until) + END AS about_until + FROM observation_evidence e + JOIN claims c ON c.claim_id = e.claim_id + WHERE e.observation_id = o.observation_id + AND e.stance = 'supports' + AND c.is_current_testimony + AND c.claim_valid_from IS NOT NULL + ) timing ON true + WHERE o.deployment_id = :deployment_id + AND o.subject_entity_id = :subject_entity_id + AND o.invalidated_at IS NULL + ORDER BY o.created_at """ ) @@ -1196,7 +1617,9 @@ def _is_later_in_total_order( _SELECT_EVIDENCE_FOR_OBS = text( """ - SELECT e.claim_id, e.doc_id, c.asserted_at + SELECT e.claim_id, e.doc_id, c.asserted_at, + c.claim_valid_kind::text AS valid_kind, + c.claim_valid_from, c.claim_valid_until FROM observation_evidence e JOIN claims c ON c.claim_id = e.claim_id WHERE e.deployment_id = :deployment_id @@ -1266,5 +1689,9 @@ def _is_later_in_total_order( ).bindparams(bindparam("features", type_=JSON)) _CLAIMS_ASSERTED = text( - "SELECT claim_id, asserted_at FROM claims WHERE claim_id = ANY(:claim_ids)" + """ + SELECT claim_id, asserted_at, claim_valid_kind::text AS valid_kind, + claim_valid_from, claim_valid_until + FROM claims WHERE claim_id = ANY(:claim_ids) + """ ) diff --git a/src/rememberstack/workers/e3.py b/src/rememberstack/workers/e3.py index 89590099..acfcbde8 100644 --- a/src/rememberstack/workers/e3.py +++ b/src/rememberstack/workers/e3.py @@ -85,8 +85,12 @@ def _run_profile_refresh(*, action: Callable[[], object], call_key: str) -> None Temperature=0.0 is part of provenance. """ -OBS_FLUSH_VERSION: Final = "e3-obs-flush-2026.08a:claim-fanout-1:entity-fanout-1" -"""Post-barrier observation flush generation (D88 §5.6; D90 entity fan-out).""" +OBS_FLUSH_VERSION: Final = ( + "e3-obs-flush-2026.09a:temporal-gate-1:claim-fanout-1:entity-fanout-1" +) +"""Post-barrier observation flush generation (D88 §5.6; D90 entity fan-out). +09a/temporal-gate-1: D106 — dated events with disjoint resolved windows never +collapse, and a dated event never becomes evidence for an undated statement.""" OBS_FLUSH_LEGACY_VERSION: Final = "e3-obs-flush-2026.08a:claim-fanout-1" """Pre-D90 version-serial obs flush component version (cutover only).""" @@ -657,7 +661,11 @@ def _handle_entity_unit( extractor_version=extractor_version, content_hash=content_hash, lane=work.lane, - obs_flush_component_version=OBS_FLUSH_VERSION, + # Report the claimed unit's own generation, not the current + # constant: a unit enqueued before an OBS_FLUSH_VERSION roll + # must complete under the generation its barrier counts, or + # the barrier never closes (D106 rollout contract). + obs_flush_component_version=work.component_version, doc_id=doc_id, ), ) diff --git a/src/tests/benchmarks/test_locomo_protocol.py b/src/tests/benchmarks/test_locomo_protocol.py index 020adddd..cda74dff 100644 --- a/src/tests/benchmarks/test_locomo_protocol.py +++ b/src/tests/benchmarks/test_locomo_protocol.py @@ -239,10 +239,10 @@ def test_current_protocol_pins_manifest_and_complete_read_plane() -> None: assert len(tool_catalog_sha256()) == 64 -def test_protocol_is_v20_and_answer_prompt_has_reasoning_and_loop_guards() -> None: +def test_protocol_is_v21_and_answer_prompt_has_reasoning_and_loop_guards() -> None: """The current identity, bounded inference, and loop discipline are locked.""" - assert PROTOCOL_NAME == "RS-LoCoMo-Full-v20" - assert DEFAULT_PROTOCOL_KEY == "full-v20" + assert PROTOCOL_NAME == "RS-LoCoMo-Full-v21" + assert DEFAULT_PROTOCOL_KEY == "full-v21" prompt = ANSWER_AGENT_PROMPT_TEMPLATE normalized_prompt = " ".join(prompt.split()) assert ( @@ -274,10 +274,10 @@ def test_protocol_is_v20_and_answer_prompt_has_reasoning_and_loop_guards() -> No def test_typed_protocol_registry_pins_answer_agent_identity_and_effort() -> None: - assert tuple(PROTOCOL_REGISTRY) == ("full-v20",) - protocol = PROTOCOL_REGISTRY["full-v20"] + assert tuple(PROTOCOL_REGISTRY) == ("full-v21",) + protocol = PROTOCOL_REGISTRY["full-v21"] - assert protocol.name == "RS-LoCoMo-Full-v20" + assert protocol.name == "RS-LoCoMo-Full-v21" assert protocol.answer_agent_model == "openai/gpt-5.6-luna" assert protocol.answer_agent_reasoning_effort == "none" assert protocol.judge_reasoning_effort == "none" @@ -315,7 +315,7 @@ def fake_prepare_run(**values: object) -> _Prepared: ) assert exit_code == 0 - assert selected == ["full-v20"] + assert selected == ["full-v21"] def test_summarize_cli_accepts_multiple_run_flags( diff --git a/src/tests/benchmarks/test_locomo_runner.py b/src/tests/benchmarks/test_locomo_runner.py index 4c34a2c7..e4b162cc 100644 --- a/src/tests/benchmarks/test_locomo_runner.py +++ b/src/tests/benchmarks/test_locomo_runner.py @@ -1047,7 +1047,7 @@ def test_answer_persists_usage_when_provider_drifts_after_tool_call() -> None: "invalid_first_step_completions", "invalid_reader_completions", ), - (("full-v20", "openai/gpt-5.6-luna", "none", 0, 2),), + (("full-v21", "openai/gpt-5.6-luna", "none", 0, 2),), ) def test_staged_mock_run_uses_prepared_protocol_and_resumes( protocol: ProtocolKey, @@ -1680,7 +1680,7 @@ def drifted_binding(request: httpx.Request) -> httpx.Response: def test_ingest_refuses_document_binding_generation_drift_before_upload( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - """Full-v20 cannot silently process with document-local T0 disabled.""" + """Full-v21 cannot silently process with document-local T0 disabled.""" _patch_prepared_inputs(monkeypatch=monkeypatch) run_dir = tmp_path / "run" prepare_run(dataset_path=tmp_path / "synthetic.json", tier="smoke", output=run_dir) @@ -1906,8 +1906,8 @@ def test_single_run_summary_json_is_unchanged( serialized = summarize_run(run_dir=run_dir).model_dump_json() assert serialized == ( - '{"protocol_name":"RS-LoCoMo-Full-v20","protocol_fingerprint":' - '"073711294b4c535ebdb51a0f7b1ddf74e314f9ca2cf110335b898c489f5ee916",' + '{"protocol_name":"RS-LoCoMo-Full-v21","protocol_fingerprint":' + '"0391b709fc76eb2cfc5748f62c6e3a294885bf2b329f1537715f6148ef455fef",' '"tier":"smoke","questions":1,"judge_correct":0,"judge_percent":0.0,' '"official_f1":0.0,"categories":[{"category":1,"questions":0,' '"judge_correct":0,"judge_percent":0.0,"official_f1":0.0},{"category":2,' @@ -2126,7 +2126,7 @@ def test_prepared_protocol_pins_current_surface_and_luna( dataset_path=tmp_path / "synthetic.json", tier="smoke", output=run_dir ) - assert prepared.protocol_name == "RS-LoCoMo-Full-v20" + assert prepared.protocol_name == "RS-LoCoMo-Full-v21" assert prepared.answer_agent_model == "openai/gpt-5.6-luna" assert prepared.answer_agent_reasoning_effort == "none" assert prepared.answer_reader_retry_budget == 2 diff --git a/src/tests/benchmarks/test_locomo_store_backup.py b/src/tests/benchmarks/test_locomo_store_backup.py index a3176754..06be91ac 100644 --- a/src/tests/benchmarks/test_locomo_store_backup.py +++ b/src/tests/benchmarks/test_locomo_store_backup.py @@ -24,7 +24,7 @@ def _run_json(path: Path) -> None: (path / "run.json").write_text( json.dumps( { - "protocol_name": "RS-LoCoMo-Full-v20", + "protocol_name": "RS-LoCoMo-Full-v21", "protocol_fingerprint": "p" * 64, "repository_revision": "r" * 40, "prepared_at": "2026-08-11T00:00:00Z", @@ -660,7 +660,7 @@ def test_restore_validates_every_archive_before_running_docker( deployment_id="57000000-0000-0000-0000-000000000001", compose_project="rememberstack", run=store_backup.RunIdentity( - protocol_name="RS-LoCoMo-Full-v20", + protocol_name="RS-LoCoMo-Full-v21", protocol_fingerprint="p" * 64, repository_revision="r" * 40, prepared_at="2026-08-11T00:00:00Z", @@ -922,7 +922,7 @@ def test_runtime_validation_uses_the_image_revision_stamp( sample_id="conv-1", deployment_id=deployment_id, run=store_backup.RunIdentity( - protocol_name="RS-LoCoMo-Full-v20", + protocol_name="RS-LoCoMo-Full-v21", protocol_fingerprint="p" * 64, repository_revision=revision, prepared_at="2026-08-11T00:00:00Z", diff --git a/src/tests/spikes/test_operational_scale.py b/src/tests/spikes/test_operational_scale.py index 4397e3e5..dcaebe0a 100644 --- a/src/tests/spikes/test_operational_scale.py +++ b/src/tests/spikes/test_operational_scale.py @@ -715,12 +715,18 @@ def _provider_neutral_batching( expected_hydration_statements = math.ceil( len(claim_ids) / INTERACTIVE_HYDRATION_BATCH_SIZE ) + # The front-loaded block and the batch's claim-timing read must each run + # once per entity batch (D43/D88). Match stable fragments of the two + # statements: the block reads `observations o` with its D106 timing + # lateral; the timing read selects `asserted_at` plus the D41 valid-time + # columns for the batch's claim ids. block_reads = sum( - "SELECT observation_id, statement, contradiction_group" in statement + "FROM observations o" in statement and "invalidated_at IS NULL" in statement for statement in observation_probe.statements ) timestamp_reads = sum( - "SELECT claim_id, asserted_at FROM claims" in statement + "SELECT claim_id, asserted_at" in statement + and "FROM claims WHERE claim_id = ANY(" in statement for statement in observation_probe.statements ) return OperationalScaleMeasurement( diff --git a/src/tests/spine/test_observation_adjudication.py b/src/tests/spine/test_observation_adjudication.py index 9ac3756c..a14ffd55 100644 --- a/src/tests/spine/test_observation_adjudication.py +++ b/src/tests/spine/test_observation_adjudication.py @@ -153,42 +153,158 @@ def _add( statement: str, engine: Engine | None = None, asserted_at: str | None = None, + event_window: tuple[str, str | None] | None = None, + doc_id: UUID | None = None, ) -> UUID: """One observation through the cascade with a fresh claim. With `asserted_at`, a real dated claim row backs the testimony (the D41 seed the boundary math reads); without it, the testimony is undated. + With `event_window`, the claim is a datable EVENT resolved to that + `[from, until]` window (`claim_valid_kind = 'event_time'`, day precision; + an `until` of `None` is a D41 open interval), the input the D106 + temporal-compatibility rung reads. """ + doc_id = doc_id or uuid4() claim_id = uuid4() if asserted_at is not None and engine is not None: - with engine.begin() as connection: - connection.execute( - text( - "INSERT INTO claims (claim_id, deployment_id, doc_id," - " chunk_id, claim_text, source_span, char_start, char_end," - " anchor_ok, window_membership_ok, extractor_version," - " asserted_at)" - " VALUES (:c, :d, :doc, :ch, :s, :s, 0, 1, true, true," - " 'test', CAST(:a AS timestamptz))" - ), - { - "c": claim_id, - "d": _DEPLOYMENT_ID, - "doc": uuid4(), - "ch": uuid4(), - "s": statement, - "a": asserted_at, - }, - ) + _claim( + engine=engine, + claim_id=claim_id, + doc_id=doc_id, + statement=statement, + asserted_at=asserted_at, + event_window=event_window, + ) return adjudicator.add_observation( deployment_id=_DEPLOYMENT_ID, subject_entity_id=entity, statement=statement, claim_id=claim_id, - doc_id=uuid4(), + doc_id=doc_id, ) +def _claim( + *, + engine: Engine, + claim_id: UUID, + doc_id: UUID, + statement: str, + asserted_at: str, + event_window: tuple[str, str | None] | None = None, +) -> None: + """Insert one dated claim row, optionally a resolved D41 event window.""" + precision = "unknown" + if event_window is not None: + precision = "open" if event_window[1] is None else "day" + with engine.begin() as connection: + connection.execute( + text( + "INSERT INTO claims (claim_id, deployment_id, doc_id," + " chunk_id, claim_text, source_span, char_start, char_end," + " anchor_ok, window_membership_ok, extractor_version," + " asserted_at, claim_valid_kind, claim_valid_precision," + " claim_valid_from, claim_valid_until)" + " VALUES (:c, :d, :doc, :ch, :s, :s, 0, 1, true, true," + " 'test', CAST(:a AS timestamptz)," + " CAST(:kind AS claim_valid_kind)," + " CAST(:precision AS claim_valid_precision)," + " CAST(:vf AS timestamptz), CAST(:vu AS timestamptz))" + ), + { + "c": claim_id, + "d": _DEPLOYMENT_ID, + "doc": doc_id, + "ch": uuid4(), + "s": statement, + "a": asserted_at, + "kind": None if event_window is None else "event_time", + "precision": precision, + "vf": None if event_window is None else event_window[0], + "vu": None if event_window is None else event_window[1], + }, + ) + + +def _collapse_happy_router(prompt: str, type_name: str) -> dict[str, object]: + """The failure the D106 rung exists to stop: a small model that reads + "won a tournament last week" twice and calls the second a re-assertion. + + It answers `evidence` for any pair about the same recurring activity — + exactly what the conv-42 run's adjudicator did to five of seven wins — + so a test passes only when the deterministic rung keeps it from mattering. + """ + if type_name != "ObservationVerdict": + raise AssertionError(f"unexpected generate call: {type_name}") + lower = prompt.lower() + if "tournament" in lower or "participa" in lower or "competing" in lower: + return {"outcome": "evidence", "confidence": 0.95} + return {"outcome": "new", "confidence": 0.9} + + +def _observations(*, engine: Engine, entity: UUID) -> list[dict[str, object]]: + """The entity's live observations, oldest window first.""" + with engine.connect() as connection: + return [ + dict(row) + for row in connection.execute( + text( + "SELECT observation_id, statement, valid_from, evidence_count" + " FROM observations" + " WHERE subject_entity_id = :e AND invalidated_at IS NULL" + " ORDER BY valid_from NULLS LAST, statement" + ), + {"e": entity}, + ).mappings() + ] + + +def _verdict_calls(provider: FakeModelProvider) -> int: + """How many verdict prompts the ladder actually bought.""" + return sum(1 for prompt in provider.generated_prompts if "EXISTING:" in prompt) + + +# Seven distinct wins as conv-42 reported them: same shape, months apart. +_SEVEN_WINS: tuple[tuple[str, str, str], ...] = ( + ( + "Nate said he won his first video game tournament last week.", + "2022-01-21T19:31:00Z", + "2022-01-14", + ), + ( + "Nate said that Nate won Nate's second tournament last week.", + "2022-05-02T11:54:00Z", + "2022-04-25", + ), + ( + "Nate just won another regional video game tournament last week.", + "2022-06-03T17:44:00Z", + "2022-05-27", + ), + ( + "Nate won Nate's fourth video game tournament on Friday.", + "2022-07-10T14:34:00Z", + "2022-07-08", + ), + ( + "Nate won an international tournament yesterday.", + "2022-08-22T10:57:00Z", + "2022-08-21", + ), + ( + "Nate said that Nate won a really big video game tournament last week.", + "2022-10-06T11:15:00Z", + "2022-09-29", + ), + ( + "Nate won the final of a big Valorant tournament last Saturday.", + "2022-11-07T20:10:00Z", + "2022-11-05", + ), +) + + def test_d90_staggered_late_arrival_resplit_shapes(database_engine: Engine) -> None: """D90 §5.5.3: A{t1,t3} first then B{t2} yields A[t1,t2), B[t2,t3), A[t3,∞).""" adjudicator, _provider = _adjudicator(engine=database_engine) @@ -674,3 +790,467 @@ def test_stance_content_never_becomes_a_fact(database_engine: Engine) -> None: ).scalar_one() assert subjects == [team] # anchored on the holder, nowhere else assert relations == 0 # no fact about Atlas was derived + + +def test_d106_seven_dated_wins_survive_as_seven_facts(database_engine: Engine) -> None: + """The conv-42 counting case: seven same-shaped wins on seven dates must + stay seven observations whatever the model says about any pair.""" + adjudicator, provider = _adjudicator( + engine=database_engine, router=_collapse_happy_router + ) + nate = _entity(engine=database_engine) + for statement, asserted_at, day in _SEVEN_WINS: + _add( + adjudicator=adjudicator, + entity=nate, + statement=statement, + engine=database_engine, + asserted_at=asserted_at, + event_window=(day, day), + ) + rows = _observations(engine=database_engine, entity=nate) + assert [row["statement"] for row in rows] == [win[0] for win in _SEVEN_WINS] + assert all(row["evidence_count"] == 1 for row in rows) + # Disjoint dated pairs still reach the date-aware model (a same-occurrence + # date dispute must be able to surface as a contradiction); its + # collapse-happy `evidence` verdicts are coerced to `new` and audited. + assert _verdict_calls(provider) > 0 + with database_engine.connect() as connection: + noops = connection.execute( + text( + "SELECT count(*) FROM observation_adjudications a" + " JOIN observations o ON o.observation_id = a.observation_id" + " WHERE o.subject_entity_id = :e AND a.outcome = 'noop'" + ), + {"e": nate}, + ).scalar_one() + assert noops == 0 + + +def test_d106_arrival_order_does_not_change_the_count(database_engine: Engine) -> None: + """Reverse (and shuffled) arrival yields the same seven facts.""" + adjudicator, _provider = _adjudicator( + engine=database_engine, router=_collapse_happy_router + ) + nate = _entity(engine=database_engine) + shuffled = ( + _SEVEN_WINS[6], + _SEVEN_WINS[2], + _SEVEN_WINS[0], + _SEVEN_WINS[4], + _SEVEN_WINS[1], + _SEVEN_WINS[5], + _SEVEN_WINS[3], + ) + for statement, asserted_at, day in shuffled: + _add( + adjudicator=adjudicator, + entity=nate, + statement=statement, + engine=database_engine, + asserted_at=asserted_at, + event_window=(day, day), + ) + rows = _observations(engine=database_engine, entity=nate) + assert len(rows) == 7 + assert [row["statement"] for row in rows] == [win[0] for win in _SEVEN_WINS] + + +def test_d106_same_event_mentioned_twice_still_collapses( + database_engine: Engine, +) -> None: + """The rung is about DIFFERENT events: a re-mention of one event (same + resolved date) still collapses to one observation with two lineages.""" + adjudicator, provider = _adjudicator( + engine=database_engine, router=_collapse_happy_router + ) + nate = _entity(engine=database_engine) + statement = "Nate won an international tournament yesterday." + for asserted_at in ("2022-08-22T10:57:00Z", "2022-08-22T18:00:00Z"): + _add( + adjudicator=adjudicator, + entity=nate, + statement=statement, + engine=database_engine, + asserted_at=asserted_at, + event_window=("2022-08-21", "2022-08-21"), + ) + rows = _observations(engine=database_engine, entity=nate) + assert len(rows) == 1 and rows[0]["evidence_count"] == 2 + assert _verdict_calls(provider) == 0 # exact re-assertion: zero LLM + + +def test_d106_identical_words_about_two_dates_are_two_events( + database_engine: Engine, +) -> None: + """ "won a tournament last week" in January and again in October is two + wins even though the strings are byte-identical.""" + adjudicator, provider = _adjudicator( + engine=database_engine, router=_collapse_happy_router + ) + nate = _entity(engine=database_engine) + statement = "Nate said that Nate won a video game tournament last week." + for asserted_at, day in ( + ("2022-01-21T19:31:00Z", "2022-01-14"), + ("2022-10-06T11:15:00Z", "2022-09-29"), + ): + _add( + adjudicator=adjudicator, + entity=nate, + statement=statement, + engine=database_engine, + asserted_at=asserted_at, + event_window=(day, day), + ) + rows = _observations(engine=database_engine, entity=nate) + assert len(rows) == 2 and all(row["evidence_count"] == 1 for row in rows) + assert _verdict_calls(provider) == 0 # identical text: decided without a model + + +def test_d106_a_vague_summary_never_absorbs_a_dated_event( + database_engine: Engine, +) -> None: + """ "has been winning a few tournaments" (undated) must not swallow the + later dated international win as evidence, whatever the model says — + and the verdict prompt must have shown the model both timelines.""" + adjudicator, provider = _adjudicator( + engine=database_engine, router=_collapse_happy_router + ) + nate = _entity(engine=database_engine) + _add( + adjudicator=adjudicator, + entity=nate, + statement="Nate said that Nate has been winning a few gaming tournaments.", + engine=database_engine, + asserted_at="2022-05-02T11:54:00Z", + ) + dated = _add( + adjudicator=adjudicator, + entity=nate, + statement="Nate won an international tournament yesterday.", + engine=database_engine, + asserted_at="2022-08-22T10:57:00Z", + event_window=("2022-08-21", "2022-08-21"), + ) + rows = _observations(engine=database_engine, entity=nate) + assert len(rows) == 2 and all(row["evidence_count"] == 1 for row in rows) + assert _verdict_calls(provider) == 1 # mixed pair: judged, then coerced + last_prompt = provider.generated_prompts[-1] + assert "said on: 2022-05-02\n is about: no specific time given" in last_prompt + assert "said on: 2022-08-22\n is about: a dated event on 2022-08-21" in last_prompt + with database_engine.connect() as connection: + features = connection.execute( + text( + "SELECT features FROM observation_adjudications" + " WHERE observation_id = :o AND outcome = 'add'" + ), + {"o": dated}, + ).scalar_one() + assert features["temporal_gate"][0]["reason"].startswith("evidence coerced to new") + + +def test_d106_a_dated_event_never_absorbs_a_later_summary( + database_engine: Engine, +) -> None: + """The mirror image: a summary arriving after a dated event coexists.""" + adjudicator, _provider = _adjudicator( + engine=database_engine, router=_collapse_happy_router + ) + nate = _entity(engine=database_engine) + _add( + adjudicator=adjudicator, + entity=nate, + statement="Nate won an international tournament yesterday.", + engine=database_engine, + asserted_at="2022-08-22T10:57:00Z", + event_window=("2022-08-21", "2022-08-21"), + ) + _add( + adjudicator=adjudicator, + entity=nate, + statement="Nate said that Nate has been winning a few gaming tournaments.", + engine=database_engine, + asserted_at="2022-10-06T11:15:00Z", + ) + assert len(_observations(engine=database_engine, entity=nate)) == 2 + + +def test_d106_boilerplate_state_never_absorbs_dated_participation( + database_engine: Engine, +) -> None: + """conv-42's "Nate is a participant." (document-header boilerplate, nine + lineages) swallowed two real tournament entries. The boilerplate may + still collapse onto itself; the dated events stay distinct facts.""" + adjudicator, _provider = _adjudicator( + engine=database_engine, router=_collapse_happy_router + ) + nate = _entity(engine=database_engine) + for asserted_at in ( + "2022-01-21T19:31:00Z", + "2022-02-07T09:00:00Z", + "2022-03-01T09:00:00Z", + ): + _add( + adjudicator=adjudicator, + entity=nate, + statement="Nate is a participant.", + engine=database_engine, + asserted_at=asserted_at, + ) + for statement, asserted_at, day in ( + ( + "Nate said that Nate is currently participating in the video game tournament again.", + "2022-03-24T15:00:00Z", + "2022-03-24", + ), + ( + "Nate said that Nate tried playing in the local Street Fighter tournament this time.", + "2022-04-25T15:00:00Z", + "2022-04-25", + ), + ): + _add( + adjudicator=adjudicator, + entity=nate, + statement=statement, + engine=database_engine, + asserted_at=asserted_at, + event_window=(day, day), + ) + rows = _observations(engine=database_engine, entity=nate) + assert len(rows) == 3 + boilerplate = next( + row for row in rows if row["statement"] == "Nate is a participant." + ) + assert boilerplate["evidence_count"] == 3 + + +def test_d106_identical_text_dated_beside_undated_coexists( + database_engine: Engine, +) -> None: + """One anchored and one unanchored extraction of the same sentence are + kept apart (a duplicate, never a silent merge), with no model call.""" + adjudicator, provider = _adjudicator( + engine=database_engine, router=_collapse_happy_router + ) + alice = _entity(engine=database_engine) + statement = "Alice said she resigned from the tournament committee yesterday." + _add( + adjudicator=adjudicator, + entity=alice, + statement=statement, + engine=database_engine, + asserted_at="2026-03-02T09:00:00Z", + event_window=("2026-03-01", "2026-03-01"), + ) + _add( + adjudicator=adjudicator, + entity=alice, + statement=statement, + engine=database_engine, + asserted_at="2026-03-09T09:00:00Z", + ) + rows = _observations(engine=database_engine, entity=alice) + assert len(rows) == 2 and all(row["evidence_count"] == 1 for row in rows) + assert _verdict_calls(provider) == 0 + + +def test_d106_open_window_is_unbounded_not_a_point(database_engine: Engine) -> None: + """A D41 open interval ``[from, ∞)`` overlaps every later day, so a later + dated re-mention collapses onto it instead of splitting off.""" + adjudicator, _provider = _adjudicator( + engine=database_engine, router=_collapse_happy_router + ) + nate = _entity(engine=database_engine) + statement = "Nate has been competing in the regional tournament circuit." + _add( + adjudicator=adjudicator, + entity=nate, + statement=statement, + engine=database_engine, + asserted_at="2026-01-02T09:00:00Z", + event_window=("2026-01-01", None), + ) + _add( + adjudicator=adjudicator, + entity=nate, + statement=statement, + engine=database_engine, + asserted_at="2026-02-16T09:00:00Z", + event_window=("2026-02-15", "2026-02-15"), + ) + rows = _observations(engine=database_engine, entity=nate) + assert len(rows) == 1 and rows[0]["evidence_count"] == 2 + + +def test_d106_overlapping_windows_of_different_precision_may_collapse( + database_engine: Engine, +) -> None: + """A year-level claim and a day-level claim inside that year overlap, so + the model's `evidence` verdict is honoured (no coercion).""" + adjudicator, provider = _adjudicator( + engine=database_engine, router=_collapse_happy_router + ) + nate = _entity(engine=database_engine) + _add( + adjudicator=adjudicator, + entity=nate, + statement="Nate said he won a video game tournament last year.", + engine=database_engine, + asserted_at="2023-02-01T09:00:00Z", + event_window=("2022-01-01", "2022-12-31"), + ) + _add( + adjudicator=adjudicator, + entity=nate, + statement="Nate won Nate's fourth video game tournament on Friday.", + engine=database_engine, + asserted_at="2022-07-10T14:34:00Z", + event_window=("2022-07-08", "2022-07-08"), + ) + rows = _observations(engine=database_engine, entity=nate) + assert len(rows) == 1 and rows[0]["evidence_count"] == 2 + assert _verdict_calls(provider) == 1 + assert ( + "is about: a dated event within 2022-01-01 to 2022-12-31" + in (provider.generated_prompts[-1]) + ) + + +def test_d106_batch_widens_the_candidate_window_as_it_absorbs( + database_engine: Engine, +) -> None: + """Within one batch the front-loaded row must not go stale: after + ``[Jan 1–10]`` absorbs an overlapping ``[Jan 10–20]`` re-mention, a + ``[Jan 20]`` re-mention still overlaps and collapses too.""" + adjudicator, provider = _adjudicator( + engine=database_engine, router=_collapse_happy_router + ) + nate = _entity(engine=database_engine) + statement = "Nate said he was competing in the winter tournament." + doc_id = uuid4() + claims = [] + for asserted_at, window in ( + ("2026-01-11T09:00:00Z", ("2026-01-01", "2026-01-10")), + ("2026-01-21T09:00:00Z", ("2026-01-10", "2026-01-20")), + ("2026-01-21T18:00:00Z", ("2026-01-20", "2026-01-20")), + ): + claim_id = uuid4() + _claim( + engine=database_engine, + claim_id=claim_id, + doc_id=doc_id, + statement=statement, + asserted_at=asserted_at, + event_window=window, + ) + claims.append(claim_id) + adjudicator.add_observations( + deployment_id=_DEPLOYMENT_ID, + subject_entity_id=nate, + assertions=tuple( + ObservationAssertion(statement=statement, claim_id=claim_id, doc_id=doc_id) + for claim_id in claims + ), + ) + rows = _observations(engine=database_engine, entity=nate) + assert len(rows) == 1 + with database_engine.connect() as connection: + evidence = connection.execute( + text("SELECT count(*) FROM observation_evidence WHERE observation_id = :o"), + {"o": rows[0]["observation_id"]}, + ).scalar_one() + assert evidence == 3 + assert _verdict_calls(provider) == 0 + + +def test_d106_disjoint_dates_may_still_contradict_one_occurrence( + database_engine: Engine, +) -> None: + """Two sources disagreeing about the date of ONE named final are a + contradiction (both stand, grouped) — not two finals, not one merged.""" + + def router(prompt: str, type_name: str) -> dict[str, object]: + assert type_name == "ObservationVerdict" + return {"outcome": "contradict", "confidence": 0.9} + + adjudicator, provider = _adjudicator(engine=database_engine, router=router) + nate = _entity(engine=database_engine) + _add( + adjudicator=adjudicator, + entity=nate, + statement="Nate won the 2022 Valorant regional final on Friday.", + engine=database_engine, + asserted_at="2022-11-07T20:10:00Z", + event_window=("2022-11-04", "2022-11-04"), + ) + _add( + adjudicator=adjudicator, + entity=nate, + statement="Nate won the 2022 Valorant regional final on Saturday.", + engine=database_engine, + asserted_at="2022-11-08T20:10:00Z", + event_window=("2022-11-05", "2022-11-05"), + ) + with database_engine.connect() as connection: + groups = ( + connection.execute( + text( + "SELECT contradiction_group FROM observations" + " WHERE subject_entity_id = :e" + ), + {"e": nate}, + ) + .scalars() + .all() + ) + assert len(groups) == 2 and groups[0] is not None and groups[0] == groups[1] + assert _verdict_calls(provider) == 1 + + +def test_d106_a_dated_event_may_still_supersede_an_undated_state( + database_engine: Engine, +) -> None: + """Mixed dating forbids only `evidence`: a dated resignation can end an + undated "is CEO" state under the ordinary cap contract.""" + + def router(prompt: str, type_name: str) -> dict[str, object]: + assert type_name == "ObservationVerdict" + return { + "outcome": "supersede", + "confidence": 0.9, + "rationale": "the resignation ends the CEO state", + } + + adjudicator, _provider = _adjudicator(engine=database_engine, router=router) + alice = _entity(engine=database_engine) + _add( + adjudicator=adjudicator, + entity=alice, + statement="Alice is the CEO of Acme.", + engine=database_engine, + asserted_at="2025-06-01T09:00:00Z", + ) + _add( + adjudicator=adjudicator, + entity=alice, + statement="Alice resigned as CEO of Acme yesterday.", + engine=database_engine, + asserted_at="2026-03-02T09:00:00Z", + event_window=("2026-03-01", "2026-03-01"), + ) + with database_engine.connect() as connection: + rows = [ + dict(row) + for row in connection.execute( + text( + "SELECT statement, valid_until FROM observations" + " WHERE subject_entity_id = :e ORDER BY valid_from" + ), + {"e": alice}, + ).mappings() + ] + assert len(rows) == 2 + assert rows[0]["statement"] == "Alice is the CEO of Acme." + assert rows[0]["valid_until"] is not None # capped by the dated event + assert rows[1]["valid_until"] is None diff --git a/src/tests/workers/test_e3_entity_obs_flush_fanout.py b/src/tests/workers/test_e3_entity_obs_flush_fanout.py index cd9014aa..58c0c7e0 100644 --- a/src/tests/workers/test_e3_entity_obs_flush_fanout.py +++ b/src/tests/workers/test_e3_entity_obs_flush_fanout.py @@ -162,3 +162,15 @@ def test_entity_obs_flush_barrier_fields() -> None: "obs_flush_component_version", ): assert name in fields + + +def test_entity_handler_reports_the_claimed_units_own_generation() -> None: + """D106 rollout: a unit enqueued before an OBS_FLUSH_VERSION roll must + complete under the generation its barrier counts, or the barrier never + closes — so the entity handler reports ``work.component_version``, not + the current constant, to ``EntityObsFlushBarrier``.""" + import inspect + + source = inspect.getsource(e3.AdjudicateObservationsHandler._handle_entity_unit) + assert "obs_flush_component_version=work.component_version" in source + assert "obs_flush_component_version=OBS_FLUSH_VERSION" not in source diff --git a/website/src/app/docs/project-status/page.mdx b/website/src/app/docs/project-status/page.mdx index 93f3ebb8..73a97746 100644 --- a/website/src/app/docs/project-status/page.mdx +++ b/website/src/app/docs/project-status/page.mdx @@ -250,8 +250,15 @@ examined, and the failure looks like the deployment being down. hypothetical may follow causal or motivational relationships in retrieved evidence even when no source states the hypothetical verbatim. It adds no retrieval, retry, reasoning-effort, or call-budget work. -- The current `RS-LoCoMo-Full-v20` also requires every distinct retrieved value +- Historical `RS-LoCoMo-Full-v20` also requires every distinct retrieved value that directly satisfies the question. It does not add a retrieval or retry. +- The current `RS-LoCoMo-Full-v21` pins the D106 observation adjudicator: + dated events with disjoint resolved windows never collapse onto or supersede + each other (a same-occurrence date dispute may contradict), and a dated + event is never `evidence` for an undated statement. Recurring + same-shaped events (seven tournament wins months apart) now survive as + distinct facts. It changes ingestion provenance only; retrieval, prompts, + budgets, and scoring are those of v20. ## What landed after v0.6.0 (in v0.7.0; historical resolver generation)