From 7cde86d75cd86660e0532a012cfae30b3a5fb58b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 00:44:20 +0000 Subject: [PATCH] test: add replayable challenge-pack fixtures Checked-in Crashlanded packs plus a unit loader that evaluates pass/fail over fixture receipts. No live RimWorld and no billed model calls. Scoring and published claims are unchanged. Co-authored-by: Jason --- .../composite_receipt/challenge.yaml | 39 ++ .../receipts/fail/summary.json | 13 + .../receipts/pass/summary.json | 14 + .../ensure_live_save/challenge.yaml | 27 ++ .../receipts/fail/preflight.json | 7 + .../receipts/pass/preflight.json | 7 + .../preflight_contract/challenge.yaml | 45 ++ .../receipts/fail/events.jsonl | 1 + .../receipts/fail/preflight.json | 7 + .../receipts/pass/events.jsonl | 1 + .../receipts/pass/preflight.json | 7 + .../research_unblock/challenge.yaml | 41 ++ .../receipts/fail/preflight.json | 17 + .../receipts/pass/preflight.json | 17 + tests/unit/challenge_pack_loader.py | 444 ++++++++++++++++++ tests/unit/test_challenge_packs.py | 150 ++++++ 16 files changed, 837 insertions(+) create mode 100644 tests/fixtures/challenge_packs/composite_receipt/challenge.yaml create mode 100644 tests/fixtures/challenge_packs/composite_receipt/receipts/fail/summary.json create mode 100644 tests/fixtures/challenge_packs/composite_receipt/receipts/pass/summary.json create mode 100644 tests/fixtures/challenge_packs/ensure_live_save/challenge.yaml create mode 100644 tests/fixtures/challenge_packs/ensure_live_save/receipts/fail/preflight.json create mode 100644 tests/fixtures/challenge_packs/ensure_live_save/receipts/pass/preflight.json create mode 100644 tests/fixtures/challenge_packs/preflight_contract/challenge.yaml create mode 100644 tests/fixtures/challenge_packs/preflight_contract/receipts/fail/events.jsonl create mode 100644 tests/fixtures/challenge_packs/preflight_contract/receipts/fail/preflight.json create mode 100644 tests/fixtures/challenge_packs/preflight_contract/receipts/pass/events.jsonl create mode 100644 tests/fixtures/challenge_packs/preflight_contract/receipts/pass/preflight.json create mode 100644 tests/fixtures/challenge_packs/research_unblock/challenge.yaml create mode 100644 tests/fixtures/challenge_packs/research_unblock/receipts/fail/preflight.json create mode 100644 tests/fixtures/challenge_packs/research_unblock/receipts/pass/preflight.json create mode 100644 tests/unit/challenge_pack_loader.py create mode 100644 tests/unit/test_challenge_packs.py diff --git a/tests/fixtures/challenge_packs/composite_receipt/challenge.yaml b/tests/fixtures/challenge_packs/composite_receipt/challenge.yaml new file mode 100644 index 0000000..475cf7a --- /dev/null +++ b/tests/fixtures/challenge_packs/composite_receipt/challenge.yaml @@ -0,0 +1,39 @@ +schema_version: 1 +id: composite_receipt +scenario: Crashlanded Survival +seed: 42 +tick_budget: 10 +mode: fixture +harness: fixture +model: fixture +scoring_version: "1.2" +expected: + - composite_receipt +pass_fail: + - judgment: composite_receipt + receipt: summary.json + match: object + all: + - path: scoring_version + eq: "1.2" + - path: scenario + eq: Crashlanded Survival + - path: random_seed + eq: 42 + - path: harness + eq: fixture + - path: model + eq: fixture + - path: ticks_run + gte: 1 + - path: ticks_run + lte: 10 + - path: final_score + type: number + - path: outcome + type: string + absent: + - reasoning_trace + - reasoning_traces + - pass_rate + - n_runs diff --git a/tests/fixtures/challenge_packs/composite_receipt/receipts/fail/summary.json b/tests/fixtures/challenge_packs/composite_receipt/receipts/fail/summary.json new file mode 100644 index 0000000..836b004 --- /dev/null +++ b/tests/fixtures/challenge_packs/composite_receipt/receipts/fail/summary.json @@ -0,0 +1,13 @@ +{ + "scoring_version": "1.1", + "scenario": "Crashlanded Survival", + "random_seed": 42, + "harness": "fixture", + "model": "fixture", + "ticks_run": 10, + "outcome": "timeout", + "final_score": "narrative", + "n_runs": 2, + "pass_rate": true, + "reasoning_traces": [{"summary": "abridged"}] +} diff --git a/tests/fixtures/challenge_packs/composite_receipt/receipts/pass/summary.json b/tests/fixtures/challenge_packs/composite_receipt/receipts/pass/summary.json new file mode 100644 index 0000000..a7f7f67 --- /dev/null +++ b/tests/fixtures/challenge_packs/composite_receipt/receipts/pass/summary.json @@ -0,0 +1,14 @@ +{ + "scoring_version": "1.2", + "scenario": "Crashlanded Survival", + "scenario_save_name": "rle_crashlanded_v1", + "random_seed": 42, + "harness": "fixture", + "model": "fixture", + "ticks_run": 10, + "max_ticks": 10, + "outcome": "timeout", + "final_score": 0.0, + "live_save_sha256": "pin-match", + "live_save_copied": false +} diff --git a/tests/fixtures/challenge_packs/ensure_live_save/challenge.yaml b/tests/fixtures/challenge_packs/ensure_live_save/challenge.yaml new file mode 100644 index 0000000..68bbc7e --- /dev/null +++ b/tests/fixtures/challenge_packs/ensure_live_save/challenge.yaml @@ -0,0 +1,27 @@ +schema_version: 1 +id: ensure_live_save +scenario: Crashlanded Survival +seed: 42 +tick_budget: 10 +mode: fixture +harness: fixture +model: fixture +scoring_version: "1.2" +expected: + - ensure_live_save +pass_fail: + - judgment: ensure_live_save + receipt: preflight.json + match: object + all: + - path: save_name + eq: rle_crashlanded_v1 + - path: live_save_sha256 + eq_path: pinned_sha256 + - path: error + absent: true + - path: copied + type: bool + absent: + - reasoning_trace + - reasoning_traces diff --git a/tests/fixtures/challenge_packs/ensure_live_save/receipts/fail/preflight.json b/tests/fixtures/challenge_packs/ensure_live_save/receipts/fail/preflight.json new file mode 100644 index 0000000..38f94ec --- /dev/null +++ b/tests/fixtures/challenge_packs/ensure_live_save/receipts/fail/preflight.json @@ -0,0 +1,7 @@ +{ + "save_name": "rle_crashlanded_v1", + "pinned_sha256": "pin-match", + "live_save_sha256": "stale-appdata", + "copied": false, + "error": "LiveSavePinError" +} diff --git a/tests/fixtures/challenge_packs/ensure_live_save/receipts/pass/preflight.json b/tests/fixtures/challenge_packs/ensure_live_save/receipts/pass/preflight.json new file mode 100644 index 0000000..18f1783 --- /dev/null +++ b/tests/fixtures/challenge_packs/ensure_live_save/receipts/pass/preflight.json @@ -0,0 +1,7 @@ +{ + "save_name": "rle_crashlanded_v1", + "pinned_sha256": "pin-match", + "live_save_sha256": "pin-match", + "copied": false, + "error": null +} diff --git a/tests/fixtures/challenge_packs/preflight_contract/challenge.yaml b/tests/fixtures/challenge_packs/preflight_contract/challenge.yaml new file mode 100644 index 0000000..867939b --- /dev/null +++ b/tests/fixtures/challenge_packs/preflight_contract/challenge.yaml @@ -0,0 +1,45 @@ +schema_version: 1 +id: preflight_contract +scenario: Crashlanded Survival +seed: 42 +tick_budget: 10 +mode: fixture +harness: fixture +model: fixture +scoring_version: "1.2" +expected: + - stockpile_delete_quarantine + - preflight_contract +pass_fail: + - judgment: stockpile_delete_quarantine + receipt: events.jsonl + match: any + where: + event_type: action_exec + all: + - path: data.action_type + eq: stockpile_delete + - path: data.success + eq: false + - path: data.error + contains: quarantined + absent: + - reasoning_trace + - reasoning_traces + - judgment: preflight_contract + receipt: preflight.json + match: object + all: + - path: growing_zone_already_covers + eq: true + - path: work_priority.Growing + eq: 1 + - path: research_target_status + eq: available + - path: tend_pair_resolved + eq: true + - path: stockpile_delete_quarantined + eq: true + absent: + - reasoning_trace + - reasoning_traces diff --git a/tests/fixtures/challenge_packs/preflight_contract/receipts/fail/events.jsonl b/tests/fixtures/challenge_packs/preflight_contract/receipts/fail/events.jsonl new file mode 100644 index 0000000..892d837 --- /dev/null +++ b/tests/fixtures/challenge_packs/preflight_contract/receipts/fail/events.jsonl @@ -0,0 +1 @@ +{"timestamp": 0.0, "tick": 1, "event_type": "action_exec", "agent": null, "data": {"action_type": "stockpile_delete", "target": null, "success": true, "error": null, "parameters": {"zone_id": 3}}} diff --git a/tests/fixtures/challenge_packs/preflight_contract/receipts/fail/preflight.json b/tests/fixtures/challenge_packs/preflight_contract/receipts/fail/preflight.json new file mode 100644 index 0000000..067e033 --- /dev/null +++ b/tests/fixtures/challenge_packs/preflight_contract/receipts/fail/preflight.json @@ -0,0 +1,7 @@ +{ + "growing_zone_already_covers": true, + "work_priority": {"Growing": 1}, + "research_target_status": "available", + "tend_pair_resolved": true, + "stockpile_delete_quarantined": false +} diff --git a/tests/fixtures/challenge_packs/preflight_contract/receipts/pass/events.jsonl b/tests/fixtures/challenge_packs/preflight_contract/receipts/pass/events.jsonl new file mode 100644 index 0000000..34479fb --- /dev/null +++ b/tests/fixtures/challenge_packs/preflight_contract/receipts/pass/events.jsonl @@ -0,0 +1 @@ +{"timestamp": 0.0, "tick": 1, "event_type": "action_exec", "agent": null, "data": {"action_type": "stockpile_delete", "target": null, "success": false, "error": "stockpile_delete is quarantined — the endpoint is missing on this RIMAPI build. Do not advertise or call it.", "parameters": {"zone_id": 3}}} diff --git a/tests/fixtures/challenge_packs/preflight_contract/receipts/pass/preflight.json b/tests/fixtures/challenge_packs/preflight_contract/receipts/pass/preflight.json new file mode 100644 index 0000000..e3291f3 --- /dev/null +++ b/tests/fixtures/challenge_packs/preflight_contract/receipts/pass/preflight.json @@ -0,0 +1,7 @@ +{ + "growing_zone_already_covers": true, + "work_priority": {"Growing": 1}, + "research_target_status": "available", + "tend_pair_resolved": true, + "stockpile_delete_quarantined": true +} diff --git a/tests/fixtures/challenge_packs/research_unblock/challenge.yaml b/tests/fixtures/challenge_packs/research_unblock/challenge.yaml new file mode 100644 index 0000000..ebda18c --- /dev/null +++ b/tests/fixtures/challenge_packs/research_unblock/challenge.yaml @@ -0,0 +1,41 @@ +schema_version: 1 +id: research_unblock +scenario: Crashlanded Survival +seed: 42 +tick_budget: 10 +mode: fixture +harness: fixture +model: fixture +scoring_version: "1.2" +expected: + - research_bench_present + - research_adapters +pass_fail: + - judgment: research_bench_present + receipt: preflight.json + match: object + all: + - path: research_bench_present + eq: true + - path: structures.0.def_name + eq: SimpleResearchBench + absent: + - reasoning_trace + - reasoning_traces + - judgment: research_adapters + receipt: preflight.json + match: object + all: + - path: adapter + eq: _adapt_research + - path: research.current_project + eq: Smithing + - path: research_target_status + eq: current + - path: research.completed + not_contains: Medieval + - path: research.available + type: list + absent: + - reasoning_trace + - reasoning_traces diff --git a/tests/fixtures/challenge_packs/research_unblock/receipts/fail/preflight.json b/tests/fixtures/challenge_packs/research_unblock/receipts/fail/preflight.json new file mode 100644 index 0000000..10c534d --- /dev/null +++ b/tests/fixtures/challenge_packs/research_unblock/receipts/fail/preflight.json @@ -0,0 +1,17 @@ +{ + "research_bench_present": false, + "structures": [ + { + "def_name": "Wall", + "position": [0, 0] + } + ], + "adapter": "_adapt_research", + "research": { + "current_project": null, + "progress": 0.0, + "completed": ["Medieval"], + "available": [] + }, + "research_target_status": "locked" +} diff --git a/tests/fixtures/challenge_packs/research_unblock/receipts/pass/preflight.json b/tests/fixtures/challenge_packs/research_unblock/receipts/pass/preflight.json new file mode 100644 index 0000000..b0e0a33 --- /dev/null +++ b/tests/fixtures/challenge_packs/research_unblock/receipts/pass/preflight.json @@ -0,0 +1,17 @@ +{ + "research_bench_present": true, + "structures": [ + { + "def_name": "SimpleResearchBench", + "position": [128, 136] + } + ], + "adapter": "_adapt_research", + "research": { + "current_project": "Smithing", + "progress": 0.0, + "completed": [], + "available": ["Smithing"] + }, + "research_target_status": "current" +} diff --git a/tests/unit/challenge_pack_loader.py b/tests/unit/challenge_pack_loader.py new file mode 100644 index 0000000..6c46d6c --- /dev/null +++ b/tests/unit/challenge_pack_loader.py @@ -0,0 +1,444 @@ +"""Load and evaluate public-proof challenge packs. + +Pack shape (schema_version 1): challenge file, expected judgment names, +and pass/fail clauses over printed receipts. Chain-of-thought keys +(``reasoning_trace``, ``reasoning_traces``) fail closed — they are +narrative, not proof. Fixture mode reads checked-in files only. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + +# Judgment names drawn from existing RLE receipts. A name is the judgment. +JUDGMENT_NAMES: frozenset[str] = frozenset({ + "research_bench_present", + "ensure_live_save", + "research_adapters", + "stockpile_delete_quarantine", + "preflight_contract", + "composite_receipt", +}) + +_COT_KEYS: frozenset[str] = frozenset({"reasoning_trace", "reasoning_traces"}) +_OPS: frozenset[str] = frozenset({ + "eq", + "ne", + "lte", + "gte", + "contains", + "not_contains", + "type", + "present", + "absent", + "eq_path", +}) +_TYPES: frozenset[str] = frozenset({"number", "string", "bool", "list", "object"}) +_MISSING = object() + +PACK_ROOT = Path(__file__).resolve().parents[1] / "fixtures" / "challenge_packs" + + +@dataclass(frozen=True) +class Clause: + """One deterministic check against a receipt object.""" + + path: str + op: str + value: Any + + +@dataclass(frozen=True) +class Predicate: + """Pass/fail rule for one expected judgment name.""" + + judgment: str + receipt: str + match: str + where: tuple[tuple[str, Any], ...] + clauses: tuple[Clause, ...] + absent: tuple[str, ...] + + +@dataclass(frozen=True) +class Challenge: + """One challenge pack: challenge fields plus predicates.""" + + id: str + scenario: str + seed: int + tick_budget: int + mode: str + harness: str + model: str + scoring_version: str + expected: tuple[str, ...] + predicates: tuple[Predicate, ...] + root: Path + + +@dataclass(frozen=True) +class EvalResult: + """Outcome of evaluating every predicate on one receipt case.""" + + passed: bool + reasons: tuple[str, ...] + + +def load_packs(root: Path | None = None) -> tuple[Challenge, ...]: + """Load every pack directory under ``root`` (sorted by id).""" + base = root if root is not None else PACK_ROOT + dirs = sorted(p for p in base.iterdir() if p.is_dir() and not p.name.startswith(".")) + return tuple(load_pack(path) for path in dirs) + + +def load_pack(path: Path) -> Challenge: + """Load and validate one pack directory.""" + challenge_path = _challenge_file(path) + raw = yaml.safe_load(challenge_path.read_text(encoding="utf-8")) + if not isinstance(raw, dict): + raise ValueError(f"{challenge_path} must be a mapping") + _require_keys(raw, challenge_path) + if raw["schema_version"] != 1: + raise ValueError(f"{path.name}: schema_version must be 1") + if raw["id"] != path.name: + raise ValueError(f"{path.name}: id {raw['id']!r} does not match the directory") + if raw["scoring_version"] != "1.2": + raise ValueError(f"{path.name}: scoring_version must be '1.2'") + mode = raw["mode"] + if mode not in ("fixture", "harness"): + raise ValueError(f"{path.name}: mode must be fixture or harness") + harness = raw["harness"] + model = raw["model"] + if not isinstance(harness, str) or not harness: + raise ValueError(f"{path.name}: harness must be a non-empty string") + if not isinstance(model, str) or not model: + raise ValueError(f"{path.name}: model must be a non-empty string") + if mode == "fixture" and (harness != "fixture" or model != "fixture"): + raise ValueError(f"{path.name}: fixture mode requires harness and model 'fixture'") + if mode == "harness" and (harness == "fixture" or model == "fixture"): + raise ValueError(f"{path.name}: harness mode requires a real harness and model") + seed = raw["seed"] + tick_budget = raw["tick_budget"] + if not isinstance(seed, int) or isinstance(seed, bool): + raise ValueError(f"{path.name}: seed must be an int") + if not isinstance(tick_budget, int) or isinstance(tick_budget, bool) or tick_budget < 1: + raise ValueError(f"{path.name}: tick_budget must be a positive int") + scenario = raw["scenario"] + if not isinstance(scenario, str) or not scenario: + raise ValueError(f"{path.name}: scenario must be a non-empty string") + expected = _expected(raw["expected"], path.name) + predicates = _predicates(raw["pass_fail"], expected, path.name) + pack = Challenge( + id=path.name, + scenario=scenario, + seed=seed, + tick_budget=tick_budget, + mode=mode, + harness=harness, + model=model, + scoring_version="1.2", + expected=expected, + predicates=predicates, + root=path, + ) + _require_receipts(pack) + return pack + + +def evaluate(pack: Challenge, case: str) -> EvalResult: + """Evaluate every predicate against ``receipts//``.""" + if case not in ("pass", "fail"): + raise ValueError("case must be 'pass' or 'fail'") + reasons: list[str] = [] + for predicate in pack.predicates: + receipt_path = pack.root / "receipts" / case / predicate.receipt + payload = _read_receipt(receipt_path, predicate.match) + found = judge(predicate, payload) + reasons.extend(f"{predicate.judgment}: {reason}" for reason in found) + return EvalResult(passed=not reasons, reasons=tuple(reasons)) + + +def judge(predicate: Predicate, payload: Any) -> tuple[str, ...]: + """Return failure reasons for one predicate. Empty means pass.""" + if predicate.match == "object": + return _judge_object(predicate, payload) + if predicate.match == "any": + return _judge_any(predicate, payload) + return (f"unknown match {predicate.match!r}",) + + +def _judge_object(predicate: Predicate, payload: Any) -> tuple[str, ...]: + if not isinstance(payload, dict): + return ("receipt is not a JSON object",) + reasons = _cot_reasons(payload) + reasons.extend(_absent_reasons(predicate, payload)) + if reasons: + return tuple(reasons) + return tuple( + reason + for clause in predicate.clauses + if (reason := _clause_reason(clause, payload)) is not None + ) + + +def _judge_any(predicate: Predicate, payload: Any) -> tuple[str, ...]: + if not isinstance(payload, list): + return ("events receipt is not a list of JSONL objects",) + eligible: list[dict[str, Any]] = [] + for line in payload: + if not isinstance(line, dict): + continue + if _COT_KEYS.intersection(line): + continue + if any(_lookup(line, path) != expected for path, expected in predicate.where): + continue + if any(key in line for key in predicate.absent): + continue + eligible.append(line) + if not eligible: + return ("no eligible events line matched",) + if any( + all(_clause_reason(clause, line) is None for clause in predicate.clauses) + for line in eligible + ): + return () + return ("no events line satisfied the clauses",) + + +def _clause_reason(clause: Clause, payload: dict[str, Any]) -> str | None: + found = _lookup(payload, clause.path) + if clause.op == "present": + if found is _MISSING or found is None: + return f"{clause.path} is absent" + return None + if clause.op == "absent": + if found is _MISSING or found is None: + return None + return f"{clause.path} is present" + if found is _MISSING: + return f"{clause.path} is missing" + if clause.op == "eq": + return None if found == clause.value else f"{clause.path} {found!r} != {clause.value!r}" + if clause.op == "ne": + return None if found != clause.value else f"{clause.path} == {clause.value!r}" + if clause.op == "eq_path": + other = _lookup(payload, str(clause.value)) + if other is _MISSING: + return f"{clause.value} is missing" + if found != other: + return f"{clause.path} {found!r} != {clause.value} {other!r}" + return None + if clause.op in ("lte", "gte"): + if not _is_number(found) or not _is_number(clause.value): + return f"{clause.path} is not numeric" + if clause.op == "lte" and found > clause.value: + return f"{clause.path} {found!r} > {clause.value!r}" + if clause.op == "gte" and found < clause.value: + return f"{clause.path} {found!r} < {clause.value!r}" + return None + if clause.op == "contains": + if _contains(found, clause.value): + return None + return f"{clause.path} does not contain {clause.value!r}" + if clause.op == "not_contains": + if not _contains(found, clause.value): + return None + return f"{clause.path} contains {clause.value!r}" + if clause.op == "type": + if _matches_type(found, str(clause.value)): + return None + return f"{clause.path} is not {clause.value}" + return f"unknown op {clause.op}" + + +def _cot_reasons(payload: dict[str, Any]) -> list[str]: + return [ + f"{key} is narrative, not a receipt" + for key in sorted(_COT_KEYS.intersection(payload)) + ] + + +def _absent_reasons(predicate: Predicate, payload: dict[str, Any]) -> list[str]: + return [f"forbidden key {key}" for key in predicate.absent if key in payload] + + +def _contains(haystack: Any, needle: Any) -> bool: + if isinstance(haystack, str): + return isinstance(needle, str) and needle in haystack + if isinstance(haystack, list): + return needle in haystack + return False + + +def _matches_type(value: Any, kind: str) -> bool: + if kind == "number": + return _is_number(value) + if kind == "string": + return isinstance(value, str) + if kind == "bool": + return isinstance(value, bool) + if kind == "list": + return isinstance(value, list) + if kind == "object": + return isinstance(value, dict) + return False + + +def _is_number(value: Any) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) + + +def _lookup(payload: Any, path: str) -> Any: + current: Any = payload + for part in path.split("."): + if isinstance(current, dict) and part in current: + current = current[part] + continue + if isinstance(current, list) and part.isdigit(): + index = int(part) + if 0 <= index < len(current): + current = current[index] + continue + return _MISSING + return current + + +def _challenge_file(path: Path) -> Path: + for name in ("challenge.yaml", "challenge.yml"): + candidate = path / name + if candidate.is_file(): + return candidate + raise ValueError(f"{path} has no challenge.yaml") + + +def _require_keys(raw: dict[str, Any], challenge_path: Path) -> None: + required = ( + "schema_version", + "id", + "scenario", + "seed", + "tick_budget", + "mode", + "harness", + "model", + "scoring_version", + "expected", + "pass_fail", + ) + missing = [key for key in required if key not in raw] + if missing: + raise ValueError(f"{challenge_path.name} missing {missing}") + + +def _expected(raw: Any, pack_id: str) -> tuple[str, ...]: + if not isinstance(raw, list) or not raw: + raise ValueError(f"{pack_id}: expected must be a non-empty list of names") + names: list[str] = [] + for item in raw: + if not isinstance(item, str) or item not in JUDGMENT_NAMES: + raise ValueError(f"{pack_id}: {item!r} is not a judgment name") + if item in names: + raise ValueError(f"{pack_id}: duplicate expected name {item}") + names.append(item) + return tuple(names) + + +def _predicates(raw: Any, expected: tuple[str, ...], pack_id: str) -> tuple[Predicate, ...]: + if not isinstance(raw, list): + raise ValueError(f"{pack_id}: pass_fail must be a list") + predicates = tuple(_predicate(item, pack_id) for item in raw) + got = tuple(item.judgment for item in predicates) + if len(got) != len(set(got)): + raise ValueError(f"{pack_id}: duplicate pass_fail judgment") + if set(got) != set(expected): + raise ValueError(f"{pack_id}: pass_fail judgments must match expected") + return predicates + + +def _predicate(raw: Any, pack_id: str) -> Predicate: + if not isinstance(raw, dict): + raise ValueError(f"{pack_id}: pass_fail entry must be a mapping") + judgment = raw.get("judgment") + receipt = raw.get("receipt") + match = raw.get("match") + if not isinstance(judgment, str) or judgment not in JUDGMENT_NAMES: + raise ValueError(f"{pack_id}: bad judgment {judgment!r}") + if not isinstance(receipt, str) or "/" in receipt or "\\" in receipt: + raise ValueError(f"{pack_id}: receipt must be a file name") + if match not in ("object", "any"): + raise ValueError(f"{pack_id}: match must be object or any") + if match == "object" and not receipt.endswith(".json"): + raise ValueError(f"{pack_id}: object receipts end with .json") + if match == "any" and not receipt.endswith(".jsonl"): + raise ValueError(f"{pack_id}: any-match receipts end with .jsonl") + where_raw = raw.get("where") or {} + if not isinstance(where_raw, dict): + raise ValueError(f"{pack_id}: where must be a mapping") + if match == "object" and where_raw: + raise ValueError(f"{pack_id}: where is only for JSONL receipts") + clauses_raw = raw.get("all") + if not isinstance(clauses_raw, list) or not clauses_raw: + raise ValueError(f"{pack_id}: {judgment} needs a non-empty all list") + absent_raw = raw.get("absent") or [] + if not isinstance(absent_raw, list) or not all(isinstance(item, str) for item in absent_raw): + raise ValueError(f"{pack_id}: absent must be a list of keys") + return Predicate( + judgment=judgment, + receipt=receipt, + match=match, + where=tuple((str(key), value) for key, value in where_raw.items()), + clauses=tuple(_clause(item, pack_id) for item in clauses_raw), + absent=tuple(absent_raw), + ) + + +def _clause(raw: Any, pack_id: str) -> Clause: + if not isinstance(raw, dict): + raise ValueError(f"{pack_id}: clause must be a mapping") + path = raw.get("path") + if not isinstance(path, str) or not path: + raise ValueError(f"{pack_id}: clause needs a path") + ops = [key for key in raw if key != "path"] + if len(ops) != 1 or ops[0] not in _OPS: + raise ValueError(f"{pack_id}: clause needs exactly one operator") + op = ops[0] + value = raw[op] + if op == "type" and value not in _TYPES: + raise ValueError(f"{pack_id}: type must be one of {sorted(_TYPES)}") + if op in ("present", "absent") and value is not True: + raise ValueError(f"{pack_id}: {op} clause value must be true") + if op == "eq_path" and not isinstance(value, str): + raise ValueError(f"{pack_id}: eq_path needs a path string") + return Clause(path=path, op=op, value=value) + + +def _require_receipts(pack: Challenge) -> None: + for case in ("pass", "fail"): + for predicate in pack.predicates: + receipt = pack.root / "receipts" / case / predicate.receipt + if not receipt.is_file(): + raise ValueError(f"{pack.id}: missing {case} receipt {predicate.receipt}") + + +def _read_receipt(path: Path, match: str) -> Any: + text = path.read_text(encoding="utf-8") + if match == "object": + try: + return json.loads(text) + except json.JSONDecodeError as exc: + return {"_unreadable": str(exc)} + rows: list[Any] = [] + for line_no, line in enumerate(text.splitlines(), start=1): + if not line.strip(): + continue + try: + rows.append(json.loads(line)) + except json.JSONDecodeError: + rows.append({"_unreadable_line": line_no}) + return rows diff --git a/tests/unit/test_challenge_packs.py b/tests/unit/test_challenge_packs.py new file mode 100644 index 0000000..47b829e --- /dev/null +++ b/tests/unit/test_challenge_packs.py @@ -0,0 +1,150 @@ +"""Challenge packs: load challenge + expected + pass/fail over fixture receipts.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import yaml + +from tests.unit.challenge_pack_loader import ( + JUDGMENT_NAMES, + PACK_ROOT, + evaluate, + judge, + load_pack, + load_packs, +) + +PACK_IDS = ( + "research_unblock", + "ensure_live_save", + "preflight_contract", + "composite_receipt", +) + + +def test_four_packs_load() -> None: + packs = load_packs(PACK_ROOT) + assert tuple(pack.id for pack in packs) == tuple(sorted(PACK_IDS)) + + +@pytest.mark.parametrize("pack_id", PACK_IDS) +def test_pack_is_crashlanded_fixture(pack_id: str) -> None: + pack = load_pack(PACK_ROOT / pack_id) + assert pack.scenario == "Crashlanded Survival" + assert pack.seed == 42 + assert pack.tick_budget == 10 + assert pack.mode == "fixture" + assert pack.harness == "fixture" + assert pack.model == "fixture" + assert pack.scoring_version == "1.2" + assert set(pack.expected) <= JUDGMENT_NAMES + assert {item.judgment for item in pack.predicates} == set(pack.expected) + + +def test_expected_names_cover_the_catalog_shapes() -> None: + packs = {pack.id: pack for pack in load_packs(PACK_ROOT)} + assert packs["research_unblock"].expected == ( + "research_bench_present", + "research_adapters", + ) + assert packs["ensure_live_save"].expected == ("ensure_live_save",) + assert packs["preflight_contract"].expected == ( + "stockpile_delete_quarantine", + "preflight_contract", + ) + assert packs["composite_receipt"].expected == ("composite_receipt",) + + +@pytest.mark.parametrize("pack_id", PACK_IDS) +def test_pass_receipts_pass(pack_id: str) -> None: + result = evaluate(load_pack(PACK_ROOT / pack_id), "pass") + assert result.passed, result.reasons + + +@pytest.mark.parametrize("pack_id", PACK_IDS) +def test_fail_receipts_fail(pack_id: str) -> None: + result = evaluate(load_pack(PACK_ROOT / pack_id), "fail") + assert not result.passed + assert result.reasons + + +def test_ensure_live_save_pin_match_versus_stale_appdata() -> None: + root = PACK_ROOT / "ensure_live_save" / "receipts" + passed = json.loads((root / "pass" / "preflight.json").read_text(encoding="utf-8")) + failed = json.loads((root / "fail" / "preflight.json").read_text(encoding="utf-8")) + assert passed["live_save_sha256"] == passed["pinned_sha256"] + assert passed["error"] is None + assert failed["live_save_sha256"] != failed["pinned_sha256"] + assert failed["error"] == "LiveSavePinError" + pack = load_pack(PACK_ROOT / "ensure_live_save") + assert evaluate(pack, "pass").passed + assert not evaluate(pack, "fail").passed + + +def test_stockpile_delete_pass_is_quarantined_event() -> None: + root = PACK_ROOT / "preflight_contract" / "receipts" + passed = json.loads((root / "pass" / "events.jsonl").read_text(encoding="utf-8")) + failed = json.loads((root / "fail" / "events.jsonl").read_text(encoding="utf-8")) + assert passed["data"]["action_type"] == "stockpile_delete" + assert passed["data"]["success"] is False + assert "quarantined" in passed["data"]["error"] + assert failed["data"]["success"] is True + + +def test_composite_receipt_rejects_n_claim_and_reasoning_trace() -> None: + root = PACK_ROOT / "composite_receipt" / "receipts" + passed = json.loads((root / "pass" / "summary.json").read_text(encoding="utf-8")) + failed = json.loads((root / "fail" / "summary.json").read_text(encoding="utf-8")) + assert passed["scoring_version"] == "1.2" + assert isinstance(passed["final_score"], (int, float)) + assert "n_runs" not in passed + assert "pass_rate" not in passed + assert "reasoning_traces" not in passed + assert "n_runs" in failed + assert "pass_rate" in failed + assert "reasoning_traces" in failed + pack = load_pack(PACK_ROOT / "composite_receipt") + predicate = pack.predicates[0] + lte = next(clause for clause in predicate.clauses if clause.op == "lte") + assert lte.value == pack.tick_budget + assert evaluate(pack, "pass").passed + assert not evaluate(pack, "fail").passed + + +def test_reasoning_traces_fail_closed_even_on_a_passing_summary() -> None: + pack = load_pack(PACK_ROOT / "composite_receipt") + summary = json.loads( + (PACK_ROOT / "composite_receipt" / "receipts" / "pass" / "summary.json").read_text( + encoding="utf-8", + ), + ) + summary["reasoning_traces"] = [{"summary": "abridged"}] + reasons = judge(pack.predicates[0], summary) + assert reasons + assert any("reasoning_traces" in reason for reason in reasons) + + +def test_unknown_judgment_name_is_rejected(tmp_path: Path) -> None: + pack_dir = tmp_path / "not_a_judgment" + pack_dir.mkdir() + (pack_dir / "challenge.yaml").write_text( + yaml.safe_dump({ + "schema_version": 1, + "id": "not_a_judgment", + "scenario": "Crashlanded Survival", + "seed": 42, + "tick_budget": 10, + "mode": "fixture", + "harness": "fixture", + "model": "fixture", + "scoring_version": "1.2", + "expected": ["invented_rate"], + "pass_fail": [], + }), + encoding="utf-8", + ) + with pytest.raises(ValueError, match="not a judgment name"): + load_pack(pack_dir)