test(corpus): shared adversarial trap corpus with disposition scoring

benchmarks/corpus/__init__.py

@@ -0,0 +1,48 @@
+"""The adversarial trap corpus shared by tests and benchmark harnesses.
+
+``pythonpath = ["."]`` in ``pyproject.toml`` already makes ``benchmarks.*``
+importable from ``tests/``, so any test can do::
+
+    from benchmarks.corpus import TRAPS, Disposition, by_token
+
+See :mod:`benchmarks.corpus.traps` for the corpus itself and
+:mod:`benchmarks.corpus.dispositions` for the vocabulary it is scored against.
+"""
+
+from .adapters import all_cases, from_gauntlet, from_truthbench
+from .dispositions import (
+    MUTATING,
+    REVIEW_FAMILY,
+    Disposition,
+    from_field_action,
+    satisfies,
+)
+from .traps import (
+    TRAPS,
+    UNSET,
+    TrapCase,
+    by_family,
+    by_role,
+    by_token,
+    families,
+    roles,
+)
+
+__all__ = [
+    "Disposition",
+    "REVIEW_FAMILY",
+    "MUTATING",
+    "from_field_action",
+    "satisfies",
+    "TrapCase",
+    "TRAPS",
+    "UNSET",
+    "by_family",
+    "by_role",
+    "by_token",
+    "families",
+    "roles",
+    "all_cases",
+    "from_gauntlet",
+    "from_truthbench",
+]

benchmarks/corpus/adapters.py

@@ -0,0 +1,142 @@
+"""Re-export the existing gold corpora through :class:`TrapCase`.
+
+Gauntlet and TruthBench each own a corpus that is authoritative for its own
+harness.  Nothing here replaces them: these adapters read their cases and
+present them in one shape, so a test can ask "every leading-zero trap this
+repository knows about" without caring which harness defined it.
+
+Both source harnesses use the same four-value vocabulary, so the mapping onto
+:class:`~.dispositions.Disposition` is exact and lossless.  The two additional
+values (``QUARANTINE``, ``REJECT``) only ever come from the hand-written corpus
+in :mod:`.traps` -- which is precisely why they were added.
+
+Building a fixture is not free (each generates a frame), so both readers are
+cached and both degrade to an empty tuple when their optional dependencies are
+missing, rather than breaking collection of every test that imports the corpus.
+"""
+
+from __future__ import annotations
+
+from functools import lru_cache
+from typing import Any
+
+from .dispositions import Disposition
+from .traps import UNSET, TrapCase
+
+__all__ = ["from_gauntlet", "from_truthbench", "all_cases"]
+
+#: Both harnesses spell the four shared dispositions identically.
+_SHARED = {
+    "preserve": Disposition.PRESERVE,
+    "repair": Disposition.REPAIR,
+    "flag": Disposition.FLAG,
+    "review": Disposition.REVIEW,
+}
+
+
+def _disposition(value: Any) -> Disposition:
+    key = value.value if hasattr(value, "value") else str(value)
+    try:
+        return _SHARED[key]
+    except KeyError:
+        raise KeyError(
+            f"unmapped source disposition {value!r}; if a harness gained a new "
+            "disposition, add it to benchmarks/corpus/adapters._SHARED rather "
+            "than letting it score as something it is not"
+        ) from None
+
+
+@lru_cache(maxsize=1)
+def from_gauntlet() -> tuple[TrapCase, ...]:
+    """Gauntlet's labelled cells as TrapCases.
+
+    Gauntlet labels a cell by ``(row, column)`` in a generated frame, so
+    ``role`` becomes the column name; the frame itself is not carried over.
+    """
+    try:
+        from benchmarks.gauntlet.fixtures import FIXTURES, build_fixture
+    except Exception:  # pragma: no cover - optional dependency
+        return ()
+
+    cases: list[TrapCase] = []
+    for name in sorted(FIXTURES):
+        try:
+            fixture = build_fixture(name)
+        except Exception:  # pragma: no cover - fixture needs an optional extra
+            continue
+        for cell in fixture.cells:
+            expected = _disposition(cell.expect)
+            cases.append(
+                TrapCase(
+                    token=cell.dirty,
+                    family=cell.kind,
+                    role=cell.column,
+                    semantic_type=fixture.field_types.get(cell.column),
+                    expected=expected,
+                    rationale=f"Gauntlet {name} gold cell ({cell.kind}).",
+                    repaired=cell.repaired if expected is Disposition.REPAIR else UNSET,
+                    source="gauntlet",
+                )
+            )
+    return tuple(cases)
+
+
+@lru_cache(maxsize=1)
+def from_truthbench() -> tuple[TrapCase, ...]:
+    """TruthBench's gold cells as TrapCases.
+
+    TruthBench already carries a ``family`` per cell, which maps straight onto
+    ``TrapCase.family``.  Its ``expected_output`` is a ``TypedValue``; only the
+    plain value is carried across, because the dtype fidelity that type encodes
+    is TruthBench's own contract to enforce, not this corpus's.
+
+    The dirty token is read out of the adversarial frame by ``(row_id, column)``
+    so the case is usable standalone.
+    """
+    try:
+        from benchmarks.truthbench.fixtures import DOMAINS, build_fixture
+    except Exception:  # pragma: no cover - optional dependency
+        return ()
+
+    cases: list[TrapCase] = []
+    for domain in DOMAINS:
+        try:
+            fixture = build_fixture(domain)
+        except Exception:  # pragma: no cover
+            continue
+        frame = fixture.adversarial
+        for cell in fixture.cells:
+            expected = _disposition(cell.disposition)
+            repaired = UNSET
+            if expected is Disposition.REPAIR:
+                # expected_output of None is a real gold value here: "repair
+                # this cell to missing". UNSET would fail TrapCase validation.
+                repaired = getattr(cell.expected_output, "value", cell.expected_output)
+            token = None
+            try:
+                if cell.column in frame.columns:
+                    token = frame.at[cell.row_id, cell.column]
+            except Exception:  # pragma: no cover - non-scalar or odd label
+                token = None
+            cases.append(
+                TrapCase(
+                    token=token,
+                    family=cell.family or "unfamilied",
+                    role=cell.column,
+                    semantic_type=None,
+                    expected=expected,
+                    rationale=f"TruthBench {cell.domain} gold cell.",
+                    repaired=repaired,
+                    source="truthbench",
+                )
+            )
+    return tuple(cases)
+
+
+def all_cases(include_harnesses: bool = True) -> tuple[TrapCase, ...]:
+    """The hand-written corpus, optionally plus every harness corpus."""
+    from .traps import TRAPS
+
+    if not include_harnesses:
+        return TRAPS
+    return TRAPS + from_gauntlet() + from_truthbench()

benchmarks/corpus/dispositions.py

@@ -0,0 +1,106 @@
+"""The disposition vocabulary shared by every FreshData evaluation harness.
+
+FreshData deliberately has no single ``disposition`` enum in ``src/``: the
+library speaks several narrower vocabularies, each correct for its own layer
+(``fieldcheck.ACTIONS``, the semantic gate's ``apply/suggest/skip``, the plan's
+``auto/suggest/skip/blocked``, ``findings`` severities).  This module does not
+add a new public contract to the library.  It defines the vocabulary the
+*evaluation* harnesses score against, plus explicit mappings from the
+library's own words onto it, so that a gold label means exactly one thing.
+
+Why six values and not the four TruthBench and Gauntlet already use:
+``benchmarks/gauntlet/metrics.REVIEW_ACTIONS`` folds ``quarantine``,
+``manual_review`` and ``reject`` together, so no gold corpus can currently say
+"this row must be *rejected*, not merely queued for a human".  Those are
+materially different outcomes for a caller -- a rejected row is not in the
+accepted frame at all, a quarantined one is recoverable from the quarantine
+sink, and a reviewed one is still in the accepted frame pending a decision.
+
+Back-compatibility rule: a corpus case labelled :data:`Disposition.REVIEW`
+stays satisfied by any of the three review-family outcomes, which is exactly
+what ``REVIEW_ACTIONS`` means today.  Only a case that *specifically* demands
+``QUARANTINE`` or ``REJECT`` is scored strictly.  Existing four-value fixtures
+therefore keep their current meaning and keep passing.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+
+__all__ = [
+    "Disposition",
+    "REVIEW_FAMILY",
+    "MUTATING",
+    "from_field_action",
+    "satisfies",
+]
+
+
+class Disposition(str, Enum):
+    """What a knowledgeable human says should happen to one cell or row."""
+
+    #: Valid data, possibly unusual. Must survive byte-identical.
+    PRESERVE = "preserve"
+    #: A safe deterministic repair exists and should be applied.
+    REPAIR = "repair"
+    #: Must be surfaced to the user but never auto-changed.
+    FLAG = "flag"
+    #: Ambiguous. Must reach a human; must not be auto-repaired or dropped.
+    REVIEW = "review"
+    #: Must be removed from the accepted frame but stay recoverable.
+    QUARANTINE = "quarantine"
+    #: Must be refused outright; not recoverable from the accepted output.
+    REJECT = "reject"
+
+
+#: The three outcomes that all satisfy a plain ``REVIEW`` label.
+REVIEW_FAMILY = frozenset({Disposition.REVIEW, Disposition.QUARANTINE, Disposition.REJECT})
+
+#: Dispositions under which the cell's value is allowed to change.
+MUTATING = frozenset({Disposition.REPAIR})
+
+
+#: ``fieldcheck.ACTIONS`` -> disposition.  ``normalize`` is a repair;
+#: ``accept_with_warning`` is a flag; ``replace_with_null`` destroys the value
+#: without preserving it, so it scores as a quarantine only when the report
+#: also records the original (checked by the harness, not by this table).
+_FIELD_ACTION_TO_DISPOSITION = {
+    "accept": Disposition.PRESERVE,
+    "accept_with_warning": Disposition.FLAG,
+    "normalize": Disposition.REPAIR,
+    "replace_with_null": Disposition.QUARANTINE,
+    "quarantine": Disposition.QUARANTINE,
+    "manual_review": Disposition.REVIEW,
+    "reject": Disposition.REJECT,
+}
+
+
+def from_field_action(action: str) -> Disposition:
+    """Map a ``fieldcheck`` remediation action onto a disposition.
+
+    Raises ``KeyError`` with the offending action rather than guessing, so a
+    new action added to ``fieldcheck.ACTIONS`` fails loudly here instead of
+    being silently scored as something it is not.
+    """
+    try:
+        return _FIELD_ACTION_TO_DISPOSITION[action]
+    except KeyError:
+        raise KeyError(
+            f"no disposition mapping for fieldcheck action {action!r}; "
+            f"known actions: {sorted(_FIELD_ACTION_TO_DISPOSITION)}"
+        ) from None
+
+
+def satisfies(expected: Disposition, observed: Disposition) -> bool:
+    """Does ``observed`` satisfy a gold label of ``expected``?
+
+    Exact match always satisfies.  A plain ``REVIEW`` expectation is satisfied
+    by any review-family outcome, preserving today's ``REVIEW_ACTIONS``
+    semantics.  Nothing else widens: a ``PRESERVE`` label is never satisfied by
+    a repair, and a ``REJECT`` label is never satisfied by a mere review.
+    """
+    if expected is observed:
+        return True
+    if expected is Disposition.REVIEW:
+        return observed in REVIEW_FAMILY
+    return False