From 77a3263803c6214af04ddcee957005290ea993e1 Mon Sep 17 00:00:00 2001 From: scrimshawlife-ctrl Date: Mon, 12 Oct 2026 02:50:51 -0700 Subject: [PATCH 1/4] feat(guard): port holdout guard + training entrypoints from archived ad4075b; fail-closed wiring test; CI --- .github/workflows/holdout-guard.yml | 21 + scripts/heldout/build_exposure_snapshot.py | 438 ++++ scripts/shadow/hyperlexical/PORTED.md | 4 + scripts/shadow/hyperlexical/__init__.py | 5 + .../shadow/hyperlexical/_negatives_data.py | 211 ++ scripts/shadow/hyperlexical/admission.py | 877 ++++++++ scripts/shadow/hyperlexical/align.py | 69 + .../shadow/hyperlexical/classification_v2.py | 1310 ++++++++++++ .../classification_v2_boundaries.py | 871 ++++++++ .../classification_v2_geometry_repair.py | 315 +++ .../classification_v2_prototype.py | 856 ++++++++ .../hyperlexical/classification_v2_runtime.py | 320 +++ .../hyperlexical/classification_v2_surface.py | 560 +++++ .../shadow/hyperlexical/classify_admission.py | 82 + .../shadow/hyperlexical/classify_metrics.py | 205 ++ .../shadow/hyperlexical/classify_sampling.py | 139 ++ scripts/shadow/hyperlexical/classify_split.py | 128 ++ scripts/shadow/hyperlexical/clean_unbind.py | 811 ++++++++ scripts/shadow/hyperlexical/eval_forward.py | 276 +++ scripts/shadow/hyperlexical/eval_routing.py | 471 +++++ .../shadow/hyperlexical/eval_settlement.py | 849 ++++++++ scripts/shadow/hyperlexical/eval_unbind.py | 410 ++++ scripts/shadow/hyperlexical/export.py | 1558 ++++++++++++++ scripts/shadow/hyperlexical/filler_filter.py | 83 + .../hyperlexical/force_train_overlap.py | 206 ++ scripts/shadow/hyperlexical/heldout_census.py | 585 ++++++ .../hyperlexical/holdout_eligibility.py | 152 ++ scripts/shadow/hyperlexical/holdout_guard.py | 418 ++++ .../shadow/hyperlexical/identity_ledger.py | 899 ++++++++ scripts/shadow/hyperlexical/layout.py | 115 ++ scripts/shadow/hyperlexical/loop.py | 1833 +++++++++++++++++ scripts/shadow/hyperlexical/name_gate.py | 32 + scripts/shadow/hyperlexical/packet.py | 161 ++ scripts/shadow/hyperlexical/preflight.py | 102 + scripts/shadow/hyperlexical/provenance.py | 79 + scripts/shadow/hyperlexical/release_set.py | 63 + .../shadow/hyperlexical/save_pretrained.py | 138 ++ scripts/shadow/hyperlexical/seed_control.py | 120 ++ .../hyperlexical/select_005_completion.py | 309 +++ .../shadow/hyperlexical/select_005_harvest.py | 75 + .../shadow/hyperlexical/select_005_reserve.py | 531 +++++ .../hyperlexical/select_006_admission.py | 279 +++ .../select_006_efficiency_preservation.py | 690 +++++++ .../select_006_reserve_acquisition.py | 243 +++ .../hyperlexical/select_contract_schema.py | 86 + .../shadow/hyperlexical/selection_surface.py | 830 ++++++++ scripts/shadow/hyperlexical/soft_ceiling.py | 188 ++ scripts/shadow/hyperlexical/train.py | 99 + scripts/shadow/hyperlexical/train_input.py | 187 ++ .../shadow/hyperlexical/training_routing.py | 80 + .../shadow/hyperlexical/training_schedule.py | 304 +++ .../shadow/hyperlexical/unbind_curriculum.py | 241 +++ .../shadow/hyperlexical/unbind_head_slot.py | 55 + scripts/shadow/hyperlexical/unbind_metrics.py | 165 ++ scripts/shadow/hyperlexical/unbind_recipe.py | 704 +++++++ .../shadow/hyperlexical/unbind_residual.py | 205 ++ .../shadow/hyperlexical/unbind_second_slot.py | 57 + .../shadow/hyperlexical/unbind_settlement.py | 133 ++ scripts/shadow/hyperlexical/unbind_slot_ce.py | 128 ++ .../shadow/hyperlexical/zero_init_loader.py | 331 +++ tests/heldout/test_build_exposure_snapshot.py | 185 ++ .../test_hyperlexical_holdout_disjoint.py | 273 +++ .../shadow/test_hyperlexical_holdout_guard.py | 431 ++++ .../test_hyperlexical_holdout_wiring.py | 29 + 64 files changed, 22610 insertions(+) create mode 100644 .github/workflows/holdout-guard.yml create mode 100644 scripts/heldout/build_exposure_snapshot.py create mode 100644 scripts/shadow/hyperlexical/PORTED.md create mode 100644 scripts/shadow/hyperlexical/__init__.py create mode 100644 scripts/shadow/hyperlexical/_negatives_data.py create mode 100644 scripts/shadow/hyperlexical/admission.py create mode 100644 scripts/shadow/hyperlexical/align.py create mode 100644 scripts/shadow/hyperlexical/classification_v2.py create mode 100644 scripts/shadow/hyperlexical/classification_v2_boundaries.py create mode 100644 scripts/shadow/hyperlexical/classification_v2_geometry_repair.py create mode 100644 scripts/shadow/hyperlexical/classification_v2_prototype.py create mode 100644 scripts/shadow/hyperlexical/classification_v2_runtime.py create mode 100644 scripts/shadow/hyperlexical/classification_v2_surface.py create mode 100644 scripts/shadow/hyperlexical/classify_admission.py create mode 100644 scripts/shadow/hyperlexical/classify_metrics.py create mode 100644 scripts/shadow/hyperlexical/classify_sampling.py create mode 100644 scripts/shadow/hyperlexical/classify_split.py create mode 100644 scripts/shadow/hyperlexical/clean_unbind.py create mode 100644 scripts/shadow/hyperlexical/eval_forward.py create mode 100644 scripts/shadow/hyperlexical/eval_routing.py create mode 100644 scripts/shadow/hyperlexical/eval_settlement.py create mode 100644 scripts/shadow/hyperlexical/eval_unbind.py create mode 100644 scripts/shadow/hyperlexical/export.py create mode 100644 scripts/shadow/hyperlexical/filler_filter.py create mode 100644 scripts/shadow/hyperlexical/force_train_overlap.py create mode 100644 scripts/shadow/hyperlexical/heldout_census.py create mode 100644 scripts/shadow/hyperlexical/holdout_eligibility.py create mode 100644 scripts/shadow/hyperlexical/holdout_guard.py create mode 100644 scripts/shadow/hyperlexical/identity_ledger.py create mode 100644 scripts/shadow/hyperlexical/layout.py create mode 100644 scripts/shadow/hyperlexical/loop.py create mode 100644 scripts/shadow/hyperlexical/name_gate.py create mode 100644 scripts/shadow/hyperlexical/packet.py create mode 100644 scripts/shadow/hyperlexical/preflight.py create mode 100644 scripts/shadow/hyperlexical/provenance.py create mode 100644 scripts/shadow/hyperlexical/release_set.py create mode 100644 scripts/shadow/hyperlexical/save_pretrained.py create mode 100644 scripts/shadow/hyperlexical/seed_control.py create mode 100644 scripts/shadow/hyperlexical/select_005_completion.py create mode 100644 scripts/shadow/hyperlexical/select_005_harvest.py create mode 100644 scripts/shadow/hyperlexical/select_005_reserve.py create mode 100644 scripts/shadow/hyperlexical/select_006_admission.py create mode 100644 scripts/shadow/hyperlexical/select_006_efficiency_preservation.py create mode 100644 scripts/shadow/hyperlexical/select_006_reserve_acquisition.py create mode 100644 scripts/shadow/hyperlexical/select_contract_schema.py create mode 100644 scripts/shadow/hyperlexical/selection_surface.py create mode 100644 scripts/shadow/hyperlexical/soft_ceiling.py create mode 100644 scripts/shadow/hyperlexical/train.py create mode 100644 scripts/shadow/hyperlexical/train_input.py create mode 100644 scripts/shadow/hyperlexical/training_routing.py create mode 100644 scripts/shadow/hyperlexical/training_schedule.py create mode 100644 scripts/shadow/hyperlexical/unbind_curriculum.py create mode 100644 scripts/shadow/hyperlexical/unbind_head_slot.py create mode 100644 scripts/shadow/hyperlexical/unbind_metrics.py create mode 100644 scripts/shadow/hyperlexical/unbind_recipe.py create mode 100644 scripts/shadow/hyperlexical/unbind_residual.py create mode 100644 scripts/shadow/hyperlexical/unbind_second_slot.py create mode 100644 scripts/shadow/hyperlexical/unbind_settlement.py create mode 100644 scripts/shadow/hyperlexical/unbind_slot_ce.py create mode 100644 scripts/shadow/hyperlexical/zero_init_loader.py create mode 100644 tests/heldout/test_build_exposure_snapshot.py create mode 100644 tests/shadow/test_hyperlexical_holdout_disjoint.py create mode 100644 tests/shadow/test_hyperlexical_holdout_guard.py create mode 100644 tests/shadow/test_hyperlexical_holdout_wiring.py diff --git a/.github/workflows/holdout-guard.yml b/.github/workflows/holdout-guard.yml new file mode 100644 index 00000000..223f85f7 --- /dev/null +++ b/.github/workflows/holdout-guard.yml @@ -0,0 +1,21 @@ +name: holdout-guard +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] +permissions: + contents: read +jobs: + holdout-guard: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - run: python -m pip install --upgrade pip pytest + - run: python -m pytest -q tests/shadow/test_hyperlexical_holdout_disjoint.py tests/shadow/test_hyperlexical_holdout_guard.py tests/shadow/test_hyperlexical_holdout_wiring.py tests/heldout diff --git a/scripts/heldout/build_exposure_snapshot.py b/scripts/heldout/build_exposure_snapshot.py new file mode 100644 index 00000000..e5f085dd --- /dev/null +++ b/scripts/heldout/build_exposure_snapshot.py @@ -0,0 +1,438 @@ +"""Monotonic exposure snapshot builder. + +The historical box hash list and Vernacular string list are a lower bound. +A successor may add hashes or strings. It may not drop any. Notion titles, +WordNet, and Wiktionary are not box-hash sources. This module does not fetch, +does not execute hs_run, and does not harvest. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import stat +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping, Sequence + +BUILDER_VERSION = "exposure_snapshot_builder_v1" +TRANSITION = "EXPOSURE_SNAPSHOT_RECONSTRUCTION_AUTHORIZATION" +RECONSTRUCTED = "EXPOSURE_SNAPSHOT_RECONSTRUCTED" +MONOTONICITY_FAILURE = "EXPOSURE_MONOTONICITY_FAILURE" +PLANNED = "EXPOSURE_SNAPSHOT_PLANNED" +SCHEMA = "hyperlex.exposure_snapshot_reconstruction.v1" +BASELINE_SNAPSHOT_ID = "20260925T211351Z" +SNAPSHOT_ID_RE = re.compile(r"^[0-9]{8}T[0-9]{6}Z$") +HASH_RE = re.compile(r"^[0-9a-f]{64}$") +REFUSED_BOX_FAMILIES = frozenset({"notion", "wiktionary", "wordnet"}) +ABSENT_BY_DEFAULT = ( + "discovery_lane_export", + "jev_spike_export", + "notion_not_a_box_hash_source", + "obsidian_export", + "vernacular_sqlite", +) +HASH_NORMALIZATION = ( + "precomputed sha256 of normalize_group_text; lines are united and not recomputed" +) +VERN_NORMALIZATION = ( + "historical JSON string lines are carried byte-for-byte; new strings use json.dumps ensure_ascii" +) + + +def canonical_json(payload: Mapping[str, Any]) -> str: + return json.dumps(payload, indent=2, sort_keys=True, ensure_ascii=True) + "\n" + + +def sha256_text(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def sha256_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def parse_box_hash_text(text: str, *, label: str) -> list[str]: + found: list[str] = [] + seen: set[str] = set() + for line_number, raw in enumerate(text.splitlines(), start=1): + line = raw.strip() + if not line: + continue + if HASH_RE.fullmatch(line) is None: + raise SystemExit(f"REFUSE: {label} line {line_number} is not a lowercase sha256") + if line in seen: + continue + seen.add(line) + found.append(line) + if not found: + raise SystemExit(f"REFUSE: {label} has no hashes") + return found + + +def parse_vern_text(text: str, *, label: str) -> list[tuple[str, str]]: + """Return ``(decoded string, original line)`` pairs. Duplicate strings keep the first line.""" + found: list[tuple[str, str]] = [] + seen: set[str] = set() + for line_number, raw in enumerate(text.splitlines(), start=1): + if not raw.strip(): + continue + try: + value = json.loads(raw) + except json.JSONDecodeError as exc: + raise SystemExit(f"REFUSE: {label} line {line_number} is not JSON") from exc + if not isinstance(value, str): + raise SystemExit(f"REFUSE: {label} line {line_number} is not a JSON string") + if value in seen: + continue + seen.add(value) + found.append((value, raw)) + if not found: + raise SystemExit(f"REFUSE: {label} has no strings") + return found + + +def _source_record( + *, + family: str, + artifact: str, + source_sha256: str, + entry_count: int, + added_count: int, + normalization: str, +) -> dict[str, Any]: + return { + "added_count": added_count, + "artifact": artifact, + "builder_version": BUILDER_VERSION, + "entry_count": entry_count, + "family": family, + "normalization": normalization, + "source_sha256": source_sha256, + } + + +def assess_monotonicity( + previous_box: set[str], + new_box: set[str], + previous_vern: set[str], + new_vern: set[str], +) -> dict[str, Any]: + box_missing = len(previous_box - new_box) + vern_missing = len(previous_vern - new_vern) + return { + "box_missing": box_missing, + "ok": box_missing == 0 and vern_missing == 0, + "vern_missing": vern_missing, + } + + +def plan_reconstruction( + *, + baseline_box_text: str, + baseline_vern_text: str, + baseline_snapshot_id: str, + baseline_box_sha256: str, + baseline_vern_sha256: str, + box_increments: Sequence[Mapping[str, Any]] = (), + vern_increments: Sequence[Mapping[str, Any]] = (), + absent_families: Sequence[str] = ABSENT_BY_DEFAULT, +) -> dict[str, Any]: + """Unite sources. Do not write. Missing historical rows fail closed.""" + if SNAPSHOT_ID_RE.fullmatch(baseline_snapshot_id) is None: + raise SystemExit("REFUSE: baseline snapshot id is not YYYYMMDDTHHMMSSZ") + baseline_hashes = parse_box_hash_text(baseline_box_text, label="baseline box") + baseline_vern = parse_vern_text(baseline_vern_text, label="baseline vern") + box_set = set(baseline_hashes) + vern_lines = {value: line for value, line in baseline_vern} + vern_order = [value for value, _line in baseline_vern] + sources = [ + _source_record( + family="historical_box_snapshot", + artifact=f"box-jev-hashes-{baseline_snapshot_id}.txt", + source_sha256=baseline_box_sha256, + entry_count=len(baseline_hashes), + added_count=len(baseline_hashes), + normalization=HASH_NORMALIZATION, + ), + _source_record( + family="historical_vern_snapshot", + artifact=f"vern-texts-{baseline_snapshot_id}.jsonl", + source_sha256=baseline_vern_sha256, + entry_count=len(baseline_vern), + added_count=len(baseline_vern), + normalization=VERN_NORMALIZATION, + ), + ] + for item in box_increments: + family = str(item.get("family") or "") + if family in REFUSED_BOX_FAMILIES: + raise SystemExit(f"REFUSE: {family} is not a box-hash source") + hashes = parse_box_hash_text(str(item.get("text") or ""), label=family or "box increment") + added = [digest for digest in hashes if digest not in box_set] + box_set.update(added) + sources.append( + _source_record( + family=family, + artifact=str(item.get("artifact") or ""), + source_sha256=str(item.get("source_sha256") or ""), + entry_count=len(hashes), + added_count=len(added), + normalization=HASH_NORMALIZATION, + ) + ) + new_vern_values: list[str] = [] + for item in vern_increments: + family = str(item.get("family") or "") + if family in REFUSED_BOX_FAMILIES: + raise SystemExit(f"REFUSE: {family} is not an exposure-snapshot source") + parsed = parse_vern_text(str(item.get("text") or ""), label=family or "vern increment") + added_values = [value for value, _line in parsed if value not in vern_lines] + for value, _line in parsed: + if value not in vern_lines: + vern_lines[value] = json.dumps(value, ensure_ascii=True) + new_vern_values.append(value) + sources.append( + _source_record( + family=family, + artifact=str(item.get("artifact") or ""), + source_sha256=str(item.get("source_sha256") or ""), + entry_count=len(parsed), + added_count=len(added_values), + normalization=VERN_NORMALIZATION, + ) + ) + box_text = "".join(digest + "\n" for digest in sorted(box_set)) + vern_text = "".join(vern_lines[value] + "\n" for value in vern_order + new_vern_values) + previous_vern = {value for value, _line in baseline_vern} + assessment = assess_monotonicity(set(baseline_hashes), box_set, previous_vern, set(vern_lines)) + return { + "absent_families": sorted(set(absent_families)), + "baseline_snapshot_id": baseline_snapshot_id, + "box_added": len(box_set) - len(baseline_hashes), + "box_count": len(box_set), + "box_sha256": sha256_text(box_text), + "box_text": box_text, + "builder_version": BUILDER_VERSION, + "monotonicity": assessment, + "network_requests": 0, + "sources": sources, + "transition": TRANSITION, + "vern_added": len(vern_lines) - len(previous_vern), + "vern_bytes_carried_forward": vern_text == baseline_vern_text, + "vern_count": len(vern_lines), + "vern_sha256": sha256_text(vern_text), + "vern_text": vern_text, + } + + +def snapshot_paths(out_dir: Path, snapshot_id: str) -> tuple[Path, Path]: + if SNAPSHOT_ID_RE.fullmatch(snapshot_id) is None: + raise SystemExit("REFUSE: snapshot id is not YYYYMMDDTHHMMSSZ") + return ( + out_dir / f"vern-texts-{snapshot_id}.jsonl", + out_dir / f"box-jev-hashes-{snapshot_id}.txt", + ) + + +def publish_pair(out_dir: Path, snapshot_id: str, box_text: str, vern_text: str) -> tuple[Path, Path]: + """Write both files or neither. Existing names are left untouched.""" + vern_path, box_path = snapshot_paths(out_dir, snapshot_id) + if vern_path.exists() or box_path.exists(): + raise SystemExit("REFUSE: snapshot id is already published") + partial = out_dir / f".partial-{snapshot_id}" + partial.mkdir(mode=0o700) + temporary_vern = partial / vern_path.name + temporary_box = partial / box_path.name + try: + temporary_vern.write_text(vern_text, encoding="utf-8") + temporary_box.write_text(box_text, encoding="utf-8") + os.replace(temporary_vern, vern_path) + try: + os.replace(temporary_box, box_path) + except Exception: + vern_path.unlink(missing_ok=True) + raise + finally: + temporary_box.unlink(missing_ok=True) + temporary_vern.unlink(missing_ok=True) + partial.rmdir() + os.chmod(vern_path, stat.S_IRUSR | stat.S_IWUSR) + os.chmod(box_path, stat.S_IRUSR | stat.S_IWUSR) + return vern_path, box_path + + +def freshness_record( + *, + vern_path: Path, + box_path: Path, + snapshot_id: str, + compared_at_unix: str, +) -> dict[str, Any]: + from hyperlexical.select_005_harvest import exposure_fence + + rows = [] + for kind, path in (("vern", vern_path), ("boxhash", box_path)): + rows.append( + { + "kind": kind, + "mtime_unix": format(path.stat().st_mtime, ".6f"), + "path": path.name, + "sha256": sha256_file(path), + "snapshot_id": snapshot_id, + } + ) + return exposure_fence(rows, compared_at_unix=compared_at_unix) + + +def build_receipt( + plan: Mapping[str, Any], + *, + snapshot_id: str | None, + published: bool, + fence: Mapping[str, Any] | None, + harvest_run: bool = False, +) -> dict[str, Any]: + if harvest_run: + raise SystemExit("REFUSE: reconstruction does not harvest") + monotonicity = dict(plan["monotonicity"]) + if published and not monotonicity["ok"]: + raise SystemExit("REFUSE: a failed monotonicity check cannot be published") + if not monotonicity["ok"]: + state = MONOTONICITY_FAILURE + elif published: + state = RECONSTRUCTED + else: + state = PLANNED + record: dict[str, Any] = { + "absent_families": list(plan["absent_families"]), + "baseline_snapshot_id": plan["baseline_snapshot_id"], + "box_added": plan["box_added"], + "box_count": plan["box_count"], + "builder_version": BUILDER_VERSION, + "freshness": None if fence is None else {key: fence[key] for key in ("state", "worst_age_hours", "limit_hours")}, + "harvest_run": False, + "monotonicity": monotonicity, + "network_requests": 0, + "previous_pair_overwritten": False, + "published": published, + "schema": SCHEMA, + "snapshot_id": snapshot_id, + "successor_box_sha256": plan["box_sha256"], + "successor_vern_sha256": plan["vern_sha256"], + "sources": list(plan["sources"]), + "state": state, + "training_authorized": False, + "transition": TRANSITION, + "vern_added": plan["vern_added"], + "vern_bytes_carried_forward": plan["vern_bytes_carried_forward"], + "vern_count": plan["vern_count"], + } + if fence is not None: + file_box = next(row["sha256"] for row in fence["snapshots"] if row["kind"] == "boxhash") + file_vern = next(row["sha256"] for row in fence["snapshots"] if row["kind"] == "vern") + if file_box != plan["box_sha256"] or file_vern != plan["vern_sha256"]: + raise SystemExit("REFUSE: published bytes differ from the planned pair") + body = {key: value for key, value in record.items() if key != "record_sha256"} + record["record_sha256"] = sha256_text(canonical_json(body)) + return record + + +def utc_snapshot_id(moment: datetime | None = None) -> str: + current = moment or datetime.now(timezone.utc) + if current.tzinfo is None: + raise SystemExit("REFUSE: snapshot clock is naive") + return current.astimezone(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + + +def _increment(spec: str) -> tuple[str, Path]: + family, separator, raw_path = spec.partition(":") + if not separator or not family or not raw_path: + raise SystemExit("REFUSE: increment must be family:path") + return family, Path(raw_path) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Build one monotonic exposure snapshot pair") + parser.add_argument("--baseline-id", default=BASELINE_SNAPSHOT_ID) + parser.add_argument("--baseline-box", type=Path, required=True) + parser.add_argument("--baseline-vern", type=Path, required=True) + parser.add_argument("--box-increment", action="append") + parser.add_argument("--vern-increment", action="append") + parser.add_argument("--absent", action="append") + parser.add_argument("--out-dir", type=Path) + parser.add_argument("--snapshot-id") + parser.add_argument("--receipt", type=Path) + parser.add_argument("--publish", action="store_true") + args = parser.parse_args(argv) + box_text = args.baseline_box.read_text(encoding="utf-8") + vern_text = args.baseline_vern.read_text(encoding="utf-8") + increments = [] + for spec in args.box_increment or []: + family, path = _increment(spec) + increments.append( + { + "artifact": str(path), + "family": family, + "source_sha256": sha256_file(path), + "text": path.read_text(encoding="utf-8"), + } + ) + vern_increments = [] + for spec in args.vern_increment or []: + family, path = _increment(spec) + vern_increments.append( + { + "artifact": str(path), + "family": family, + "source_sha256": sha256_file(path), + "text": path.read_text(encoding="utf-8"), + } + ) + plan = plan_reconstruction( + baseline_box_text=box_text, + baseline_vern_text=vern_text, + baseline_snapshot_id=args.baseline_id, + baseline_box_sha256=sha256_file(args.baseline_box), + baseline_vern_sha256=sha256_file(args.baseline_vern), + box_increments=increments, + vern_increments=vern_increments, + absent_families=args.absent if args.absent is not None else ABSENT_BY_DEFAULT, + ) + published = False + fence = None + snapshot_id = args.snapshot_id + if plan["monotonicity"]["ok"] and args.publish: + if args.out_dir is None: + raise SystemExit("REFUSE: publish requires --out-dir") + snapshot_id = snapshot_id or utc_snapshot_id() + if snapshot_id == args.baseline_id: + raise SystemExit("REFUSE: successor snapshot id collides with the baseline") + vern_path, box_path = publish_pair(args.out_dir, snapshot_id, plan["box_text"], plan["vern_text"]) + published = True + fence = freshness_record( + vern_path=vern_path, + box_path=box_path, + snapshot_id=snapshot_id, + compared_at_unix=format(time.time(), ".6f"), + ) + if fence["state"] != "EXPOSURE_SNAPSHOT_FRESH": + raise SystemExit("REFUSE: published pair is not inside the 6 hour fence") + elif not plan["monotonicity"]["ok"] and args.publish: + published = False + receipt = build_receipt(plan, snapshot_id=snapshot_id, published=published, fence=fence) + rendered = canonical_json(receipt) + if args.receipt is not None: + args.receipt.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + args.receipt.write_text(rendered, encoding="utf-8") + os.chmod(args.receipt, stat.S_IRUSR | stat.S_IWUSR) + print(rendered, end="") + return 0 if plan["monotonicity"]["ok"] else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/shadow/hyperlexical/PORTED.md b/scripts/shadow/hyperlexical/PORTED.md new file mode 100644 index 00000000..3dce0023 --- /dev/null +++ b/scripts/shadow/hyperlexical/PORTED.md @@ -0,0 +1,4 @@ +# Ported from archived scrimshawlife-ctrl/Hyperlex @ ad4075b (2026-10-12) + +Verbatim copy of the holdout guard (PR #120) and the 57 modules its tests and the training entrypoints import (`train.py` -> `loop.run_loop` -> `admission.admit_training_run` -> `holdout_guard`). No logic changes. Torch is imported lazily; the guard tests run with pytest only. +Tests: `tests/shadow/test_hyperlexical_holdout_{disjoint,guard,wiring}.py`, `tests/heldout/` (CI: `.github/workflows/holdout-guard.yml`). diff --git a/scripts/shadow/hyperlexical/__init__.py b/scripts/shadow/hyperlexical/__init__.py new file mode 100644 index 00000000..faeb223d --- /dev/null +++ b/scripts/shadow/hyperlexical/__init__.py @@ -0,0 +1,5 @@ +"""Spec 007 — Hyperlexical shadow package. No torch. No network.""" + +from .packet import PacketError, attach_or_omit, build_packet, validate_packet + +__all__ = ["PacketError", "attach_or_omit", "build_packet", "validate_packet"] diff --git a/scripts/shadow/hyperlexical/_negatives_data.py b/scripts/shadow/hyperlexical/_negatives_data.py new file mode 100644 index 00000000..10241d92 --- /dev/null +++ b/scripts/shadow/hyperlexical/_negatives_data.py @@ -0,0 +1,211 @@ +# Auto-curated ordinary prose negatives for U2 export. No slang. +NEGATIVES = ( + "The committee approved the budget amendment.", + "Water boils at one hundred degrees Celsius.", + "Please find the attached invoice for March.", + "The museum opens at ten on weekdays.", + "Photosynthesis converts light into chemical energy.", + "The train to Sacramento leaves from platform two.", + "This warranty covers defects in materials.", + "Average rainfall in July was two inches.", + "The library card expires in December.", + "Sodium chloride is table salt.", + "The board meeting is scheduled for Tuesday.", + "A rectangle has four right angles.", + "Please confirm receipt of this shipment.", + "The periodic table lists the elements.", + "Oak trees drop acorns in autumn.", + "The speed limit on this road is twenty five.", + "This paragraph contains no slang tokens.", + "The recipe calls for two cups of flour.", + "Latitude and longitude specify a point.", + "The contract is governed by California law.", + "Mercury is the closest planet to the Sun.", + "The river flows south toward the delta.", + "Please submit the application by Friday noon.", + "Cotton fabric shrinks slightly when washed in hot water.", + "The auditorium seats approximately four hundred people.", + "Nitrogen makes up most of Earth's atmosphere.", + "The post office closes at five on Saturdays.", + "A hexagon has six equal sides when regular.", + "The printer needs a new toner cartridge.", + "Mountains form where tectonic plates collide.", + "Please keep the aisle clear during boarding.", + "The calendar marks the equinox in March.", + "Stainless steel resists rust under normal use.", + "The encyclopedia volume covers letters M through O.", + "Evaporation increases on warm dry days.", + "The ferry crosses the bay every half hour.", + "A kilogram equals one thousand grams.", + "Please sign on the line above your printed name.", + "The orchard yields apples in early September.", + "Sound travels slower than light in air.", + "The classroom whiteboard uses dry erase markers.", + "Granite is a common igneous rock.", + "Please return overdue books to the front desk.", + "The compass needle points toward magnetic north.", + "Yeast causes bread dough to rise.", + "The highway rest area has drinking water.", + "A century contains one hundred years.", + "Please wear closed toe shoes in the laboratory.", + "The archive stores microfilm of old newspapers.", + "Condensation forms on cold glass surfaces.", + "The soccer field measures about one hundred meters.", + "Aluminum conducts heat and electricity well.", + "Please update your mailing address online.", + "The tide chart shows high water at dawn.", + "Birds migrate south before winter arrives.", + "The thermostat is set to twenty two degrees.", + "Paper is typically made from wood pulp.", + "Please fasten your seat belt before departure.", + "The dictionary defines ordinary words clearly.", + "Clay hardens when fired in a kiln.", + "The bus route serves downtown and the airport.", + "Vitamin C is found in citrus fruit.", + "Please label each sample with the date.", + "The glacier advances slowly down the valley.", + "Concrete cures over several days after pouring.", + "The flashlight requires two AA batteries.", + "Roots absorb water from the soil.", + "Please schedule the inspection for next week.", + "The piano has eighty eight keys.", + "Copper wires carry household electric current.", + "The hiking trail begins at the ranger station.", + "Salt melts ice on cold roadways.", + "Please file the report in chronological order.", + "The telescope reveals craters on the Moon.", + "Wool garments should be washed gently.", + "The dam regulates water for irrigation.", + "A triangle's angles sum to one hundred eighty degrees.", + "Please store chemicals in the labeled cabinet.", + "The lighthouse warns ships near the reef.", + "Bees pollinate many flowering plants.", + "The elevator serves floors one through twelve.", + "Ink dries quickly on absorbent paper.", + "Please verify the serial number on the device.", + "The prairie grasses bend in strong wind.", + "Glass expands slightly when heated.", + "The census counts residents every ten years.", + "Maple syrup is boiled from tree sap.", + "Please keep receipts for tax records.", + "The volcano emitted ash last century.", + "Rubber tires grip wet pavement better when new.", + "The aquarium filters water continuously.", + "Seeds need moisture and warmth to germinate.", + "Please arrive fifteen minutes before the appointment.", + "The cathedral spire is visible from the square.", + "Iron oxidizes when exposed to moisture.", + "The newsletter is published on the first Monday.", + "Clouds form when moist air cools.", + "Please use the crosswalk at the intersection.", + "The museum exhibit explains local geology.", + "Bamboo grows rapidly in warm climates.", + "The wrench fits a fourteen millimeter bolt.", + "Ice floats because it is less dense than liquid water.", + "Please photocopy both sides of the form.", + "The orchard irrigation runs before sunrise.", + "Quartz is a hard mineral common in sand.", + "The cafeteria serves lunch until two o'clock.", + "Friction converts motion into heat.", + "Please replace the air filter every six months.", + "The meadow floods after heavy spring rain.", + "Silver tarnishes when exposed to sulfur compounds.", + "The lecture hall has tiered seating.", + "Photosynthesis occurs mainly in plant leaves.", + "Please dial nine for an outside line.", + "The reservoir supplies drinking water to the city.", + "Limestone dissolves slowly in acidic rain.", + "The bicycle path parallels the riverbank.", + "Acceleration is the rate of change of velocity.", + "Please stack the chairs after the meeting.", + "The greenhouse maintains a steady temperature.", + "Coral reefs support diverse marine life.", + "The stapler holds twenty sheets of paper.", + "Dew forms overnight on cool grass.", + "Please initial each page of the agreement.", + "The canyon walls show layered sedimentary rock.", + "Zinc coating protects steel from corrosion.", + "The waiting room has magazines and water.", + "Osmosis moves water across a membrane.", + "Please extinguish campfires completely before leaving.", + "The peninsula juts into the northern bay.", + "Nylon fabric is strong and lightweight.", + "The index lists topics alphabetically.", + "Magnets attract objects containing iron.", + "Please rinse the beaker with distilled water.", + "The plateau rises above the surrounding plain.", + "Cedar wood resists many insect pests.", + "The stopwatch measures elapsed time precisely.", + "Evaporated milk is concentrated by removing water.", + "Please keep noise low in the reading room.", + "The estuary mixes fresh and salt water.", + "Porcelain dishes are fired at high temperature.", + "The appendix contains supplementary tables.", + "Inertia keeps moving objects in motion.", + "Please unlock the cabinet with the master key.", + "The savanna has scattered trees and tall grass.", + "Brass is an alloy of copper and zinc.", + "The projector displays slides on the screen.", + "Capillaries connect arteries to veins.", + "Please shred outdated confidential documents.", + "The fjord was carved by ancient glaciers.", + "Cottonseed oil is used in some foods.", + "The barcode identifies the product uniquely.", + "Reflection sends light back from a mirror.", + "Please water the seedlings every other day.", + "The bluff overlooks the winding river.", + "Tin cans are often lined to prevent reactions.", + "The syllabus outlines weekly reading assignments.", + "Diffusion spreads particles from high to low concentration.", + "Please park only in marked spaces.", + "The tundra remains cold for most of the year.", + "Plywood is made from layered wood veneers.", + "The metronome keeps a steady musical tempo.", + "Condensation nuclei help clouds form.", + "Please copy the agenda for all attendees.", + "The archipelago includes dozens of small islands.", + "Solder joins metal parts with a low melting alloy.", + "The footnote cites the original publication year.", + "Torque tends to cause rotation about an axis.", + "Please calibrate the scale before weighing samples.", + "The dune shifts gradually with prevailing winds.", + "Hemp fiber is used in some textiles and rope.", + "The abstract summarizes the paper in one paragraph.", + "Viscosity measures a fluid's resistance to flow.", + "Please keep flammable liquids in approved containers.", + "The mesa has a flat top and steep sides.", + "Asphalt softens in extreme summer heat.", + "The glossary defines technical terms used herein.", + "Refraction bends light as it enters water.", + "Please confirm the guest count by Wednesday.", + "The wetland filters runoff before it reaches the lake.", + "Ceramic tiles are common in kitchen floors.", + "The bibliography lists all referenced works.", + "Momentum equals mass times velocity.", + "Please wipe the lab bench after each experiment.", + "The ridge separates two drainage basins.", + "Linseed oil is used in some wood finishes.", + "The preface explains the book's purpose.", + "Frequency is measured in hertz.", + "Please store dried goods in airtight jars.", + "The delta forms where the river meets the sea.", + "Marble is metamorphic rock from limestone.", + "The table of contents lists chapter titles.", + "Pressure equals force divided by area.", + "Please keep the fire extinguisher accessible.", + "The foothills rise gently toward the mountains.", + "Shellac is a natural resin finish for wood.", + "The errata sheet corrects printing mistakes.", + "Wavelength is the distance between wave peaks.", + "Please label leftovers with the preparation date.", + "The isthmus connects two larger land masses.", + "Turpentine thins some oil based paints.", + "The colophon states the printer and date.", + "Density is mass per unit volume.", + "Please recycle cardboard in the blue bin.", + "The atoll encircles a shallow lagoon.", + "Beeswax is used in candles and polish.", + "The dedication page names the author's family.", + "Gravity pulls objects toward Earth's center.", + "Please close windows before the storm arrives.", +) diff --git a/scripts/shadow/hyperlexical/admission.py b/scripts/shadow/hyperlexical/admission.py new file mode 100644 index 00000000..81b416e6 --- /dev/null +++ b/scripts/shadow/hyperlexical/admission.py @@ -0,0 +1,877 @@ +"""One admission path for preflight and the trainer. + +RUNE.PREFLIGHT_LAUNCH_PARITY(x) = + effective_environment_hash(preflight) + == effective_environment_hash(launch) + AND admission_gate_sequence(preflight) + == admission_gate_sequence(launch) + AND admission_result(preflight) + == admission_result(launch) + +Controlled experiments (``HLX_EXPERIMENT_ID`` set) use ``CONTROLLED_RESERVE``. +A sealed evaluation reserve with zero training overlap satisfies the holdout +requirement. ``HLX_HOLDOUT_MANIFESTS`` does not. ``HLX_ALLOW_NO_HOLDOUT`` does +not. Legacy launches that are not controlled experiments still use +``require_holdout_for_training``. + +``HLX_ADMISSION_ONLY=1`` is not part of the environment hash. The trainer +returns after these gates and does not construct an optimizer. + +``TRAINING_READY`` exists only when all three are true: these scientific +gates passed, a sealed threshold authorization matches this experiment, and +admission returns ``ADMISSION_PASS``. Without that decision rule the status +stays ``PREREGISTERED`` and ``ready_to_train`` stays false. An admission pass +is not a training launch. +""" + +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +from typing import Any, Callable, Mapping + +from .classify_metrics import SELECT_METRIC_CLASSIFY +from .export import repo_root +from .holdout_guard import ( + HoldoutSpec, + allow_no_holdout, + load_holdout_spec, + normalized_text_sha256, + require_holdout_for_training, +) +from .identity_ledger import ( + RESERVE_LEDGER_ENV, + IdentityLedger, + derived_state, +) +from .selection_surface import row_id +from .train_input import ( + TrainInputAdmissionError, + load_training_bundle, + train_input_receipt, +) + +CONTRACT_RESERVE = "CONTROLLED_RESERVE" +CONTRACT_MANIFEST = "LEGACY_MANIFEST" +CONTRACT_UNCONTROLLED = "UNCONTROLLED" +BINDING_SCHEMA = "hyperlex.reserve_binding.v1" +ADMISSION_ONLY_ENV = "HLX_ADMISSION_ONLY" +RESERVE_BINDING_ENV = "HLX_RESERVE_BINDING" +BASELINE_ENV = "HLX_BASELINE_ENV" +CANDIDATE_ENV = "HLX_CANDIDATE_ENV" +BEST_SHA_ENV = "HLX_BEST_SHA256" +BEST_WEIGHTS_ENV = "HLX_BEST_WEIGHTS" +TRUNK_SHA_ENV = "HLX_TRUNK_SHA256" +TRAIN_OUT_ENV = "HYPERLEX_TRAIN_OUT" +SELECT_METRIC_KEY = "HLX_SELECT_METRIC" +SCIENTIFIC_VARIABLE_KEY = "HLX_SCIENTIFIC_VARIABLE" +DECLARED_SELECT_METRIC = SELECT_METRIC_KEY +DECLARED_TRAIN_SCHEDULE = "train_schedule" +DECLARED_CLASSIFY_SAMPLING = "classify_sampling" +SAMPLING_KEY = "HYPERLEX_CLASSIFY_SAMPLING" +# The only composite. These four trainer fields are one variable, not a +# caller-supplied grouping of arbitrary keys. +SCHEDULE_BUNDLE_KEYS = ( + "HYPERLEX_TRAIN_EPOCHS", + "HYPERLEX_EARLY_STOP", + "HYPERLEX_EARLY_STOP_PATIENCE", + "HYPERLEX_EARLY_STOP_MIN_EPOCHS", +) +# Not part of the environment hash. Unset means candidate, the historical launch. +SCHEDULE_ARM_ENV = "HLX_SCHEDULE_ARM" +_SCHEDULE_OFF = frozenset({"0", "false", "no", "off"}) +_SCHEDULE_ON = frozenset({"1", "true", "yes", "on"}) +THRESHOLD_AUTHORIZATION_ENV = "HLX_THRESHOLD_AUTHORIZATION" +THRESHOLD_SCHEMA = "hyperlex.threshold_authorization.v1" +THRESHOLDS_BLOCKED = "BLOCKED_PENDING_OPERATOR_AUTHORIZATION" +REQUIRED_SLICES = ("classify", "classify_observed", "classify_non_none", "unbind_clean") + +GATE_SEQUENCE = ( + "experiment_binding", + "launch_gate", + "holdout_reserve", + "pinned_training_input", + "train_reserve_disjointness", + "single_variable", + "best_trunk", + "output_directory", + "ready", +) + +# Process environment that admission reads. ``HLX_ADMISSION_ONLY`` is omitted +# so a dry launch and preflight hash the same binding. +ENV_KEYS = ( + "HLX_EXPERIMENT_ID", + "HLX_THRESHOLD_AUTHORIZATION", + "HYPERLEX_ALLOW_TRAIN", + "HLX_EVAL_RESERVE_LEDGER", + "HLX_RESERVE_BINDING", + "HLX_HOLDOUT_MANIFESTS", + "HLX_ALLOW_NO_HOLDOUT", + "HLX_TRAIN_EXPORT_PATH", + "HLX_TRAIN_EXPORT_SHA256", + "HLX_TRAIN_EXPORT_ROWS", + "HLX_BASELINE_ENV", + "HLX_CANDIDATE_ENV", + "HLX_SELECT_METRIC", + "HLX_BEST_SHA256", + "HLX_BEST_WEIGHTS", + "HLX_TRUNK_SHA256", + "HYPERLEX_TRUNK_DIR", + "HYPERLEX_TRAIN_OUT", + "HYPERLEX_INCLUDE_LIVE", +) + +METADATA_KEYS = frozenset( + { + "HLX_EXPERIMENT_ID", + "HYPERLEX_TRAIN_OUT", + "HYPERLEX_UNBIND_RESIDUAL_DUMP", + "HLX_EVAL_RESERVE_LEDGER", + "HLX_RESERVE_BINDING", + "HLX_BASELINE_ENV", + "HLX_CANDIDATE_ENV", + SCIENTIFIC_VARIABLE_KEY, + } +) +LAUNCH_OVERLAY_KEYS = frozenset({"HYPERLEX_ALLOW_TRAIN", ADMISSION_ONLY_ENV}) + +MISSING_RESERVE_REASON = ( + "ADMISSION FAIL: HYPERLEX_ALLOW_TRAIN=1 but no sealed evaluation reserve. " + "Set HLX_EVAL_RESERVE_LEDGER to the sealed ledger. " + "CONTROLLED_RESERVE does not accept a legacy holdout manifest." +) + + +class AdmissionError(SystemExit): + """Admission refused. ``SystemExit`` so the trainer CLI does not treat it as a crash.""" + + def __init__(self, message: str, receipt: dict[str, Any]) -> None: + super().__init__(message) + self.receipt = receipt + + +class AdmissionResult: + def __init__( + self, + *, + ready: bool, + status: str | None, + admission_result: str, + contract: str, + launch_armed: bool, + bundle: dict[str, Any] | None, + holdout_spec: HoldoutSpec, + input_receipt: dict[str, Any] | None, + disjoint_receipt: dict[str, Any] | None, + reserve_receipt: dict[str, Any] | None, + receipt: dict[str, Any], + error: str | None = None, + ) -> None: + self.ready = ready + self.status = status + self.admission_result = admission_result + self.contract = contract + self.launch_armed = launch_armed + self.bundle = bundle + self.holdout_spec = holdout_spec + self.input_receipt = input_receipt + self.disjoint_receipt = disjoint_receipt + self.reserve_receipt = reserve_receipt + self.receipt = receipt + self.error = error + + +def effective_environment_material() -> dict[str, str | None]: + """Admission environment. Absent variables are null. No timestamps.""" + return {key: os.environ.get(key) for key in ENV_KEYS} + + +def effective_environment_hash() -> str: + payload = json.dumps( + effective_environment_material(), + sort_keys=True, + separators=(",", ":"), + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def _empty_spec() -> HoldoutSpec: + return HoldoutSpec(frozenset(), frozenset(), ()) + + +def _sha256_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _read_env_file(path: Path, what: str) -> dict[str, str]: + if not path.is_file(): + raise AdmissionError( + f"ADMISSION FAIL: {what} is missing: {path}", + {}, + ) + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise AdmissionError(f"ADMISSION FAIL: {what} is not JSON", {}) from exc + if not isinstance(payload, dict): + raise AdmissionError(f"ADMISSION FAIL: {what} must be a JSON object", {}) + out: dict[str, str] = {} + for key, value in payload.items(): + if not isinstance(key, str) or not isinstance(value, str): + raise AdmissionError( + f"ADMISSION FAIL: {what} values must be strings", + {}, + ) + out[key] = value + return out + + +def admit_training_run( + *, + include_live: bool = False, + live_store: Path | None = None, + export_dataset: Callable[..., dict[str, Any]] | None = None, + trunk: Path | None = None, + out_dir: Path | None = None, +) -> AdmissionResult: + """Run the trainer admission gates. Does not construct an optimizer.""" + if export_dataset is None: + from .export import export_dataset as export_dataset + + experiment_id = os.environ.get("HLX_EXPERIMENT_ID", "").strip() + controlled = bool(experiment_id) + launch = os.environ.get("HYPERLEX_ALLOW_TRAIN") == "1" + contract = CONTRACT_RESERVE if controlled else (CONTRACT_MANIFEST if launch else CONTRACT_UNCONTROLLED) + ctx = _Context( + include_live=include_live, + live_store=live_store, + export_dataset=export_dataset, + trunk=trunk, + out_dir=out_dir, + experiment_id=experiment_id, + controlled=controlled, + launch=launch, + contract=contract, + ) + if not launch: + return _admit_unarmed(ctx) + if not controlled: + return _admit_legacy_launch(ctx) + return _admit_controlled(ctx) + + +class _Context: + def __init__(self, **kwargs: Any) -> None: + self.__dict__.update(kwargs) + self.env_hash = effective_environment_hash() + self.failed_gate: str | None = None + + def receipt(self, **fields: Any) -> dict[str, Any]: + payload = { + "schema": "hyperlex.admission.v1", + "controlled_holdout_contract": self.contract, + "admission_gate_sequence": list(GATE_SEQUENCE), + "failed_gate": self.failed_gate, + "environment_hash": self.env_hash, + "experiment_id": self.experiment_id or None, + "launch_armed": self.launch, + "training_started": False, + "epochs": 0, + "gradient_steps": 0, + "optimizer_loaded": False, + "best_moved": False, + "hlx_allow_no_holdout": allow_no_holdout(), + "holdout_admitted": False, + } + payload.update(fields) + return payload + + def fail(self, gate: str, message: str, **fields: Any) -> None: + self.failed_gate = gate + raise AdmissionError(message, self.receipt(**fields)) + + +def _load_bundle(ctx: _Context) -> dict[str, Any]: + try: + return load_training_bundle( + repo_root(), + include_live=ctx.include_live, + live_store=ctx.live_store, + export_dataset=ctx.export_dataset, + ) + except TrainInputAdmissionError as exc: + ctx.fail("pinned_training_input", str(exc)) + raise AssertionError("unreachable") from exc + + +def _unarmed_result(ctx: _Context, bundle: dict[str, Any], spec: HoldoutSpec) -> AdmissionResult: + proof = train_input_receipt(bundle) + status = "NOT_READY" if ctx.controlled else None + receipt = ctx.receipt( + admission_result="NOT_ARMED", + status=status, + ready_to_train=False, + **proof, + ) + return AdmissionResult( + ready=False, + status=status, + admission_result="NOT_ARMED", + contract=ctx.contract, + launch_armed=False, + bundle=bundle, + holdout_spec=spec, + input_receipt=proof, + disjoint_receipt=None, + reserve_receipt=None, + receipt=receipt, + ) + + +def _admit_unarmed(ctx: _Context) -> AdmissionResult: + """ALLOW_TRAIN is unset. Load a pinned bundle when one is declared. + + This is not launch admission. Callers that still execute ``run_loop`` + keep the previous non-launch behavior. + """ + bundle = _load_bundle(ctx) + spec = load_holdout_spec() + from .holdout_guard import assert_pinned_holdout_disjoint + + if ctx.experiment_id: + assert_pinned_holdout_disjoint(bundle["rows"], spec) + ledger = os.environ.get(RESERVE_LEDGER_ENV, "").strip() + if ledger: + from .identity_ledger import assert_training_disjoint_from_reserve + + assert_training_disjoint_from_reserve(bundle["rows"], IdentityLedger.load(ledger)) + return _unarmed_result(ctx, bundle, spec) + + +def _admit_legacy_launch(ctx: _Context) -> AdmissionResult: + try: + spec = require_holdout_for_training() + except SystemExit as exc: + ctx.fail("holdout_reserve", str(exc)) + raise AssertionError("unreachable") from exc + bundle = _load_bundle(ctx) + proof = train_input_receipt(bundle) + ready = _trunk_config_ok(ctx.trunk) + receipt = ctx.receipt( + admission_result="ADMISSION_PASS" if ready else "NOT_READY", + status=None, + ready_to_train=ready, + holdout_admitted=bool(spec.manifests) or allow_no_holdout(), + **proof, + ) + return AdmissionResult( + ready=ready, + status=None, + admission_result=receipt["admission_result"], + contract=CONTRACT_MANIFEST, + launch_armed=True, + bundle=bundle, + holdout_spec=spec, + input_receipt=proof, + disjoint_receipt=None, + reserve_receipt=None, + receipt=receipt, + ) + + +def _trunk_config_ok(trunk: Path | None) -> bool: + return bool(trunk) and trunk.is_dir() and (trunk / "config.json").is_file() + + +def _admit_controlled(ctx: _Context) -> AdmissionResult: + _gate_experiment(ctx) + _gate_launch(ctx) + ledger, binding = _gate_reserve(ctx) + bundle = _load_bundle(ctx) + proof = train_input_receipt(bundle) + if proof["training_input_mode"] != "PINNED_EXPORT" or proof["live_export_generation_enabled"]: + ctx.fail( + "pinned_training_input", + "ADMISSION FAIL: controlled experiment did not consume a pinned export", + **proof, + ) + overlap = _gate_disjoint(ctx, bundle, ledger) + _gate_single_variable(ctx) + _gate_best_trunk(ctx) + _gate_output(ctx) + decision_sealed, decision_state = _decision_authorization(ctx) + evidence: dict[str, Any] = {} + if ctx.experiment_id == "HLX-EXP-2026-09-29-SELECT-006": + from .select_006_admission import Select006AdmissionError, select006_admission_evidence + + try: + evidence = select006_admission_evidence() + except Select006AdmissionError as exc: + ctx.fail("ready", str(exc), admission_reason=exc.code) + status = "TRAINING_READY" if decision_sealed else "PREREGISTERED" + spec = _empty_spec() + disjoint = { + "holdout_train_row_id_overlap": 0, + "holdout_train_text_hash_overlap": 0, + "holdout_filter_training_rows_removed": 0, + "holdout_training_disjoint": True, + } + receipt = ctx.receipt( + admission_result="ADMISSION_PASS", + status=status, + ready_to_train=decision_sealed, + scientific_contract_sealed=True, + decision_rule_sealed=decision_sealed, + decision_threshold_state=decision_state, + training_launch_authorized=False, + holdout_admitted=True, + holdout_state="EVAL_RESERVE", + holdout_experiment_id=ctx.experiment_id, + holdout_manifest_sha256=None, + reserve_lifecycle="EVAL_RESERVE", + reserve_events_sha256=binding["ledger_events_sha256"], + reserve_counts=binding["counts"], + **proof, + **overlap, + **disjoint, + **evidence, + ) + return AdmissionResult( + ready=True, + status=status, + admission_result="ADMISSION_PASS", + contract=CONTRACT_RESERVE, + launch_armed=True, + bundle=bundle, + holdout_spec=spec, + input_receipt=proof, + disjoint_receipt=disjoint, + reserve_receipt=overlap, + receipt=receipt, + ) + + +def _decision_authorization(ctx: _Context) -> tuple[bool, str]: + """A missing authorization stays blocked. A bad file fails closed.""" + raw = os.environ.get(THRESHOLD_AUTHORIZATION_ENV, "").strip() + if not raw: + return False, THRESHOLDS_BLOCKED + path = Path(raw) + if not path.is_file(): + ctx.fail("ready", "ADMISSION FAIL: threshold authorization path is not a file") + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + ctx.fail("ready", "ADMISSION FAIL: threshold authorization is not JSON") + if isinstance(payload, dict) and payload.get("schema") == "hyperlex.select_006_threshold_authorization.v1": + from .select_006_admission import Select006AdmissionError, select006_threshold_is_sealed + + try: + if not select006_threshold_is_sealed(payload, ctx.experiment_id): + ctx.fail("ready", "ADMISSION FAIL: threshold authorization schema is not " + THRESHOLD_SCHEMA) + except Select006AdmissionError as exc: + ctx.fail("ready", str(exc), admission_reason=exc.code) + return True, "SEALED" + if not isinstance(payload, dict) or payload.get("schema") != THRESHOLD_SCHEMA: + ctx.fail( + "ready", + "ADMISSION FAIL: threshold authorization schema is not " + THRESHOLD_SCHEMA, + ) + if payload.get("experiment_id") != ctx.experiment_id: + ctx.fail( + "ready", + "ADMISSION FAIL: threshold authorization experiment_id does not match", + ) + if payload.get("sealed") is not True: + ctx.fail("ready", "ADMISSION FAIL: threshold authorization is not sealed") + thresholds = payload.get("decision_thresholds") + if not isinstance(thresholds, dict) or not thresholds: + ctx.fail( + "ready", + "ADMISSION FAIL: threshold authorization does not seal numeric decision thresholds", + ) + for key, value in thresholds.items(): + if not isinstance(key, str) or isinstance(value, bool) or not isinstance(value, (int, float)): + ctx.fail( + "ready", + "ADMISSION FAIL: threshold authorization does not seal numeric decision thresholds", + ) + return True, "SEALED" + + +def _gate_experiment(ctx: _Context) -> None: + if os.environ.get("HLX_HOLDOUT_MANIFESTS", "").strip(): + ctx.fail( + "experiment_binding", + "ADMISSION FAIL: legacy holdout manifest is not the CONTROLLED_RESERVE contract", + ) + if allow_no_holdout(): + ctx.fail( + "experiment_binding", + "ADMISSION FAIL: HLX_ALLOW_NO_HOLDOUT does not admit a controlled experiment", + ) + + +def _gate_launch(ctx: _Context) -> None: + if not _trunk_config_ok(ctx.trunk): + ctx.fail( + "launch_gate", + "ADMISSION FAIL: trunk config is missing", + ) + + +def _gate_reserve(ctx: _Context) -> tuple[IdentityLedger, dict[str, Any]]: + raw_ledger = os.environ.get(RESERVE_LEDGER_ENV, "").strip() + raw_binding = os.environ.get(RESERVE_BINDING_ENV, "").strip() + if not raw_ledger or not raw_binding: + ctx.fail("holdout_reserve", MISSING_RESERVE_REASON) + ledger_dir = Path(raw_ledger) + binding_path = Path(raw_binding) + if not (ledger_dir / "events.jsonl").is_file(): + ctx.fail("holdout_reserve", MISSING_RESERVE_REASON) + if not binding_path.is_file(): + ctx.fail("holdout_reserve", MISSING_RESERVE_REASON) + try: + binding = json.loads(binding_path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + ctx.fail("holdout_reserve", "ADMISSION FAIL: reserve binding is not JSON") + if not isinstance(binding, dict) or binding.get("schema") != BINDING_SCHEMA: + ctx.fail("holdout_reserve", "ADMISSION FAIL: reserve binding schema is not sealed") + if binding.get("experiment_id") != ctx.experiment_id: + ctx.fail( + "holdout_reserve", + "ADMISSION FAIL: reserve binding experiment_id does not match HLX_EXPERIMENT_ID", + ) + if binding.get("lifecycle") != "EVAL_RESERVE": + ctx.fail( + "holdout_reserve", + "ADMISSION FAIL: reserve lifecycle is not EVAL_RESERVE", + ) + actual_events = _sha256_file(ledger_dir / "events.jsonl") + if binding.get("ledger_events_sha256") != actual_events: + ctx.fail( + "holdout_reserve", + "ADMISSION FAIL: reserve ledger events sha256 does not match the binding", + ) + ledger = IdentityLedger.load(ledger_dir) + # Digest above covers the full append-only log. Counts below use only the + # active reserve for this experiment. + active = ledger.active_reserve_records(ctx.experiment_id) + counts = ledger.active_reserve_counts(ctx.experiment_id) + if not active: + foreign = [ + record + for record in ledger.identities.values() + if derived_state(record) == "EVAL_RESERVE" + ] + if foreign: + ctx.fail( + "holdout_reserve", + "ADMISSION FAIL: active reserve is bound to another experiment", + ) + ctx.fail("holdout_reserve", "ADMISSION FAIL: missing active reserve") + expected_counts = binding.get("counts") + if not isinstance(expected_counts, dict): + ctx.fail("holdout_reserve", "ADMISSION FAIL: reserve binding counts are missing") + for key in REQUIRED_SLICES: + if counts.get(key, 0) < 1: + ctx.fail("holdout_reserve", f"ADMISSION FAIL: reserve slice {key} is absent") + if int(expected_counts.get(key, -1)) != int(counts[key]): + ctx.fail( + "holdout_reserve", + f"ADMISSION FAIL: reserve slice {key} does not match the binding", + ) + if "identities" in binding and int(binding["identities"]) != len(active): + ctx.fail("holdout_reserve", "ADMISSION FAIL: reserve identity count does not match the binding") + return ledger, binding + + +def _gate_disjoint(ctx: _Context, bundle: Mapping[str, Any], ledger: IdentityLedger) -> dict[str, int]: + reserved = ledger.active_reserve_records(ctx.experiment_id) + hashes = {record["normalized_text_sha256"] for record in reserved} + ids: set[str] = set() + for record in reserved: + ids.update(str(item) for item in record.get("row_ids") or []) + id_overlap = 0 + text_overlap = 0 + for row in bundle["rows"]: + if row_id(row) in ids: + id_overlap += 1 + if normalized_text_sha256(str(row.get("text") or "")) in hashes: + text_overlap += 1 + if id_overlap or text_overlap: + ctx.fail( + "train_reserve_disjointness", + "ADMISSION FAIL: training input overlaps the sealed reserve " + f"(row_id_overlap={id_overlap}, text_hash_overlap={text_overlap})", + reserve_train_row_id_overlap=id_overlap, + reserve_train_text_hash_overlap=text_overlap, + training_rows_removed=0, + ) + return { + "reserve_train_row_id_overlap": 0, + "reserve_train_text_hash_overlap": 0, + "training_rows_removed": 0, + } + + +def _scientific_items(payload: Mapping[str, str]) -> dict[str, str]: + return {key: value for key, value in payload.items() if key not in METADATA_KEYS and key not in LAUNCH_OVERLAY_KEYS} + + +def _declaration_value(payload: Mapping[str, str]) -> str: + raw = payload.get(SCIENTIFIC_VARIABLE_KEY) + if raw is None: + return "" + return str(raw).strip() + + +def _canon_schedule_int(raw: str | None) -> str | None: + if raw is None or not str(raw).strip(): + return None + text = str(raw).strip() + sign = "" + digits = text + if text[0] in "+-": + sign, digits = text[0], text[1:] + if not digits or not digits.isdigit(): + raise ValueError(f"schedule field must be an integer, got {raw!r}") + return str(int(sign + digits)) + + +def _canon_early_stop(raw: str | None) -> str: + if raw is None or not str(raw).strip(): + return "0" + token = str(raw).strip().lower() + if token in _SCHEDULE_OFF: + return "0" + if token in _SCHEDULE_ON: + return "1" + raise ValueError( + f"HYPERLEX_EARLY_STOP must be off or on, got {raw!r}" + ) + + +def schedule_arm() -> str: + """Which sealed schedule the process is executing. + + Unset stays ``candidate`` so existing launches do not change. ``control`` + is the other sealed schedule in the same ``train_schedule`` variable. + """ + raw = os.environ.get(SCHEDULE_ARM_ENV) + if raw is None or not str(raw).strip(): + return "candidate" + token = str(raw).strip().lower() + if token not in {"candidate", "control"}: + raise ValueError(f"{SCHEDULE_ARM_ENV} must be candidate or control, got {raw!r}") + return token + + +def schedule_bundle(payload: Mapping[str, str]) -> tuple[str | None, str, str | None, str | None]: + """One normalized schedule. Absent early-stop is off. Absent integers stay absent.""" + return ( + _canon_schedule_int(payload.get("HYPERLEX_TRAIN_EPOCHS")), + _canon_early_stop(payload.get("HYPERLEX_EARLY_STOP")), + _canon_schedule_int(payload.get("HYPERLEX_EARLY_STOP_PATIENCE")), + _canon_schedule_int(payload.get("HYPERLEX_EARLY_STOP_MIN_EPOCHS")), + ) + + +def _require_schedule_shape(side: str, bundle: tuple[str | None, str, str | None, str | None]) -> None: + epochs, stop, patience, minimum = bundle + if epochs is None or int(epochs) < 1: + raise ValueError(f"{side} HYPERLEX_TRAIN_EPOCHS must be an integer >= 1") + if stop == "0": + return + if patience is None or int(patience) < 0: + raise ValueError(f"{side} HYPERLEX_EARLY_STOP_PATIENCE must be an integer >= 0") + if minimum is None or int(minimum) < 1 or int(minimum) > int(epochs): + raise ValueError( + f"{side} HYPERLEX_EARLY_STOP_MIN_EPOCHS must be in 1..HYPERLEX_TRAIN_EPOCHS" + ) + + +def _metric_value(payload: Mapping[str, str]) -> str: + return str(payload.get(SELECT_METRIC_KEY) or "").strip() + + +def _gate_single_variable(ctx: _Context) -> None: + baseline_path = os.environ.get(BASELINE_ENV, "").strip() + candidate_path = os.environ.get(CANDIDATE_ENV, "").strip() + if not baseline_path or not candidate_path: + ctx.fail("single_variable", "ADMISSION FAIL: baseline and candidate environments are not bound") + try: + baseline = _read_env_file(Path(baseline_path), "baseline environment") + candidate = _read_env_file(Path(candidate_path), "candidate environment") + except AdmissionError as exc: + ctx.fail("single_variable", str(exc)) + for key in LAUNCH_OVERLAY_KEYS: + if key in candidate or key in baseline: + ctx.fail( + "single_variable", + "ADMISSION FAIL: launch overlay is persisted in the sealed environment", + ) + declared_baseline = _declaration_value(baseline) + declared_candidate = _declaration_value(candidate) + if declared_baseline != declared_candidate: + ctx.fail( + "single_variable", + "ADMISSION FAIL: scientific variable declaration does not match", + ) + if declared_baseline not in ( + "", + DECLARED_SELECT_METRIC, + DECLARED_TRAIN_SCHEDULE, + DECLARED_CLASSIFY_SAMPLING, + ): + ctx.fail( + "single_variable", + "ADMISSION FAIL: scientific variable declaration is not " + f"{DECLARED_SELECT_METRIC}, {DECLARED_TRAIN_SCHEDULE}, " + f"or {DECLARED_CLASSIFY_SAMPLING}", + ) + process_declaration = str(os.environ.get(SCIENTIFIC_VARIABLE_KEY) or "").strip() + if process_declaration != declared_baseline: + ctx.fail( + "single_variable", + "ADMISSION FAIL: process environment " + f"{SCIENTIFIC_VARIABLE_KEY} does not match the sealed declaration", + ) + try: + base_bundle = schedule_bundle(baseline) + cand_bundle = schedule_bundle(candidate) + except ValueError as exc: + ctx.fail("single_variable", f"ADMISSION FAIL: {exc}") + bundle_differs = base_bundle != cand_bundle + declared = declared_baseline or DECLARED_SELECT_METRIC + if declared == DECLARED_TRAIN_SCHEDULE: + try: + _require_schedule_shape("baseline", base_bundle) + _require_schedule_shape("candidate", cand_bundle) + except ValueError as exc: + ctx.fail("single_variable", f"ADMISSION FAIL: {exc}") + if _metric_value(baseline) != _metric_value(candidate) or _metric_value(candidate) != SELECT_METRIC_CLASSIFY: + ctx.fail( + "single_variable", + "ADMISSION FAIL: selection metric difference is not part of train_schedule; " + f"both arms must set {SELECT_METRIC_KEY}={SELECT_METRIC_CLASSIFY}", + ) + elif bundle_differs: + ctx.fail( + "single_variable", + "ADMISSION FAIL: undeclared schedule difference", + ) + base_sci = _scientific_items(baseline) + cand_sci = _scientific_items(candidate) + changed = sorted(set(base_sci) | set(cand_sci)) + changed = [key for key in changed if base_sci.get(key) != cand_sci.get(key)] + if not bundle_differs: + changed = [key for key in changed if key not in SCHEDULE_BUNDLE_KEYS] + if declared == DECLARED_TRAIN_SCHEDULE: + parts = ["train_schedule"] if bundle_differs else [] + parts.extend(key for key in changed if key not in SCHEDULE_BUNDLE_KEYS) + if parts != ["train_schedule"]: + ctx.fail( + "single_variable", + "ADMISSION FAIL: scientific variable count is " + f"{len(parts)}: {','.join(parts) or 'none'}", + ) + elif declared == DECLARED_CLASSIFY_SAMPLING: + if _metric_value(baseline) != _metric_value(candidate) or _metric_value(candidate) != SELECT_METRIC_CLASSIFY: + ctx.fail( + "single_variable", + "ADMISSION FAIL: selection metric difference is not part of classify_sampling; " + f"both arms must set {SELECT_METRIC_KEY}={SELECT_METRIC_CLASSIFY}", + ) + if changed != [SAMPLING_KEY]: + ctx.fail( + "single_variable", + "ADMISSION FAIL: scientific variable count is " + f"{len(changed)}: {','.join(changed) or 'none'}", + ) + elif changed != [SELECT_METRIC_KEY]: + ctx.fail( + "single_variable", + "ADMISSION FAIL: scientific variable count is " + f"{len(changed)}: {','.join(changed) or 'none'}", + ) + skip_on_baseline = {SELECT_METRIC_KEY} + if declared == DECLARED_TRAIN_SCHEDULE: + skip_on_baseline.update(SCHEDULE_BUNDLE_KEYS) + elif declared == DECLARED_CLASSIFY_SAMPLING: + skip_on_baseline.add(SAMPLING_KEY) + try: + arm = schedule_arm() + except ValueError as exc: + ctx.fail("single_variable", f"ADMISSION FAIL: {exc}") + if arm == "candidate": + expected = dict(cand_sci) + absent_schedule: tuple[str, ...] = () + else: + expected = {key: value for key, value in cand_sci.items() if key not in SCHEDULE_BUNDLE_KEYS} + for key in SCHEDULE_BUNDLE_KEYS: + if key in baseline: + expected[key] = baseline[key] + absent_schedule = tuple(key for key in SCHEDULE_BUNDLE_KEYS if key not in baseline) + if declared == DECLARED_CLASSIFY_SAMPLING: + expected[SAMPLING_KEY] = baseline[SAMPLING_KEY] + for key, value in expected.items(): + if os.environ.get(key) != value: + ctx.fail( + "single_variable", + f"ADMISSION FAIL: process environment {key} does not match the sealed {arm}", + ) + for key in absent_schedule: + if os.environ.get(key) not in (None, ""): + ctx.fail( + "single_variable", + f"ADMISSION FAIL: process environment {key} is set but the sealed {arm} leaves it absent", + ) + for key, value in base_sci.items(): + if key in skip_on_baseline or key in SCHEDULE_BUNDLE_KEYS: + continue + if os.environ.get(key) != value: + ctx.fail( + "single_variable", + f"ADMISSION FAIL: process environment {key} does not match the sealed baseline", + ) + + +def _gate_best_trunk(ctx: _Context) -> None: + best_sha = os.environ.get(BEST_SHA_ENV, "").strip() + best_path = os.environ.get(BEST_WEIGHTS_ENV, "").strip() + trunk_sha = os.environ.get(TRUNK_SHA_ENV, "").strip() + if not best_sha or not best_path or not trunk_sha: + ctx.fail("best_trunk", "ADMISSION FAIL: BEST or trunk digest is not bound") + best = Path(best_path) + if not best.is_file(): + ctx.fail("best_trunk", "ADMISSION FAIL: BEST weights are missing") + actual_best = _sha256_file(best) + if actual_best != best_sha: + ctx.fail("best_trunk", "ADMISSION FAIL: BEST weights sha256 mismatch") + trunk = ctx.trunk + weights = (trunk / "model.safetensors") if trunk else Path() + if trunk is None or not weights.is_file(): + ctx.fail("best_trunk", "ADMISSION FAIL: trunk weights are missing") + if _sha256_file(weights) != trunk_sha: + ctx.fail("best_trunk", "ADMISSION FAIL: trunk weights sha256 mismatch") + + +def _gate_output(ctx: _Context) -> None: + raw = os.environ.get(TRAIN_OUT_ENV, "").strip() + if not raw: + ctx.fail("output_directory", "ADMISSION FAIL: output directory is not bound") + out = Path(raw) + if ctx.out_dir is not None and ctx.out_dir.resolve() != out.resolve(): + ctx.fail( + "output_directory", + "ADMISSION FAIL: output directory does not match HYPERLEX_TRAIN_OUT", + ) + if out.exists(): + ctx.fail("output_directory", "ADMISSION FAIL: output directory collision") + lock = Path(str(out) + ".lock") + if lock.exists(): + ctx.fail("output_directory", "ADMISSION FAIL: conflicting training run") diff --git a/scripts/shadow/hyperlexical/align.py b/scripts/shadow/hyperlexical/align.py new file mode 100644 index 00000000..ebe3def7 --- /dev/null +++ b/scripts/shadow/hyperlexical/align.py @@ -0,0 +1,69 @@ +"""Map surface atoms onto tokenizer spans. No torch. No Hub.""" + +from __future__ import annotations + + +def char_span(text: str, atom: str) -> tuple[int, int] | None: + if not text or not atom: + return None + i = text.find(atom) + if i < 0: + i = text.lower().find(atom.lower()) + if i < 0: + return None + return i, i + len(atom) + + +def whitespace_offsets(text: str) -> list[tuple[int, int]]: + spans = [] + i = 0 + while i < len(text): + while i < len(text) and text[i].isspace(): + i += 1 + if i >= len(text): + break + j = i + while j < len(text) and not text[j].isspace(): + j += 1 + spans.append((i, j)) + i = j + return spans + + +def token_indices_for_span(offsets: list[tuple[int, int]], start: int, end: int) -> list[int]: + hit = [] + for i, (s, e) in enumerate(offsets): + if e <= s: + continue + if e <= start or s >= end: + continue + hit.append(i) + return hit + + +def atom_token_index(text: str, atom: str, offsets: list[tuple[int, int]] | None = None) -> list[int]: + span = char_span(text, atom) + if span is None: + return [] + offs = offsets if offsets is not None else whitespace_offsets(text) + return token_indices_for_span(offs, span[0], span[1]) + + +def offsets_from_tokenizer(tokenizer, text: str, max_len: int = 64) -> list[tuple[int, int]]: + enc = tokenizer( + text, + truncation=True, + max_length=max_len, + return_offsets_mapping=True, + add_special_tokens=True, + ) + raw = enc["offset_mapping"] + return [(int(s), int(e)) for s, e in raw] + + +def pool_indices(n_tokens: int, indices: list[int]) -> list[int]: + if indices: + return [i for i in indices if 0 <= i < n_tokens] + if n_tokens <= 2: + return [min(1, n_tokens - 1)] + return [1] diff --git a/scripts/shadow/hyperlexical/classification_v2.py b/scripts/shadow/hyperlexical/classification_v2.py new file mode 100644 index 00000000..721ca421 --- /dev/null +++ b/scripts/shadow/hyperlexical/classification_v2.py @@ -0,0 +1,1310 @@ +"""Sealed Classification v2 contract. No training and no checkpoint writes. + +Normative text: ``specs/007-hyperlexical-model/classification-architecture-v2.md``. + +The family head has one row for every active production family. ``none``, +``ABSTAIN``, ``AMBIGUOUS``, and legacy near-matches are not rows. Provenance +weights are frozen. Class weights are a pure function of the training split +and are undefined while any active family has zero positive support. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import statistics +import struct +from typing import Any, Mapping, Sequence + +from hyperlexical.eval_settlement import ACTIVE_FAMILIES, CANDIDATE_FAMILIES +from hyperlexical.layout import FAMILIES + +ARCHITECTURE_ID = "HYPERLEX_CLASSIFICATION_ARCHITECTURE_V2" +STATE = "PREREGISTERED" +SCHEMA = "hyperlex.classification.v2" +PACKET_SCHEMA = "hyperlex.jev.decision_packet.v1" +ADAPTER_SCHEMA = "hyperlex.classification.v2.row_mapping.v1" +VOCABULARY_ID_HISTORICAL = "hyperlex.active_families.v1" +VOCABULARY_ID_FORWARD = "hyperlex.active_families.v2.forward_merge_pair_ad_ss" + +BEST_REFERENCE_SHA256 = ( + "9fba0f66b1d5de6492470f53577d1447bfac1d29b9ac03869268abb70bbd97f6" +) + +HISTORICAL_ACTIVE_FAMILY_VOCABULARY: tuple[str, ...] = ( + "gaming-meta", + "betting-sharp", + "crypto-degen", + "internet-slang", + "memetic", + "social-status", + "relationship-dating", + "approval-disapproval", + "conflict-aggression", + "technology-ai", + "workplace-career", + "sports-competition", + "music-entertainment", + "fashion-aesthetic", + "regional-cultural", + "spiritual-mystic", + "identity-affiliation", + "politics-civic", + "ai-native", +) + + +def _forward_active_family_vocabulary() -> tuple[str, ...]: + """18-family forward vocab (AD+SS → social-evaluation). Historical tuple untouched.""" + out: list[str] = [] + inserted = False + for family in HISTORICAL_ACTIVE_FAMILY_VOCABULARY: + if family in {"approval-disapproval", "social-status"}: + if not inserted: + out.append("social-evaluation") + inserted = True + continue + out.append(family) + if not inserted: + out.append("social-evaluation") + return tuple(out) + + +FORWARD_ONTOLOGY = os.environ.get("HLX_V2_FORWARD_ONTOLOGY") == "1" +ACTIVE_FAMILY_VOCABULARY: tuple[str, ...] = ( + _forward_active_family_vocabulary() + if FORWARD_ONTOLOGY + else HISTORICAL_ACTIVE_FAMILY_VOCABULARY +) +VOCABULARY_ID = VOCABULARY_ID_FORWARD if FORWARD_ONTOLOGY else VOCABULARY_ID_HISTORICAL +if not FORWARD_ONTOLOGY and tuple(ACTIVE_FAMILIES) != ACTIVE_FAMILY_VOCABULARY: + raise RuntimeError("ACTIVE_FAMILIES drifted from the sealed v2 vocabulary") + +V1_HEAD = tuple(FAMILIES) +V1_NONE_CLASS = "none" +LEGACY_HEADS = tuple(name for name in V1_HEAD if name != V1_NONE_CLASS and name not in ACTIVE_FAMILY_VOCABULARY) +EXACT_COPY_FAMILIES = tuple(name for name in V1_HEAD if name in ACTIVE_FAMILY_VOCABULARY) +FORBIDDEN_NEAR_MATCHES = ( + ("workplace-corp", "workplace-career"), + ("political-status", "politics-civic"), + ("brainrot-aura", "memetic"), + ("kinship-address", "relationship-dating"), +) + +V2_FAMILY = "FAMILY" +V2_NONE = "NONE" +V2_ABSTAIN = "ABSTAIN" +V2_AMBIGUOUS = "AMBIGUOUS" +DECISIONS = (V2_FAMILY, V2_NONE, V2_ABSTAIN, V2_AMBIGUOUS) +# Canonical Classification v2 family decision is training-side retrieval +# (HYPERLEX_FAMILY_RETRIEVAL_DECISION_V1). The residual 18-way family head remains +# for diagnostic/compatibility/research only and is not product output. +CANONICAL_FAMILY_DECISION = "retrieval_mean_top_m_cosine" +RESIDUAL_FAMILY_HEAD_ROLE = "diagnostic_compatibility_research_only" +APPLICABILITY_NONE = "NONE" +APPLICABILITY_PRESENT = "FAMILY_PRESENT" +APPLICABILITY = (APPLICABILITY_NONE, APPLICABILITY_PRESENT) +AMBIGUITY_EMISSION = "DISABLED_PENDING_GOLD" +PROVENANCE = ("OBSERVED", "INFERRED", "UNKNOWN") +LABEL_CLASS = ("OBSERVED", "INFERRED") +SETTLEMENT_ABSTAIN_TOKEN = "none" + +JEV_OFF = "OFF" +JEV_SHADOW = "SHADOW" +JEV_GATED = "GATED" +JEV_MODES_IMPLEMENTED = (JEV_OFF, JEV_SHADOW) +PROHIBITED_SURFACES = frozenset( + {"held_out", "evaluation_reserve", "settlement", "measurement"} +) +PROBABILITY_SUM_TOLERANCE = 1e-12 +TEMPERATURE_GRID = tuple(i / 100 for i in range(5, 501)) + +PROVENANCE_WEIGHTS: dict[str, float] = { + "observed_non_none_applicability": 1.00, + "observed_non_none_family": 1.00, + "inferred_non_none_applicability": 0.50, + "inferred_non_none_family": 0.50, + "observed_none_applicability": 1.00, + "inferred_none_applicability": 0.25, +} +FAMILY_WEIGHT_FLOOR = 0.50 +FAMILY_WEIGHT_CAP = 2.00 +SELECTION_WEIGHTS = (0.50, 0.25, 0.25) + +V2_SCHEDULE = { + "max_epochs": 12, + "minimum_epochs": 4, + "early_stopping_patience": 4, + "improvement": "strict", + "ties": "keep_earlier", + "restore_best": True, + "last_trainable": 2, + "learning_rate": 2e-5, + "batch_size": 8, + "gradient_accumulation": 1, + "max_len": 64, + "filler_filter": "strict", + "unbind_curriculum": False, + "unbind_every_n": 1, + "unbind_loss_weight": 1.0, + "unbind_primary": "mixed", +} +TELEMETRY_FIELDS = ( + "total_loss", + "applicability_loss", + "family_loss", + "unbind_loss", + "applicability_macro_f1", + "none_precision", + "none_recall", + "none_f1", + "family_present_precision", + "family_present_recall", + "family_present_f1", + "active_family_macro_f1", + "observed_active_family_macro_f1", + "exact_copy_family_macro_f1", + "prototype_family_macro_f1", + "per_family", + "predicted_none_rate", + "family_emission_rate", + "selection_score", + "learning_rate", + "global_step", + "checkpoint_identity", + "observed_slice", + "inferred_slice", +) + +OPERATOR_AUTHORIZATION = { + "present": True, + "authorizes_architecture": True, + "authorizes_training_before_ready": False, + "authorizes_best_promotion": False, + "statement": ( + "Operator sealed Classification v2 on 2026-09-29. " + "One end-to-end training run is the next action after READY. " + "BEST promotion stays a later explicit action." + ), +} + + +class ClassificationContractError(ValueError): + def __init__(self, reason: str, detail: Any = None) -> None: + self.reason = reason + self.detail = detail + super().__init__(reason if detail is None else f"{reason}: {detail}") + + +class JevContractError(ClassificationContractError): + pass + + +class ActiveFamilyWithoutTrainingSupport(ClassificationContractError): + def __init__(self, families: Sequence[str]) -> None: + self.families = tuple(families) + super().__init__("ACTIVE_FAMILY_WITHOUT_TRAINING_SUPPORT", list(self.families)) + + +def canonical_json(payload: Mapping[str, Any]) -> str: + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def sha256_text(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def vocabulary_sha256() -> str: + return sha256_text("\n".join(ACTIVE_FAMILY_VOCABULARY)) + + +def _unit(value: Any) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise JevContractError("malformed_probability") + number = float(value) + if not math.isfinite(number) or number < 0.0 or number > 1.0: + raise JevContractError("malformed_probability") + return number + + +def _finite(value: Any, reason: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ClassificationContractError(reason) + number = float(value) + if not math.isfinite(number): + raise ClassificationContractError(reason) + return number + + +def decision_seal() -> dict[str, Any]: + body = { + "active_family_vocabulary": list(ACTIVE_FAMILY_VOCABULARY), + "ambiguity_emission": AMBIGUITY_EMISSION, + "applicability_balance": "inverse_square_root_mean_one", + "applicability_surface_balance": "four_cell_inverse_support_mean_one", + "exact_copy_families": list(EXACT_COPY_FAMILIES), + "family_weight_clip": [FAMILY_WEIGHT_FLOOR, FAMILY_WEIGHT_CAP], + "family_weight_rule": "sqrt(median/effective)_then_mean_one_then_clip", + "forbidden_near_matches": [list(pair) for pair in FORBIDDEN_NEAR_MATCHES], + "legacy_heads": list(LEGACY_HEADS), + "new_row_initialization": "semantic_prototype", + "provenance_weights": PROVENANCE_WEIGHTS, + "schedule": V2_SCHEDULE, + "selection_score": "0.50*active_family_macro_f1+0.25*applicability_macro_f1+0.25*observed_active_family_macro_f1", + "applicability_surface_f1_min": 0.80, + "none_surface_gap_abs_max": 0.10, + "residualized_length_correlation_abs_max": 0.30, + "surface_rule": "hyperlex.classification.v2.surface.v1", + "surface_shortcut_abs_correlation_max": 0.30, + "family_canonical_logits": "learned_residual", + "family_fusion_alpha": 1.0, + "family_fusion_beta": 1.0, + "family_fusion_rule": "population_zscore(cosine/tau)+population_zscore(residual)", + "family_geometry_hard_negative_multiplier": 2.0, + "family_geometry_lambda": 0.5, + "family_geometry_repair": "HYPERLEX_FAMILY_GEOMETRY_REPAIR_V1", + "family_geometry_tau": 0.10, + "family_hard_negative_multiplier": 2.0, + "family_prototype_lambda": 0.5, + "family_prototype_tau": 0.10, + "family_prototypes": "frozen_loss_structure_only", + "surface_shortcut_diagnostic": "gold_conditional_residualized", + "vocabulary_id": VOCABULARY_ID, + } + return {"sha256": sha256_text(canonical_json(body)), "body": body} + + +def architecture_state() -> dict[str, Any]: + seal = decision_seal() + return { + "architecture_id": ARCHITECTURE_ID, + "state": STATE, + "authorizes_training": False, + "authorizes_preregistration": True, + "authorizes_gated_jev": False, + "moves_best": False, + "best_reference_sha256": BEST_REFERENCE_SHA256, + "select_006": "SETTLED_PASS", + "select_007": "SETTLED_FAIL", + "reopens_select_006": False, + "reopens_select_007": False, + "decision_seal_sha256": seal["sha256"], + "active_family_count": len(ACTIVE_FAMILY_VOCABULARY), + "ambiguity_emission": AMBIGUITY_EMISSION, + "jev_default": JEV_OFF, + "operator_authorization": dict(OPERATOR_AUTHORIZATION), + } + + +def classification_version(raw: str | None = None) -> str: + if raw is None: + raw = os.environ.get("HYPERLEX_CLASSIFICATION") + token = (raw or "v1").strip().lower() + if token in {"", "v1", "1"}: + return "v1" + if token in {"v2", "2"}: + return "v2" + raise ClassificationContractError("classification_version_invalid") + + +def gated_authorized(_evidence: Mapping[str, Any] | None = None) -> bool: + return False + + +def family_index() -> dict[str, int]: + return {name: index for index, name in enumerate(ACTIVE_FAMILY_VOCABULARY)} + + +def mapping_plan(source_labels: Sequence[str]) -> list[dict[str, Any]]: + """Exact-name plan. Near-matches stay unmapped and are not family rows.""" + source = tuple(source_labels) + if len(source) != len(set(source)): + raise ClassificationContractError("duplicate_source_label") + source_of = {label: index for index, label in enumerate(source)} + rows: list[dict[str, Any]] = [] + for family in ACTIVE_FAMILY_VOCABULARY: + source_index = source_of.get(family) + if source_index is None: + rows.append( + { + "active_family": family, + "source_row": None, + "mapping_status": "ZERO_INIT", + "initialization_status": "exact_zero", + } + ) + continue + rows.append( + { + "active_family": family, + "source_row": family, + "source_index": source_index, + "mapping_status": "EXACT_COPY", + "initialization_status": "copied", + } + ) + return rows + + +def _f32_hash(values: Sequence[float]) -> str: + raw = b"".join(struct.pack(" dict[str, Any]: + """Copy exact rows and set every other active-family row to exact zero. + + ``source_weight[row][hidden]``. No random values are retained. + """ + plan = mapping_plan(source_labels) + if len(source_weight) != len(source_labels) or len(source_bias) != len(source_labels): + raise ClassificationContractError("source_shape_mismatch") + hidden = len(source_weight[0]) if source_weight else 0 + if hidden < 1 or any(len(row) != hidden for row in source_weight): + raise ClassificationContractError("source_shape_mismatch") + weight: list[list[float]] = [] + bias: list[float] = [] + witness: list[dict[str, Any]] = [] + for item in plan: + if item["mapping_status"] == "EXACT_COPY": + copied_weight = [float(value) for value in source_weight[item["source_index"]]] + copied_bias = float(source_bias[item["source_index"]]) + if copied_weight != [float(value) for value in source_weight[item["source_index"]]]: + raise ClassificationContractError("copy_mismatch") + else: + copied_weight = [0.0] * hidden + copied_bias = 0.0 + if item["mapping_status"] != "EXACT_COPY" and any(value != 0.0 for value in copied_weight): + raise ClassificationContractError("nonzero_new_row") + if item["mapping_status"] != "EXACT_COPY" and copied_bias != 0.0: + raise ClassificationContractError("nonzero_new_row") + weight.append(copied_weight) + bias.append(copied_bias) + witness.append( + { + **item, + "weight_sha256": _f32_hash(copied_weight), + "bias_sha256": _f32_hash((copied_bias,)), + } + ) + for pair in FORBIDDEN_NEAR_MATCHES: + if any(row["source_row"] == pair[0] and row["active_family"] == pair[1] for row in witness): + raise ClassificationContractError("legacy_near_match") + if any(row["active_family"] in {V1_NONE_CLASS, V2_ABSTAIN, V2_AMBIGUOUS} for row in witness): + raise ClassificationContractError("forbidden_family_row") + return { + "schema": ADAPTER_SCHEMA, + "vocabulary_id": VOCABULARY_ID, + "vocabulary_sha256": vocabulary_sha256(), + "historical_artifact_mutated": False, + "target_rows": len(weight), + "mapped_families": [row["active_family"] for row in witness if row["mapping_status"] == "EXACT_COPY"], + "zero_initialized_families": [row["active_family"] for row in witness if row["mapping_status"] == "ZERO_INIT"], + "legacy_not_mapped": list(LEGACY_HEADS), + "legacy_remap": {}, + "rows": witness, + "weight": weight, + "bias": bias, + "status": "PASS", + } + + +def _class_of(row: Mapping[str, Any]) -> str: + klass = row.get("class") + if klass not in LABEL_CLASS: + raise ClassificationContractError("provenance_class_missing") + return str(klass) + + +def _reject_isolated(row: Mapping[str, Any]) -> None: + if row.get("split") == "test": + raise ClassificationContractError("evaluation_isolation") + surface = row.get("surface") + if surface in PROHIBITED_SURFACES or row.get("evaluation_reserve") or row.get("held_out"): + raise ClassificationContractError("evaluation_isolation") + + +def support_audit(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Training-split counts. Legacy lineages do not fill an active family.""" + counts = { + name: {"OBSERVED": 0, "INFERRED": 0} + for name in (*ACTIVE_FAMILY_VOCABULARY, V1_NONE_CLASS) + } + legacy = {name: {"OBSERVED": 0, "INFERRED": 0} for name in LEGACY_HEADS} + for row in rows: + _reject_isolated(row) + if row.get("split") not in (None, "train"): + raise ClassificationContractError("support_requires_training_split") + lineage = row.get("lineage") + klass = _class_of(row) + if lineage in counts: + counts[str(lineage)][klass] += 1 + elif lineage in legacy: + legacy[str(lineage)][klass] += 1 + else: + raise ClassificationContractError("unknown_lineage", lineage) + families = [] + missing: list[str] = [] + for name in ACTIVE_FAMILY_VOCABULARY: + observed = counts[name]["OBSERVED"] + inferred = counts[name]["INFERRED"] + effective = observed + 0.5 * inferred + if effective == 0: + missing.append(name) + families.append( + { + "family": name, + "observed": observed, + "inferred": inferred, + "effective_support": effective, + } + ) + observed_active = sum(item["observed"] for item in families) + inferred_active = sum(item["inferred"] for item in families) + return { + "families": families, + "missing_support": missing, + "legacy_excluded": [ + { + "lineage": name, + "observed": legacy[name]["OBSERVED"], + "inferred": legacy[name]["INFERRED"], + } + for name in LEGACY_HEADS + ], + "none": { + "observed": counts[V1_NONE_CLASS]["OBSERVED"], + "inferred": counts[V1_NONE_CLASS]["INFERRED"], + "effective_support": ( + counts[V1_NONE_CLASS]["OBSERVED"] + + PROVENANCE_WEIGHTS["inferred_none_applicability"] * counts[V1_NONE_CLASS]["INFERRED"] + ), + }, + "family_present_effective": ( + observed_active * PROVENANCE_WEIGHTS["observed_non_none_applicability"] + + inferred_active * PROVENANCE_WEIGHTS["inferred_non_none_applicability"] + ), + } + + +_RESERVE_STATES = frozenset({"EVAL_RESERVE", "EVAL_SPENT", "EVAL_ABANDONED"}) + + +def reserve_positive_census(identities: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Count settled positives that cannot fill v2 training support. + + Evaluation-reserved, spent, and abandoned identities stay out of the + training split. This census does not copy them and does not read row text. + """ + counts = { + name: {"OBSERVED": 0, "INFERRED": 0, "training_eligible": 0} + for name in ACTIVE_FAMILY_VOCABULARY + } + for identity in identities: + excluded = bool(identity.get("evaluation_reserved")) or identity.get("state") in _RESERVE_STATES + for label in identity.get("labels") or []: + lineage = label.get("lineage") + if lineage not in counts: + continue + if not excluded: + counts[lineage]["training_eligible"] += 1 + continue + klass = label.get("class") + if klass in ("OBSERVED", "INFERRED"): + counts[lineage][klass] += 1 + absent = [ + name + for name in ACTIVE_FAMILY_VOCABULARY + if counts[name]["OBSERVED"] == 0 + and counts[name]["INFERRED"] == 0 + and counts[name]["training_eligible"] == 0 + ] + return { + "schema": "hyperlex.classification.v2.reserve_exclusion.v1", + "counts": counts, + "training_eligible_rows": sum(item["training_eligible"] for item in counts.values()), + "usable_for_training": False, + "no_settled_positive": absent, + } + + +def family_loss_weights(audit: Mapping[str, Any]) -> dict[str, float]: + missing = list(audit["missing_support"]) + if missing: + raise ActiveFamilyWithoutTrainingSupport(missing) + effective = [float(item["effective_support"]) for item in audit["families"]] + median = float(statistics.median(effective)) + raw = [math.sqrt(median / count) for count in effective] + mean = sum(raw) / len(raw) + weights = {} + for item, value in zip(audit["families"], raw): + scaled = value / mean + if scaled < FAMILY_WEIGHT_FLOOR: + scaled = FAMILY_WEIGHT_FLOOR + elif scaled > FAMILY_WEIGHT_CAP: + scaled = FAMILY_WEIGHT_CAP + weights[item["family"]] = scaled + return weights + + +def applicability_class_weights(audit: Mapping[str, Any]) -> dict[str, float]: + present = float(audit["family_present_effective"]) + none = float(audit["none"]["effective_support"]) + if present <= 0.0 or none <= 0.0: + raise ClassificationContractError("applicability_support_missing") + raw_present = 1.0 / math.sqrt(present) + raw_none = 1.0 / math.sqrt(none) + mean = (raw_present + raw_none) / 2.0 + return { + APPLICABILITY_PRESENT: raw_present / mean, + APPLICABILITY_NONE: raw_none / mean, + } + + +def freeze_training_contract(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + audit = support_audit(rows) + if audit["missing_support"]: + raise ActiveFamilyWithoutTrainingSupport(audit["missing_support"]) + family_weights = family_loss_weights(audit) + applicability = applicability_class_weights(audit) + from .classification_v2_surface import surface_cell_weights + + return { + "status": "FROZEN", + "provenance_weights": dict(PROVENANCE_WEIGHTS), + "family_weights": family_weights, + "applicability_weights": applicability, + "surface_cell_weights": surface_cell_weights(rows, PROVENANCE_WEIGHTS), + "audit": audit, + "reserve_rows_used": 0, + } + + + +def _surface_of(row: Mapping[str, Any]) -> str | None: + if "text" not in row: + return None + from .classification_v2_surface import surface_form + + return surface_form(str(row.get("text") or "")) + + +def _applicability_multiplier( + row: Mapping[str, Any], + contract: Mapping[str, Any], + target: str, +) -> float | None: + """Provenance authority inside a surface cell. + + Rows that predate surface metadata keep the two-class applicability weight. + Ambiguous text is masked out of the applicability objective. + """ + provenance = contract["provenance_weights"] + klass = _class_of(row) + if target == APPLICABILITY_NONE: + key = "observed_none_applicability" if klass == "OBSERVED" else "inferred_none_applicability" + else: + prefix = "observed_non_none" if klass == "OBSERVED" else "inferred_non_none" + key = prefix + "_applicability" + base = provenance[key] + if "text" not in row: + return base * contract["applicability_weights"][target] + from .classification_v2_surface import SURFACE_AMBIGUOUS, cell_name, surface_form + + form = surface_form(str(row.get("text") or "")) + if form == SURFACE_AMBIGUOUS: + return None + weights = contract.get("surface_cell_weights") or {} + name = cell_name(target, form) + if name not in weights: + raise ClassificationContractError("applicability_surface_cell_missing", name) + return base * weights[name] + + +def example_loss(row: Mapping[str, Any], contract: Mapping[str, Any]) -> dict[str, Any]: + """Per-row multipliers. ``None`` means that term is masked off.""" + _reject_isolated(row) + lineage = row.get("lineage") + klass = _class_of(row) + provenance = contract["provenance_weights"] + plan: dict[str, Any] = { + "applicability_target": None, + "applicability_weight": None, + "family_target": None, + "family_weight": None, + "unbind_loss": "separate_unchanged", + "trained_class_abstain": False, + "trained_class_ambiguous": False, + } + if lineage == V1_NONE_CLASS: + plan["applicability_target"] = APPLICABILITY_NONE + plan["applicability_weight"] = _applicability_multiplier(row, contract, APPLICABILITY_NONE) + plan["surface"] = _surface_of(row) + return plan + if lineage in ACTIVE_FAMILY_VOCABULARY: + prefix = "observed_non_none" if klass == "OBSERVED" else "inferred_non_none" + plan["applicability_target"] = APPLICABILITY_PRESENT + plan["applicability_weight"] = _applicability_multiplier(row, contract, APPLICABILITY_PRESENT) + plan["surface"] = _surface_of(row) + plan["family_target"] = lineage + plan["family_weight"] = provenance[prefix + "_family"] * contract["family_weights"][lineage] + return plan + if lineage in LEGACY_HEADS: + plan["excluded"] = "legacy_not_remapped" + return plan + raise ClassificationContractError("unknown_lineage", lineage) + + +def epoch_selection_metrics( + rows: Sequence[Mapping[str, Any]], + applicability_predictions: Sequence[str], + family_predictions: Sequence[str], +) -> dict[str, Any]: + """Argmax metrics for one validation pass. Legacy rows are left out.""" + if not (len(rows) == len(applicability_predictions) == len(family_predictions)): + raise ClassificationContractError("metric_length_mismatch") + app_gold: list[str] = [] + app_pred: list[str] = [] + fam_gold: list[str] = [] + fam_pred: list[str] = [] + obs_gold: list[str] = [] + obs_pred: list[str] = [] + observed_app_gold: list[str] = [] + observed_app_pred: list[str] = [] + inferred_app_gold: list[str] = [] + inferred_app_pred: list[str] = [] + for row, app, fam in zip(rows, applicability_predictions, family_predictions): + lineage = row.get("lineage") + if lineage not in ACTIVE_FAMILY_VOCABULARY and lineage != V1_NONE_CLASS: + continue + gold_app = APPLICABILITY_NONE if lineage == V1_NONE_CLASS else APPLICABILITY_PRESENT + app_gold.append(gold_app) + app_pred.append(app) + klass = row.get("class") + if klass == "OBSERVED": + observed_app_gold.append(gold_app) + observed_app_pred.append(app) + elif klass == "INFERRED": + inferred_app_gold.append(gold_app) + inferred_app_pred.append(app) + if lineage in ACTIVE_FAMILY_VOCABULARY: + fam_gold.append(str(lineage)) + fam_pred.append(fam) + if klass == "OBSERVED": + obs_gold.append(str(lineage)) + obs_pred.append(fam) + app = prf_table(app_gold, app_pred, APPLICABILITY) + fam = prf_table(fam_gold, fam_pred, ACTIVE_FAMILY_VOCABULARY) + obs = prf_table(obs_gold, obs_pred, ACTIVE_FAMILY_VOCABULARY) + per_label = fam["per_label"] + + def _group_macro(names: Sequence[str]) -> float | None: + scored = [float(per_label[name]["f1"]) for name in names if per_label[name]["support"]] + if not scored: + return None + return sum(scored) / len(scored) + + copied = [name for name in ACTIVE_FAMILY_VOCABULARY if name in EXACT_COPY_FAMILIES] + prototyped = [name for name in ACTIVE_FAMILY_VOCABULARY if name not in EXACT_COPY_FAMILIES] + score = None + if None not in (app["macro_f1"], fam["macro_f1"], obs["macro_f1"]): + score = selection_score(float(fam["macro_f1"]), float(app["macro_f1"]), float(obs["macro_f1"])) + n = len(app_pred) + none_stats = app["per_label"][APPLICABILITY_NONE] + present_stats = app["per_label"][APPLICABILITY_PRESENT] + return { + "selection_score": score, + "applicability_macro_f1": app["macro_f1"], + "active_family_macro_f1": fam["macro_f1"], + "observed_active_family_macro_f1": obs["macro_f1"], + "exact_copy_family_macro_f1": _group_macro(copied), + "prototype_family_macro_f1": _group_macro(prototyped), + "none_precision": none_stats["precision"], + "none_recall": none_stats["recall"], + "none_f1": none_stats["f1"], + "family_present_precision": present_stats["precision"], + "family_present_recall": present_stats["recall"], + "family_present_f1": present_stats["f1"], + "per_family": fam["per_label"], + "predicted_none_rate": None if n == 0 else app_pred.count(APPLICABILITY_NONE) / n, + "family_emission_rate": None if n == 0 else app_pred.count(APPLICABILITY_PRESENT) / n, + "observed_slice": prf_table(observed_app_gold, observed_app_pred, APPLICABILITY), + "inferred_slice": prf_table(inferred_app_gold, inferred_app_pred, APPLICABILITY), + } + + +def selection_score( + active_family_macro_f1: float, + applicability_macro_f1: float, + observed_active_family_macro_f1: float, +) -> float: + parts = ( + _finite(active_family_macro_f1, "selection_term_invalid"), + _finite(applicability_macro_f1, "selection_term_invalid"), + _finite(observed_active_family_macro_f1, "selection_term_invalid"), + ) + return sum(weight * value for weight, value in zip(SELECTION_WEIGHTS, parts)) + + +def _prf(gold: int, predicted: int, hit: int) -> dict[str, float | None]: + if gold == 0 and predicted == 0: + return {"precision": None, "recall": None, "f1": None, "support": 0} + precision = hit / predicted if predicted else 0.0 + recall = hit / gold if gold else 0.0 + f1 = 0.0 if precision + recall == 0 else 2 * precision * recall / (precision + recall) + return {"precision": precision, "recall": recall, "f1": f1, "support": gold, "predicted": predicted} + + +def prf_table(golds: Sequence[str], preds: Sequence[str], labels: Sequence[str]) -> dict[str, Any]: + if len(golds) != len(preds): + raise ClassificationContractError("metric_length_mismatch") + rows = {} + scored: list[float] = [] + for label in labels: + gold = sum(1 for item in golds if item == label) + predicted = sum(1 for item in preds if item == label) + hit = sum(1 for left, right in zip(golds, preds) if left == label and right == label) + stats = _prf(gold, predicted, hit) + rows[label] = stats + if gold: + scored.append(float(stats["f1"])) + macro = None if not scored else sum(scored) / len(scored) + return {"per_label": rows, "macro_f1": macro, "labels_with_support": len(scored)} + + +def _softmax(logits: Sequence[float], temperature: float) -> list[float]: + scaled = [value / temperature for value in logits] + peak = max(scaled) + exps = [math.exp(value - peak) for value in scaled] + total = sum(exps) + return [value / total for value in exps] + + +def _mean_nll(rows: Sequence[tuple[Sequence[float], int]], temperature: float) -> float: + """Mean NLL. Log-sum-exp stays finite when a class probability underflows.""" + total = 0.0 + for logits, label in rows: + if label < 0 or label >= len(logits): + raise ClassificationContractError("calibration_label_invalid") + scaled = [value / temperature for value in logits] + peak = max(scaled) + log_partition = math.log(sum(math.exp(value - peak) for value in scaled)) + total -= scaled[label] - peak - log_partition + return total / len(rows) + + +def fit_temperature(rows: Sequence[tuple[Sequence[float], int]], *, surface: str = "validation") -> float: + if surface != "validation": + raise ClassificationContractError("calibration_surface_forbidden") + if not rows: + raise ClassificationContractError("calibration_empty") + best_t = TEMPERATURE_GRID[0] + best_nll = _mean_nll(rows, best_t) + for temperature in TEMPERATURE_GRID[1:]: + nll = _mean_nll(rows, temperature) + closer = abs(temperature - 1.0) < abs(best_t - 1.0) + tied = abs(nll - best_nll) <= 1e-12 + better = nll < best_nll - 1e-12 + if better or (tied and (closer or (abs(abs(temperature - 1.0) - abs(best_t - 1.0)) <= 1e-15 and temperature < best_t))): + best_t = temperature + best_nll = nll + return best_t + + +def applicability_threshold(probabilities: Sequence[float], labels: Sequence[int]) -> float: + """Maximize balanced accuracy. Ties: closest to 0.5, then the lower threshold.""" + if len(probabilities) != len(labels) or not probabilities: + raise ClassificationContractError("calibration_empty") + if any(label not in (0, 1) for label in labels): + raise ClassificationContractError("calibration_label_invalid") + if any(label == 0 for label in labels) is False or any(label == 1 for label in labels) is False: + raise ClassificationContractError("calibration_class_missing") + candidates = sorted(set(_unit(value) for value in probabilities).union({0.0, 1.0})) + best: tuple[float, float, float] | None = None + chosen = candidates[0] + for threshold in candidates: + predictions = [1 if probability >= threshold else 0 for probability in probabilities] + recalls = [] + for label in (0, 1): + gold = [index for index, value in enumerate(labels) if value == label] + hit = sum(1 for index in gold if predictions[index] == label) + recalls.append(hit / len(gold)) + balanced = 0.5 * (recalls[0] + recalls[1]) + rank = (balanced, -abs(threshold - 0.5), -threshold) + if best is None or rank > best: + best = rank + chosen = threshold + return chosen + + +def family_emit_threshold(confidences: Sequence[float], correct: Sequence[bool]) -> float: + """Largest proper subset whose selective accuracy preserves the unfiltered accuracy. + + Threshold 0 means no confidence abstention is added. + """ + if len(confidences) != len(correct) or not confidences: + raise ClassificationContractError("calibration_empty") + checked = [_unit(value) for value in confidences] + flags = [bool(value) for value in correct] + unfiltered = sum(1 for value in flags if value) / len(flags) + best: tuple[float, float, float] | None = None + chosen: float | None = None + for threshold in sorted(set(checked)): + emitted = [(confidence, flag) for confidence, flag in zip(checked, flags) if confidence >= threshold] + if len(emitted) in (0, len(flags)): + continue + selective = sum(1 for _, flag in emitted if flag) / len(emitted) + if selective + 1e-12 < unfiltered: + continue + coverage = len(emitted) / len(flags) + rank = (coverage, selective, -threshold) + if best is None or rank > best: + best = rank + chosen = threshold + return 0.0 if chosen is None else chosen + + +def _argmax_vocabulary(values: Sequence[float]) -> int: + best_index = 0 + best_value = values[0] + for index, value in enumerate(values[1:], start=1): + if value > best_value: + best_index = index + best_value = value + return best_index + + +def freeze_calibration( + *, + applicability_rows: Sequence[tuple[Sequence[float], int]], + family_rows: Sequence[tuple[Sequence[float], int]], + surface: str = "validation", + checkpoint_identity: str | None = None, +) -> dict[str, Any]: + """Temperature and thresholds from the restored checkpoint's validation logits. + + Training rows and the evaluation reserve are not inputs. + """ + if surface != "validation": + raise ClassificationContractError("calibration_surface_forbidden") + applicability_temperature = fit_temperature(applicability_rows, surface=surface) + probabilities = [ + _softmax(logits, applicability_temperature)[1] for logits, _label in applicability_rows + ] + labels = [int(label) for _logits, label in applicability_rows] + threshold = applicability_threshold(probabilities, labels) + family_temperature = fit_temperature(family_rows, surface=surface) + confidences: list[float] = [] + correct: list[bool] = [] + width = len(ACTIVE_FAMILY_VOCABULARY) + for logits, label in family_rows: + if label < 0 or label >= width or len(logits) != width: + raise ClassificationContractError("calibration_label_invalid") + distribution = _softmax(logits, family_temperature) + predicted = _argmax_vocabulary(distribution) + confidences.append(distribution[predicted]) + correct.append(predicted == label) + emit = family_emit_threshold(confidences, correct) + return { + "schema": "hyperlex.classification.v2.calibration.v1", + "surface": "validation", + "reserve_used": False, + "training_rows_used": False, + "applicability_temperature": applicability_temperature, + "applicability_threshold": threshold, + "family_temperature": family_temperature, + "family_emit_threshold": emit, + "n_applicability": len(applicability_rows), + "n_family": len(family_rows), + "checkpoint_identity": checkpoint_identity, + } + + +def decide_v2( + *, + p_family_present: float, + applicability_threshold_value: float, + family_distribution: Mapping[str, float], + family_emit_threshold_value: float, +) -> dict[str, Any]: + present = _unit(p_family_present) + app_threshold = _unit(applicability_threshold_value) + emit_threshold = _unit(family_emit_threshold_value) + if set(family_distribution) != set(ACTIVE_FAMILY_VOCABULARY): + raise ClassificationContractError("family_distribution_incomplete") + if V1_NONE_CLASS in family_distribution: + raise ClassificationContractError("none_is_not_a_family") + checked = {name: _unit(family_distribution[name]) for name in ACTIVE_FAMILY_VOCABULARY} + if abs(sum(checked.values()) - 1.0) > 1e-6: + raise ClassificationContractError("distribution_not_normalized") + ordered = sorted(checked.items(), key=lambda item: (-item[1], ACTIVE_FAMILY_VOCABULARY.index(item[0]))) + margin = ordered[0][1] - ordered[1][1] + ambiguous_candidate = margin == 0.0 + if present < app_threshold: + decision, family, stage = V2_ABSTAIN, None, "Q1_APPLICABILITY" + elif present <= 0.5: + decision, family, stage = V2_NONE, None, "Q2_NONE" + else: + family_name, confidence = ordered[0] + if confidence < emit_threshold: + decision, family, stage = V2_ABSTAIN, None, "Q5_FAMILY_CONFIDENCE" + else: + decision, family, stage = V2_FAMILY, family_name, "Q4_FAMILY" + if decision == V2_AMBIGUOUS: + raise ClassificationContractError("ambiguous_emission_disabled") + return { + "decision": decision, + "family": family, + "stage": stage, + "margin": margin, + "ambiguous_candidate": ambiguous_candidate, + "ambiguity_emission": AMBIGUITY_EMISSION, + "family_confidence": ordered[0][1], + } + + +def build_result( + *, + p_family_present: float, + applicability_threshold_value: float, + family_distribution: Mapping[str, float], + family_emit_threshold_value: float, + confidence: float, + provenance_context: str, +) -> dict[str, Any]: + if provenance_context not in PROVENANCE: + raise ClassificationContractError("provenance_context_invalid") + decided = decide_v2( + p_family_present=p_family_present, + applicability_threshold_value=applicability_threshold_value, + family_distribution=family_distribution, + family_emit_threshold_value=family_emit_threshold_value, + ) + return { + "schema": SCHEMA, + "decision": decided["decision"], + "family": decided["family"], + "applicability": APPLICABILITY_PRESENT if _unit(p_family_present) > 0.5 else APPLICABILITY_NONE, + "applicability_score": _unit(p_family_present), + "applicability_threshold": _unit(applicability_threshold_value), + "family_distribution": {name: _unit(family_distribution[name]) for name in ACTIVE_FAMILY_VOCABULARY}, + "family_confidence": decided["family_confidence"], + "family_emit_threshold": _unit(family_emit_threshold_value), + "confidence": _unit(confidence), + "margin": decided["margin"], + "ambiguous_candidate": decided["ambiguous_candidate"], + "ambiguity_emission": AMBIGUITY_EMISSION, + "provenance_context": provenance_context, + "stage": decided["stage"], + "brier": None, + "forecast_eligible": False, + "jev": { + "mode": JEV_OFF, + "invoked": False, + "decision_packet_ref": None, + "agreement": None, + "rejection": None, + }, + } + + +def validate_telemetry(record: Mapping[str, Any]) -> None: + missing = [field for field in TELEMETRY_FIELDS if field not in record] + if missing: + raise ClassificationContractError("telemetry_incomplete", missing) + for slice_name in ("observed_slice", "inferred_slice"): + if not isinstance(record[slice_name], Mapping): + raise ClassificationContractError("telemetry_incomplete", slice_name) + if record.get("brier_head") is not None and "brier_head" in record: + raise ClassificationContractError("brier_head_forbidden") + + +def packet_body(packet: Mapping[str, Any]) -> dict[str, Any]: + return {key: packet[key] for key in packet if key != "output_hash"} + + +def packet_output_hash(packet: Mapping[str, Any]) -> str: + return sha256_text(canonical_json(packet_body(packet))) + + +FORBIDDEN_PACKET_KEYS = frozenset( + {"api_key", "best", "canonical_family", "observed_label", "reserve_gold", "semantic_family"} +) +PACKET_REQUIRED = ( + "decision_type", + "candidate_set", + "probabilities", + "confidence", + "provider", + "model", + "schema_version", + "prompt_schema_version", + "timestamp", + "input_hash", + "output_hash", +) +DECISION_TYPES = ("choice", "score", "route") +ROUTE_CHOICES = ("accept_automated", "human_review", "additional_evidence") + + +def _hash64(value: Any, reason: str) -> str: + if not isinstance(value, str) or len(value) != 64: + raise JevContractError(reason) + try: + int(value, 16) + except ValueError as exc: + raise JevContractError(reason) from exc + return value + + +def validate_decision_packet(packet: Mapping[str, Any], *, eligible: Sequence[str]) -> dict[str, Any]: + if not isinstance(packet, Mapping): + raise JevContractError("packet_not_object") + if FORBIDDEN_PACKET_KEYS.intersection(packet): + raise JevContractError("forbidden_authority_field") + if any(key not in packet for key in PACKET_REQUIRED): + raise JevContractError("packet_field_missing") + if packet.get("schema_version") != PACKET_SCHEMA: + raise JevContractError("schema_mismatch") + decision_type = packet.get("decision_type") + if decision_type not in DECISION_TYPES: + raise JevContractError("decision_type_invalid") + _unit(packet.get("confidence")) + _hash64(packet.get("input_hash"), "input_hash_invalid") + _hash64(packet.get("output_hash"), "output_hash_invalid") + if packet_output_hash(packet) != packet["output_hash"]: + raise JevContractError("output_hash_mismatch") + for key in ("provider", "model", "prompt_schema_version", "timestamp"): + if not isinstance(packet.get(key), str) or not str(packet.get(key)).strip(): + raise JevContractError("packet_field_missing") + candidates = packet.get("candidate_set") + probabilities = packet.get("probabilities") + if not isinstance(candidates, list) or not isinstance(probabilities, dict): + raise JevContractError("packet_field_missing") + eligible_set = set(eligible) + if not eligible_set.issubset(set(ACTIVE_FAMILY_VOCABULARY)): + raise ClassificationContractError("eligible_family_invalid") + if decision_type == "choice": + if not candidates or len(candidates) != len(set(candidates)): + raise JevContractError("candidate_set_invalid") + if any(name not in eligible_set for name in candidates): + raise JevContractError("candidate_outside_eligible") + if set(probabilities) != set(candidates): + raise JevContractError("probability_keys_mismatch") + if "abstain_probability" not in packet: + raise JevContractError("packet_field_missing") + checked = {name: _unit(probabilities[name]) for name in candidates} + abstain = _unit(packet.get("abstain_probability")) + if abs(sum(checked.values()) + abstain - 1.0) > 1e-6: + raise JevContractError("not_normalized") + selected = packet.get("selected") + if selected is not None and selected not in candidates: + raise JevContractError("candidate_outside_eligible") + elif decision_type == "score": + expected = {"confidence", "ambiguity", "escalation_need"} + if set(probabilities) != expected: + raise JevContractError("probability_keys_mismatch") + for key in expected: + _unit(probabilities[key]) + else: + if not candidates or any(name not in ROUTE_CHOICES for name in candidates): + raise JevContractError("candidate_set_invalid") + if set(probabilities) != set(candidates): + raise JevContractError("probability_keys_mismatch") + checked = {name: _unit(probabilities[name]) for name in candidates} + if abs(sum(checked.values()) - 1.0) > 1e-6: + raise JevContractError("not_normalized") + return dict(packet) + + +def jev_call_count(*, surface: str, mode: str) -> int: + if mode == JEV_GATED or mode not in JEV_MODES_IMPLEMENTED: + raise JevContractError("gated_not_authorized") + if surface in PROHIBITED_SURFACES or mode == JEV_OFF: + return 0 + return 1 + + +def integrate_jev( + *, + canonical: Mapping[str, Any], + mode: str, + surface: str, + eligible: Sequence[str], + packet: Mapping[str, Any] | None = None, + provider_status: str = "ok", +) -> dict[str, Any]: + if mode == JEV_GATED or mode not in JEV_MODES_IMPLEMENTED: + raise JevContractError("gated_not_authorized") + result = json.loads(canonical_json(canonical)) + decision = result.get("decision") + family = result.get("family") + result["jev"] = { + "mode": mode, + "invoked": False, + "decision_packet_ref": None, + "agreement": None, + "rejection": None, + } + if surface in PROHIBITED_SURFACES: + result["jev"]["rejection"] = "exposure_prohibited" + elif mode == JEV_SHADOW and provider_status != "ok": + result["jev"]["rejection"] = "provider_unavailable" + elif mode == JEV_SHADOW and packet is not None: + try: + checked = validate_decision_packet(packet, eligible=eligible) + except JevContractError as exc: + result["jev"]["rejection"] = exc.reason + else: + jev_choice = checked.get("selected") if checked.get("decision_type") == "choice" else None + hyper_family = family if decision == V2_FAMILY else None + result["jev"]["invoked"] = True + result["jev"]["decision_packet_ref"] = checked["output_hash"] + result["jev"]["agreement"] = { + "family_agreement": None + if checked.get("decision_type") != "choice" + else hyper_family == jev_choice and hyper_family is not None, + "hyperlex_abstain_jev_choice": decision == V2_ABSTAIN and jev_choice is not None, + "hyperlex_choice_jev_null": decision == V2_FAMILY and jev_choice is None, + "adopted": False, + } + if result.get("decision") != decision or result.get("family") != family: + raise JevContractError("canonical_mutated") + return result + + +def map_legacy_jevgate(*, jevgate: bool, no_jevgate: bool, v2_jev_mode: str | None) -> dict[str, Any]: + requested = (v2_jev_mode or JEV_OFF).upper() + if requested == JEV_GATED or requested not in JEV_MODES_IMPLEMENTED: + raise JevContractError("gated_not_authorized") + return { + "legacy_gate": "jevgate-1", + "legacy_enabled": False if no_jevgate else bool(jevgate), + "authorizes_v2_gated": False, + "v2_mode": requested, + "legacy_flag_changes_v2_canonical": False, + "required_for_v2_training": False, + } + + +def require_v2_schedule(schedule: Mapping[str, Any]) -> None: + expected = { + "max_epochs": V2_SCHEDULE["max_epochs"], + "minimum_epochs": V2_SCHEDULE["minimum_epochs"], + "patience": V2_SCHEDULE["early_stopping_patience"], + "improvement": "strict", + "ties": "keep_earlier", + "restore_best": True, + "early_stopping": True, + } + for key, value in expected.items(): + if schedule.get(key) != value: + raise ClassificationContractError("v2_schedule_mismatch", key) + + +def readiness( + rows: Sequence[Mapping[str, Any]], + *, + loader_status: str, + random_new_rows: bool = False, + prototype_report: Mapping[str, Any] | None = None, + validation_report: Mapping[str, Any] | None = None, + definition_report: Mapping[str, Any] | None = None, + surface_report: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """One audit. Incomplete families are returned together. Training stays off.""" + audit = support_audit(rows) + missing = list(audit["missing_support"]) + family_weights: dict[str, float] | None = None + applicability: dict[str, float] | None = None + if not missing: + family_weights = family_loss_weights(audit) + applicability = applicability_class_weights(audit) + elif audit["family_present_effective"] > 0 and audit["none"]["effective_support"] > 0: + applicability = applicability_class_weights(audit) + prototype_pass = bool(prototype_report and prototype_report.get("pass")) + validation_pass = bool(validation_report and validation_report.get("pass")) + definition_pass = bool(definition_report and definition_report.get("pass")) + copied_pass = bool(prototype_report and prototype_report.get("copied_rows_match")) + checks = { + "all_active_family_rows_present": True, + "all_active_families_have_positive_training_support": not missing, + "all_new_rows_have_semantic_prototypes": prototype_pass, + "all_copied_rows_match_source": copied_pass, + "all_families_have_validation_support": validation_pass, + "definition_string_rule_pass": definition_pass, + "prototype_witness_pass": prototype_pass and bool(prototype_report and prototype_report.get("determinism") == "pass"), + "family_weight_table_frozen": family_weights is not None and not missing, + "applicability_weight_table_frozen": applicability is not None and not missing, + "applicability_surface_cells_populated": surface_report is None or bool(surface_report.get("pass")), + "representation_lineage_isolated": surface_report is None or not surface_report.get("representation_leaks"), + "provenance_weights_frozen": True, + "loader_witness_pass": loader_status == "PASS", + "no_random_new_rows": not random_new_rows, + "loss_masks_pass": True, + "calibration_procedure_frozen": True, + "selection_metric_frozen": True, + "evaluation_contract_frozen": True, + "telemetry_contract_pass": ( + "exact_copy_family_macro_f1" in TELEMETRY_FIELDS + and "prototype_family_macro_f1" in TELEMETRY_FIELDS + ), + "training_evaluation_isolation_pass": True, + "operator_authorization_present": OPERATOR_AUTHORIZATION["present"] is True, + "ambiguous_emission_disabled": AMBIGUITY_EMISSION == "DISABLED_PENDING_GOLD", + "jev_required_for_readiness": False, + "historical_best_unchanged": True, + } + ready = all( + checks[key] + for key in checks + if key not in {"jev_required_for_readiness", "ambiguous_emission_disabled"} + ) + blockers: list[str] = [] + if missing: + blockers.append("ACTIVE_FAMILY_WITHOUT_TRAINING_SUPPORT") + if not definition_pass: + blockers.append("DEFINITION_STRING_RULE") + if not prototype_pass or checks["prototype_witness_pass"] is False: + blockers.append("FAMILY_PROTOTYPE_UNAVAILABLE") + if not validation_pass: + blockers.append("VALIDATION_FAMILY_SUPPORT_INSUFFICIENT") + if surface_report is not None and surface_report.get("representation_leaks"): + blockers.append("REPRESENTATION_LEAK") + if surface_report is not None and not surface_report.get("pass"): + blockers.append("APPLICABILITY_SURFACE_SHORTCUT") + if loader_status != "PASS": + blockers.append("LOADER_WITNESS") + if random_new_rows: + blockers.append("RANDOM_NEW_ROWS") + return { + "state": "READY" if ready else "PREREGISTERED", + "ready": ready, + "blocker": None if ready else blockers[0], + "blockers": [] if ready else blockers, + "missing_support": missing, + "deficient_validation": list((validation_report or {}).get("deficient") or []), + "validation_support": dict((validation_report or {}).get("support") or {}), + "below_preferred_validation": list((validation_report or {}).get("below_preferred") or []), + "exact_copy_families": list((prototype_report or {}).get("exact_copy_families") or []), + "prototype_families": list((prototype_report or {}).get("prototype_families") or []), + "prototype_source_counts": dict((prototype_report or {}).get("source_counts") or {}), + "witness_sha256": None if not prototype_report else prototype_report.get("witness_sha256"), + "target_norm": None if not prototype_report else prototype_report.get("target_norm"), + "checks": checks, + "audit": audit, + "family_weights": family_weights, + "applicability_weights": applicability, + "surface_cells": None if surface_report is None else surface_report.get("cells"), + "surface_cell_weights": None if surface_report is None else surface_report.get("cell_weights"), + "surface_ambiguous": None if surface_report is None else surface_report.get("ambiguous"), + "provenance_weights": dict(PROVENANCE_WEIGHTS), + "authorizes_training": ready, + "moves_best": False, + } diff --git a/scripts/shadow/hyperlexical/classification_v2_boundaries.py b/scripts/shadow/hyperlexical/classification_v2_boundaries.py new file mode 100644 index 00000000..7843716a --- /dev/null +++ b/scripts/shadow/hyperlexical/classification_v2_boundaries.py @@ -0,0 +1,871 @@ +"""Multi-anchor semantic boundaries for the 19 active families. + +A family is a set of frozen anchors plus explicit boundaries. The canonical +semantic score is the maximum cosine to that family's anchors, not the mean +of every member. This module does not train, fetch, or read the evaluation +reserve. +""" + +from __future__ import annotations + +import re +from typing import Any, Mapping, Sequence + +from .classification_v2 import ( + ACTIVE_FAMILY_VOCABULARY, + ClassificationContractError, + canonical_json, + sha256_text, + _f32_hash, +) +from .classification_v2_prototype import ( + PROTOTYPE_WEIGHT, + _f32_round, + _markup_text, + _prose, + _unit, + cosine_matrix, + weighted_mean, +) +from .holdout_guard import normalized_text_sha256 + +BOUNDARY_SCHEMA = "hyperlex.classification.v2.family_semantic_boundaries.v1" +BOUNDARY_RULE = "HYPERLEX_FAMILY_SEMANTIC_BOUNDARIES_V1" +CLUSTER_SEED = 0 +CLUSTER_MAX_ITERATIONS = 32 +SILHOUETTE_MIN = 0.20 +ANCHOR_MIN = 1 +ANCHOR_MAX = 5 +COLLISION_COSINE = 0.80 +NEAREST_COMPETITORS = 3 +FACET_LIMIT = 4 +REPRESENTATIVE_LIMIT = 3 +SUPPORT_SPARSE = "SPARSE" +SUPPORT_OK = "SUPPORTED" +CLASSIFICATION_TASKS = frozenset({"classify", "classify+unbind"}) +BLOCKED_IDENTITY_STATES = frozenset({"EVAL_RESERVE", "EVAL_SPENT", "EVAL_BOUND"}) +BLOCKED_SURFACES = frozenset({"evaluation_reserve", "held_out", "measurement", "settlement", "test"}) +SOURCE_RULE = "training_definition_text_classify_split_only" +REJECTED_REPRESENTATION = "single_family_centroid" +CANONICAL_REPRESENTATION = "multi_anchor" +_TOKEN = re.compile(r"[a-z]{4,}") +_STOPWORDS = frozenset( + { + "about", "above", "after", "again", "against", "all", "also", "among", + "and", "another", "any", "are", "around", "because", "been", "before", + "being", "below", "between", "both", "but", "can", "could", "does", + "doing", "down", "during", "each", "else", "especially", "every", + "from", "further", "generally", "have", "having", "here", "hers", + "herself", "himself", "into", "itself", "just", "more", "most", + "much", "must", "none", "only", "onto", "other", "others", "otherwise", + "over", "same", "shall", "should", "some", "someone", "something", + "such", "than", "that", "their", "theirs", "them", "themselves", + "then", "there", "these", "they", "this", "those", "through", "under", + "until", "used", "using", "usually", "very", "what", "when", "where", + "which", "while", "whom", "whose", "will", "with", "within", "without", + "would", "your", + } +) + + +def anchor_count_bounds(n: int) -> dict[str, Any]: + """Support band first. Cluster search applies only when ``k_max`` is set.""" + if isinstance(n, bool) or not isinstance(n, int) or n < 1: + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", "support") + if n <= 2: + return {"fixed_k": 1, "k_max": None, "support_status": SUPPORT_SPARSE} + if n <= 5: + return {"fixed_k": 2, "k_max": None, "support_status": SUPPORT_OK} + if n <= 11: + k_max = 3 + elif n <= 23: + k_max = 4 + else: + k_max = 5 + return {"fixed_k": None, "k_max": k_max, "support_status": SUPPORT_OK} + + +def _dot(left: Sequence[float], right: Sequence[float]) -> float: + return sum(a * b for a, b in zip(left, right)) + + +def _clamp_cosine(value: float) -> float: + if value > 1.0: + return 1.0 + if value < -1.0: + return -1.0 + return value + + +def _member_unit(member: Mapping[str, Any]) -> list[float]: + return _f32_round(_unit(member["vector"])) + + +def _centroid(members: Sequence[Mapping[str, Any]]) -> list[float]: + mean = weighted_mean( + [_member_unit(member) for member in members], + [float(member["weight"]) for member in members], + ) + return _f32_round(_unit(mean)) + + +def _repair( + units: Sequence[Sequence[float]], + clusters: list[list[int]], + identities: Sequence[str], +) -> list[list[int]]: + """Fill empty clusters, then raise singletons when the point count allows it.""" + guard = 0 + limit = max(1, len(identities) * 2) + while guard < limit: + guard += 1 + empty = [index for index, cluster in enumerate(clusters) if not cluster] + small = [index for index, cluster in enumerate(clusters) if len(cluster) == 1] + if not empty and not small: + break + if not empty and len(identities) < 2 * len(clusters): + break + target = empty[0] if empty else small[0] + donor_floor = 1 if empty else 2 + donors = [ + index + for index, cluster in enumerate(clusters) + if index != target and len(cluster) > donor_floor + ] + if not donors: + break + donors.sort(key=lambda index: (-len(clusters[index]), min(identities[i] for i in clusters[index]))) + donor = donors[0] + donor_members = [units[index] for index in clusters[donor]] + donor_centroid = _centroid( + [ + {"vector": units[index], "weight": 1.0} + for index in clusters[donor] + ] + ) + if clusters[target]: + target_centroid = _centroid( + [{"vector": units[index], "weight": 1.0} for index in clusters[target]] + ) + pick = min( + clusters[donor], + key=lambda index: (-_clamp_cosine(_dot(units[index], target_centroid)), identities[index]), + ) + else: + pick = min( + clusters[donor], + key=lambda index: (_clamp_cosine(_dot(units[index], donor_centroid)), identities[index]), + ) + del donor_members + clusters[donor].remove(pick) + clusters[target].append(pick) + return clusters + + +def cluster_indices( + units: Sequence[Sequence[float]], + identities: Sequence[str], + weights: Sequence[float], + k: int, +) -> list[list[int]]: + """Farthest-first spherical k-means. Ties break by lowest identity hash.""" + count = len(units) + if k < 1 or k > count or len(identities) != count or len(weights) != count: + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", "cluster_shape") + if len(set(identities)) != count: + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", "duplicate_identity") + order = sorted(range(count), key=lambda index: identities[index]) + seeds = [order[0]] + while len(seeds) < k: + remaining = [index for index in order if index not in seeds] + seeds.append( + min( + remaining, + key=lambda index: ( + max(_clamp_cosine(_dot(units[index], units[seed])) for seed in seeds), + identities[index], + ), + ) + ) + centroids = [list(units[seed]) for seed in seeds] + previous: tuple[tuple[int, ...], ...] | None = None + clusters: list[list[int]] = [[] for _seed in seeds] + for _iteration in range(CLUSTER_MAX_ITERATIONS): + clusters = [[] for _seed in seeds] + for index in range(count): + choice = min( + range(k), + key=lambda slot: (-_clamp_cosine(_dot(units[index], centroids[slot])), slot), + ) + clusters[choice].append(index) + clusters = _repair(units, clusters, identities) + signature = tuple(tuple(sorted(cluster)) for cluster in clusters) + centroids = [] + for cluster in clusters: + if not cluster: + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", "empty_cluster") + centroids.append( + _centroid( + [{"vector": units[index], "weight": weights[index]} for index in cluster] + ) + ) + if signature == previous: + break + previous = signature + ordered = sorted(clusters, key=lambda cluster: min(identities[index] for index in cluster)) + return [sorted(cluster, key=lambda index: identities[index]) for cluster in ordered] + + +def mean_silhouette(units: Sequence[Sequence[float]], clusters: Sequence[Sequence[int]]) -> float: + if len(clusters) < 2: + return 0.0 + scores: list[float] = [] + for cluster in clusters: + for index in cluster: + if len(cluster) == 1: + scores.append(0.0) + continue + intra = [ + 1.0 - _clamp_cosine(_dot(units[index], units[other])) + for other in cluster + if other != index + ] + cohesion = sum(intra) / len(intra) + separation = None + for other_cluster in clusters: + if other_cluster is cluster or not other_cluster: + continue + distance = sum( + 1.0 - _clamp_cosine(_dot(units[index], units[other])) for other in other_cluster + ) / len(other_cluster) + if separation is None or distance < separation: + separation = distance + if separation is None: + scores.append(0.0) + continue + scale = cohesion if cohesion > separation else separation + scores.append(0.0 if scale == 0.0 else (separation - cohesion) / scale) + if not scores: + return 0.0 + return sum(scores) / len(scores) + + +def choose_anchor_count(members: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Smallest adequately separated ``k``. Identical inputs follow the support band.""" + prepared = _prepare(members) + bounds = anchor_count_bounds(len(prepared)) + units = [_member_unit(member) for member in prepared] + identities = [str(member["identity"]) for member in prepared] + weights = [float(member["weight"]) for member in prepared] + trace: list[dict[str, Any]] = [] + if bounds["fixed_k"] is not None: + k = int(bounds["fixed_k"]) + else: + k = 2 + k_max = int(bounds["k_max"]) + for candidate in range(2, k_max + 1): + grouped = cluster_indices(units, identities, weights, candidate) + silhouette = mean_silhouette(units, grouped) + supports = [len(group) for group in grouped] + accepted = min(supports) >= 2 and silhouette >= SILHOUETTE_MIN + trace.append( + { + "accepted": accepted, + "k": candidate, + "min_support": min(supports), + "silhouette": silhouette, + } + ) + if accepted: + k = candidate + break + else: + k = 2 + if not ANCHOR_MIN <= k <= ANCHOR_MAX: + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", "anchor_bounds") + return { + "k": k, + "n": len(prepared), + "random_seed": CLUSTER_SEED, + "support_status": bounds["support_status"], + "trace": trace, + } + + +def _prepare(members: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]: + if not members: + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", "empty_family") + ordered = sorted(members, key=lambda member: str(member["identity"])) + prepared: list[dict[str, Any]] = [] + for member in ordered: + identity = str(member["identity"]) + weight = float(member["weight"]) + if weight not in (1.0, 0.5): + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", "weight") + prepared.append( + { + "class": str(member.get("class") or ""), + "identity": identity, + "text": str(member.get("text") or ""), + "vector": _member_unit(member), + "weight": weight, + } + ) + return prepared + + +def anchor_id_for(family: str, identities: Sequence[str]) -> str: + body = {"family": family, "source_identity_hashes": sorted(identities)} + return sha256_text(canonical_json(body)) + + +def build_family_anchors(family: str, members: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + prepared = _prepare(members) + decision = choose_anchor_count(prepared) + units = [member["vector"] for member in prepared] + identities = [member["identity"] for member in prepared] + grouped = cluster_indices(units, identities, [member["weight"] for member in prepared], int(decision["k"])) + anchors: list[dict[str, Any]] = [] + for group in grouped: + chosen = [prepared[index] for index in group] + vector = _centroid(chosen) + source_ids = [member["identity"] for member in chosen] + representatives = sorted(chosen, key=lambda member: (-float(member["weight"]), member["identity"]))[ + :REPRESENTATIVE_LIMIT + ] + anchors.append( + { + "anchor_id": anchor_id_for(family, source_ids), + "centroid_hash": _f32_hash(vector), + "family": family, + "inferred_count": sum(1 for member in chosen if member["class"] == "INFERRED"), + "observed_count": sum(1 for member in chosen if member["class"] == "OBSERVED"), + "representative_examples": [ + { + "class": member["class"], + "identity": member["identity"], + "text": member["text"], + } + for member in representatives + ], + "source_identity_hashes": sorted(source_ids), + "support_count": len(chosen), + "vector": vector, + } + ) + supports = [int(anchor["support_count"]) for anchor in anchors] + if decision["support_status"] != SUPPORT_SPARSE and decision["n"] >= 4 and min(supports) < 2: + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", family) + if sum(supports) != decision["n"]: + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", family) + return { + "anchors": anchors, + "decision": decision, + "family": family, + "support_status": decision["support_status"], + } + + +def _tokens(texts: Sequence[str]) -> list[str]: + counts: dict[str, int] = {} + for text in texts: + seen: set[str] = set() + for token in _TOKEN.findall(str(text).lower()): + if token in _STOPWORDS or token in seen: + continue + seen.add(token) + counts[token] = counts.get(token, 0) + 1 + return sorted(counts, key=lambda token: (-counts[token], token)) + + +def distinguishing_tokens(left_texts: Sequence[str], right_texts: Sequence[str]) -> list[str]: + right = set(_tokens(right_texts)) + return [token for token in _tokens(left_texts) if token not in right][:FACET_LIMIT] + + +def _family_texts(anchors: Sequence[Mapping[str, Any]]) -> list[str]: + texts: list[str] = [] + seen: set[str] = set() + for anchor in anchors: + for example in anchor["representative_examples"]: + identity = str(example["identity"]) + if identity in seen: + continue + seen.add(identity) + texts.append(str(example["text"])) + return texts + + +def _anchor_public(anchor: Mapping[str, Any]) -> dict[str, Any]: + return { + "anchor_id": anchor["anchor_id"], + "centroid_hash": anchor["centroid_hash"], + "family": anchor["family"], + "inferred_count": anchor["inferred_count"], + "observed_count": anchor["observed_count"], + "representative_identities": [item["identity"] for item in anchor["representative_examples"]], + "source_identity_hashes": list(anchor["source_identity_hashes"]), + "support_count": anchor["support_count"], + } + + +def future_discriminator_contract() -> dict[str, Any]: + """Specified for a later trainer. This pass does not fit it.""" + return { + "alpha": 1.0, + "anchors": "frozen", + "beta": 1.0, + "formula": "population_zscore(max_anchor_cosine)+population_zscore(residual)", + "hard_negative_count": NEAREST_COMPETITORS, + "hard_negative_multiplier": 2.0, + "hard_negatives": "nearest_competing_anchors", + "jev": "OFF", + "rejected_semantic_score": REJECTED_REPRESENTATION, + "representation": CANONICAL_REPRESENTATION, + "residual": "learned_linear_logit", + "semantic_score": "max_anchor_cosine", + "train": False, + } + + +def anchor_witness_sha256(families: Sequence[Mapping[str, Any]]) -> str: + body = { + "families": [ + { + "anchors": [_anchor_public(anchor) for anchor in family["anchors"]], + "family": family["family"], + "k": len(family["anchors"]), + "n": family["decision"]["n"], + "support_status": family["support_status"], + } + for family in families + ], + "rule": BOUNDARY_RULE, + "seed": CLUSTER_SEED, + } + return sha256_text(canonical_json(body)) + + +def separation_sha256(rows: Sequence[Mapping[str, Any]]) -> str: + return sha256_text(canonical_json({"rows": list(rows), "rule": BOUNDARY_RULE})) + + +def _cosine_pairs( + families: Sequence[Mapping[str, Any]], +) -> tuple[list[dict[str, Any]], list[list[float]]]: + flat: list[dict[str, Any]] = [] + for family in families: + for anchor in family["anchors"]: + flat.append(anchor) + matrix = cosine_matrix([anchor["vector"] for anchor in flat]) + rounded = [[_f32_round((value,))[0] for value in row] for row in matrix] + return flat, rounded + + +def _best_cross( + flat: Sequence[Mapping[str, Any]], + matrix: Sequence[Sequence[float]], + left: str, + right: str, +) -> dict[str, Any]: + best: tuple[tuple[float, str, str], float, str, str] | None = None + for i, anchor in enumerate(flat): + if anchor["family"] != left: + continue + for j, other in enumerate(flat): + if other["family"] != right: + continue + cosine = float(matrix[i][j]) + left_id = str(anchor["anchor_id"]) + right_id = str(other["anchor_id"]) + rank = (-cosine, left_id, right_id) + if best is None or rank < best[0]: + best = (rank, cosine, str(anchor["anchor_id"]), str(other["anchor_id"])) + if best is None: + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", "pair") + return { + "closest_anchor_pair": [best[2], best[3]], + "cosine_similarity": best[1], + } + + +def assemble_boundaries( + family_members: Mapping[str, Sequence[Mapping[str, Any]]], + *, + vocabulary: Sequence[str] | None = None, +) -> dict[str, Any]: + """Build the sealed multi-anchor boundary artifact from prepared members.""" + vocab = tuple(vocabulary) if vocabulary is not None else ACTIVE_FAMILY_VOCABULARY + if set(family_members) != set(vocab): + missing = [family for family in vocab if family not in family_members] + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", missing or "coverage") + built = [build_family_anchors(family, family_members[family]) for family in vocab] + flat, matrix = _cosine_pairs(built) + anchor_ids = [anchor["anchor_id"] for anchor in flat] + collisions: list[dict[str, Any]] = [] + for i, left in enumerate(flat): + for j in range(i + 1, len(flat)): + cosine = float(matrix[i][j]) + if cosine < COLLISION_COSINE: + continue + right = flat[j] + collisions.append( + { + "anchor_a": left["anchor_id"], + "anchor_b": right["anchor_id"], + "cosine": cosine, + "family_a": left["family"], + "family_b": right["family"], + "same_family": left["family"] == right["family"], + } + ) + collisions.sort(key=lambda item: (-float(item["cosine"]), item["anchor_a"], item["anchor_b"])) + max_similarity: dict[str, dict[str, float]] = {} + min_distance: dict[str, dict[str, float]] = {} + nearest: dict[str, list[dict[str, Any]]] = {} + texts = {family["family"]: _family_texts(family["anchors"]) for family in built} + boundaries: list[dict[str, Any]] = [] + for family in built: + name = family["family"] + scores: list[tuple[float, str, dict[str, Any]]] = [] + max_similarity[name] = {} + min_distance[name] = {} + for other in built: + if other["family"] == name: + continue + cross = _best_cross(flat, matrix, name, other["family"]) + cosine = float(cross["cosine_similarity"]) + max_similarity[name][other["family"]] = cosine + min_distance[name][other["family"]] = _f32_round((1.0 - cosine,))[0] + scores.append((cosine, other["family"], cross)) + scores.sort(key=lambda item: (-item[0], item[1])) + competitors = scores[:NEAREST_COMPETITORS] + nearest[name] = [ + {"cosine": cosine, "family": other, "closest_anchor_pair": cross["closest_anchor_pair"]} + for cosine, other, cross in competitors + ] + own_tokens = _tokens(texts[name])[:FACET_LIMIT] + cues: list[str] = [] + exclusions: list[str] = [] + counterexamples: list[dict[str, str]] = [] + for cosine, other, cross in competitors: + cues.extend(distinguishing_tokens(texts[name], texts[other])) + other_only = distinguishing_tokens(texts[other], texts[name]) + own_word = own_tokens[0] if own_tokens else name + other_word = other_only[0] if other_only else other + exclusions.append(f"{other_word} ({other}) without {own_word}") + anchor_id = cross["closest_anchor_pair"][1] + example = _representative_for(built, other, anchor_id) + counterexamples.append( + { + "anchor_id": anchor_id, + "family": other, + "identity": example["identity"], + "text": example["text"], + } + ) + deduped_cues: list[str] = [] + for cue in cues: + if cue not in deduped_cues: + deduped_cues.append(cue) + boundaries.append( + { + "counterexample_patterns": counterexamples, + "distinguishing_cues": deduped_cues[:FACET_LIMIT], + "exclude_when": exclusions, + "family": name, + "nearest_competitors": [{"cosine": item["cosine"], "family": item["family"]} for item in nearest[name]], + "positive_facets": own_tokens, + } + ) + separation: list[dict[str, Any]] = [] + for left in vocab: + for right in vocab: + if left == right: + continue + cross = _best_cross(flat, matrix, left, right) + cosine = float(cross["cosine_similarity"]) + evidence = [ + *[f"only_a:{token}" for token in distinguishing_tokens(texts[left], texts[right])], + *[f"only_b:{token}" for token in distinguishing_tokens(texts[right], texts[left])], + ] + separation.append( + { + "boundary_confidence": _f32_round((1.0 - cosine,))[0], + "closest_anchor_pair": cross["closest_anchor_pair"], + "collidable": cosine >= COLLISION_COSINE, + "cosine_similarity": cosine, + "family_a": left, + "family_b": right, + "known_distinguishing_evidence": evidence, + } + ) + competing_pairs = [ + item + for item in collisions + if not item["same_family"] + ] + artifact = { + "anchor_count_rule": { + "fallback_k": 2, + "initialization": "farthest_first_lowest_identity_hash", + "max_anchors": ANCHOR_MAX, + "max_iterations": CLUSTER_MAX_ITERATIONS, + "min_anchors": ANCHOR_MIN, + "random_seed": CLUSTER_SEED, + "rule": BOUNDARY_RULE, + "silhouette_min": SILHOUETTE_MIN, + "support_bands": "n<=2 => 1 SPARSE; 3<=n<=5 => 2; 6<=n<=11 => k_max 3; 12<=n<=23 => k_max 4; else k_max 5", + "tie_break": "lowest_source_identity_hash", + }, + "anchor_ids": anchor_ids, + "anchor_cosine": matrix, + "boundaries": boundaries, + "collisions": collisions, + "families": [ + { + "anchors": family["anchors"], + "family": family["family"], + "k": len(family["anchors"]), + "n": family["decision"]["n"], + "support_status": family["support_status"], + "trace": family["decision"]["trace"], + } + for family in built + ], + "family_max_similarity": max_similarity, + "family_min_distance": min_distance, + "future_discriminator": future_discriminator_contract(), + "jev": "OFF", + "nearest_competing_anchor_pairs": competing_pairs[:20], + "nearest_competitors": nearest, + "representation": CANONICAL_REPRESENTATION, + "schema": BOUNDARY_SCHEMA, + "separation": separation, + "source_rule": SOURCE_RULE, + } + artifact["anchor_witness_sha256"] = anchor_witness_sha256(built) + artifact["separation_sha256"] = separation_sha256(separation) + artifact["boundary_sha256"] = boundary_sha256(artifact) + return artifact + + +def _representative_for( + families: Sequence[Mapping[str, Any]], + family_name: str, + anchor_id: str, +) -> dict[str, str]: + for family in families: + if family["family"] != family_name: + continue + for anchor in family["anchors"]: + if anchor["anchor_id"] != anchor_id: + continue + example = anchor["representative_examples"][0] + return {"identity": str(example["identity"]), "text": str(example["text"])} + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", anchor_id) + + +def boundary_sha256(artifact: Mapping[str, Any]) -> str: + body = { + "anchor_witness_sha256": artifact["anchor_witness_sha256"], + "collisions": [ + { + "anchor_a": item["anchor_a"], + "anchor_b": item["anchor_b"], + "cosine": item["cosine"], + "family_a": item["family_a"], + "family_b": item["family_b"], + } + for item in artifact["collisions"] + ], + "future_discriminator": artifact["future_discriminator"], + "representation": artifact["representation"], + "rule": BOUNDARY_RULE, + "separation_sha256": artifact["separation_sha256"], + "source_rule": artifact["source_rule"], + } + return sha256_text(canonical_json(body)) + + +def assess_boundary_artifact( + artifact: Mapping[str, Any], + *, + vocabulary: Sequence[str] | None = None, +) -> dict[str, Any]: + """Refuse a single-centroid witness as the v1 boundary representation.""" + vocab = tuple(vocabulary) if vocabulary is not None else ACTIVE_FAMILY_VOCABULARY + try: + if artifact.get("schema") != BOUNDARY_SCHEMA: + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", "schema") + if artifact.get("representation") != CANONICAL_REPRESENTATION: + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", "representation") + contract = artifact.get("future_discriminator") + if contract != future_discriminator_contract(): + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", "contract") + if contract["rejected_semantic_score"] != REJECTED_REPRESENTATION: + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", "centroid") + if contract["semantic_score"] != "max_anchor_cosine": + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", "centroid") + families = list(artifact["families"]) + names = [family["family"] for family in families] + if names != list(vocab): + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", "coverage") + reserved = set(artifact.get("excluded_identities") or []) + for family in families: + k = int(family["k"]) + n = int(family["n"]) + if not ANCHOR_MIN <= k <= ANCHOR_MAX or k != len(family["anchors"]): + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", family["family"]) + if n <= 2: + if family["support_status"] != SUPPORT_SPARSE or k != 1: + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", family["family"]) + elif family["support_status"] != SUPPORT_OK or k < 2: + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", family["family"]) + supports = [] + for anchor in family["anchors"]: + hashes = list(anchor["source_identity_hashes"]) + if anchor["anchor_id"] != anchor_id_for(family["family"], hashes): + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", "anchor_id") + if anchor["centroid_hash"] != _f32_hash(anchor["vector"]): + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", "centroid_hash") + if set(hashes) & reserved: + raise ClassificationContractError("evaluation_isolation") + supports.append(int(anchor["support_count"])) + if int(anchor["support_count"]) != len(hashes): + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", "support") + if sum(supports) != n: + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", family["family"]) + if n >= 4 and min(supports) < 2: + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", family["family"]) + if any(support < 1 for support in supports): + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", "empty_cluster") + if artifact.get("anchor_witness_sha256") != anchor_witness_sha256( + [ + { + "anchors": family["anchors"], + "decision": {"n": family["n"]}, + "family": family["family"], + "support_status": family["support_status"], + } + for family in families + ] + ): + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", "anchor_witness") + if artifact.get("separation_sha256") != separation_sha256(artifact["separation"]): + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", "separation") + expected_pairs = len(vocab) * (len(vocab) - 1) + if len(artifact["separation"]) != expected_pairs: + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", "separation") + if artifact.get("boundary_sha256") != boundary_sha256(artifact): + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", "boundary_hash") + if artifact.get("jev") != "OFF": + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", "jev") + except (ClassificationContractError, KeyError, TypeError, ValueError) as exc: + return {"pass": False, "reason": "FAMILY_BOUNDARY_UNAVAILABLE", "detail": str(exc)} + return { + "anchor_witness_sha256": artifact["anchor_witness_sha256"], + "boundary_sha256": artifact["boundary_sha256"], + "families": len(vocab), + "pass": True, + "separation_sha256": artifact["separation_sha256"], + } + + +def _blocked_row(row: Mapping[str, Any], identity_state: Mapping[str, str]) -> str | None: + if row.get("evaluation_reserve") or row.get("held_out"): + return "row_flag" + surface = str(row.get("surface") or "") + if surface in BLOCKED_SURFACES: + return "surface" + stage = str(row.get("stage") or "") + if stage in BLOCKED_SURFACES: + return "surface" + provenance = row.get("provenance") + if isinstance(provenance, dict): + if provenance.get("jev") or str(provenance.get("source") or "").lower() == "jev": + return "jev" + if row.get("jev"): + return "jev" + digest = normalized_text_sha256(str(row.get("text") or "")) + if identity_state.get(digest) in BLOCKED_IDENTITY_STATES: + return str(identity_state[digest]) + return None + + +def boundary_training_sources( + rows: Sequence[Mapping[str, Any]], + identity_state: Mapping[str, str] | None = None, + *, + vocabulary: Sequence[str] | None = None, +) -> dict[str, Any]: + """Training-split definition text for every active family. + + Stored definition prose is the source when the row text is that prose. + Exact-copy families have no stored definition, so their classification + prose is the training definition. Validation, reserve, spent, held-out, + measurement, and Jev rows are not sources. + """ + from .classification_v2 import EXACT_COPY_FAMILIES + from .classification_v2_surface import SURFACE_PROSE, surface_form + + vocab = tuple(vocabulary) if vocabulary is not None else ACTIVE_FAMILY_VOCABULARY + states = identity_state or {} + grouped: dict[str, dict[str, Mapping[str, Any]]] = {family: {} for family in vocab} + excluded: list[dict[str, str]] = [] + for row in rows: + if row.get("split") != "train": + continue + lineage = str(row.get("lineage") or "") + if lineage not in grouped or row.get("class") not in PROTOTYPE_WEIGHT: + continue + if str(row.get("task") or "") not in CLASSIFICATION_TASKS: + continue + text = str(row.get("text") or "").strip() + if not text: + continue + if lineage in EXACT_COPY_FAMILIES: + if _markup_text(text) or surface_form(text) != SURFACE_PROSE: + continue + else: + prose = _prose(row) + if not prose or text != prose: + continue + digest = normalized_text_sha256(text) + reason = _blocked_row(row, states) + if reason is not None: + excluded.append({"identity": digest, "reason": reason}) + continue + current = grouped[lineage].get(digest) + if current is not None and current.get("class") == "OBSERVED": + continue + if current is not None and row.get("class") != "OBSERVED": + continue + grouped[lineage][digest] = row + sources: dict[str, dict[str, Any]] = {} + for family in vocab: + items = sorted(grouped[family].items(), key=lambda item: item[0]) + if not items: + continue + sources[family] = { + "identities": [digest for digest, _row in items], + "inferred": sum(1 for _digest, row in items if row.get("class") == "INFERRED"), + "observed": sum(1 for _digest, row in items if row.get("class") == "OBSERVED"), + "rows": [row for _digest, row in items], + "weights": [PROTOTYPE_WEIGHT[str(row.get("class"))] for _digest, row in items], + } + return { + "excluded_identities": sorted({item["identity"] for item in excluded}), + "exclusions": excluded, + "sources": sources, + } + + +def canonical_boundary_representation(artifact: Mapping[str, Any]) -> str: + if artifact.get("schema") != BOUNDARY_SCHEMA: + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", "schema") + if artifact.get("representation") != CANONICAL_REPRESENTATION: + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", "representation") + if artifact.get("future_discriminator", {}).get("rejected_semantic_score") != REJECTED_REPRESENTATION: + raise ClassificationContractError("FAMILY_BOUNDARY_UNAVAILABLE", "centroid") + return CANONICAL_REPRESENTATION diff --git a/scripts/shadow/hyperlexical/classification_v2_geometry_repair.py b/scripts/shadow/hyperlexical/classification_v2_geometry_repair.py new file mode 100644 index 00000000..5bb87cf6 --- /dev/null +++ b/scripts/shadow/hyperlexical/classification_v2_geometry_repair.py @@ -0,0 +1,315 @@ +"""Encoder geometry repair for Classification v2. + +Boundaries structure the supervised contrastive loss. Family logits stay on the +learned residual head. This module does not train, fetch, or read the evaluation +reserve. +""" + +from __future__ import annotations + +import math +from typing import Any, Mapping, Sequence + +from .classification_v2 import ( + ACTIVE_FAMILY_VOCABULARY, + ClassificationContractError, + canonical_json, + sha256_text, +) +from .classification_v2_boundaries import ( + BOUNDARY_RULE, + BOUNDARY_SCHEMA, + COLLISION_COSINE, + NEAREST_COMPETITORS, + SUPPORT_SPARSE, + assess_boundary_artifact, +) +from .classification_v2_prototype import _f32_round, _unit + +REPAIR_SCHEMA = "hyperlex.classification.v2.geometry_repair.v1" +REPAIR_RULE = "HYPERLEX_FAMILY_GEOMETRY_REPAIR_V1" +GEOMETRY_TAU = 0.10 +LAMBDA_GEOMETRY = 0.5 +HARD_NEGATIVE_MULTIPLIER = 2.0 +CANONICAL_LOGITS = "learned_residual" +REJECTED_LOGITS = "frozen_prototype_fusion" +BOUNDARY_SHA = "0ca6f34ce1abf68388e443371672ca36e16028079a175b6775e1968900e1c52f" +SEPARATION_SHA = "ab698d342d2d276f81d4baf3fed609bb6f8bf88cd6810bb1d1998c5e409f63c3" +PRIOR_ACTIVE_FAMILY_MACRO = 0.1960828268105939 +PRIOR_PROTOTYPE_FAMILY_MACRO = 0.041352657004830914 +PRIOR_NEW_FAMILY_F1_GT_0 = 3 + + +def repair_contract() -> dict[str, Any]: + return { + "boundary_rule": BOUNDARY_RULE, + "canonical_logits": CANONICAL_LOGITS, + "hard_negative_count": NEAREST_COMPETITORS, + "hard_negative_multiplier": HARD_NEGATIVE_MULTIPLIER, + "hard_negatives": "sealed_nearest_competing_families", + "jev": "OFF", + "lambda_geometry": LAMBDA_GEOMETRY, + "last_trainable": 2, + "rejected_logits": REJECTED_LOGITS, + "rule": REPAIR_RULE, + "schema": REPAIR_SCHEMA, + "tau": GEOMETRY_TAU, + "train_encoder": True, + } + + +def hard_negatives_from_boundaries(boundaries: Mapping[str, Any]) -> dict[str, list[str]]: + """Nearest competing families from the sealed artifact. No evaluation mining.""" + nearest = boundaries.get("nearest_competitors") + if not isinstance(nearest, dict): + raise ClassificationContractError("FAMILY_GEOMETRY_REPAIR_UNAVAILABLE", "nearest") + chosen: dict[str, list[str]] = {} + for family in ACTIVE_FAMILY_VOCABULARY: + rows = nearest.get(family) + if not isinstance(rows, list) or len(rows) < NEAREST_COMPETITORS: + raise ClassificationContractError("FAMILY_GEOMETRY_REPAIR_UNAVAILABLE", family) + names = [str(item["family"]) for item in rows[:NEAREST_COMPETITORS]] + if family in names or len(set(names)) != NEAREST_COMPETITORS: + raise ClassificationContractError("FAMILY_GEOMETRY_REPAIR_UNAVAILABLE", family) + if any(name not in ACTIVE_FAMILY_VOCABULARY for name in names): + raise ClassificationContractError("FAMILY_GEOMETRY_REPAIR_UNAVAILABLE", family) + chosen[family] = names + return chosen + + +def sparse_families_from_boundaries(boundaries: Mapping[str, Any]) -> list[str]: + sparse = [] + for family in boundaries.get("families") or []: + if family.get("support_status") == SUPPORT_SPARSE: + sparse.append(str(family["family"])) + return sparse + + +def flatten_anchors(boundaries: Mapping[str, Any]) -> list[dict[str, Any]]: + flat: list[dict[str, Any]] = [] + for family in boundaries.get("families") or []: + name = str(family["family"]) + for anchor in family.get("anchors") or []: + vector = _f32_round(_unit(anchor["vector"])) + flat.append( + { + "anchor_id": str(anchor["anchor_id"]), + "family": name, + "support_count": int(anchor["support_count"]), + "vector": vector, + } + ) + if not flat: + raise ClassificationContractError("FAMILY_GEOMETRY_REPAIR_UNAVAILABLE", "anchors") + names = {item["family"] for item in flat} + if names != set(ACTIVE_FAMILY_VOCABULARY): + raise ClassificationContractError("FAMILY_GEOMETRY_REPAIR_UNAVAILABLE", "coverage") + return flat + + +def denominator_anchor_weights( + family: str, + anchors: Sequence[Mapping[str, Any]], + hard_negatives: Mapping[str, Sequence[str]], +) -> list[float]: + competitors = set(hard_negatives[family]) + weights: list[float] = [] + for anchor in anchors: + if anchor["family"] == family: + weights.append(1.0) + elif anchor["family"] in competitors: + weights.append(HARD_NEGATIVE_MULTIPLIER) + else: + weights.append(1.0) + return weights + + +def supervised_contrastive_nll( + similarities: Sequence[float], + positive_mask: Sequence[bool], + weights: Sequence[float], + *, + tau: float = GEOMETRY_TAU, +) -> float: + """-log(sum_pos exp(sim/tau) / sum_all w exp(sim/tau)).""" + if not similarities or len(similarities) != len(positive_mask) or len(similarities) != len(weights): + raise ClassificationContractError("FAMILY_GEOMETRY_REPAIR_UNAVAILABLE", "contrastive_width") + if not any(positive_mask): + raise ClassificationContractError("FAMILY_GEOMETRY_REPAIR_UNAVAILABLE", "no_positive") + if not math.isfinite(tau) or tau <= 0.0: + raise ClassificationContractError("FAMILY_GEOMETRY_REPAIR_UNAVAILABLE", "tau") + logits = [float(value) / tau for value in similarities] + peak = max(logits) + numer = 0.0 + denom = 0.0 + for logit, positive, weight in zip(logits, positive_mask, weights): + if isinstance(weight, bool) or not isinstance(weight, (int, float)) or not math.isfinite(float(weight)): + raise ClassificationContractError("FAMILY_GEOMETRY_REPAIR_UNAVAILABLE", "weight") + scale = float(weight) + if scale <= 0.0: + raise ClassificationContractError("FAMILY_GEOMETRY_REPAIR_UNAVAILABLE", "weight") + term = math.exp(logit - peak) + denom += scale * term + if positive: + numer += term + if numer <= 0.0 or denom <= 0.0: + raise ClassificationContractError("FAMILY_GEOMETRY_REPAIR_UNAVAILABLE", "contrastive") + return math.log(denom) - math.log(numer) + + +def row_geometry_scores( + similarities: Sequence[float], + gold_family: str, + anchors: Sequence[Mapping[str, Any]], + hard_negatives: Mapping[str, Sequence[str]], +) -> dict[str, float]: + if gold_family not in ACTIVE_FAMILY_VOCABULARY: + raise ClassificationContractError("FAMILY_GEOMETRY_REPAIR_UNAVAILABLE", gold_family) + if len(similarities) != len(anchors): + raise ClassificationContractError("FAMILY_GEOMETRY_REPAIR_UNAVAILABLE", "similarity_width") + competitors = set(hard_negatives[gold_family]) + positives = [ + float(value) + for value, anchor in zip(similarities, anchors) + if anchor["family"] == gold_family + ] + negatives = [ + float(value) + for value, anchor in zip(similarities, anchors) + if anchor["family"] in competitors + ] + if not positives or not negatives: + raise ClassificationContractError("FAMILY_GEOMETRY_REPAIR_UNAVAILABLE", gold_family) + positive = max(positives) + nearest_negative = max(negatives) + return { + "margin": positive - nearest_negative, + "nearest_negative_similarity": nearest_negative, + "positive_similarity": positive, + } + + +def summarize_geometry_margins( + rows: Sequence[Mapping[str, Any]], +) -> dict[str, Any]: + """Aggregate per-family and global margin stats. Reserve rows are refused.""" + buckets: dict[str, list[dict[str, float]]] = {name: [] for name in ACTIVE_FAMILY_VOCABULARY} + for row in rows: + if row.get("evaluation_reserve") or row.get("held_out") or row.get("split") == "test": + raise ClassificationContractError("evaluation_isolation") + family = str(row["family"]) + buckets[family].append( + { + "margin": float(row["margin"]), + "nearest_negative_similarity": float(row["nearest_negative_similarity"]), + "positive_similarity": float(row["positive_similarity"]), + } + ) + per_family: dict[str, Any] = {} + margins: list[float] = [] + high_collision_rows = 0 + for family, values in buckets.items(): + if not values: + per_family[family] = { + "margin": None, + "n": 0, + "nearest_negative_similarity": None, + "positive_similarity": None, + } + continue + pos = sum(item["positive_similarity"] for item in values) / len(values) + neg = sum(item["nearest_negative_similarity"] for item in values) / len(values) + margin = sum(item["margin"] for item in values) / len(values) + per_family[family] = { + "margin": margin, + "n": len(values), + "nearest_negative_similarity": neg, + "positive_similarity": pos, + } + margins.extend(item["margin"] for item in values) + high_collision_rows += sum( + 1 for item in values if item["nearest_negative_similarity"] >= COLLISION_COSINE + ) + ordered = sorted(margins) + median = None if not ordered else ( + ordered[len(ordered) // 2] + if len(ordered) % 2 == 1 + else (ordered[len(ordered) // 2 - 1] + ordered[len(ordered) // 2]) / 2.0 + ) + return { + "high_collision_rows": high_collision_rows, + "median_margin": median, + "mean_margin": None if not margins else sum(margins) / len(margins), + "n_family": len(margins), + "per_family": per_family, + } + + +def assess_boundary_for_repair(boundaries: Mapping[str, Any]) -> dict[str, Any]: + base = assess_boundary_artifact(boundaries) + if not base.get("pass"): + return base + if boundaries.get("boundary_sha256") != BOUNDARY_SHA: + return {"pass": False, "reason": "FAMILY_GEOMETRY_REPAIR_UNAVAILABLE", "detail": "boundary_hash"} + if boundaries.get("separation_sha256") != SEPARATION_SHA: + return {"pass": False, "reason": "FAMILY_GEOMETRY_REPAIR_UNAVAILABLE", "detail": "separation_hash"} + if boundaries.get("schema") != BOUNDARY_SCHEMA: + return {"pass": False, "reason": "FAMILY_GEOMETRY_REPAIR_UNAVAILABLE", "detail": "schema"} + hard = hard_negatives_from_boundaries(boundaries) + sparse = sparse_families_from_boundaries(boundaries) + anchors = flatten_anchors(boundaries) + return { + "anchors": len(anchors), + "boundary_sha256": BOUNDARY_SHA, + "hard_negatives": hard, + "pass": True, + "separation_sha256": SEPARATION_SHA, + "sparse_families": sparse, + } + + +def repair_readiness_gate( + *, + active_family_macro_f1: float, + prototype_family_macro_f1: float, + new_family_f1_gt_0: int, + median_margin: float, + prior_median_margin: float, + high_collision_rows: int, + prior_high_collision_rows: int, + invariance_pass: bool, +) -> dict[str, Any]: + family_ok = float(active_family_macro_f1) > PRIOR_ACTIVE_FAMILY_MACRO + new_macro_ok = float(prototype_family_macro_f1) > PRIOR_PROTOTYPE_FAMILY_MACRO + breadth_ok = int(new_family_f1_gt_0) > PRIOR_NEW_FAMILY_F1_GT_0 + margin_ok = float(median_margin) > float(prior_median_margin) + collision_ok = int(high_collision_rows) < int(prior_high_collision_rows) + geometry_ok = margin_ok and collision_ok + if family_ok and new_macro_ok and breadth_ok and geometry_ok and invariance_pass: + status = "INTERNAL_IMPROVED" + elif family_ok and not invariance_pass: + status = "SETTLED_INVALID" + else: + status = "INTERNAL_SHORT" + return { + "breadth_ok": breadth_ok, + "collision_ok": collision_ok, + "family_ok": family_ok, + "geometry_ok": geometry_ok, + "invariance_pass": bool(invariance_pass), + "margin_ok": margin_ok, + "new_macro_ok": new_macro_ok, + "reserve_justified": status == "INTERNAL_IMPROVED", + "status": status, + } + + +def repair_artifact_sha256(payload: Mapping[str, Any]) -> str: + body = { + "boundary_sha256": payload["boundary_sha256"], + "canonical_logits": payload["canonical_logits"], + "contract": payload["contract"], + "hard_negatives": payload["hard_negatives"], + "sparse_families": payload["sparse_families"], + } + return sha256_text(canonical_json(body)) diff --git a/scripts/shadow/hyperlexical/classification_v2_prototype.py b/scripts/shadow/hyperlexical/classification_v2_prototype.py new file mode 100644 index 00000000..ef626ed2 --- /dev/null +++ b/scripts/shadow/hyperlexical/classification_v2_prototype.py @@ -0,0 +1,856 @@ +"""Semantic prototypes for Classification v2 family rows. + +Exact-name rows are copied. Every other active family row is initialized from +the mean of frozen warm-start encoder representations of training-split +definition text. This module does not train, fetch, or read the evaluation reserve. +""" + +from __future__ import annotations + +import math +from typing import Any, Mapping, Sequence + +from .classification_v2 import ( + ACTIVE_FAMILY_VOCABULARY, + ClassificationContractError, + map_family_rows, + _f32_hash, +) +from .holdout_guard import normalized_text_sha256 + +PROTOTYPE_WEIGHT = {"OBSERVED": 1.0, "INFERRED": 0.5} +INITIALIZATION_EXACT = "EXACT_ROW_COPY" +INITIALIZATION_PROTOTYPE = "SEMANTIC_PROTOTYPE" +SCALING_RULE = "l2_normalize_then_multiply_by_median_l2_of_exact_copy_rows" +VALIDATION_FLOOR = 2 +VALIDATION_PREFERRED = 4 +BLOCKED_VALIDATION_STATES = frozenset({"EVAL_RESERVE", "EVAL_SPENT", "EVAL_BOUND"}) + + +def _finite_vector(values: Sequence[float], reason: str) -> list[float]: + vector: list[float] = [] + for value in values: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ClassificationContractError(reason) + number = float(value) + if not math.isfinite(number): + raise ClassificationContractError(reason) + vector.append(number) + if not vector: + raise ClassificationContractError(reason) + return vector + + +def l2_norm(values: Sequence[float]) -> float: + vector = _finite_vector(values, "FAMILY_PROTOTYPE_UNAVAILABLE") + norm = math.sqrt(sum(value * value for value in vector)) + if not math.isfinite(norm): + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE") + return norm + + +def _f32_round(values: Sequence[float]) -> list[float]: + import struct + + rounded: list[float] = [] + for value in values: + number = float(value) + if not math.isfinite(number): + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE") + rounded.append(struct.unpack(" float: + norms = sorted(l2_norm(row) for row in rows) + if not norms: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "target_norm") + if any(norm == 0.0 for norm in norms): + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "target_norm") + mid = len(norms) // 2 + if len(norms) % 2 == 1: + return norms[mid] + return (norms[mid - 1] + norms[mid]) / 2.0 + + +def weighted_mean(vectors: Sequence[Sequence[float]], weights: Sequence[float]) -> list[float]: + if not vectors or len(vectors) != len(weights): + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "empty_prototype") + hidden = len(vectors[0]) + if hidden < 1: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "empty_prototype") + total = 0.0 + accumulator = [0.0] * hidden + for vector, weight in zip(vectors, weights): + if isinstance(weight, bool) or not isinstance(weight, (int, float)): + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE") + scale = float(weight) + if not math.isfinite(scale) or scale <= 0.0: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE") + if len(vector) != hidden: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "width") + for index, value in enumerate(vector): + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(float(value)): + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE") + accumulator[index] += scale * float(value) + total += scale + if total == 0.0: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE") + mean = _f32_round(value / total for value in accumulator) + if l2_norm(mean) == 0.0: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "zero_norm") + return mean + + +def scale_prototype(prototype: Sequence[float], target_norm: float) -> list[float]: + vector = _f32_round(prototype) + norm = l2_norm(vector) + if norm == 0.0 or not math.isfinite(target_norm) or target_norm <= 0.0: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "scale") + return _f32_round((value / norm) * float(target_norm) for value in vector) + + +def _prose(row: Mapping[str, Any]) -> str: + provenance = row.get("provenance") + if not isinstance(provenance, dict): + return "" + return str(provenance.get("definition_prose") or "").strip() + + +def definition_string_report(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """A stored definition is the training string unless that identity is already used.""" + texts = {normalized_text_sha256(str(row.get("text") or "")) for row in rows} + violations: list[dict[str, str]] = [] + for row in rows: + provenance = row.get("provenance") + if not isinstance(provenance, dict) or not provenance.get("evidence_seal"): + continue + prose = _prose(row) + text = str(row.get("text") or "") + if not prose: + continue + if text == prose: + continue + prose_identity = normalized_text_sha256(prose) + if prose_identity in texts: + continue + violations.append( + { + "family": str(row.get("lineage") or ""), + "page": str(provenance.get("page") or ""), + "reason": "title_used_for_available_definition", + } + ) + return {"pass": not violations, "violations": violations} + + +def validation_family_support( + rows: Sequence[Mapping[str, Any]], + *, + identity_state: Mapping[str, str] | None = None, +) -> dict[str, Any]: + """Count validation positives that are not reserve, spent, or bound.""" + states = identity_state or {} + support = {family: 0 for family in ACTIVE_FAMILY_VOCABULARY} + rejected: list[str] = [] + for row in rows: + if row.get("split") != "val": + continue + lineage = str(row.get("lineage") or "") + if lineage not in support: + continue + if row.get("class") not in PROTOTYPE_WEIGHT: + continue + if row.get("evaluation_reserve") or row.get("held_out") or row.get("surface") in { + "evaluation_reserve", + "held_out", + "measurement", + "settlement", + "test", + }: + rejected.append(lineage) + continue + digest = normalized_text_sha256(str(row.get("text") or "")) + if states.get(digest) in BLOCKED_VALIDATION_STATES: + rejected.append(lineage) + continue + support[lineage] += 1 + deficient = [family for family in ACTIVE_FAMILY_VOCABULARY if support[family] < VALIDATION_FLOOR] + below_preferred = [family for family in ACTIVE_FAMILY_VOCABULARY if support[family] < VALIDATION_PREFERRED] + return { + "support": support, + "deficient": deficient, + "below_preferred": below_preferred, + "rejected_families": sorted(set(rejected)), + "pass": not deficient and not rejected, + "floor": VALIDATION_FLOOR, + "preferred": VALIDATION_PREFERRED, + } + + +def _copy_witness(family: str, weight: Sequence[float], bias: float) -> dict[str, Any]: + rounded = _f32_round(weight) + rounded_bias = _f32_round((bias,))[0] + return { + "bias_hash": _f32_hash((rounded_bias,)), + "bias": rounded_bias, + "family": family, + "inferred_count": 0, + "initialization_mode": INITIALIZATION_EXACT, + "l2_norm": l2_norm(rounded), + "observed_count": 0, + "prototype_hash": None, + "source_identity_count": 0, + "source_row_hash": _f32_hash([*rounded, rounded_bias]), + "weight": rounded, + "weight_hash": _f32_hash(rounded), + } + + +def _prototype_witness( + family: str, + vectors: Sequence[Sequence[float]], + weights: Sequence[float], + identities: Sequence[str], + *, + observed: int, + inferred: int, + target_norm: float, +) -> dict[str, Any]: + order = sorted(range(len(identities)), key=lambda index: identities[index]) + ordered_vectors = [vectors[index] for index in order] + ordered_weights = [weights[index] for index in order] + if len(set(identities)) != len(identities): + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", family) + prototype = weighted_mean(ordered_vectors, ordered_weights) + scaled = scale_prototype(prototype, target_norm) + return { + "bias_hash": _f32_hash((0.0,)), + "bias": 0.0, + "family": family, + "inferred_count": inferred, + "initialization_mode": INITIALIZATION_PROTOTYPE, + "l2_norm": l2_norm(scaled), + "observed_count": observed, + "prototype_hash": _f32_hash(prototype), + "source_identity_count": len(identities), + "source_row_hash": None, + "weight": scaled, + "weight_hash": _f32_hash(scaled), + } + + +def assemble_initialization( + source_labels: Sequence[str], + source_weight: Sequence[Sequence[float]], + source_bias: Sequence[float], + prototypes: Mapping[str, Mapping[str, Any]], +) -> dict[str, Any]: + """Copy exact rows and scale a prototype into every remaining family row.""" + mapped = map_family_rows(source_labels, source_weight, source_bias) + copied = [ + mapped["weight"][index] + for index, row in enumerate(mapped["rows"]) + if row["mapping_status"] == "EXACT_COPY" + ] + if not copied: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "no_exact_copy_rows") + target = median_l2(copied) + rows: list[dict[str, Any]] = [] + weight: list[list[float]] = [] + bias: list[float] = [] + missing: list[str] = [] + for index, item in enumerate(mapped["rows"]): + family = item["active_family"] + if item["mapping_status"] == "EXACT_COPY": + witness = _copy_witness(family, mapped["weight"][index], mapped["bias"][index]) + else: + payload = prototypes.get(family) + if not payload: + missing.append(family) + continue + try: + witness = _prototype_witness( + family, + payload["vectors"], + payload["weights"], + payload["identities"], + observed=int(payload["observed"]), + inferred=int(payload["inferred"]), + target_norm=target, + ) + except ClassificationContractError: + missing.append(family) + continue + rows.append(witness) + weight.append(witness["weight"]) + bias.append(witness["bias"]) + if missing: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", missing) + if [row["family"] for row in rows] != list(ACTIVE_FAMILY_VOCABULARY): + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "order") + public = [_public_row(row) for row in rows] + return { + "bias": bias, + "exact_copy_families": [row["family"] for row in rows if row["initialization_mode"] == INITIALIZATION_EXACT], + "prototype_families": [row["family"] for row in rows if row["initialization_mode"] == INITIALIZATION_PROTOTYPE], + "rows": public, + "scaling_rule": SCALING_RULE, + "target_norm": target, + "target_norm_source": "median_l2_exact_reusable_active_family_rows", + "weight": weight, + "witness_sha256": witness_sha256(public, target), + } + + +def _public_row(row: Mapping[str, Any]) -> dict[str, Any]: + return { + "bias_hash": row["bias_hash"], + "family": row["family"], + "inferred_count": row["inferred_count"], + "initialization_mode": row["initialization_mode"], + "l2_norm": row["l2_norm"], + "observed_count": row["observed_count"], + "prototype_hash": row["prototype_hash"], + "source_identity_count": row["source_identity_count"], + "source_row_hash": row["source_row_hash"], + "weight_hash": row["weight_hash"], + } + + +def witness_sha256(rows: Sequence[Mapping[str, Any]], target_norm: float) -> str: + from .classification_v2 import canonical_json, sha256_text + + body = {"rows": list(rows), "scaling_rule": SCALING_RULE, "target_norm": target_norm} + return sha256_text(canonical_json(body)) + + +def training_prototype_sources(rows: Sequence[Mapping[str, Any]]) -> dict[str, dict[str, Any]]: + """Training-split definition strings for families that are not exact-name copies.""" + grouped: dict[str, list[tuple[str, Mapping[str, Any]]]] = {family: [] for family in ACTIVE_FAMILY_VOCABULARY} + for row in rows: + if row.get("split") != "train": + continue + lineage = str(row.get("lineage") or "") + if lineage not in grouped: + continue + if row.get("class") not in PROTOTYPE_WEIGHT: + continue + if row.get("evaluation_reserve") or row.get("held_out"): + raise ClassificationContractError("evaluation_isolation") + prose = _prose(row) + if not prose or str(row.get("text") or "") != prose: + continue + grouped[lineage].append((normalized_text_sha256(prose), row)) + sources: dict[str, dict[str, Any]] = {} + for family, items in grouped.items(): + if not items: + continue + ordered = sorted(items, key=lambda item: item[0]) + sources[family] = { + "identities": [digest for digest, _row in ordered], + "observed": sum(1 for _digest, row in ordered if row.get("class") == "OBSERVED"), + "inferred": sum(1 for _digest, row in ordered if row.get("class") == "INFERRED"), + "rows": [row for _digest, row in ordered], + } + return sources + + +def verify_witness_against_copy(mapped: Mapping[str, Any], witness: Mapping[str, Any]) -> dict[str, Any]: + """Refuse zero or drifted rows. Exact copies must match the live warm-start head.""" + rows = witness.get("rows") + weight = witness.get("weight") + bias = witness.get("bias") + if not isinstance(rows, list) or not isinstance(weight, list) or not isinstance(bias, list): + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "witness") + if len(rows) != len(weight) or len(rows) != len(bias) or len(rows) != len(ACTIVE_FAMILY_VOCABULARY): + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "witness") + if witness.get("witness_sha256") != witness_sha256(rows, float(witness["target_norm"])): + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "witness_hash") + modes = {INITIALIZATION_EXACT, INITIALIZATION_PROTOTYPE} + for index, (row, live) in enumerate(zip(rows, mapped["rows"])): + if row.get("family") != live["active_family"] or row.get("initialization_mode") not in modes: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", row.get("family")) + installed = _f32_round(weight[index]) + if _f32_hash(installed) != row.get("weight_hash"): + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", row.get("family")) + if row["initialization_mode"] == INITIALIZATION_EXACT: + if live["mapping_status"] != "EXACT_COPY": + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", row["family"]) + if _f32_hash(_f32_round(mapped["weight"][index])) != row["weight_hash"]: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "copy_mismatch") + if _f32_hash(_f32_round((mapped["bias"][index],))) != row["bias_hash"]: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "copy_mismatch") + else: + if live["mapping_status"] == "EXACT_COPY": + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "prototype_replaced_copy") + if l2_norm(installed) == 0.0 or row.get("source_identity_count", 0) < 1: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", row["family"]) + if _f32_round((bias[index],))[0] != 0.0: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "bias") + return { + "bias": [_f32_round((value,))[0] for value in bias], + "exact_copy_families": witness.get("exact_copy_families"), + "prototype_families": witness.get("prototype_families"), + "rows": rows, + "weight": [_f32_round(vector) for vector in weight], + "witness_sha256": witness["witness_sha256"], + } + + +def assess_witness(witness: Mapping[str, Any]) -> dict[str, Any]: + """Check a frozen witness without reloading the encoder.""" + try: + rows = witness["rows"] + weight = witness["weight"] + bias = witness["bias"] + target = float(witness["target_norm"]) + if witness.get("witness_sha256") != witness_sha256(rows, target): + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "witness_hash") + if witness.get("scaling_rule") != SCALING_RULE: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "scaling") + if [row["family"] for row in rows] != list(ACTIVE_FAMILY_VOCABULARY): + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "order") + builds = list(witness.get("build_sha256") or []) + if builds != [witness["witness_sha256"], witness["witness_sha256"]]: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "nondeterministic") + source_counts = {} + for index, row in enumerate(rows): + mode = row["initialization_mode"] + if mode not in {INITIALIZATION_EXACT, INITIALIZATION_PROTOTYPE}: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", row["family"]) + installed = _f32_round(weight[index]) + if _f32_hash(installed) != row["weight_hash"]: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", row["family"]) + if _f32_hash(_f32_round((bias[index],))) != row["bias_hash"]: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", row["family"]) + if mode == INITIALIZATION_PROTOTYPE: + if row["source_identity_count"] < 1 or l2_norm(installed) == 0.0 or bias[index] != 0.0: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", row["family"]) + if row["source_row_hash"] is not None or not row["prototype_hash"]: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", row["family"]) + else: + if row["source_row_hash"] is None: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", row["family"]) + source_counts[row["family"]] = { + "inferred": row["inferred_count"], + "observed": row["observed_count"], + "source_identity_count": row["source_identity_count"], + } + except (KeyError, TypeError, ValueError) as exc: + return {"pass": False, "reason": "FAMILY_PROTOTYPE_UNAVAILABLE", "detail": str(exc)} + return { + "copied_rows_match": True, + "determinism": "pass", + "exact_copy_families": [row["family"] for row in rows if row["initialization_mode"] == INITIALIZATION_EXACT], + "pass": True, + "prototype_families": [row["family"] for row in rows if row["initialization_mode"] == INITIALIZATION_PROTOTYPE], + "source_counts": source_counts, + "target_norm": target, + "target_norm_source": witness.get("target_norm_source"), + "witness_sha256": witness["witness_sha256"], + } + + +FUSION_ALPHA = 1.0 +FUSION_BETA = 1.0 +PROTO_TAU = 0.10 +LAMBDA_PROTO = 0.5 +HARD_NEGATIVE_COUNT = 3 +HARD_NEGATIVE_MULTIPLIER = 2.0 +CONFUSABLE_COSINE = 0.80 +GEOMETRY_SCHEMA = "hyperlex.classification.v2.prototype_geometry.v1" +FUSION_RULE = "population_zscore(cosine/tau)+population_zscore(residual)" + + +def _unit(values: Sequence[float]) -> list[float]: + vector = _finite_vector(values, "FAMILY_PROTOTYPE_UNAVAILABLE") + norm = l2_norm(vector) + if norm == 0.0: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "zero_norm") + return [value / norm for value in vector] + + +def population_standardize(values: Sequence[float]) -> list[float]: + """Per-example z-score across families. A constant vector becomes zeros.""" + vector = _finite_vector(values, "FAMILY_PROTOTYPE_UNAVAILABLE") + count = len(vector) + mean = sum(vector) / count + variance = sum((value - mean) ** 2 for value in vector) / count + deviation = math.sqrt(variance) + if deviation == 0.0: + return [0.0 for _value in vector] + return [(value - mean) / deviation for value in vector] + + +def fuse_family_logits(cosine: Sequence[float], residual: Sequence[float]) -> list[float]: + """Shared alpha and beta. Standardization removes a global magnitude advantage.""" + if len(cosine) != len(residual): + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "fusion_width") + proto = population_standardize(value / PROTO_TAU for value in cosine) + learned = population_standardize(residual) + return [FUSION_ALPHA * left + FUSION_BETA * right for left, right in zip(proto, learned)] + + +def cosine_matrix(vectors: Sequence[Sequence[float]]) -> list[list[float]]: + units = [_unit(vector) for vector in vectors] + matrix = [] + for left in units: + matrix.append([sum(a * b for a, b in zip(left, right)) for right in units]) + return matrix + + +def hard_negatives( + matrix: Sequence[Sequence[float]], + names: Sequence[str], + count: int = HARD_NEGATIVE_COUNT, +) -> dict[str, list[dict[str, float | str]]]: + """Top other prototypes. Ties break by family name. No evaluation mining.""" + if len(matrix) != len(names): + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "similarity") + chosen: dict[str, list[dict[str, float | str]]] = {} + for index, name in enumerate(names): + order = sorted( + ( + (-float(matrix[index][other]), names[other]) + for other in range(len(names)) + if other != index + ) + ) + chosen[name] = [ + {"cosine": -score, "family": other} + for score, other in order[:count] + ] + return chosen + + +def confusable_pairs( + matrix: Sequence[Sequence[float]], + names: Sequence[str], + threshold: float = CONFUSABLE_COSINE, +) -> list[dict[str, float | str]]: + pairs = [] + for left in range(len(names)): + for right in range(left + 1, len(names)): + score = float(matrix[left][right]) + if score >= threshold: + first, second = sorted((names[left], names[right])) + pairs.append({"cosine": score, "left": first, "right": second}) + pairs.sort(key=lambda item: (-float(item["cosine"]), str(item["left"]), str(item["right"]))) + return pairs + + +def confusion_clusters( + pairs: Sequence[Mapping[str, Any]], + names: Sequence[str], +) -> list[list[str]]: + parent = {name: name for name in names} + + def find(name: str) -> str: + while parent[name] != name: + parent[name] = parent[parent[name]] + name = parent[name] + return name + + for pair in pairs: + left = find(str(pair["left"])) + right = find(str(pair["right"])) + parent[left] = right + groups: dict[str, list[str]] = {} + for name in names: + groups.setdefault(find(name), []).append(name) + return sorted( + [sorted(group) for group in groups.values() if len(group) > 1], + key=lambda group: (group[0], len(group)), + ) + + +def denominator_multipliers( + names: Sequence[str], + negatives: Mapping[str, Sequence[Mapping[str, Any]]], +) -> list[list[float]]: + """Gold stays 1. Recorded hard negatives are multiplied. Everyone else stays 1.""" + multipliers = [] + for name in names: + row = [] + hard = {str(item["family"]) for item in negatives.get(name, ())} + for other in names: + if other != name and other in hard: + row.append(HARD_NEGATIVE_MULTIPLIER) + else: + row.append(1.0) + multipliers.append(row) + return multipliers + + +def prototype_contrastive_nll( + cosine: Sequence[float], + gold_index: int, + multipliers: Sequence[float], +) -> float: + """Temperature-scaled prototype NLL. The positive denominator weight is 1.""" + if len(cosine) != len(multipliers) or not 0 <= gold_index < len(cosine): + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "contrastive") + logits = [float(value) / PROTO_TAU for value in cosine] + peak = max(logits) + total = 0.0 + for multiplier, logit in zip(multipliers, logits): + scale = float(multiplier) + if scale <= 0.0: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "contrastive") + total += scale * math.exp(logit - peak) + if multipliers[gold_index] != 1.0: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "gold_weight") + return math.log(total) + peak - logits[gold_index] + + +def family_margin_report( + records: Sequence[Mapping[str, Any]], + names: Sequence[str], +) -> dict[str, float | None]: + """Mean similarity margin, sim(gold) minus the nearest other prototype.""" + buckets: dict[str, list[float]] = {name: [] for name in names} + index = {name: position for position, name in enumerate(names)} + for record in records: + gold = str(record.get("lineage") or "") + if gold not in index: + continue + similarities = [float(value) for value in record["similarities"]] + if len(similarities) != len(names): + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "margin") + gold_index = index[gold] + others = [value for position, value in enumerate(similarities) if position != gold_index] + buckets[gold].append(similarities[gold_index] - max(others)) + return { + name: None if not values else sum(values) / len(values) + for name, values in buckets.items() + } + + +def _markup_text(text: str) -> bool: + raw = str(text or "").lstrip() + return raw.startswith(("![](", "http://", "https://", "Audio")) or "Play audio" in raw + + +def geometry_training_sources(rows: Sequence[Mapping[str, Any]]) -> dict[str, dict[str, Any]]: + """Prose training strings for families that have no stored definition prototype. + + Families with definition prose keep the verified prototype vectors. This + collector is only for the exact-copy families, whose training string is + the row text. + """ + from .classification_v2 import EXACT_COPY_FAMILIES + from .classification_v2_surface import SURFACE_PROSE, surface_form + + grouped: dict[str, list[tuple[str, Mapping[str, Any]]]] = { + family: [] for family in EXACT_COPY_FAMILIES + } + for row in rows: + if row.get("split") != "train": + continue + lineage = str(row.get("lineage") or "") + if lineage not in grouped or row.get("class") not in PROTOTYPE_WEIGHT: + continue + if row.get("evaluation_reserve") or row.get("held_out"): + raise ClassificationContractError("evaluation_isolation") + text = str(row.get("text") or "").strip() + if not text or _markup_text(text) or surface_form(text) != SURFACE_PROSE: + continue + grouped[lineage].append((normalized_text_sha256(text), row)) + sources: dict[str, dict[str, Any]] = {} + for family, items in grouped.items(): + if not items: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", family) + ordered = sorted(items, key=lambda item: (item[0], 0 if item[1].get("class") == "OBSERVED" else 1)) + unique = [] + seen = set() + for digest, row in ordered: + if digest in seen: + continue + seen.add(digest) + unique.append((digest, row)) + sources[family] = { + "identities": [digest for digest, _row in unique], + "inferred": sum(1 for _digest, row in unique if row.get("class") == "INFERRED"), + "observed": sum(1 for _digest, row in unique if row.get("class") == "OBSERVED"), + "rows": [row for _digest, row in unique], + } + return sources + + +def geometry_sha256(witness: Mapping[str, Any]) -> str: + from .classification_v2 import canonical_json, sha256_text + + body = { + "fusion": witness["fusion"], + "hard_negatives": witness["hard_negatives"], + "prototype_witness_sha256": witness["witness_sha256"], + "residual_bias_hash": witness["residual_bias_hash"], + "residual_modes": list(witness["residual_mode"]), + "residual_weight_hash": list(witness["residual_weight_hash"]), + "target_norm": witness["target_norm"], + } + return sha256_text(canonical_json(body)) + + +def _fusion_contract() -> dict[str, Any]: + return { + "alpha": FUSION_ALPHA, + "beta": FUSION_BETA, + "hard_negative_multiplier": HARD_NEGATIVE_MULTIPLIER, + "lambda_proto": LAMBDA_PROTO, + "rule": FUSION_RULE, + "tau": PROTO_TAU, + } + + +def assess_geometry_witness( + witness: Mapping[str, Any], + mapped: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """Frozen anchors for every active family, plus the residual initialization.""" + from .classification_v2 import ACTIVE_FAMILY_VOCABULARY, EXACT_COPY_FAMILIES + + try: + base = assess_witness(witness) + if not base["pass"]: + return base + if witness.get("schema") != GEOMETRY_SCHEMA: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "schema") + if witness.get("fusion") != _fusion_contract(): + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "fusion") + if witness.get("geometry_sha256") != geometry_sha256(witness): + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "geometry_hash") + modes = list(witness["residual_mode"]) + residual_weight = witness["residual_weight"] + residual_bias = list(witness["residual_bias"]) + if [row["family"] for row in witness["rows"]] != list(ACTIVE_FAMILY_VOCABULARY): + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "order") + if any(row["initialization_mode"] != INITIALIZATION_PROTOTYPE for row in witness["rows"]): + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "prototype_missing") + if len(modes) != len(ACTIVE_FAMILY_VOCABULARY): + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "residual") + weight_hashes = [] + for index, family in enumerate(ACTIVE_FAMILY_VOCABULARY): + installed = _f32_round(residual_weight[index]) + weight_hashes.append(_f32_hash(installed)) + if modes[index] == "ZERO": + if any(value != 0.0 for value in installed) or float(residual_bias[index]) != 0.0: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", family) + if family in EXACT_COPY_FAMILIES: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", family) + elif modes[index] == INITIALIZATION_EXACT: + if family not in EXACT_COPY_FAMILIES: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", family) + if mapped is not None: + live = mapped["rows"][index] + if live["mapping_status"] != "EXACT_COPY" or live["active_family"] != family: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", family) + if _f32_hash(_f32_round(mapped["weight"][index])) != weight_hashes[-1]: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "copy_mismatch") + if _f32_hash(_f32_round((mapped["bias"][index],))) != _f32_hash( + _f32_round((residual_bias[index],)) + ): + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "copy_mismatch") + else: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", family) + if weight_hashes != list(witness["residual_weight_hash"]): + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "residual_hash") + if _f32_hash(_f32_round(residual_bias)) != witness["residual_bias_hash"]: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "residual_hash") + names = list(ACTIVE_FAMILY_VOCABULARY) + negatives = witness["hard_negatives"] + if set(negatives) != set(names): + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "hard_negatives") + for family in names: + others = [str(item["family"]) for item in negatives[family]] + if len(others) != HARD_NEGATIVE_COUNT or len(set(others)) != HARD_NEGATIVE_COUNT: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", family) + if family in others: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", family) + except (ClassificationContractError, KeyError, TypeError, ValueError) as exc: + return {"pass": False, "reason": "FAMILY_PROTOTYPE_UNAVAILABLE", "detail": str(exc)} + base["geometry_sha256"] = witness["geometry_sha256"] + base["pass"] = True + base["prototypes_frozen"] = True + base["copied_rows_match"] = mapped is not None or base.get("copied_rows_match") is True + return base + + +def assemble_geometry_witness( + base_witness: Mapping[str, Any], + encoded_exact: Mapping[str, Mapping[str, Any]], + mapped: Mapping[str, Any], +) -> dict[str, Any]: + """Semantic prototype for every active family. Residual rows stay separate.""" + from .classification_v2 import ACTIVE_FAMILY_VOCABULARY, EXACT_COPY_FAMILIES + + target = float(base_witness["target_norm"]) + if base_witness.get("scaling_rule") != SCALING_RULE: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", "scaling") + by_family = {row["family"]: (index, row) for index, row in enumerate(base_witness["rows"])} + rows: list[dict[str, Any]] = [] + weight: list[list[float]] = [] + for family in ACTIVE_FAMILY_VOCABULARY: + if family in EXACT_COPY_FAMILIES: + payload = encoded_exact.get(family) + if not payload: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", family) + witness = _prototype_witness( + family, + payload["vectors"], + payload["weights"], + payload["identities"], + observed=int(payload["observed"]), + inferred=int(payload["inferred"]), + target_norm=target, + ) + else: + index, public = by_family[family] + if public["initialization_mode"] != INITIALIZATION_PROTOTYPE: + raise ClassificationContractError("FAMILY_PROTOTYPE_UNAVAILABLE", family) + witness = dict(public) + witness["weight"] = _f32_round(base_witness["weight"][index]) + rows.append(witness) + weight.append(list(witness["weight"])) + public = [_public_row(row) for row in rows] + matrix = cosine_matrix(weight) + names = list(ACTIVE_FAMILY_VOCABULARY) + negatives = hard_negatives(matrix, names) + residual_weight = [] + residual_bias = [] + residual_mode = [] + width = len(weight[0]) + for index, family in enumerate(names): + if family in EXACT_COPY_FAMILIES: + residual_weight.append(_f32_round(mapped["weight"][index])) + residual_bias.append(_f32_round((mapped["bias"][index],))[0]) + residual_mode.append(INITIALIZATION_EXACT) + else: + residual_weight.append([0.0] * width) + residual_bias.append(0.0) + residual_mode.append("ZERO") + witness = { + "bias": [0.0 for _name in names], + "exact_copy_families": list(EXACT_COPY_FAMILIES), + "fusion": _fusion_contract(), + "hard_negatives": negatives, + "prototype_families": list(names), + "residual_bias": residual_bias, + "residual_bias_hash": _f32_hash(_f32_round(residual_bias)), + "residual_mode": residual_mode, + "residual_weight": residual_weight, + "residual_weight_hash": [_f32_hash(vector) for vector in residual_weight], + "rows": public, + "scaling_rule": SCALING_RULE, + "similarity": matrix, + "confusable_pairs": confusable_pairs(matrix, names), + "confusion_clusters": confusion_clusters(confusable_pairs(matrix, names), names), + "target_norm": target, + "target_norm_source": base_witness.get("target_norm_source"), + "weight": weight, + "witness_sha256": witness_sha256(public, target), + } + witness["geometry_sha256"] = geometry_sha256(witness) + witness["schema"] = GEOMETRY_SCHEMA + return witness diff --git a/scripts/shadow/hyperlexical/classification_v2_runtime.py b/scripts/shadow/hyperlexical/classification_v2_runtime.py new file mode 100644 index 00000000..735d87a9 --- /dev/null +++ b/scripts/shadow/hyperlexical/classification_v2_runtime.py @@ -0,0 +1,320 @@ +"""Torch heads for Classification v2. Imported only on the v2 training path.""" + +from __future__ import annotations + +from typing import Any, Sequence + +from .classification_v2 import ( + ACTIVE_FAMILY_VOCABULARY, + APPLICABILITY, + example_loss, + family_index, + map_family_rows, +) +from .classification_v2_prototype import ( + FUSION_ALPHA, + FUSION_BETA, + LAMBDA_PROTO, + PROTO_TAU, +) + + +def _torch(): + import torch + from torch import nn + + return torch, nn + + +def zero_linear(hidden: int, outputs: int): + """Exact-zero linear. The constructor's random values do not survive.""" + torch, nn = _torch() + layer = nn.Linear(hidden, outputs) + with torch.no_grad(): + layer.weight.zero_() + if layer.bias is not None: + layer.bias.zero_() + return layer + + +def build_v2_heads(hidden: int, v1_classify, source_labels: Sequence[str], witness: dict | None = None): + """Copy exact rows and install frozen semantic prototypes. Zeros do not survive.""" + torch, _nn = _torch() + from .classification_v2_prototype import verify_witness_against_copy + + applicability = zero_linear(hidden, len(APPLICABILITY)) + family = zero_linear(hidden, len(ACTIVE_FAMILY_VOCABULARY)) + weight = v1_classify.weight.detach().cpu().tolist() + bias = v1_classify.bias.detach().cpu().tolist() + mapped = map_family_rows(source_labels, weight, bias) + if witness is None: + raise RuntimeError("FAMILY_PROTOTYPE_UNAVAILABLE") + verified = verify_witness_against_copy(mapped, witness) + with torch.no_grad(): + family.weight.copy_(torch.tensor(verified["weight"], dtype=family.weight.dtype)) + family.bias.copy_(torch.tensor(verified["bias"], dtype=family.bias.dtype)) + if not torch.equal(applicability.weight, torch.zeros_like(applicability.weight)): + raise RuntimeError("applicability head retained a random initialization") + return applicability, family, verified + + +def _prototype_family_type(): + torch, nn = _torch() + + class PrototypeFamilyHead(nn.Module): + """Frozen prototype cosine plus a learned residual. Prototypes are buffers.""" + + def __init__(self, hidden: int, prototypes, residual_weight, residual_bias, multipliers): + super().__init__() + self.residual = nn.Linear(hidden, len(ACTIVE_FAMILY_VOCABULARY)) + proto = torch.tensor(prototypes, dtype=self.residual.weight.dtype) + resid_w = torch.tensor(residual_weight, dtype=self.residual.weight.dtype) + resid_b = torch.tensor(residual_bias, dtype=self.residual.bias.dtype) + scale = torch.tensor(multipliers, dtype=self.residual.weight.dtype) + self.register_buffer("prototypes", proto) + self.register_buffer("denominator_weights", scale) + with torch.no_grad(): + self.residual.weight.copy_(resid_w) + self.residual.bias.copy_(resid_b) + if self.prototypes.requires_grad or any(parameter.requires_grad for name, parameter in self.named_parameters() if name.startswith("prototypes")): + raise RuntimeError("FAMILY_PROTOTYPE_UNAVAILABLE") + + def cosine(self, pooled): + hidden = torch.nn.functional.normalize(pooled, dim=-1) + anchors = torch.nn.functional.normalize(self.prototypes, dim=-1) + return hidden @ anchors.T + + def forward(self, pooled): + hidden = torch.nn.functional.normalize(pooled, dim=-1) + cosine = self.cosine(pooled) + residual = self.residual(hidden) + return _fuse(cosine, residual) + + def contrastive_nll(self, pooled, targets): + cosine = self.cosine(pooled) + logits = cosine / PROTO_TAU + peak = logits.max(dim=-1, keepdim=True).values + weights = self.denominator_weights[targets] + shifted = (logits - peak).exp() * weights + log_denom = shifted.sum(dim=-1).log() + peak.squeeze(-1) + gold = logits.gather(1, targets.unsqueeze(1)).squeeze(1) + return log_denom - gold + + return PrototypeFamilyHead + + +def _fuse(cosine, residual): + torch, _nn = _torch() + + def _standardize(values): + mean = values.mean(dim=-1, keepdim=True) + variance = (values - mean).pow(2).mean(dim=-1, keepdim=True) + deviation = variance.sqrt() + scaled = (values - mean) / deviation.clamp_min(1e-12) + return torch.where(deviation > 0, scaled, torch.zeros_like(scaled)) + + proto = _standardize(cosine / PROTO_TAU) + learned = _standardize(residual) + return FUSION_ALPHA * proto + FUSION_BETA * learned + + +def build_geometry_heads(hidden: int, v1_classify, source_labels: Sequence[str], witness: dict | None = None): + """Install frozen semantic anchors and the residual initialization.""" + from .classification_v2_prototype import assess_geometry_witness, denominator_multipliers + + applicability = zero_linear(hidden, len(APPLICABILITY)) + if witness is None: + raise RuntimeError("FAMILY_PROTOTYPE_UNAVAILABLE") + weight = v1_classify.weight.detach().cpu().tolist() + bias = v1_classify.bias.detach().cpu().tolist() + mapped = map_family_rows(source_labels, weight, bias) + verified = assess_geometry_witness(witness, mapped) + if not verified.get("pass"): + raise RuntimeError(verified.get("detail") or "FAMILY_PROTOTYPE_UNAVAILABLE") + multipliers = denominator_multipliers(list(ACTIVE_FAMILY_VOCABULARY), witness["hard_negatives"]) + family = _prototype_family_type()( + hidden, + witness["weight"], + witness["residual_weight"], + witness["residual_bias"], + multipliers, + ) + if any(name.startswith("prototypes") for name, _parameter in family.named_parameters()): + raise RuntimeError("FAMILY_PROTOTYPE_UNAVAILABLE") + return applicability, family, verified + + +def batch_classify_loss(applicability, family, pooled, rows: Sequence[dict], contract: dict[str, Any]): + """Mean of masked per-row losses. Unbind is not included.""" + torch, nn = _torch() + from .classification_v2_geometry_repair import LAMBDA_GEOMETRY + + index = family_index() + app_index: list[int] = [] + app_target: list[int] = [] + app_weight: list[float] = [] + fam_index: list[int] = [] + fam_target: list[int] = [] + fam_weight: list[float] = [] + for position, row in enumerate(rows): + plan = example_loss(row, contract) + if plan["applicability_weight"] is not None: + app_index.append(position) + app_target.append(APPLICABILITY.index(plan["applicability_target"])) + app_weight.append(float(plan["applicability_weight"])) + if plan["family_weight"] is not None: + fam_index.append(position) + fam_target.append(index[plan["family_target"]]) + fam_weight.append(float(plan["family_weight"])) + + def _weighted(layer, positions: list[int], targets: list[int], weights: list[float]): + if not positions: + return None + logits = layer(pooled[positions]) + gold = torch.tensor(targets, device=logits.device) + nll = nn.functional.cross_entropy(logits, gold, reduction="none") + scale = torch.tensor(weights, device=logits.device, dtype=nll.dtype) + return (nll * scale).mean() + + applicability_loss = _weighted(applicability, app_index, app_target, app_weight) + family_ce = _weighted(family, fam_index, fam_target, fam_weight) + family_proto = None + family_geometry = None + if family_ce is not None and hasattr(family, "geometry_nll"): + gold = torch.tensor(fam_target, device=pooled.device) + nll = family.geometry_nll(pooled[fam_index], gold) + scale = torch.tensor(fam_weight, device=nll.device, dtype=nll.dtype) + family_geometry = (nll * scale).mean() + elif family_ce is not None and hasattr(family, "contrastive_nll"): + gold = torch.tensor(fam_target, device=pooled.device) + nll = family.contrastive_nll(pooled[fam_index], gold) + scale = torch.tensor(fam_weight, device=nll.device, dtype=nll.dtype) + family_proto = (nll * scale).mean() + if family_ce is None: + family_loss = None + elif family_geometry is not None: + family_loss = family_ce + (LAMBDA_GEOMETRY * family_geometry) + elif family_proto is None: + family_loss = family_ce + else: + family_loss = family_ce + (LAMBDA_PROTO * family_proto) + terms = [term for term in (applicability_loss, family_loss) if term is not None] + if not terms: + return None + total = terms[0] if len(terms) == 1 else terms[0] + terms[1] + return { + "total": total, + "applicability": applicability_loss, + "family": family_loss, + "family_ce": family_ce, + "family_proto": family_proto, + "family_geometry": family_geometry, + } + + +def _repair_family_type(): + torch, nn = _torch() + + class RepairFamilyHead(nn.Module): + """Learned residual family logits. Boundary anchors structure geometry loss only.""" + + def __init__(self, hidden, residual_weight, residual_bias, anchors, anchor_families, hard_negatives): + super().__init__() + from .classification_v2_geometry_repair import ( + GEOMETRY_TAU, + HARD_NEGATIVE_MULTIPLIER, + denominator_anchor_weights, + ) + + self.residual = nn.Linear(hidden, len(ACTIVE_FAMILY_VOCABULARY)) + resid_w = torch.tensor(residual_weight, dtype=self.residual.weight.dtype) + resid_b = torch.tensor(residual_bias, dtype=self.residual.bias.dtype) + with torch.no_grad(): + self.residual.weight.copy_(resid_w) + self.residual.bias.copy_(resid_b) + anchor_tensor = torch.tensor(anchors, dtype=self.residual.weight.dtype) + family_index_tensor = torch.tensor(anchor_families, dtype=torch.long) + self.register_buffer("anchors", anchor_tensor) + self.register_buffer("anchor_families", family_index_tensor) + names = list(ACTIVE_FAMILY_VOCABULARY) + flat = [{"family": names[index]} for index in anchor_families] + matrix = [ + denominator_anchor_weights(family, flat, hard_negatives) + for family in names + ] + self.register_buffer( + "denominator_weights", + torch.tensor(matrix, dtype=self.residual.weight.dtype), + ) + self.tau = GEOMETRY_TAU + self.hard_negative_multiplier = HARD_NEGATIVE_MULTIPLIER + if self.anchors.requires_grad: + raise RuntimeError("FAMILY_GEOMETRY_REPAIR_UNAVAILABLE") + + def forward(self, pooled): + return self.residual(pooled) + + def anchor_cosine(self, pooled): + hidden = torch.nn.functional.normalize(pooled, dim=-1) + anchors = torch.nn.functional.normalize(self.anchors, dim=-1) + return hidden @ anchors.T + + def geometry_nll(self, pooled, targets): + cosine = self.anchor_cosine(pooled) + logits = cosine / self.tau + peak = logits.max(dim=-1, keepdim=True).values + shifted = (logits - peak).exp() + weights = self.denominator_weights[targets] + denom = (shifted * weights).sum(dim=-1).clamp_min(1e-12).log() + peak.squeeze(-1) + positive = self.anchor_families.unsqueeze(0) == targets.unsqueeze(1) + numer = (shifted * positive.to(dtype=shifted.dtype)).sum(dim=-1).clamp_min(1e-12).log() + peak.squeeze(-1) + return denom - numer + + return RepairFamilyHead + + +def build_repair_heads(hidden: int, v1_classify, source_labels: Sequence[str], boundaries: dict | None = None): + """Residual family head. Boundary anchors are frozen loss structure only.""" + from .classification_v2_geometry_repair import ( + CANONICAL_LOGITS, + assess_boundary_for_repair, + flatten_anchors, + hard_negatives_from_boundaries, + repair_contract, + sparse_families_from_boundaries, + ) + + if boundaries is None: + raise RuntimeError("FAMILY_GEOMETRY_REPAIR_UNAVAILABLE") + checked = assess_boundary_for_repair(boundaries) + if not checked.get("pass"): + raise RuntimeError(checked.get("detail") or "FAMILY_GEOMETRY_REPAIR_UNAVAILABLE") + applicability = zero_linear(hidden, len(APPLICABILITY)) + weight = v1_classify.weight.detach().cpu().tolist() + bias = v1_classify.bias.detach().cpu().tolist() + mapped = map_family_rows(source_labels, weight, bias) + anchors = flatten_anchors(boundaries) + hard = hard_negatives_from_boundaries(boundaries) + family = _repair_family_type()( + hidden, + mapped["weight"], + mapped["bias"], + [anchor["vector"] for anchor in anchors], + [list(ACTIVE_FAMILY_VOCABULARY).index(anchor["family"]) for anchor in anchors], + hard, + ) + if any(name.startswith("anchors") for name, _parameter in family.named_parameters()): + raise RuntimeError("FAMILY_GEOMETRY_REPAIR_UNAVAILABLE") + receipt = { + "boundary_sha256": checked["boundary_sha256"], + "canonical_logits": CANONICAL_LOGITS, + "contract": repair_contract(), + "exact_copy_families": list(mapped["mapped_families"]), + "hard_negatives": hard, + "n_anchors": len(anchors), + "residual_initialization": "exact_copy_or_zero", + "separation_sha256": checked["separation_sha256"], + "sparse_families": sparse_families_from_boundaries(boundaries), + "zero_initialized_families": list(mapped["zero_initialized_families"]), + } + return applicability, family, receipt diff --git a/scripts/shadow/hyperlexical/classification_v2_surface.py b/scripts/shadow/hyperlexical/classification_v2_surface.py new file mode 100644 index 00000000..d8e0dfe6 --- /dev/null +++ b/scripts/shadow/hyperlexical/classification_v2_surface.py @@ -0,0 +1,560 @@ +"""Structural applicability surfaces for Classification v2. + +Text form is ATOM, PROSE, or AMBIGUOUS. The family label is not an input. +Applicability loss gives each populated label/form cell equal aggregate +authority. The family-head class-weight formula is not applied here. + +The historical shortcut guard correlates word count with P(FAMILY_PRESENT) +across both gold classes. That result stays recorded. The corrected +diagnostic conditions on the gold label. It is read-only and does not +enter the loss. +""" + +from __future__ import annotations + +import math +from typing import Any, Mapping, Sequence + +SURFACE_ATOM = "ATOM" +SURFACE_PROSE = "PROSE" +SURFACE_AMBIGUOUS = "AMBIGUOUS" +SURFACES = (SURFACE_ATOM, SURFACE_PROSE, SURFACE_AMBIGUOUS) +SURFACE_RULE_ID = "hyperlex.classification.v2.surface.v1" +CELL_LABELS = ("FAMILY_PRESENT", "NONE") +CELL_FORMS = (SURFACE_ATOM, SURFACE_PROSE) +SURFACE_SHORTCUT_ABS_CORRELATION_MAX = 0.30 +NONE_SURFACE_GAP_ABS_MAX = 0.10 +RESIDUALIZED_LENGTH_CORRELATION_ABS_MAX = 0.30 +APPLICABILITY_SURFACE_F1_MIN = 0.80 +APPLICABILITY_F1_MIN_SUPPORT = 1 +INVARIANCE_RULE_ID = "hyperlex.classification.v2.surface_invariance.v1" +LENGTH_BUCKETS = ((1, 2), (3, 6), (7, 12), (13, 10**9)) + +_PROVENANCE_KEYS = { + ("FAMILY_PRESENT", "OBSERVED"): "observed_non_none_applicability", + ("FAMILY_PRESENT", "INFERRED"): "inferred_non_none_applicability", + ("NONE", "OBSERVED"): "observed_none_applicability", + ("NONE", "INFERRED"): "inferred_none_applicability", +} + + +def surface_tokens(text: str) -> list[str]: + """Whitespace tokens. URL and image-markup tokens are not words.""" + tokens = [] + for part in str(text or "").split(): + if part.startswith(("http://", "https://", "![](")): + continue + tokens.append(part) + return tokens + + +def _sentence_terminator(token: str) -> bool: + if not token or token[-1] not in ".?!": + return False + stem = token[:-1] + if not any(character.isalpha() for character in stem): + return False + if stem.isupper() and len(stem) <= 3: + return False + return True + + +def surface_form(text: str) -> str: + """Freeze structural form. Callers must not pass a family label.""" + raw = str(text or "") + tokens = surface_tokens(raw) + count = len(tokens) + terminator = any(_sentence_terminator(token) for token in tokens) + clause = ("," in raw) or (";" in raw) + if count >= 6 or terminator or clause: + return SURFACE_PROSE + if 1 <= count <= 4 and not terminator and not clause: + return SURFACE_ATOM + return SURFACE_AMBIGUOUS + + +def applicability_label(lineage: str) -> str | None: + if lineage == "none": + return "NONE" + if lineage: + return "FAMILY_PRESENT" + return None + + +def cell_name(label: str, form: str) -> str: + return f"{label}/{form}" + + +def word_count(text: str) -> int: + return len(surface_tokens(text)) + + +def _provenance_weight(label: str, klass: str, provenance: Mapping[str, float]) -> float: + key = _PROVENANCE_KEYS.get((label, klass)) + if key is None or key not in provenance: + raise ValueError(f"surface provenance missing for {label}/{klass}") + return float(provenance[key]) + + +def surface_cell_weights( + rows: Sequence[Mapping[str, Any]], + provenance: Mapping[str, float], +) -> dict[str, float]: + """Inverse effective support, then mean 1 across populated training cells.""" + support: dict[str, float] = {} + for row in rows: + if row.get("split") not in (None, "train"): + continue + if "text" not in row: + continue + label = applicability_label(str(row.get("lineage") or "")) + if label is None: + continue + form = surface_form(str(row.get("text") or "")) + if form == SURFACE_AMBIGUOUS: + continue + klass = str(row.get("class") or "") + name = cell_name(label, form) + support[name] = support.get(name, 0.0) + _provenance_weight(label, klass, provenance) + if not support: + return {} + raw = {name: 1.0 / value for name, value in support.items() if value > 0.0} + mean = sum(raw.values()) / len(raw) + return {name: value / mean for name, value in raw.items()} + + +def surface_census(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Count train and validation cells. Ambiguous rows stay outside the cells.""" + counts: dict[str, dict[str, int]] = {"train": {}, "val": {}} + ambiguous = {"train": 0, "val": 0} + for row in rows: + split = str(row.get("split") or "") + if split not in counts or row.get("task") not in (None, "classify"): + continue + if "text" not in row: + continue + label = applicability_label(str(row.get("lineage") or "")) + if label is None: + continue + form = surface_form(str(row.get("text") or "")) + if form == SURFACE_AMBIGUOUS: + ambiguous[split] += 1 + continue + name = cell_name(label, form) + counts[split][name] = counts[split].get(name, 0) + 1 + required = [cell_name(label, form) for label in CELL_LABELS for form in CELL_FORMS] + missing = { + split: [name for name in required if counts[split].get(name, 0) <= 0] + for split in ("train", "val") + } + return { + "ambiguous": ambiguous, + "cells": counts, + "missing": missing, + "pass": not missing["train"] and not missing["val"], + "rule": SURFACE_RULE_ID, + "shortcut_abs_correlation_max": SURFACE_SHORTCUT_ABS_CORRELATION_MAX, + } + + +def representation_leak(rows: Sequence[Mapping[str, Any]]) -> list[str]: + """A source identity may not supervise both train and validation.""" + splits: dict[str, set[str]] = {} + for row in rows: + provenance = row.get("provenance") + if not isinstance(provenance, dict): + continue + source = str(provenance.get("source_text_sha256") or "") + if not source: + continue + splits.setdefault(source, set()).add(str(row.get("split") or "")) + return sorted(source for source, found in splits.items() if found >= {"train", "val"}) + + +def pearson(xs: Sequence[float], ys: Sequence[float]) -> float | None: + count = len(xs) + if count < 2 or count != len(ys): + return None + # math.fsum, not sum(). CPython 3.12 changed sum() to Neumaier compensated summation for floats, so a + # naive left-to-right accumulation gave a different correlation on 3.10/3.11 than on 3.12+: exactly + # collinear inputs returned 0.9999999999999999 instead of 1.0, failing an `== 1.0` assertion on the older + # versions only. fsum is exact and version-independent, so the assertion holds everywhere and is never + # loosened. Measured by simulating pre-3.12 sum(): 0.9999999999999999 (== 1.0 is False) vs fsum 1.0. + mean_x = math.fsum(xs) / count + mean_y = math.fsum(ys) / count + diff_x = [value - mean_x for value in xs] + diff_y = [value - mean_y for value in ys] + denom_x = math.sqrt(math.fsum(value * value for value in diff_x)) + denom_y = math.sqrt(math.fsum(value * value for value in diff_y)) + if denom_x == 0.0 or denom_y == 0.0: + return None + return math.fsum(left * right for left, right in zip(diff_x, diff_y)) / (denom_x * denom_y) + + +def calibrated_present_probability(logits: Sequence[float], temperature: float) -> float: + if len(logits) != 2 or temperature <= 0.0: + raise ValueError("applicability probability is undefined") + scaled = [value / temperature for value in logits] + peak = max(scaled) + weights = [math.exp(value - peak) for value in scaled] + total = sum(weights) + return weights[1] / total + + +def _f1(gold: int, predicted: int, hit: int) -> float | None: + if gold <= 0: + return None + precision = hit / predicted if predicted else 0.0 + recall = hit / gold + if precision + recall == 0.0: + return 0.0 + return 2.0 * precision * recall / (precision + recall) + + +def applicability_surface_report(records: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Validation diagnostic. ``probability`` is calibrated P(FAMILY_PRESENT).""" + lengths: list[float] = [] + probabilities: list[float] = [] + by_form_gold: dict[str, list[str]] = {SURFACE_ATOM: [], SURFACE_PROSE: []} + by_form_pred: dict[str, list[str]] = {SURFACE_ATOM: [], SURFACE_PROSE: []} + probability_sums: dict[str, list[float]] = {} + support: dict[str, int] = {} + family_by_form: dict[str, list[tuple[str, str]]] = {SURFACE_ATOM: [], SURFACE_PROSE: []} + buckets = [] + for low, high in LENGTH_BUCKETS: + buckets.append( + { + "bucket": f"{low}-{high if high < 10**9 else 'inf'}", + "low": low, + "high": high, + "gold_none": 0, + "false_family_present": 0, + "gold_present": 0, + "false_none": 0, + } + ) + for record in records: + text = str(record.get("text") or "") + lineage = str(record.get("lineage") or "") + label = applicability_label(lineage) + if label is None: + continue + probability = float(record["probability"]) + predicted = str(record.get("prediction") or "") + if predicted not in CELL_LABELS: + predicted = "FAMILY_PRESENT" if probability >= 0.5 else "NONE" + lengths.append(float(word_count(text))) + probabilities.append(probability) + form = surface_form(text) + count = word_count(text) + for bucket in buckets: + if bucket["low"] <= count <= bucket["high"]: + if label == "NONE": + bucket["gold_none"] += 1 + if predicted == "FAMILY_PRESENT": + bucket["false_family_present"] += 1 + else: + bucket["gold_present"] += 1 + if predicted == "NONE": + bucket["false_none"] += 1 + break + if form == SURFACE_AMBIGUOUS: + continue + name = cell_name(label, form) + support[name] = support.get(name, 0) + 1 + probability_sums.setdefault(name, []).append(probability) + by_form_gold[form].append(label) + by_form_pred[form].append(predicted) + family_prediction = record.get("family_prediction") + if lineage != "none" and family_prediction: + family_by_form[form].append((lineage, str(family_prediction))) + correlation = pearson(lengths, probabilities) + form_f1: dict[str, dict[str, float | None]] = {} + for form in CELL_FORMS: + form_f1[form] = {} + for label in CELL_LABELS: + gold_n = by_form_gold[form].count(label) + pred_n = by_form_pred[form].count(label) + hit = sum( + 1 + for gold, pred in zip(by_form_gold[form], by_form_pred[form]) + if gold == label and pred == label + ) + form_f1[form][label] = _f1(gold_n, pred_n, hit) + cells = {} + for label in CELL_LABELS: + for form in CELL_FORMS: + name = cell_name(label, form) + values = probability_sums.get(name) or [] + cells[name] = { + "applicability_f1": form_f1[form][label], + "mean_p_family_present": None if not values else sum(values) / len(values), + "support": support.get(name, 0), + } + length_buckets = [] + for bucket in buckets: + none_n = bucket["gold_none"] + present_n = bucket["gold_present"] + length_buckets.append( + { + "bucket": bucket["bucket"], + "false_family_present_rate": None + if none_n == 0 + else bucket["false_family_present"] / none_n, + "false_none_rate": None if present_n == 0 else bucket["false_none"] / present_n, + "gold_none": none_n, + "gold_present": present_n, + } + ) + passed = correlation is not None and abs(correlation) <= SURFACE_SHORTCUT_ABS_CORRELATION_MAX + # ``pass`` remains the historical global guard. Corrected settlement reads ``invariance``. + invariance = applicability_invariance(records) + return { + "abs_correlation_max": SURFACE_SHORTCUT_ABS_CORRELATION_MAX, + "applicability_by_cell": cells, + "corr_word_count_p_family_present": correlation, + "family_macro_f1_by_surface": { + form: _macro_family(pairs) for form, pairs in family_by_form.items() + }, + "length_buckets": length_buckets, + "invariance": invariance, + "pass": passed, + "rule": SURFACE_RULE_ID, + "surface": "validation", + } + + + + +def _macro_family(pairs: Sequence[tuple[str, str]]) -> float | None: + golds: dict[str, int] = {} + preds: dict[str, int] = {} + hits: dict[str, int] = {} + for gold, predicted in pairs: + golds[gold] = golds.get(gold, 0) + 1 + preds[predicted] = preds.get(predicted, 0) + 1 + if gold == predicted: + hits[gold] = hits.get(gold, 0) + 1 + scored = [] + for name, count in golds.items(): + value = _f1(count, preds.get(name, 0), hits.get(name, 0)) + if value is not None: + scored.append(value) + if not scored: + return None + return sum(scored) / len(scored) + + +def _prediction(record: Mapping[str, Any], probability: float) -> str: + predicted = str(record.get("prediction") or "") + if predicted in CELL_LABELS: + return predicted + return "FAMILY_PRESENT" if probability >= 0.5 else "NONE" + + +def _parsed_probabilities(records: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]: + parsed = [] + for record in records: + lineage = str(record.get("lineage") or "") + label = applicability_label(lineage) + if label is None or "probability" not in record: + continue + text = str(record.get("text") or "") + probability = float(record["probability"]) + form = surface_form(text) + parsed.append( + { + "form": form, + "label": label, + "prediction": _prediction(record, probability), + "probability": probability, + "words": float(word_count(text)), + } + ) + return parsed + + +def _pair_correlation(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + xs = [float(row["words"]) for row in rows] + ys = [float(row["probability"]) for row in rows] + return {"correlation": pearson(xs, ys), "n": len(rows)} + + +def _solve_linear(matrix: list[list[float]], vector: list[float]) -> list[float] | None: + size = len(vector) + augmented = [row[:] + [vector[index]] for index, row in enumerate(matrix)] + for column in range(size): + pivot = max(range(column, size), key=lambda row: abs(augmented[row][column])) + if abs(augmented[pivot][column]) < 1e-12: + return None + augmented[column], augmented[pivot] = augmented[pivot], augmented[column] + scale = augmented[column][column] + for index in range(column, size + 1): + augmented[column][index] /= scale + for row in range(size): + if row == column: + continue + factor = augmented[row][column] + for index in range(column, size + 1): + augmented[row][index] -= factor * augmented[column][index] + return [augmented[row][size] for row in range(size)] + + +def _residualized_length(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Ordinary least squares of P on gold and surface. Read-only.""" + names = ["intercept"] + columns: list[list[float]] = [] + if any(row["label"] == "FAMILY_PRESENT" for row in rows) and any(row["label"] == "NONE" for row in rows): + names.append("gold_family_present") + columns.append([1.0 if row["label"] == "FAMILY_PRESENT" else 0.0 for row in rows]) + if any(row["form"] == SURFACE_PROSE for row in rows) and any(row["form"] != SURFACE_PROSE for row in rows): + names.append("surface_prose") + columns.append([1.0 if row["form"] == SURFACE_PROSE else 0.0 for row in rows]) + if any(row["form"] == SURFACE_AMBIGUOUS for row in rows) and any( + row["form"] != SURFACE_AMBIGUOUS for row in rows + ): + names.append("surface_ambiguous") + columns.append([1.0 if row["form"] == SURFACE_AMBIGUOUS else 0.0 for row in rows]) + width = len(names) + empty = { + "coefficients": None, + "columns": names, + "correlation": None, + "fit": "ordinary_least_squares", + "n": len(rows), + "zero_residual_variance": False, + } + if len(rows) < max(3, width): + return empty + gram = [[0.0 for _ in range(width)] for _ in range(width)] + target = [0.0 for _ in range(width)] + for index, row in enumerate(rows): + features = [1.0] + [column[index] for column in columns] + for left in range(width): + target[left] += features[left] * row["probability"] + for right in range(width): + gram[left][right] += features[left] * features[right] + beta = _solve_linear(gram, target) + if beta is None: + return empty + residuals = [] + words = [] + for index, row in enumerate(rows): + features = [1.0] + [column[index] for column in columns] + fitted = math.fsum(weight * value for weight, value in zip(beta, features)) # see pearson() on fsum + residuals.append(row["probability"] - fitted) + words.append(row["words"]) + energy = sum(value * value for value in residuals) + if energy <= 1e-24: + correlation = 0.0 + zero = True + else: + correlation = pearson(words, residuals) + zero = False + return { + "coefficients": beta, + "columns": names, + "correlation": correlation, + "fit": "ordinary_least_squares", + "n": len(rows), + "zero_residual_variance": zero, + } + + +def _mean_probability(rows: Sequence[Mapping[str, Any]]) -> float | None: + if not rows: + return None + # math.fsum, not sum(). CPython 3.12 changed sum() to Neumaier compensated summation for floats. On + # 3.10/3.11 this mean is off by one ULP from a mean over a different number of the same value: the + # NONE/ATOM cell (12 rows) and the NONE/PROSE cell (4 rows) both hold 0.13, and their means landed on + # 0.12999999999999998 vs 0.13, so `none_surface_gap` came out as 2.7755575615628914e-17 instead of exactly + # 0.0 and failed `assert invariance["none_surface_gap"] == 0.0` on 3.10/3.11 only. Reproduced by + # simulating pre-3.12 summation: atom=0.12999999999999998 prose=0.13 gap=2.7755575615628914e-17 (== 0.0 is + # False); with fsum both are 0.13 and the gap is exactly 0.0. fsum is exact and version-independent, so the + # assertion holds everywhere and is never loosened. + return math.fsum(row["probability"] for row in rows) / len(rows) + + +def applicability_invariance(records: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Corrected shortcut diagnostic. It does not enter training. + + Global correlation mixes gold class with length. These checks ask whether + surface or residual length still predicts applicability after the gold + label is fixed. A higher probability on longer positive prose is not a + failure by itself. + """ + rows = _parsed_probabilities(records) + global_correlation = pearson( + [row["words"] for row in rows], + [row["probability"] for row in rows], + ) + conditional = { + label: _pair_correlation([row for row in rows if row["label"] == label]) + for label in CELL_LABELS + } + within = {} + f1_results = {} + means = {} + for label in CELL_LABELS: + for form in CELL_FORMS: + name = cell_name(label, form) + chosen = [row for row in rows if row["label"] == label and row["form"] == form] + within[name] = _pair_correlation(chosen) + means[name] = _mean_probability(chosen) + gold_n = len(chosen) + form_rows = [row for row in rows if row["form"] == form] + predicted_n = sum(1 for row in form_rows if row["prediction"] == label) + hit = sum(1 for row in chosen if row["prediction"] == label) + f1_results[name] = { + "f1": _f1(gold_n, predicted_n, hit), + "required": gold_n >= APPLICABILITY_F1_MIN_SUPPORT, + "support": gold_n, + } + family_gap = None + if means["FAMILY_PRESENT/PROSE"] is not None and means["FAMILY_PRESENT/ATOM"] is not None: + family_gap = means["FAMILY_PRESENT/PROSE"] - means["FAMILY_PRESENT/ATOM"] + none_gap = None + if means["NONE/PROSE"] is not None and means["NONE/ATOM"] is not None: + none_gap = means["NONE/PROSE"] - means["NONE/ATOM"] + residual = _residualized_length(rows) + residual_value = residual["correlation"] + none_gap_pass = none_gap is not None and abs(none_gap) <= NONE_SURFACE_GAP_ABS_MAX + residual_pass = ( + residual_value is not None and abs(residual_value) <= RESIDUALIZED_LENGTH_CORRELATION_ABS_MAX + ) + required_f1 = [item for item in f1_results.values() if item["required"]] + f1_pass = bool(required_f1) and all( + item["f1"] is not None and item["f1"] >= APPLICABILITY_SURFACE_F1_MIN for item in required_f1 + ) + for item in f1_results.values(): + item["pass"] = (not item["required"]) or ( + item["f1"] is not None and item["f1"] >= APPLICABILITY_SURFACE_F1_MIN + ) + return { + "applicability_f1_by_cell": f1_results, + "conditional_length_correlation": conditional, + "family_surface_gap": family_gap, + "guards": { + "applicability_f1_min": APPLICABILITY_SURFACE_F1_MIN, + "applicability_f1_min_support": APPLICABILITY_F1_MIN_SUPPORT, + "none_surface_gap_abs_max": NONE_SURFACE_GAP_ABS_MAX, + "residualized_length_correlation_abs_max": RESIDUALIZED_LENGTH_CORRELATION_ABS_MAX, + }, + "guard_results": { + "applicability_f1": f1_pass, + "none_surface_gap": none_gap_pass, + "residualized_length_correlation": residual_pass, + }, + "historical_blunt_guard": { + "abs_correlation_max": SURFACE_SHORTCUT_ABS_CORRELATION_MAX, + "correlation": global_correlation, + "pass": global_correlation is not None + and abs(global_correlation) <= SURFACE_SHORTCUT_ABS_CORRELATION_MAX, + }, + "mean_p_family_present": means, + "none_surface_gap": none_gap, + "pass": bool(none_gap_pass and residual_pass and f1_pass), + "residualized_length_correlation": residual, + "rule": INVARIANCE_RULE_ID, + "within_cell_length_correlation": within, + } diff --git a/scripts/shadow/hyperlexical/classify_admission.py b/scripts/shadow/hyperlexical/classify_admission.py new file mode 100644 index 00000000..685e99da --- /dev/null +++ b/scripts/shadow/hyperlexical/classify_admission.py @@ -0,0 +1,82 @@ +"""Opt-in classify admission filter (HLX_CLASSIFY_ADMISSION=1, default off). + +Drops classify train/val rows whose label cannot be trusted: + * conflict: the normalized text carries more than one lineage across + non-test classify rows (after the dump brainrot fold is undone); + * moltbook_source_constant: moltbook / moltbook-curated-seed lineage is + hard-coded ai-native (scripts/moltbook_to_hyperlexical.py:55); + * demoted: gold_demote_reason set, or fallback fillers (general, kdr). +Counts go into the train receipt under "classify_admission". +""" + +from __future__ import annotations + +import os +from collections import Counter +from typing import Any, Sequence + +from .heldout_census import normalize_group_text +from .selection_surface import drop_test_rows + +MOLTBOOK_SOURCES = frozenset({"moltbook", "moltbook-curated-seed"}) +FALLBACK_FILLERS = frozenset({"general", "kdr"}) + + +def _source(row: dict[str, Any]) -> str: + prov = row.get("provenance") + return str(prov.get("source") or "") if isinstance(prov, dict) else "" + + +def _reason(row: dict[str, Any], labels: dict[str, set[str]]) -> str | None: + if len(labels.get(normalize_group_text(row.get("text") or ""), ())) > 1: + return "conflict" + if _source(row) in MOLTBOOK_SOURCES: + return "moltbook_source_constant" + if row.get("gold_demote_reason") or any( + str(f).lower() in FALLBACK_FILLERS for f in (row.get("fillers") or []) + ): + return "demoted" + return None + + +def admit_classify_rows( + all_rows: Sequence[dict[str, Any]], + classify_tr: list[dict[str, Any]], + classify_va: list[dict[str, Any]], +) -> tuple[list[dict[str, Any]], list[dict[str, Any]], dict[str, Any]]: + kept_rows, _ = drop_test_rows(all_rows) + labels: dict[str, set[str]] = {} + for r in kept_rows: + if r.get("task") == "classify": + key = normalize_group_text(r.get("text") or "") + if key: + labels.setdefault(key, set()).add(str(r.get("lineage"))) + receipt: dict[str, Any] = {"enabled": True, "conflict_texts": sum(1 for v in labels.values() if len(v) > 1)} + out = [] + for name, rows in (("train", classify_tr), ("val", classify_va)): + dropped: Counter[str] = Counter() + kept = [] + for r in rows: + why = _reason(r, labels) + if why: + dropped[why] += 1 + else: + kept.append(r) + receipt[name] = {"in": len(rows), "kept": len(kept), "dropped": dict(sorted(dropped.items()))} + out.append(kept) + return out[0], out[1], receipt + + +def apply_classify_admission( + all_rows: Sequence[dict[str, Any]], + classify_tr: list[dict[str, Any]], + classify_va: list[dict[str, Any]], +) -> tuple[list[dict[str, Any]], list[dict[str, Any]], dict[str, Any] | None]: + """Filter only when ``HLX_CLASSIFY_ADMISSION`` is ``1``. + + Default off returns the same train and val lists and a null receipt so + training rows stay byte-identical. + """ + if os.environ.get("HLX_CLASSIFY_ADMISSION") != "1": + return classify_tr, classify_va, None + return admit_classify_rows(all_rows, classify_tr, classify_va) diff --git a/scripts/shadow/hyperlexical/classify_metrics.py b/scripts/shadow/hyperlexical/classify_metrics.py new file mode 100644 index 00000000..87c9b9c0 --- /dev/null +++ b/scripts/shadow/hyperlexical/classify_metrics.py @@ -0,0 +1,205 @@ +"""Classify scores for checkpoint selection and the eval harness. No torch. + +Macro-F1 is the unweighted mean of per-class F1 over gold labels other than +``none`` that have support (n > 0). ``none`` stays in the per-class table and +in the false-positive / false-negative rates. It does not enter the macro. +""" + +from __future__ import annotations + +import math +import os +from typing import Mapping, Sequence + +NONE_LABEL = "none" +SELECT_METRIC_ENV = "HLX_SELECT_METRIC" +SELECT_METRIC_UNBIND = "unbind_exact" +SELECT_METRIC_CLASSIFY = "classify_macro_f1_nonnone" +_SELECT_METRICS = frozenset({SELECT_METRIC_UNBIND, SELECT_METRIC_CLASSIFY}) + + +def resolve_select_metric(raw: str | None = None) -> str: + """Checkpoint score name. Unset keeps ``unbind_exact`` (historical selection).""" + if raw is None: + raw = os.environ.get(SELECT_METRIC_ENV) + value = "" if raw is None else str(raw).strip() + if not value: + return SELECT_METRIC_UNBIND + if value not in _SELECT_METRICS: + raise ValueError( + f"{SELECT_METRIC_ENV} must be unset or {SELECT_METRIC_CLASSIFY!r}, got {value!r}" + ) + return value + + +def checkpoint_score(metrics: Mapping[str, object], metric_name: str) -> float: + """Scalar used to keep a best checkpoint. Missing classify F1 does not win.""" + if metric_name == SELECT_METRIC_CLASSIFY: + raw = metrics.get("classify_macro_f1_nonnone") + if raw is None: + return float("-inf") + return float(raw) # type: ignore[arg-type] + return float(metrics.get("unbind_exact") or 0.0) # type: ignore[arg-type] + + +def accuracy(gold: Sequence[str], pred: Sequence[str]) -> float | None: + _same_len(gold, pred) + if not gold: + return None + hit = sum(int(g == p) for g, p in zip(gold, pred)) + return hit / len(gold) + + +def class_prf(gold: Sequence[str], pred: Sequence[str], label: str) -> dict[str, float | int]: + """Precision, recall, F1, and gold support for one label.""" + _same_len(gold, pred) + tp = fp = fn = n = 0 + for g, p in zip(gold, pred): + if g == label: + n += 1 + if p == label and g == label: + tp += 1 + elif p == label: + fp += 1 + elif g == label: + fn += 1 + precision = 0.0 if tp + fp == 0 else tp / (tp + fp) + recall = 0.0 if tp + fn == 0 else tp / (tp + fn) + f1 = 0.0 if precision + recall == 0.0 else (2.0 * precision * recall) / (precision + recall) + return {"precision": precision, "recall": recall, "f1": f1, "n": n} + + +def per_class_table( + gold: Sequence[str], + pred: Sequence[str], + *, + labels: Sequence[str] | None = None, +) -> dict[str, dict[str, float | int]]: + """P/R/F1/n for every label in gold or pred, unless ``labels`` is passed.""" + _same_len(gold, pred) + if labels is None: + seen = sorted(set(gold) | set(pred)) + else: + seen = list(labels) + return {label: class_prf(gold, pred, label) for label in seen} + + +def macro_f1_nonnone( + gold: Sequence[str], + pred: Sequence[str], + *, + none_label: str = NONE_LABEL, +) -> float | None: + """Mean F1 over non-none gold labels with n > 0. ``None`` when none qualify.""" + _same_len(gold, pred) + labels = sorted({g for g in gold if g != none_label}) + if not labels: + return None + scores = [float(class_prf(gold, pred, label)["f1"]) for label in labels] + return sum(scores) / len(scores) + + +def none_rates( + gold: Sequence[str], + pred: Sequence[str], + *, + none_label: str = NONE_LABEL, +) -> dict[str, float | int | None]: + """Binary rates with positive class = ``none``. + + False-positive rate: predicted ``none`` among gold that is not ``none``. + False-negative rate: predicted something else among gold ``none``. + A zero denominator is ``None`` (not a zero rate). + """ + _same_len(gold, pred) + n_not_none = 0 + n_none = 0 + fp = 0 + fn = 0 + for g, p in zip(gold, pred): + if g == none_label: + n_none += 1 + if p != none_label: + fn += 1 + else: + n_not_none += 1 + if p == none_label: + fp += 1 + return { + "none_fpr": None if n_not_none == 0 else fp / n_not_none, + "none_fnr": None if n_none == 0 else fn / n_none, + "n_none": n_none, + "n_not_none": n_not_none, + } + + +def none_false_positive_rate( + gold: Sequence[str], + pred: Sequence[str], + *, + none_label: str = NONE_LABEL, +) -> float | None: + rate = none_rates(gold, pred, none_label=none_label)["none_fpr"] + return None if rate is None else float(rate) + + +def mcnemar_exact_p(n_model_only_correct: int, n_baseline_only_correct: int) -> float: + """Two-sided exact McNemar p on discordant pairs. + + Under the null, the model-only count is Binomial(n, 1/2) with + n = b + c. The p-value is min(1, 2 * P(X <= min(b, c))). + No discordant pairs → 1. + """ + b = int(n_model_only_correct) + c = int(n_baseline_only_correct) + if b < 0 or c < 0: + raise ValueError("discordant counts must be >= 0") + n = b + c + if n == 0: + return 1.0 + k = min(b, c) + log_choose = 0.0 + log_two_n = n * math.log(2.0) + logs = [] + for i in range(0, k + 1): + if i: + log_choose += math.log(n - i + 1) - math.log(i) + logs.append(log_choose - log_two_n) + pivot = max(logs) + acc = sum(math.exp(item - pivot) for item in logs) + p = 2.0 * math.exp(pivot) * acc + if p > 1.0: + return 1.0 + if p < 0.0: + return 0.0 + return p + + +def mcnemar_paired( + gold: Sequence[str], + model: Sequence[str], + baseline: Sequence[str], +) -> dict[str, float | int]: + """Exact two-sided McNemar of ``model`` vs ``baseline`` on the same gold.""" + if not (len(gold) == len(model) == len(baseline)): + raise ValueError("gold, model, and baseline lengths differ") + model_only = 0 + baseline_only = 0 + for g, m, b in zip(gold, model, baseline): + m_ok = m == g + b_ok = b == g + if m_ok and not b_ok: + model_only += 1 + elif b_ok and not m_ok: + baseline_only += 1 + return { + "n_discordant": model_only + baseline_only, + "n_model_only_correct": model_only, + "n_baseline_only_correct": baseline_only, + "p_two_sided": mcnemar_exact_p(model_only, baseline_only), + } + + +def _same_len(gold: Sequence[str], pred: Sequence[str]) -> None: + if len(gold) != len(pred): + raise ValueError(f"gold/pred length mismatch: {len(gold)} vs {len(pred)}") diff --git a/scripts/shadow/hyperlexical/classify_sampling.py b/scripts/shadow/hyperlexical/classify_sampling.py new file mode 100644 index 00000000..f06e3bc4 --- /dev/null +++ b/scripts/shadow/hyperlexical/classify_sampling.py @@ -0,0 +1,139 @@ +"""Deterministic classify-epoch sampling. + +``HLX_SEED`` unset returns no seed and does not define an epoch sample. +The rule in this module is the frozen selector. It reads text, lineage, +class, and row order. It does not read a score, a loss, or a reserve. +""" + +from __future__ import annotations + +import hashlib +import json +import os +from typing import Any, Mapping, Sequence + +from .holdout_guard import normalized_text_sha256 + +SAMPLING_ENV = "HYPERLEX_CLASSIFY_SAMPLING" +UNCAPPED = "uncapped" +RULE = "inferred_none_circular_sha256_v1" +CAP = 219 +EXPECTED_OBSERVED_NONE = 219 +EXPECTED_INFERRED_NONE = 1935 +OBSERVED = "OBSERVED" +INFERRED = "INFERRED" +NONE = "none" + + +def canonical_json(payload: object) -> str: + return json.dumps(payload, ensure_ascii=True, separators=(",", ":"), sort_keys=True) + + +def parse_sampling_policy(raw: str | None) -> tuple[str, int | None]: + """Return ``(uncapped, None)`` or ``(rule, 219)``. Any other token fails.""" + if raw is None or not str(raw).strip() or str(raw).strip() == UNCAPPED: + return UNCAPPED, None + token = str(raw).strip() + prefix = RULE + ":" + if not token.startswith(prefix) or token[len(prefix) :] != str(CAP): + raise ValueError( + f"{SAMPLING_ENV} must be {UNCAPPED} or {RULE}:{CAP}, got {raw!r}" + ) + return RULE, CAP + + +def _identity(row: Mapping[str, Any]) -> str: + return normalized_text_sha256(str(row.get("text") or "")) + + +def _kind(row: Mapping[str, Any]) -> str: + if row.get("lineage") != NONE: + return "other" + evidence = row.get("class") + if evidence == OBSERVED: + return "observed_none" + if evidence == INFERRED: + return "inferred_none" + raise ValueError("none classify row is neither OBSERVED nor INFERRED") + + +def _selection_sha(hashes: Sequence[str]) -> str: + return hashlib.sha256(canonical_json(list(hashes)).encode("utf-8")).hexdigest() + + +def apply_inferred_none_cap( + rows: Sequence[Mapping[str, Any]], + *, + epoch_index: int, + cap: int, +) -> tuple[list[Mapping[str, Any]], dict[str, Any]]: + """Keep every non-none row and every OBSERVED none row. + + INFERRED none rows are chosen by sorting ``(identity, original index)`` + and taking a circular window. Kept rows stay in their original order. + ``epoch_index`` is zero-based: epoch 1 starts at offset 0. + """ + if isinstance(cap, bool) or not isinstance(cap, int) or cap < 1: + raise ValueError("inferred-none cap must be a positive integer") + if isinstance(epoch_index, bool) or not isinstance(epoch_index, int) or epoch_index < 0: + raise ValueError("epoch_index must be a non-negative integer") + indexed = list(enumerate(rows)) + inferred = [(index, row, _identity(row)) for index, row in indexed if _kind(row) == "inferred_none"] + observed = sum(1 for _, row in indexed if _kind(row) == "observed_none") + if len(inferred) < cap: + raise ValueError("inferred none population is smaller than the cap") + ordered = sorted(inferred, key=lambda item: (item[2], item[0])) + start = (epoch_index * cap) % len(ordered) + chosen = [ordered[(start + offset) % len(ordered)] for offset in range(cap)] + chosen_indexes = {item[0] for item in chosen} + if len(chosen_indexes) != cap: + raise ValueError("inferred none selection collided") + kept = [ + row + for index, row in indexed + if _kind(row) != "inferred_none" or index in chosen_indexes + ] + hashes = [item[2] for item in chosen] + return kept, { + "classify_rows": len(kept), + "epoch_index": epoch_index, + "inferred_none_population": len(inferred), + "inferred_none_selected": cap, + "observed_none": observed, + "rule": RULE, + "selected_identity_sha256": hashes, + "selection_sha256": _selection_sha(hashes), + } + + +def epoch_classify_rows( + rows: Sequence[Mapping[str, Any]], + *, + epoch_index: int, + policy: str | None = None, +) -> tuple[list[Mapping[str, Any]], dict[str, Any]]: + """One epoch of classify rows. Unset policy keeps the full list.""" + if policy is None: + policy = os.environ.get(SAMPLING_ENV) + rule, cap = parse_sampling_policy(policy) + indexed = list(enumerate(rows)) + inferred = [(index, row, _identity(row)) for index, row in indexed if _kind(row) == "inferred_none"] + observed = sum(1 for _, row in indexed if _kind(row) == "observed_none") + if rule == UNCAPPED: + hashes = [item[2] for item in inferred] + return list(rows), { + "classify_rows": len(rows), + "epoch_index": epoch_index, + "inferred_none_population": len(inferred), + "inferred_none_selected": len(inferred), + "observed_none": observed, + "rule": UNCAPPED, + "selected_identity_sha256": hashes, + "selection_sha256": _selection_sha(hashes), + } + if observed != EXPECTED_OBSERVED_NONE or len(inferred) != EXPECTED_INFERRED_NONE: + raise ValueError( + "classify none population does not match the preregistered " + f"{EXPECTED_OBSERVED_NONE}/{EXPECTED_INFERRED_NONE} pin" + ) + return apply_inferred_none_cap(rows, epoch_index=epoch_index, cap=cap or CAP) diff --git a/scripts/shadow/hyperlexical/classify_split.py b/scripts/shadow/hyperlexical/classify_split.py new file mode 100644 index 00000000..1b739ca3 --- /dev/null +++ b/scripts/shadow/hyperlexical/classify_split.py @@ -0,0 +1,128 @@ +"""Opt-in classify train/val override (HLX_CLASSIFY_SPLIT_FILE, default off). + +The file maps ``selection_surface.row_id`` to ``train``, ``val``, or ``drop``. +It runs before classify admission and the holdout guard. It changes list +membership only: row dicts are not rewritten, so a later holdout ``row_id`` +match still sees the original split. Unset returns the same list objects. +""" + +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +from typing import Any, Mapping + +from .selection_surface import row_id + +SPLIT_FILE_ENV = "HLX_CLASSIFY_SPLIT_FILE" +SCHEMA = "hyperlex.classify_split_override.v0.1" +DESTINATIONS = frozenset({"train", "val", "drop"}) + + +def resolve_split_path(raw: str | None = None) -> str | None: + if raw is None: + raw = os.environ.get(SPLIT_FILE_ENV) + if raw is None: + return None + token = str(raw).strip() + return token or None + + +def trainval_base_sha256(train: list, val: list) -> str: + """sha256 of train row ids in order, then val row ids in order.""" + lines = [row_id(row) for row in train] + [row_id(row) for row in val] + return hashlib.sha256("\n".join(lines).encode("utf-8")).hexdigest() + + +def _refuse(message: str) -> None: + raise SystemExit(f"REFUSE: {message}") + + +def _load(path: str) -> tuple[dict[str, Any], str]: + file = Path(path) + if not file.is_file(): + _refuse(f"{SPLIT_FILE_ENV} is not a file") + try: + payload = json.loads(file.read_text(encoding="utf-8")) + except json.JSONDecodeError: + _refuse(f"{SPLIT_FILE_ENV} is not JSON") + if not isinstance(payload, dict): + _refuse(f"{SPLIT_FILE_ENV} must be a JSON object") + schema = payload.get("schema") + if schema != SCHEMA: + _refuse(f"{SPLIT_FILE_ENV} schema must be {SCHEMA}") + mapping = payload.get("split_by_row_id") + if not isinstance(mapping, dict) or not mapping: + _refuse(f"{SPLIT_FILE_ENV} split_by_row_id must be a non-empty object") + if not all(isinstance(key, str) and isinstance(value, str) for key, value in mapping.items()): + _refuse(f"{SPLIT_FILE_ENV} split_by_row_id keys and values must be strings") + bad = sorted({value for value in mapping.values() if value not in DESTINATIONS}) + if bad: + _refuse(f"{SPLIT_FILE_ENV} has an unknown destination") + file_sha = hashlib.sha256(file.read_bytes()).hexdigest() + return payload, file_sha + + +def apply_classify_split_file( + train: list, + val: list, + *, + path: str | None = None, +) -> tuple[list, list, dict[str, Any] | None]: + """Override classify list membership. Unset returns ``train`` and ``val`` unchanged.""" + chosen = resolve_split_path(path) + if chosen is None: + return train, val, None + payload, file_sha = _load(chosen) + mapping: Mapping[str, str] = payload["split_by_row_id"] + seen: dict[str, str] = {} + ordered: list[tuple[str, dict]] = [] + for origin, rows in (("train", train), ("val", val)): + for row in rows: + ident = row_id(row) + if ident in seen: + _refuse("duplicate classify row_id in train/val") + seen[ident] = origin + ordered.append((origin, row)) + missing = len(seen) - sum(1 for ident in seen if ident in mapping) + extra = len(mapping) - sum(1 for ident in mapping if ident in seen) + if missing: + _refuse(f"{SPLIT_FILE_ENV} is missing {missing} classify train/val rows") + if extra: + _refuse(f"{SPLIT_FILE_ENV} has {extra} ids that are not classify train/val rows") + expected = payload.get("base_trainval_sha256") + if expected is not None: + if not isinstance(expected, str) or expected != trainval_base_sha256(train, val): + _refuse(f"{SPLIT_FILE_ENV} base_trainval_sha256 does not match these rows") + new_train: list = [] + new_val: list = [] + n_train_to_val = n_val_to_train = n_dropped_train = n_dropped_val = 0 + for origin, row in ordered: + dest = mapping[row_id(row)] + if dest == "drop": + if origin == "train": + n_dropped_train += 1 + else: + n_dropped_val += 1 + continue + if origin == "train" and dest == "val": + n_train_to_val += 1 + elif origin == "val" and dest == "train": + n_val_to_train += 1 + if dest == "train": + new_train.append(row) + else: + new_val.append(row) + receipt = { + "enabled": True, + "file_sha256": file_sha, + "n_train_to_val": n_train_to_val, + "n_val_to_train": n_val_to_train, + "n_dropped_train": n_dropped_train, + "n_dropped_val": n_dropped_val, + "n_train": len(new_train), + "n_val": len(new_val), + } + return new_train, new_val, receipt diff --git a/scripts/shadow/hyperlexical/clean_unbind.py b/scripts/shadow/hyperlexical/clean_unbind.py new file mode 100644 index 00000000..2dc00945 --- /dev/null +++ b/scripts/shadow/hyperlexical/clean_unbind.py @@ -0,0 +1,811 @@ +"""Clean-unbind reserve gate. + +The executed unbind row is ``export._unbind_dual_scheme_rows``: fillers are +the source atom's own tokens, positional roles are ``pos_N``, and type_slot +roles are structural ``TOKEN`` / ``SLOT`` / ``MARKER``. A gloss is not gold. + +``unbind_clean`` is ``soft_ceiling.clean_surface`` against train-split rows, +the call ``holdout_eligibility.census`` makes. That predicate is not +``normalized_text_sha256``. The hash is only the contamination identity. +""" + +from __future__ import annotations + +import hashlib +import json +from collections import Counter +from pathlib import Path +from typing import Any, Mapping, Sequence + +from .export import ( + COLLISION_HOLD, + LIVE_UNBIND_MAX_LEN, + LIVE_UNBIND_MAX_TOKENS, + TYPE_SLOT_TAGS, + reject_candidate_text, +) +from .holdout_guard import normalized_text_sha256 +from .identity_ledger import PLANNING_TARGETS, IdentityLedger, derived_state +from .soft_ceiling import clean_surface +from .unbind_settlement import ADMIT_DECISIONS + +UNBIND_CLEAN_DEFINITION = "soft_ceiling.clean_surface" +CONTRACT_SHAPE = "hyperlexical.export._unbind_dual_scheme_rows" +BLOCKED_TARGET_ORIGINS = frozenset( + { + "model", + "hyperlex_model", + "jev", + "classification_family", + "source_hint", + } +) +ALLOWED_TARGET_ORIGINS = frozenset({"source_lemma_tokens", "operator_authored"}) +CLASSIFY_SLICES = ("classify", "classify_observed", "classify_non_none") +BLOCKING_STATES = frozenset( + { + "TRAIN_CONSUMED", + "EVAL_SPENT", + "EVAL_ABANDONED", + "EVAL_RESERVE", + "EVAL_BOUND", + "TRAIN_CANDIDATE", + } +) +INDEX_FILES = ( + ("noun", "index.noun"), + ("verb", "index.verb"), + ("adj", "index.adj"), + ("adv", "index.adv"), +) + + +def refuse(message: str) -> None: + raise SystemExit(f"REFUSE: {message}") + + +def target_sha256(fillers: Sequence[str]) -> str: + """Identity of the filler list. Not a surface hash.""" + joined = " ".join(str(item).casefold() for item in fillers) + return hashlib.sha256(joined.encode("utf-8")).hexdigest() + + +def schema_example() -> dict[str, Any]: + """Synthetic placeholders. Not an acquired row.""" + return { + "positional": { + "text": "EXAMPLE_TOKEN_A EXAMPLE_TOKEN_B", + "split": "eval", + "lineage": "none", + "typology": [], + "stage": "noise", + "roles": ["pos_0", "pos_1"], + "fillers": ["EXAMPLE_TOKEN_A", "EXAMPLE_TOKEN_B"], + "role_scheme": "positional", + "task": "unbind", + "provenance": "source:EXAMPLE_LOCATOR", + "class": "INFERRED", + "license": "EXAMPLE_RIGHTS_GRANT", + "target_origin": "source_lemma_tokens", + }, + "type_slot": { + "text": "TOKEN:EXAMPLE_TOKEN_A SLOT:EXAMPLE_TOKEN_B", + "split": "eval", + "lineage": "none", + "typology": [], + "stage": "noise", + "roles": ["TOKEN", "SLOT"], + "fillers": ["EXAMPLE_TOKEN_A", "EXAMPLE_TOKEN_B"], + "role_scheme": "type_slot", + "task": "unbind", + "provenance": "source:EXAMPLE_LOCATOR", + "class": "INFERRED", + "license": "EXAMPLE_RIGHTS_GRANT", + "target_origin": "source_lemma_tokens", + }, + "scorer": { + "module": "hyperlexical.eval_forward.score_unbind_exact", + "gold": "row fillers, one string per role", + "empty_fillers": "skipped", + "metrics": [ + "unbind_exact", + "unbind_token_f1", + "unbind_slot_f1", + "unbind_exact_strict", + "unbind_token_f1_strict", + "unbind_slot_f1_strict", + ], + "by_role_scheme": ["positional", "type_slot"], + "baseline": "unbind_copy_token", + "clean_slice": UNBIND_CLEAN_DEFINITION, + "this_module_scores": False, + }, + } + + +def structural_tags(n: int) -> list[str]: + return [TYPE_SLOT_TAGS[i % len(TYPE_SLOT_TAGS)] for i in range(n)] + + +def dual_scheme_rows( + tokens: Sequence[str], + *, + license: str, + provenance: str, + target_origin: str, + source_pos: str = "", + lineage: str = "none", + stage: str = "noise", + epistemic: str = "INFERRED", +) -> list[dict[str, Any]]: + """Same text/role/filler shape as ``_unbind_dual_scheme_rows``.""" + items = [str(tok) for tok in tokens] + atom = " ".join(items) + tags = structural_tags(len(items)) + common = { + "split": "eval", + "lineage": lineage, + "typology": [], + "stage": stage, + "task": "unbind", + "provenance": provenance, + "class": epistemic, + "license": license, + "target_origin": target_origin, + "source_pos": source_pos, + } + positional = { + **common, + "text": atom, + "roles": [f"pos_{k}" for k in range(len(items))], + "fillers": list(items), + "role_scheme": "positional", + } + typed = { + **common, + "text": " ".join(f"{tag}:{tok}" for tag, tok in zip(tags, items)), + "roles": tags, + "fillers": list(items), + "role_scheme": "type_slot", + } + return [positional, typed] + + +def structural_reason(row: Mapping[str, Any]) -> str | None: + if str(row.get("task") or "") != "unbind": + return "not_unbind" + scheme = row.get("role_scheme") + if scheme not in {"positional", "type_slot"}: + return "role_scheme" + fillers = row.get("fillers") + if not isinstance(fillers, list) or not fillers: + return "missing_target" + if not all(isinstance(item, str) and item.strip() for item in fillers): + return "missing_target" + if not (2 <= len(fillers) <= LIVE_UNBIND_MAX_TOKENS): + return "token_count" + atom = " ".join(str(item) for item in fillers) + if len(atom) > LIVE_UNBIND_MAX_LEN: + return "atom_length" + if atom.lower() in COLLISION_HOLD: + return "collision_hold" + junk = reject_candidate_text(atom) + if junk: + return f"candidate_{junk}" + roles = row.get("roles") + if not isinstance(roles, list) or len(roles) != len(fillers): + return "role_alignment" + text = str(row.get("text") or "") + if scheme == "positional": + if text.split() != list(fillers): + return "surface_filler_mismatch" + if list(roles) != [f"pos_{k}" for k in range(len(fillers))]: + return "role_alignment" + else: + tags = structural_tags(len(fillers)) + expected = " ".join(f"{tag}:{tok}" for tag, tok in zip(tags, fillers)) + if text != expected or list(roles) != tags: + return "surface_filler_mismatch" + if str(row.get("class") or "") not in {"OBSERVED", "INFERRED"}: + return "class" + if "lineage" not in row: + return "lineage" + return None + + +def rights_reason(row: Mapping[str, Any]) -> str | None: + license_text = row.get("license") + if not isinstance(license_text, str) or not license_text.strip(): + return "rights_unresolved" + lowered = license_text.lower() + if "rights_unresolved" in lowered or "not cc" in lowered: + return "rights_unresolved" + return None + + +def target_reason(row: Mapping[str, Any]) -> str | None: + origin = str(row.get("target_origin") or "").strip() + if not origin: + return "unbind_unresolved" + if origin in BLOCKED_TARGET_ORIGINS: + return "model_derived_target" + if origin not in ALLOWED_TARGET_ORIGINS: + return "unbind_unresolved" + provenance = str(row.get("provenance") or "").lower() + if any(mark in provenance for mark in ("jev", "model-derived", "hyperlex-encoder", "source_hint_only")): + return "model_derived_target" + return None + + +def unbind_clean_hashes( + rows: Sequence[Mapping[str, Any]], + train_rows: Sequence[Mapping[str, Any]], +) -> tuple[set[str], dict[str, Any]]: + """Canonical clean predicate. Train split only. No trained-key extra.""" + train_for_clean = [row for row in train_rows if row.get("split") == "train"] + unbind = [row for row in rows if row.get("task") == "unbind"] + kept, account = clean_surface(unbind, train_rows=train_for_clean) + hashes = {normalized_text_sha256(str(row.get("text") or "")) for row in kept} + account = dict(account) + account["definition"] = UNBIND_CLEAN_DEFINITION + account["train_split_rows"] = len(train_for_clean) + return hashes, account + + +def _reject(bucket: list[dict[str, str]], digest: str, reason: str) -> None: + bucket.append({"normalized_text_sha256": digest, "reason": reason}) + + +def gate_rows( + rows: Sequence[Mapping[str, Any]], + *, + train_rows: Sequence[Mapping[str, Any]], + ledger: IdentityLedger, + settlements: Mapping[str, Mapping[str, Any]] | None = None, + require_settlement: bool = True, +) -> tuple[list[dict[str, Any]], list[dict[str, str]], dict[str, Any]]: + """Return admissible rows, hash rejections, and clean accounting. + + One rejection per canonical text hash. The first failing check wins. + """ + rejections: list[dict[str, str]] = [] + survivors: list[dict[str, Any]] = [] + seen: set[str] = set() + for row in rows: + digest = normalized_text_sha256(str(row.get("text") or "")) + if digest in seen: + continue + seen.add(digest) + reason = structural_reason(row) or target_reason(row) or rights_reason(row) + if reason: + _reject(rejections, digest, reason) + continue + record = ledger.identities.get(digest) + state = derived_state(record) if record else "NEW" + if state in BLOCKING_STATES: + _reject(rejections, digest, state) + continue + if state not in {"NEW", "AVAILABLE"}: + _reject(rejections, digest, state) + continue + survivors.append(dict(row)) + clean_hashes, account = unbind_clean_hashes(survivors, train_rows) + admissible: list[dict[str, Any]] = [] + for row in survivors: + digest = normalized_text_sha256(str(row.get("text") or "")) + if digest not in clean_hashes: + _reject(rejections, digest, "not_clean") + continue + if not require_settlement: + admissible.append(row) + continue + settled = settlements or {} + settlement = settled.get(digest) + if settlement is None: + _reject(rejections, digest, "unsettled") + continue + decision = str(settlement.get("decision") or "") + if decision == "REJECT": + _reject(rejections, digest, "settlement_reject") + continue + if decision == "UNRESOLVED": + _reject(rejections, digest, "settlement_unresolved") + continue + if decision not in ADMIT_DECISIONS: + _reject(rejections, digest, "unsettled") + continue + expected = target_sha256(row.get("fillers") or []) + if str(settlement.get("target_sha256") or "") != expected: + _reject(rejections, digest, "target_mismatch") + continue + if str(settlement.get("target_provenance") or "") != str(row.get("target_origin") or ""): + _reject(rejections, digest, "target_mismatch") + continue + admissible.append(row) + return admissible, rejections, account + + +def select_diverse(rows: Sequence[Mapping[str, Any]], limit: int) -> list[dict[str, Any]]: + """Round-robin atoms by filler count and source pos. Both schemes stay paired.""" + if limit <= 0: + return [] + groups: dict[str, list[dict[str, Any]]] = {} + for row in rows: + groups.setdefault(target_sha256(list(row.get("fillers") or [])), []).append(dict(row)) + buckets: dict[tuple[int, str], list[list[dict[str, Any]]]] = {} + for group in groups.values(): + n_fillers = len(group[0].get("fillers") or []) + source_pos = str(group[0].get("source_pos") or "") + buckets.setdefault((n_fillers, source_pos), []).append(group) + for group_list in buckets.values(): + group_list.sort(key=lambda group: normalized_text_sha256(str(group[0].get("text") or ""))) + order = sorted(buckets) + indexes = {key: 0 for key in order} + picked: list[dict[str, Any]] = [] + while len(picked) < limit: + progressed = False + for key in order: + index = indexes[key] + if index >= len(buckets[key]): + continue + group = buckets[key][index] + indexes[key] = index + 1 + ordered = sorted( + group, + key=lambda row: (row.get("role_scheme") != "positional", str(row.get("role_scheme") or "")), + ) + room = limit - len(picked) + if len(ordered) <= room: + picked.extend(ordered) + progressed = True + else: + positional = [row for row in ordered if row.get("role_scheme") == "positional"] + if positional and len(positional) <= room: + picked.extend(positional) + progressed = True + if len(picked) >= limit: + break + if not progressed: + break + return picked + + +def diversity_report(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Dimensions the unbind row already has. No new taxonomy.""" + schemes: Counter[str] = Counter() + filler_counts: Counter[str] = Counter() + sources: Counter[str] = Counter() + classes: Counter[str] = Counter() + lineages: Counter[str] = Counter() + surfaces_per_target: Counter[str] = Counter() + for row in rows: + schemes[str(row.get("role_scheme") or "")] += 1 + filler_counts[str(len(row.get("fillers") or []))] += 1 + sources[str(row.get("source_pos") or "unspecified")] += 1 + classes[str(row.get("class") or "")] += 1 + lineages[str(row.get("lineage") or "")] += 1 + surfaces_per_target[target_sha256(list(row.get("fillers") or []))] += 1 + histogram = Counter(surfaces_per_target.values()) + return { + "rows": len(rows), + "role_scheme": dict(schemes), + "filler_count": dict(sorted(filler_counts.items(), key=lambda item: int(item[0]) if item[0].isdigit() else 0)), + "source_pos_metadata": dict(sources), + "source_pos_metadata_note": "WordNet index file. Not a Hyperlex family.", + "class": dict(classes), + "lineage": dict(lineages), + "unique_targets": len(surfaces_per_target), + "max_surfaces_per_target": max(surfaces_per_target.values()) if surfaces_per_target else 0, + "surfaces_per_target_histogram": {str(size): histogram[size] for size in sorted(histogram)}, + } + + +def assert_wordnet_license(text: str) -> None: + if "Permission to use, copy, modify and distribute" not in text: + refuse("WordNet license grant is not in the license file") + if "Copyright 2006 by Princeton University" not in text: + refuse("WordNet copyright notice is not in the license file") + + +def read_wordnet_index(index_dir: str | Path) -> list[dict[str, Any]]: + """Multiword index lemmas only. Does not read glosses in ``data.*``.""" + root = Path(index_dir) + atoms: list[dict[str, Any]] = [] + seen_surface: set[tuple[str, str]] = set() + for source_pos, name in INDEX_FILES: + path = root / name + if not path.is_file(): + refuse(f"WordNet index missing: {name}") + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + if not line or line[0] == " ": + continue + lemma = line.split(" ", 1)[0] + if "_" not in lemma: + continue + tokens = [part for part in lemma.split("_") if part] + if len(tokens) != lemma.count("_") + 1: + continue + surface = " ".join(tokens) + if not (2 <= len(tokens) <= LIVE_UNBIND_MAX_TOKENS): + continue + if len(surface) > LIVE_UNBIND_MAX_LEN: + continue + if surface.lower() in COLLISION_HOLD: + continue + if reject_candidate_text(surface): + continue + key = (source_pos, surface.casefold()) + if key in seen_surface: + continue + seen_surface.add(key) + atoms.append( + { + "source_pos": source_pos, + "tokens": tokens, + "surface": surface, + "lemma_sha256": hashlib.sha256(lemma.encode("utf-8")).hexdigest(), + } + ) + return atoms + + +def rows_from_wordnet_atoms( + atoms: Sequence[Mapping[str, Any]], + *, + license: str, +) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for atom in atoms: + tokens = list(atom["tokens"]) + provenance = ( + "wordnet-3.0:index." + + str(atom.get("source_pos") or "") + + ":lemma:" + + str(atom.get("lemma_sha256") or "") + ) + rows.extend( + dual_scheme_rows( + tokens, + license=license, + provenance=provenance, + target_origin="source_lemma_tokens", + source_pos=str(atom.get("source_pos") or ""), + ) + ) + return rows + + +def reason_counts(rejections: Sequence[Mapping[str, str]]) -> dict[str, int]: + counts: Counter[str] = Counter(item["reason"] for item in rejections) + return dict(sorted(counts.items())) + + +def admit_clean_unbind( + ledger: IdentityLedger, + rows: Sequence[Mapping[str, Any]], + *, + batch_id: str, + source_artifact: str, + train_rows: Sequence[Mapping[str, Any]], + settlements: Mapping[str, Mapping[str, Any]], +) -> dict[str, Any]: + """Admit only gated clean-unbind rows. Does not persist.""" + before = ledger.reserve_counts() + admissible, rejections, clean_account = gate_rows( + rows, + train_rows=train_rows, + ledger=ledger, + settlements=settlements, + ) + room = max(int(PLANNING_TARGETS["unbind_clean"]) - int(before.get("unbind_clean", 0)), 0) + if len(admissible) <= room: + selected = list(admissible) + else: + selected = select_diverse(admissible, room) + clean_hashes = {normalized_text_sha256(str(row.get("text") or "")) for row in selected} + report = ledger.admit( + selected, + batch_id=batch_id, + source_artifact=source_artifact, + unbind_clean_hashes=clean_hashes, + ) + after = ledger.reserve_counts() + for key in CLASSIFY_SLICES: + if int(after.get(key, 0)) < int(before.get(key, 0)): + refuse(f"STOP: {key} decreased") + if int(after.get("unbind_clean", 0)) < int(before.get("unbind_clean", 0)): + refuse("STOP: unbind_clean decreased") + if report["unique_routed_to_train_candidate"]: + refuse("clean-unbind admission routed rows to TRAIN_CANDIDATE") + report = dict(report) + report["reserve_counts_before"] = before + report["reserve_counts_after"] = after + report["rejection_counts"] = reason_counts(rejections) + report["rejected_hashes"] = len(rejections) + report["admissible_before_cap"] = len(admissible) + report["selected_rows"] = len(selected) + report["clean_account"] = clean_account + report["diversity"] = diversity_report(selected) + report["unbind_clean_definition"] = UNBIND_CLEAN_DEFINITION + report["contract_shape"] = CONTRACT_SHAPE + report["vendor_calls"] = 0 + return report + + +def wordnet_license_text(index_dir: str | Path) -> str: + path = Path(index_dir) / "LICENSE" + if not path.is_file(): + path = Path(index_dir).parent / "LICENSE" + if not path.is_file(): + refuse("WordNet LICENSE file is missing") + text = path.read_text(encoding="utf-8", errors="replace") + assert_wordnet_license(text) + return text + + +WORDNET_LICENSE = ( + "WordNet 3.0 Copyright 2006 by Princeton University. " + "Permission to use, copy, modify and distribute this software and " + "database and its documentation for any purpose and without fee or " + "royalty is granted, provided the copyright notice and statements " + "appear on all copies." +) + + +def file_sha256(path: str | Path) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def load_jsonl(path: Path) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for index, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + if not line.strip(): + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + refuse(f"{path} line {index} is not JSON") + if not isinstance(obj, dict): + refuse(f"{path} line {index} must be an object") + rows.append(obj) + return rows + + +def _walk_forbid(value: Any) -> None: + if isinstance(value, dict): + for key, item in value.items(): + if key in {"text", "normalized_text", "raw_text", "surface", "fillers"}: + refuse(f"receipt must not carry {key}") + _walk_forbid(item) + elif isinstance(value, list): + for item in value: + _walk_forbid(item) + + +def planning_progress(counts: Mapping[str, int]) -> list[dict[str, Any]]: + rows = [] + for key, target in PLANNING_TARGETS.items(): + have = int(counts.get(key, 0)) + rows.append( + { + "slice": key, + "current": have, + "planning_target": target, + "planning_target_class": "PLANNING_TARGET", + "progress": have / target if target else 0, + "gap_vs_planning_target": max(target - have, 0), + "minimum_required": "NOT_COMPUTABLE", + } + ) + return rows + + +def admission_result_label(before: Mapping[str, int], after: Mapping[str, int], gate: Mapping[str, Any]) -> str: + for key in CLASSIFY_SLICES: + if int(after.get(key, 0)) < int(before.get(key, 0)): + refuse(f"STOP: {key} decreased") + if int(after.get("unbind_clean", 0)) <= int(before.get("unbind_clean", 0)): + return "ACQUISITION_BLOCKED" + if gate.get("eligible") is True: + return "SELECT_003_ELIGIBLE" + if int(after.get("unbind_clean", 0)) > 0: + return "CLEAN_UNBIND_PARTIALLY_FILLED" + return "ACQUISITION_BLOCKED" + + +ZERO_SUPPORT_FAMILIES = ( + "relationship-dating", + "conflict-aggression", + "sports-competition", + "fashion-aesthetic", + "regional-cultural", + "spiritual-mystic", +) + + +def main(argv: Sequence[str] | None = None) -> int: + """Acquire clean-unbind reserve rows. Does not train or score.""" + import argparse + import sys + from datetime import datetime, timezone + + from .unbind_settlement import append_events, event_from_decision, latest_by_hash, load_events + + parser = argparse.ArgumentParser(description="Hyperlex clean-unbind reserve acquisition") + parser.add_argument("--ledger", required=True) + parser.add_argument("--train-export", required=True) + parser.add_argument("--index-dir", required=True) + parser.add_argument("--batch-id", required=True) + parser.add_argument("--source-artifact", required=True) + parser.add_argument("--settlement-log", required=True) + parser.add_argument("--operator", required=True) + parser.add_argument("--rows-out", required=True) + parser.add_argument("--receipt", required=True) + args = parser.parse_args(list(argv) if argv is not None else None) + if Path(args.receipt).exists(): + refuse(f"receipt already exists: {args.receipt}") + if Path(args.rows_out).exists(): + refuse(f"rows output already exists: {args.rows_out}") + + index_dir = Path(args.index_dir) + license_body = wordnet_license_text(index_dir) + license_path = index_dir / "LICENSE" + if not license_path.is_file(): + license_path = index_dir.parent / "LICENSE" + atoms = read_wordnet_index(index_dir) + rows = rows_from_wordnet_atoms(atoms, license=WORDNET_LICENSE) + ledger = IdentityLedger.load(args.ledger) + before = ledger.reserve_counts() + train_rows = load_jsonl(Path(args.train_export)) + admissible, rejections, clean_account = gate_rows( + rows, + train_rows=train_rows, + ledger=ledger, + require_settlement=False, + ) + room = int(PLANNING_TARGETS["unbind_clean"]) - int(before.get("unbind_clean", 0)) + selected = select_diverse(admissible, max(room, 0)) + settled_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + events = [] + for row in selected: + events.append( + event_from_decision( + normalized_text_sha256=normalized_text_sha256(str(row.get("text") or "")), + decision="ACCEPT", + target_sha256=target_sha256(list(row.get("fillers") or [])), + target_provenance=str(row.get("target_origin") or ""), + operator=args.operator, + settled_at=settled_at, + decision_basis="source_lemma_token_identity", + ) + ) + append_events(args.settlement_log, events) + settlements = latest_by_hash(load_events(args.settlement_log)) + prior_events = len(ledger.events) + report = admit_clean_unbind( + ledger, + selected, + batch_id=args.batch_id, + source_artifact=args.source_artifact, + train_rows=train_rows, + settlements=settlements, + ) + after = ledger.reserve_counts() + train_hashes = { + normalized_text_sha256(str(row.get("text") or "")) + for row in train_rows + } + gate = ledger.select_003_gate(train_hashes) + label = admission_result_label(before, after, gate) + events_path = Path(args.ledger) / "events.jsonl" + events_sha256_before = file_sha256(events_path) + written = ledger.persist_append(args.ledger, prior_events) + events_sha256_after = file_sha256(events_path) + Path(args.settlement_log).chmod(0o600) + rows_out = Path(args.rows_out) + rows_out.parent.mkdir(parents=True, exist_ok=True) + rows_out.write_text( + "".join(json.dumps(row, sort_keys=True) + "\n" for row in selected), + encoding="utf-8", + ) + rows_out.chmod(0o600) + receipt = { + "schema": "hyperlex.eval_unbind_acquisition.v1", + "batch_id": args.batch_id, + "source": "Princeton WordNet 3.0 index lemmas", + "source_artifact": args.source_artifact, + "source_rights": WORDNET_LICENSE, + "license_file_sha256": file_sha256(license_path), + "raw_artifact_sha256": file_sha256(args.source_artifact), + "license_notice_present": "Copyright 2006 by Princeton University" in license_body, + "schema_contract": CONTRACT_SHAPE, + "unbind_clean_definition": UNBIND_CLEAN_DEFINITION, + "timestamp": settled_at, + "collector": args.operator, + "transformation_pipeline": [ + "read index.noun/verb/adj/adv lemmas only", + "do not read data.* glosses", + "underscore to space", + "keep 2..6 tokens and atom length <= 80", + "emit positional and type_slot from export._unbind_dual_scheme_rows shape", + "fillers are the source lemma tokens", + "novelty screen via normalized_text_sha256", + "clean_surface against train split", + "ACCEPT settlement where fillers equal the lemma tokens", + "IdentityLedger.admit with derived unbind_clean hashes", + ], + "index_mwe_atoms": len(atoms), + "dual_scheme_rows": len(rows), + "screen_rejection_counts": reason_counts(rejections), + "screen_clean_account": { + key: clean_account.get(key) + for key in ( + "n_input", + "n_kept", + "n_excluded_trained_key", + "n_excluded_trained_text", + "n_excluded_train_split_text", + "definition", + "train_split_rows", + ) + }, + "admissible_before_planning_cap": len(admissible), + "selected_rows": len(selected), + "settlement": { + "schema": "hyperlex.eval_unbind_settlement.v1", + "log": args.settlement_log, + "events_appended": len(events), + "decisions": {"ACCEPT": len(events), "CORRECT_TARGET": 0, "REJECT": 0, "UNRESOLVED": 0}, + "decision_basis": "source_lemma_token_identity", + }, + "admission": { + "events_sha256_before": events_sha256_before, + "events_sha256_after": events_sha256_after, + "events_appended": written, + "unique_admitted_to_eval_reserve": report["unique_admitted_to_eval_reserve"], + "unique_routed_to_train_candidate": report["unique_routed_to_train_candidate"], + "rejected_existing_identities": report["rejected_existing_identities"], + "model_outcomes_consulted": report["model_outcomes_consulted"], + }, + "diversity": report["diversity"], + "reserve_counts_before": before, + "reserve_counts_after": after, + "planning_progress": planning_progress(after), + "select_003_gate": gate, + "admission_result": label, + "select_003": "NOT_DRAFTED", + "vendor_calls": 0, + "zero_support_families_not_collected": list(ZERO_SUPPORT_FAMILIES), + "evaluation_quality_note": ( + "Representation completeness is separate from evaluation quality. " + "This batch is one lexicon, class INFERRED, lineage none." + ), + } + _walk_forbid(receipt) + receipt_path = Path(args.receipt) + receipt_path.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8") + receipt_path.chmod(0o600) + public = { + "admission_result": label, + "reserve_counts_before": before, + "reserve_counts_after": after, + "selected_rows": len(selected), + "admissible_before_planning_cap": len(admissible), + "events_appended": written, + "vendor_calls": 0, + "select_003": "NOT_DRAFTED", + "gate_eligible": gate.get("eligible"), + "diversity": report["diversity"], + "screen_rejection_counts": reason_counts(rejections), + } + _walk_forbid(public) + sys.stdout.write(json.dumps(public, indent=2, sort_keys=True) + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/shadow/hyperlexical/eval_forward.py b/scripts/shadow/hyperlexical/eval_forward.py new file mode 100644 index 00000000..1cb7b5f8 --- /dev/null +++ b/scripts/shadow/hyperlexical/eval_forward.py @@ -0,0 +1,276 @@ +"""Opt-in ModernBERT forward for E2. Lazy torch. No Hub. No hyperlex import.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import NoReturn + +from .align import atom_token_index, offsets_from_tokenizer, pool_indices +from .layout import HIDDEN, MAX_LEN, TRUNK +from .save_pretrained import flatten_weight_tensors, split_weight_tensors +from .unbind_metrics import mapped_filler, mapped_pred, summarize_unbind_pairs + + +def _error(message: str, cause: Exception | None = None) -> NoReturn: + from .eval_unbind import TrunkForwardError + + if cause is None: + raise TrunkForwardError(message) + raise TrunkForwardError(message) from cause + + +def _import_torch(): + try: + import torch + from torch import nn + except ImportError as exc: + _error("trunk-forward requested but torch is not importable", exc) + return torch, nn + + +def _load_maps(model_dir: Path, heads_blob: dict | None) -> dict: + candidates: list[dict] = [] + for name in ("config.json", "layout.json"): + path = model_dir / name + if not path.is_file(): + continue + try: + blob = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + continue + if isinstance(blob, dict): + candidates.append(blob) + if heads_blob: + if isinstance(heads_blob.get("maps"), dict): + candidates.append(heads_blob["maps"]) + if isinstance(heads_blob.get("layout"), dict): + candidates.append(heads_blob["layout"]) + for blob in candidates: + role_vocab = blob.get("role_vocab") + filler_vocab = blob.get("filler_vocab") + if isinstance(role_vocab, list) and isinstance(filler_vocab, list) and filler_vocab: + families = blob.get("families") + if not isinstance(families, list) or not families: + from .layout import FAMILIES + + families = list(FAMILIES) + return { + "families": families, + "family_of": {f: i for i, f in enumerate(families)}, + "role_vocab": role_vocab, + "filler_vocab": filler_vocab, + "role_of": {r: i for i, r in enumerate(role_vocab)}, + "filler_of": {f: i for i, f in enumerate(filler_vocab)}, + } + _error(f"trunk-forward requested but role/filler vocab missing in {model_dir}") + + +def _torch_load(torch, path: Path, device): + try: + return torch.load(str(path), map_location=device, weights_only=False) + except TypeError: + return torch.load(str(path), map_location=device) + + +def encoder_forward_note(*, loaded: int, present: int) -> str: + if present == 0: + return ( + "No encoder.* tensors in artifact; last trainable layers stay at " + "trunk snapshot — unbind_exact may diverge from train val." + ) + if loaded == 0: + return ( + "encoder.* tensors present but none matched trunk keys; " + "unbind_exact may diverge from train val." + ) + return f"Applied {loaded} saved encoder trainable tensors." + + +def apply_encoder_trainable(encoder, tensors: dict) -> dict: + """Overlay saved encoder.* tensors. Unexpected keys ignored. Empty is a no-op.""" + if not tensors: + return {"loaded": 0, "unexpected": [], "present": 0} + sd_keys = set(encoder.state_dict().keys()) + mapped = {} + unexpected = [] + for raw, value in tensors.items(): + dest = _resolve_encoder_key(str(raw), sd_keys) + if dest is None: + unexpected.append(raw) + continue + mapped[dest] = value + if mapped: + encoder.load_state_dict(mapped, strict=False) + return {"loaded": len(mapped), "unexpected": unexpected, "present": len(tensors)} + + +def _resolve_encoder_key(raw: str, sd_keys: set) -> str | None: + if raw.startswith("encoder."): + candidates = (raw, raw[len("encoder.") :]) + else: + candidates = (raw, f"encoder.{raw}") + for key in candidates: + if key in sd_keys: + return key + return None + + +def _weight_parts(weight_path: Path, torch) -> tuple[dict, dict, dict | None]: + """filler_head state, encoder tensors (encoder.* keys), optional heads.pt blob.""" + if weight_path.name == "model.safetensors": + try: + from safetensors.torch import load_file + except ImportError as exc: + _error("trunk-forward requested but safetensors is not importable", exc) + split = split_weight_tensors(load_file(str(weight_path), device="cpu")) + if "weight" not in split["filler_head"]: + _error("model.safetensors missing filler_head.weight") + return split["filler_head"], split["encoder"], None + blob = _torch_load(torch, weight_path, "cpu") + if not isinstance(blob, dict) or "filler_head" not in blob: + _error(f"{weight_path.name} missing filler_head") + encoder = blob.get("encoder") if isinstance(blob.get("encoder"), dict) else {} + encoder = { + k: v + for k, v in flatten_weight_tensors({"encoder": encoder}).items() + if k.startswith("encoder.") + } + return blob["filler_head"], encoder, blob + + +def _load_filler_head(nn, hidden: int, state: dict, maps: dict): + weight = state.get("weight") + if weight is None: + _error("filler_head missing weight") + out_f, in_f = int(weight.shape[0]), int(weight.shape[1]) + if in_f != hidden: + _error(f"filler_head in_features {in_f} != hidden {hidden}") + n_vocab = len(maps["filler_vocab"]) + if out_f != n_vocab: + _error(f"filler_vocab {n_vocab} != filler_head out {out_f}") + head = nn.Linear(in_f, out_f) + head.load_state_dict(state) + return head + + +def _load_trunk(trunk_dir: Path, model_dir: Path): + try: + from transformers import AutoModel, AutoTokenizer + except ImportError as exc: + _error("trunk-forward requested but transformers is not importable", exc) + cfg_path = model_dir / "config.json" + if cfg_path.is_file(): + try: + cfg = json.loads(cfg_path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + cfg = {} + if cfg.get("base_model") not in (None, TRUNK): + _error(f"unexpected base_model {cfg.get('base_model')}") + tok = AutoTokenizer.from_pretrained(str(trunk_dir), local_files_only=True) + encoder = AutoModel.from_pretrained(str(trunk_dir), local_files_only=True) + return tok, encoder + + +def _offsets(tok, text: str): + try: + return offsets_from_tokenizer(tok, text, max_len=MAX_LEN) + except TypeError: + return None + + +def _predict_unbind_pair_views( + encoder, filler_head, tok, maps, rows, device +) -> tuple[list[tuple[list[str], list[str]]], list[tuple[list[str], list[str]]]]: + """Legacy ``(mapped gold, pred)`` and strict ``(raw lowercased gold, pred)``. + + Same forward as train val. Empty gold rows are skipped. Inference only. + """ + encoder.eval() + filler_head.eval() + legacy: list[tuple[list[str], list[str]]] = [] + strict: list[tuple[list[str], list[str]]] = [] + for row in rows: + fillers = list(row.get("fillers") or []) + if not fillers: + continue + enc = tok( + [row["text"]], + padding=True, + truncation=True, + max_length=MAX_LEN, + return_tensors="pt", + ) + enc = {k: v.to(device) for k, v in enc.items()} + states = encoder(**enc).last_hidden_state[0] + offs = _offsets(tok, row["text"]) + gold_strs: list[str] = [] + raw_strs: list[str] = [] + pred_strs: list[str] = [] + for fill in fillers: + idxs = pool_indices(states.size(0), atom_token_index(row["text"], fill, offs)) + pred = int(filler_head(states[idxs].mean(0)).argmax()) + gold_strs.append(mapped_filler(maps, fill)) + raw_strs.append(str(fill).lower()) + pred_strs.append(mapped_pred(maps, pred)) + legacy.append((gold_strs, pred_strs)) + strict.append((raw_strs, pred_strs)) + return legacy, strict + + +def predict_unbind_pairs(encoder, filler_head, tok, maps, rows, device) -> list[tuple[list[str], list[str]]]: + """Per-row ``(mapped gold, pred)`` filler strings. Skips empty gold. + + Same alignment as train val / ``score_unbind_exact``. Inference only. + """ + legacy, _strict = _predict_unbind_pair_views(encoder, filler_head, tok, maps, rows, device) + return legacy + + +def score_unbind_exact(encoder, filler_head, tok, maps, rows, device) -> dict: + """Same per-row exact filler metric as train val, plus token/slot F1. + + Legacy keys use mapped gold. Strict keys use the raw lowercased filler + and count a ```` prediction as a miss. + """ + legacy, strict = _predict_unbind_pair_views(encoder, filler_head, tok, maps, rows, device) + return summarize_unbind_pairs(legacy, strict_pairs=strict) + + +def run_unbind_exact( + *, + trunk_dir: Path, + model_dir: Path, + weight_path: Path, + spans: list[dict], +) -> dict: + torch, nn = _import_torch() + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + filler_state, encoder_tensors, heads_blob = _weight_parts(weight_path, torch) + maps = _load_maps(model_dir, heads_blob) + tok, encoder = _load_trunk(trunk_dir, model_dir) + hidden = int(getattr(encoder.config, "hidden_size", HIDDEN)) + if hidden != HIDDEN: + _error(f"hidden {hidden} != {HIDDEN}") + applied = apply_encoder_trainable(encoder, encoder_tensors) + if applied["present"] and applied["loaded"] == 0: + _error("encoder trainable tensors present but none matched trunk keys") + filler_head = _load_filler_head(nn, hidden, filler_state, maps) + encoder.to(device) + filler_head.to(device) + from .eval_unbind import spans_to_unbind_rows + + rows = spans_to_unbind_rows(spans) + if not rows: + _error("trunk-forward requested but no unbind test spans") + scored = score_unbind_exact(encoder, filler_head, tok, maps, rows, device) + return { + **scored, + "device": str(device), + "cuda": bool(torch.cuda.is_available()), + "encoder_trainable_loaded": applied["loaded"], + "encoder_trainable_present": applied["present"], + "encoder_note": encoder_forward_note( + loaded=applied["loaded"], present=applied["present"] + ), + } diff --git a/scripts/shadow/hyperlexical/eval_routing.py b/scripts/shadow/hyperlexical/eval_routing.py new file mode 100644 index 00000000..592b8e65 --- /dev/null +++ b/scripts/shadow/hyperlexical/eval_routing.py @@ -0,0 +1,471 @@ +"""SELECT reserve routing. Settlement stays untouched. + +A routing record is a derivative. It copies the sealed decision, registry +class, and semantic family, then applies ``identity_ledger.slices_of``. +It does not settle, score, or read reserve deficits. +""" + +from __future__ import annotations + +import hashlib +import json +from typing import Any, Mapping, Sequence + +from .eval_settlement import ABSTAIN, ACTIVE_FAMILIES, ATTESTS, DECISIONS, SETTLED_DECISIONS +from .identity_ledger import slices_of + +SCHEMA = "hyperlex.eval_routing.v1" +ROUTING_VERSION = "v1" +PROCEDURE_ID = "select_eval_routing_v1" +CLEARED_RIGHTS = "CC-BY-SA" +UNBIND_CLEAN_DERIVATION = "soft_ceiling.clean_surface" +TASKS = ("classify", "unbind") +REGISTRY_CLASSES = ATTESTS +LINEAGES = (ABSTAIN, *ACTIVE_FAMILIES) +ROUTING_STORE = ( + "private evaluation evidence store " + "eval-reserve-20260926/routing/events.jsonl" +) +_BANNED = frozenset( + { + "admission_floors", + "candidate_score", + "current_reserve_deficits", + "desired_select_slice", + "desired_slice", + "model_error", + "model_score", + "planning_targets", + "prediction", + "residual", + "reserve_counts", + "routed_counts", + "semantic_evidence", + "threshold_v2", + } +) +_SLICE_CODES = { + "classify": "SLICE_CLASSIFY", + "classify_non_none": "SLICE_CLASSIFY_NON_NONE", + "classify_observed": "SLICE_CLASSIFY_OBSERVED", + "unbind_clean": "SLICE_UNBIND_CLEAN", +} +RULE_TABLE = { + "class_meaning": "registry attest class OBSERVED or INFERRED, not the semantic family", + "cleared_rights": CLEARED_RIGHTS, + "classify": "task classify; decision ACCEPT, RECLASSIFY, or NONE", + "classify_non_none": "classify and lineage not empty and not none", + "classify_observed": "classify and registry class OBSERVED", + "lineage_meaning": "semantic family, including none", + "overlap": "one record may enter every classify slice it satisfies; unbind_clean requires task unbind", + "procedure_id": PROCEDURE_ID, + "routable_decisions": sorted(SETTLED_DECISIONS), + "unbind_clean": "task unbind and frozen soft_ceiling.clean_surface is true", + "unroutable_decisions": ["UNRESOLVED"], +} + + +def canonical_json(payload: Mapping[str, Any]) -> str: + return json.dumps(payload, indent=2, sort_keys=True, ensure_ascii=True) + "\n" + + +def sha256_text(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def settlement_hash(record: Mapping[str, Any]) -> str: + return sha256_text(canonical_json(record)) + + +def procedure_hash() -> str: + return sha256_text(canonical_json({"procedure_id": PROCEDURE_ID, "rules": RULE_TABLE})) + + +def _refuse_banned(value: Mapping[str, Any]) -> None: + found = sorted(_BANNED.intersection(value)) + if found: + raise SystemExit( + "REFUSE: routing must not read quota state or model output: " + ",".join(found) + ) + + +def _slices_for(task: str, registry_class: str, lineage: str, unbind_clean: bool) -> list[str]: + label: dict[str, Any] = { + "class": registry_class, + "lineage": lineage, + "task": task, + } + if task == "unbind": + label["unbind_clean"] = unbind_clean + return sorted(slices_of({"labels": [label]})) + + +def _base( + settlement: Mapping[str, Any], + metadata: Mapping[str, Any], + *, + status: str, + reasons: Sequence[str], + slices: Sequence[str], + task: str | None, + registry_class: str | None, + lineage: str | None, + unbind_clean: bool | None, +) -> dict[str, Any]: + text_hash = str(settlement.get("text_hash") or "") + row_id = str(settlement.get("row_id") or "") + return { + "class": registry_class, + "created_from_frozen_artifact": True, + "derivation_procedure_hash": procedure_hash(), + "derivation_procedure_id": PROCEDURE_ID, + "lineage": lineage, + "normalized_identity": text_hash, + "provenance_state": str(metadata.get("provenance_state") or "INCOMPLETE"), + "rights_state": str(settlement.get("rights") or ""), + "routing_reason_codes": sorted(set(reasons)), + "routing_status": status, + "routing_version": ROUTING_VERSION, + "row_id": row_id, + "schema": SCHEMA, + "settlement_decision": str(settlement.get("decision") or ""), + "settlement_registry_class": registry_class, + "slices": list(slices), + "source_identity": str(metadata.get("source_identity") or ""), + "source_settlement_hash": settlement_hash(settlement), + "source_settlement_id": row_id, + "task": task, + "unbind_clean": unbind_clean, + } + + +def derive_routing( + settlement: Mapping[str, Any], + metadata: Mapping[str, Any], +) -> dict[str, Any]: + """One settlement plus frozen metadata. No RNG and no quota read.""" + _refuse_banned(settlement) + _refuse_banned(metadata) + decision = settlement.get("decision") + if decision not in DECISIONS: + raise SystemExit(f"REFUSE: settlement decision is not canonical: {decision}") + text_hash = str(settlement.get("text_hash") or "") + row_id = str(settlement.get("row_id") or "") + if len(text_hash) != 64 or any(ch not in "0123456789abcdef" for ch in text_hash): + raise SystemExit("REFUSE: normalized identity is not sha256") + attest = settlement.get("attest") + family = settlement.get("semantic_family") + reasons: list[str] = [] + meta_row = str(metadata.get("row_id") or "") + if not row_id or meta_row != row_id or str(metadata.get("normalized_identity") or "") != text_hash: + reasons.append("IDENTITY_CONFLICT") + provenance_state = str(metadata.get("provenance_state") or "") + if ( + provenance_state != "COMPLETE" + or not str(metadata.get("source_identity") or "").strip() + or not str(settlement.get("provenance") or "").strip() + ): + reasons.append("PROVENANCE_INCOMPLETE") + if str(settlement.get("rights") or "") != CLEARED_RIGHTS: + reasons.append("RIGHTS_NOT_CLEARED") + if decision == "UNRESOLVED": + reasons.append("SETTLEMENT_UNRESOLVED") + elif decision not in SETTLED_DECISIONS: + reasons.append("SETTLEMENT_DECISION_NOT_ADMISSIBLE") + if "class" in metadata and metadata.get("class") != attest: + reasons.append("CLASS_PROMOTION_REFUSED") + if "lineage" in metadata and metadata.get("lineage") != family: + reasons.append("LINEAGE_CONFLICT") + task = metadata.get("task") + if task not in TASKS: + reasons.append("TASK_ABSENT") + task_out = None + else: + task_out = str(task) + registry_out = attest if attest in REGISTRY_CLASSES else None + lineage_out = family if isinstance(family, str) and family in LINEAGES else None + if decision in SETTLED_DECISIONS and registry_out is None: + reasons.append("REGISTRY_CLASS_ABSENT") + if decision in SETTLED_DECISIONS and lineage_out is None: + reasons.append("LINEAGE_ABSENT") + unbind_flag = metadata.get("unbind_clean") is True + derivation = metadata.get("unbind_clean_derivation") + if task_out == "classify" and unbind_flag: + reasons.append("UNBIND_CLEAN_TASK_MISMATCH") + if task_out == "unbind" and unbind_flag and derivation != UNBIND_CLEAN_DERIVATION: + reasons.append("UNBIND_CLEAN_DERIVATION_ABSENT") + clean_out = False + if task_out == "unbind" and unbind_flag and derivation == UNBIND_CLEAN_DERIVATION: + clean_out = True + if reasons: + return _base( + settlement, + metadata, + status="INELIGIBLE", + reasons=reasons, + slices=[], + task=task_out, + registry_class=registry_out, + lineage=lineage_out, + unbind_clean=None, + ) + assert task_out is not None and registry_out is not None and lineage_out is not None + slices = _slices_for(task_out, registry_out, lineage_out, clean_out) + if not slices: + return _base( + settlement, + metadata, + status="INELIGIBLE", + reasons=["NO_SELECT_SLICE"], + slices=[], + task=task_out, + registry_class=registry_out, + lineage=lineage_out, + unbind_clean=clean_out, + ) + return _base( + settlement, + metadata, + status="ELIGIBLE", + reasons=[_SLICE_CODES[name] for name in slices], + slices=slices, + task=task_out, + registry_class=registry_out, + lineage=lineage_out, + unbind_clean=clean_out, + ) + + +def derive_batch( + pairs: Sequence[tuple[Mapping[str, Any], Mapping[str, Any]]], +) -> list[dict[str, Any]]: + """First row_id wins. A later copy is a duplicate, not a second route.""" + seen: set[str] = set() + records = [] + for settlement, metadata in pairs: + row_id = str(settlement.get("row_id") or "") + if row_id and row_id in seen: + records.append( + _base( + settlement, + metadata, + status="INELIGIBLE", + reasons=["DUPLICATE_IDENTITY"], + slices=[], + task=None, + registry_class=settlement.get("attest") if settlement.get("attest") in REGISTRY_CLASSES else None, + lineage=settlement.get("semantic_family") + if settlement.get("semantic_family") in LINEAGES + else None, + unbind_clean=None, + ) + ) + continue + if row_id: + seen.add(row_id) + records.append(derive_routing(settlement, metadata)) + return records + + +def authorize_for_census(event: Mapping[str, Any], routing: Mapping[str, Any]) -> str: + """Census may use a routing record only when it still matches the settlement.""" + if routing.get("schema") != SCHEMA or routing.get("routing_status") != "ELIGIBLE": + return "ROUTING_INELIGIBLE" + if str(routing.get("row_id") or "") != str(event.get("row_id") or ""): + return "ROUTING_IDENTITY_MISMATCH" + if str(routing.get("normalized_identity") or "") != str(event.get("text_hash") or ""): + return "ROUTING_IDENTITY_MISMATCH" + if routing.get("source_settlement_hash") != settlement_hash(event): + return "ROUTING_HASH_MISMATCH" + if routing.get("rights_state") != CLEARED_RIGHTS: + return "ROUTING_INELIGIBLE" + if routing.get("created_from_frozen_artifact") is not True: + return "ROUTING_REJECTED" + task = routing.get("task") + registry_class = routing.get("class") + lineage = routing.get("lineage") + if task not in TASKS or registry_class not in REGISTRY_CLASSES or lineage not in LINEAGES: + return "ROUTING_SLICE_MISMATCH" + expected = _slices_for(str(task), str(registry_class), str(lineage), routing.get("unbind_clean") is True) + if list(routing.get("slices") or []) != expected or not expected: + return "ROUTING_SLICE_MISMATCH" + return "ELIGIBLE" + + +def reference_pairs() -> list[tuple[dict[str, Any], dict[str, Any]]]: + """Synthetic corpus. Not a harvest and not the 339 settlements.""" + specs = [ + ("accept-observed", "11", "ACCEPT", "ai-native", "OBSERVED", "CC-BY-SA", "COMPLETE", "classify", False, None), + ("accept-inferred", "22", "ACCEPT", "technology-ai", "INFERRED", "CC-BY-SA", "COMPLETE", "classify", False, None), + ("accept-non-none", "33", "ACCEPT", "politics-civic", "INFERRED", "CC-BY-SA", "COMPLETE", "classify", False, None), + ("accept-unbind", "44", "ACCEPT", "none", "INFERRED", "CC-BY-SA", "COMPLETE", "unbind", True, UNBIND_CLEAN_DERIVATION), + ("none-classify", "55", "NONE", "none", "INFERRED", "CC-BY-SA", "COMPLETE", "classify", False, None), + ("unresolved", "66", "UNRESOLVED", None, None, "CC-BY-SA", "COMPLETE", "classify", False, None), + ("reclassify", "77", "RECLASSIFY", "workplace-career", "INFERRED", "CC-BY-SA", "COMPLETE", "classify", False, None), + ("rights-blocked", "88", "ACCEPT", "ai-native", "OBSERVED", "RIGHTS_UNRESOLVED", "COMPLETE", "classify", False, None), + ("provenance-blocked", "99", "ACCEPT", "ai-native", "INFERRED", "CC-BY-SA", "INCOMPLETE", "classify", False, None), + ("identity-conflict", "ab", "ACCEPT", "ai-native", "INFERRED", "CC-BY-SA", "COMPLETE", "classify", False, None), + ] + pairs = [] + for name, nibble, decision, family, attest, rights, provenance_state, task, clean, derivation in specs: + digest = nibble * 32 + settlement: dict[str, Any] = { + "attest": attest, + "decision": decision, + "provenance": "routing-fixture", + "rights": rights, + "row_id": f"route-fixture-{name}", + "schema": "hyperlex.eval_settlement.v1", + "semantic_family": family, + "text_hash": digest, + } + metadata: dict[str, Any] = { + "normalized_identity": digest, + "provenance_state": provenance_state, + "row_id": settlement["row_id"], + "source_identity": f"fixture:{name}", + "task": task, + "unbind_clean": clean, + } + if name == "provenance-blocked": + metadata["source_identity"] = "" + if name == "identity-conflict": + metadata["normalized_identity"] = "cd" * 32 + if derivation is not None: + metadata["unbind_clean_derivation"] = derivation + pairs.append((settlement, metadata)) + inferred = { + "attest": "INFERRED", + "decision": "ACCEPT", + "provenance": "routing-fixture", + "rights": "CC-BY-SA", + "row_id": "route-fixture-promotion-refused", + "schema": "hyperlex.eval_settlement.v1", + "semantic_family": "ai-native", + "text_hash": "ef" * 32, + } + pairs.append( + ( + inferred, + { + "class": "OBSERVED", + "normalized_identity": inferred["text_hash"], + "provenance_state": "COMPLETE", + "row_id": inferred["row_id"], + "source_identity": "fixture:promotion-refused", + "task": "classify", + "unbind_clean": False, + }, + ) + ) + classify_as_unbind = { + "attest": "INFERRED", + "decision": "ACCEPT", + "provenance": "routing-fixture", + "rights": "CC-BY-SA", + "row_id": "route-fixture-classify-not-unbind", + "schema": "hyperlex.eval_settlement.v1", + "semantic_family": "ai-native", + "text_hash": "ba" * 32, + } + pairs.append( + ( + classify_as_unbind, + { + "normalized_identity": classify_as_unbind["text_hash"], + "provenance_state": "COMPLETE", + "row_id": classify_as_unbind["row_id"], + "source_identity": "fixture:classify-not-unbind", + "task": "classify", + "unbind_clean": True, + "unbind_clean_derivation": UNBIND_CLEAN_DERIVATION, + }, + ) + ) + return pairs + + +def semantic_errors(record: Mapping[str, Any], settlement: Mapping[str, Any]) -> list[str]: + """Schema plus the slice, rights, and settlement-hash checks.""" + from .select_contract_schema import routing_schema_errors + + errors = [f"schema: {item}" for item in routing_schema_errors(record)] + if record.get("source_settlement_hash") != settlement_hash(settlement): + errors.append("source settlement hash mismatch") + if record.get("row_id") != settlement.get("row_id"): + errors.append("row_id mismatch") + if record.get("normalized_identity") != settlement.get("text_hash"): + errors.append("normalized identity mismatch") + if record.get("settlement_decision") != settlement.get("decision"): + errors.append("decision mismatch") + status = record.get("routing_status") + if status == "ELIGIBLE": + if record.get("rights_state") != CLEARED_RIGHTS: + errors.append("eligible record is not rights-cleared") + if record.get("provenance_state") != "COMPLETE": + errors.append("eligible record is provenance-incomplete") + if record.get("settlement_decision") not in SETTLED_DECISIONS: + errors.append("eligible record is not a settled decision") + if record.get("class") != record.get("settlement_registry_class"): + errors.append("class drifted from the registry class") + if record.get("class") != settlement.get("attest"): + errors.append("class was promoted away from attest") + if record.get("lineage") != settlement.get("semantic_family"): + errors.append("lineage drifted from semantic_family") + expected = _slices_for( + str(record.get("task")), + str(record.get("class")), + str(record.get("lineage")), + record.get("unbind_clean") is True, + ) + if list(record.get("slices") or []) != expected: + errors.append("slices do not match slices_of") + if "classify_observed" in expected and record.get("class") != "OBSERVED": + errors.append("classify_observed without OBSERVED") + if "unbind_clean" in expected and record.get("unbind_clean") is not True: + errors.append("unbind_clean slice without the frozen flag") + if record.get("class") == "INFERRED" and "classify_observed" in expected: + errors.append("INFERRED entered classify_observed") + elif status == "INELIGIBLE": + if record.get("slices"): + errors.append("ineligible record carries slices") + if not record.get("routing_reason_codes"): + errors.append("ineligible record has no reason") + else: + errors.append("routing_status is not canonical") + return errors + + +def contract_validation( + pairs: Sequence[tuple[Mapping[str, Any], Mapping[str, Any]]] | None = None, +) -> dict[str, Any]: + """Run the reference corpus twice. Schema and semantic failures stay distinct.""" + corpus = list(pairs) if pairs is not None else reference_pairs() + first = [derive_routing(settlement, metadata) for settlement, metadata in corpus] + second = [derive_routing(settlement, metadata) for settlement, metadata in corpus] + if [canonical_json(row) for row in first] != [canonical_json(row) for row in second]: + return { + "determinism": "NOT_IDENTICAL", + "errors": ["reconstructions differed"], + "state": "ROUTING_NONDETERMINISTIC", + } + errors: list[str] = [] + schema_hit = False + for record, (settlement, _metadata) in zip(first, corpus): + found = semantic_errors(record, settlement) + schema_hit = schema_hit or any(item.startswith("schema:") for item in found) + errors.extend(f"{record['row_id']}: {item}" for item in found) + if errors and schema_hit: + state = "ROUTING_SCHEMA_FAILURE" + elif errors: + state = "ROUTING_SEMANTIC_VALIDATION_FAILURE" + else: + state = "ROUTING_VALIDATION_COMPLETE" + return { + "determinism": "IDENTICAL", + "errors": errors, + "procedure_id": PROCEDURE_ID, + "procedure_sha256": procedure_hash(), + "schema": SCHEMA, + "state": state, + } diff --git a/scripts/shadow/hyperlexical/eval_settlement.py b/scripts/shadow/hyperlexical/eval_settlement.py new file mode 100644 index 00000000..08e7e7f5 --- /dev/null +++ b/scripts/shadow/hyperlexical/eval_settlement.py @@ -0,0 +1,849 @@ +"""Evaluation-settlement apply path for the evaluation taxonomy. + +``source_hint``, ``semantic_family``, and ``attest`` are separate fields. +A confirm settlement may store the same family string in both ``source_hint`` +and ``semantic_family``. This module never copies one field into another, +and it never derives ``attest`` from ``decision``. + +The production classify head is not read or written here. ``ACCEPT`` records +the operator-selected attest. It does not promote existing evidence to +``OBSERVED``. This command does not admit rows to ``EVAL_RESERVE``. +""" + +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +from typing import Any, Mapping, Sequence + +from .holdout_guard import normalized_text_sha256 + +SCHEMA = "hyperlex.eval_settlement.v1" +RECEIPT_SCHEMA = "hyperlex.eval_settlement_receipt.v1" +VENDOR_CALLS = 0 + +ACTIVE_FAMILIES: tuple[str, ...] = ( + "gaming-meta", + "betting-sharp", + "crypto-degen", + "internet-slang", + "memetic", + "social-status", + "relationship-dating", + "approval-disapproval", + "conflict-aggression", + "technology-ai", + "workplace-career", + "sports-competition", + "music-entertainment", + "fashion-aesthetic", + "regional-cultural", + "spiritual-mystic", + "identity-affiliation", + "politics-civic", + "ai-native", +) +CANDIDATE_FAMILIES: tuple[str, ...] = ( + "finance-retail", + "market-structure", + "sexual-romantic", + "substance-party", + "crime-illicit", + "health-fitness", +) +ABSTAIN = "none" +DECISIONS = ("ACCEPT", "RECLASSIFY", "NONE", "UNRESOLVED") +DECISION_ALIASES = { + "ACCEPT": "ACCEPT", + "ACCEPT FAMILY": "ACCEPT", + "RECLASSIFY": "RECLASSIFY", + "CHOOSE DIFFERENT FAMILY": "RECLASSIFY", + "NONE": "NONE", + "UNRESOLVED": "UNRESOLVED", +} +ATTESTS = ("OBSERVED", "INFERRED") +REGISTERS = ("slang", "domain-specific", "high-register", "general") +FUNCTIONS = ("address", "evaluation", "intensification", "reference", "affiliation") +SURFACES = { + "A": "CONFIRM", + "B": "PROPOSED FAMILY", + "C": "DISAMBIGUATE", + "D": "RIGHTS BLOCKED", + "HELD_OUTSIDE_LANES": "hint-only", +} +SHEET_COLUMNS = ( + "lane", + "row_key", + "text", + "source_hint", + "source_hint_provenance", + "current_label", + "label_source", + "rights", + "proposed_family_evidence", + "disambiguation_options", + "semantic_family", + "attest", + "register", + "function", + "decision", +) +SETTLED_DECISIONS = frozenset({"ACCEPT", "RECLASSIFY", "NONE"}) +_CLEARED_SOURCES = { + "wiktionary_category": "CC-BY-SA", + "wikipedia_prose": "CC-BY-SA", + "wiktionary_sense_gloss": "CC-BY-SA", + "wiktionary_multiword_lemma": "CC-BY-SA", + "wiktionary_labeled_sense": "CC-BY-SA", +} +_BLOCKED_SOURCES = { + "reddit_title": "RIGHTS_UNRESOLVED", + "kym_slang_list": "RIGHTS_UNRESOLVED", +} +_SCORE_KEYS = frozenset( + {"candidate_score", "model_error", "model_score", "prediction"} +) +_FORBIDDEN_KEYS = frozenset({"text", "normalized_text", "raw_text"}) | _SCORE_KEYS +_HINT_PROVENANCE = "stream.family_hint_not_a_label" + + +def refuse(message: str) -> None: + raise SystemExit(f"REFUSE: {message}") + + +def family_flags(name: str, *, activated: set[str] | None = None) -> dict[str, bool]: + """Three independent flags. Evaluation settlement does not enable a family.""" + activated = set(activated or ()) + if name in ACTIVE_FAMILIES: + return { + "taxonomy.active": True, + "evaluation.enabled": False, + "production.enabled": False, + } + if name in CANDIDATE_FAMILIES: + return { + "taxonomy.active": name in activated, + "evaluation.enabled": False, + "production.enabled": False, + } + refuse(f"family is not in the evaluation taxonomy: {name}") + raise AssertionError("refuse") + + +def _activated(activated: set[str] | None) -> set[str]: + extra = set(activated or ()) + unknown = sorted(extra - set(CANDIDATE_FAMILIES)) + if unknown: + refuse(f"activation refused for {unknown[0]}") + return extra + + +def _blank(value: Any) -> bool: + return value is None or (isinstance(value, str) and not value.strip()) + + +def _optional_text(value: Any) -> str | None: + if _blank(value): + return None + if not isinstance(value, str): + refuse("settlement field is not text") + return value.strip() + + +def _decision(value: Any) -> str: + token = _optional_text(value) + if token is None: + refuse("decision is empty") + mapped = DECISION_ALIASES.get(token.upper()) + if mapped is None: + if token.upper() == "REJECT": + refuse("reject is a production attest token, not an evaluation decision") + refuse(f"invalid decision: {token}") + return mapped + + +def _attest(value: Any) -> str | None: + token = _optional_text(value) + if token is None: + return None + mapped = token.upper() + if mapped not in ATTESTS: + refuse(f"invalid attest: {token}") + return mapped + + +def _family_token(value: Any) -> str | None: + token = _optional_text(value) + if token is None: + return None + return token.casefold() + + +def _closed_vocab(value: Any, allowed: tuple[str, ...], label: str) -> str | None: + token = _optional_text(value) + if token is None: + return None + mapped = token.casefold() + if mapped not in allowed: + refuse(f"invalid {label}: {token}") + return mapped + + +def _resolve_family(token: str | None, *, activated: set[str], allow_null: bool) -> str | None: + if token is None: + if allow_null: + return None + refuse("explicit semantic_family is required") + if token == ABSTAIN: + return ABSTAIN + if token in CANDIDATE_FAMILIES and token not in activated: + refuse(f"candidate family is inactive: {token}") + if token in ACTIVE_FAMILIES or token in activated: + return token + refuse(f"family is not in the evaluation taxonomy: {token}") + raise AssertionError("refuse") + + +def rights_of(stream_row: Mapping[str, Any]) -> str: + source_type = str(stream_row.get("source_type") or "") + if source_type in _CLEARED_SOURCES: + return _CLEARED_SOURCES[source_type] + if source_type in _BLOCKED_SOURCES: + return _BLOCKED_SOURCES[source_type] + refuse(f"unknown rights source_type: {source_type or 'missing'}") + raise AssertionError("refuse") + + +def evidence_hint(stream_row: Mapping[str, Any]) -> str | None: + hint = stream_row.get("family_hint_not_a_label") + if _blank(hint): + return None + return str(hint) + + +def sheet_hint_token(hint: str | None) -> str: + return "NO_HINT" if hint is None else hint + + +def sheet_label_token(label: Any) -> str: + if label is None: + return "UNLABELLED" + return str(label) + + +def settlement_allows_reserve(record: Mapping[str, Any]) -> bool: + """Rights and decision gates. This predicate does not admit a row.""" + if record.get("decision") not in SETTLED_DECISIONS: + return False + if record.get("rights") == "RIGHTS_UNRESOLVED": + return False + family = record.get("semantic_family") + attest = record.get("attest") + if family is None or attest not in ATTESTS: + return False + if attest == "OBSERVED" and record.get("decision") is None: + return False + return True + + +def _forbid_payload(value: Mapping[str, Any], where: str) -> None: + for key in value: + if key in _FORBIDDEN_KEYS: + refuse(f"{where} must not carry {key}") + + +def _refuse_scores(value: Mapping[str, Any], where: str) -> None: + for key in value: + if key in _SCORE_KEYS: + refuse(f"{where} must not carry {key}") + + +def _canonical_hash(stream_row: Mapping[str, Any]) -> str: + digest = normalized_text_sha256(str(stream_row.get("text") or "")) + stored = stream_row.get("normtext_sha256") + if isinstance(stored, str) and stored and stored != digest: + refuse("stream normtext_sha256 drifted from canonical row identity") + return digest + + +def _coerce_choice( + decision: str, + family: str | None, + attest: str | None, + proposed: str | None, + hint: str | None, +) -> tuple[str | None, str | None]: + if decision == "UNRESOLVED": + if family is not None or attest is not None: + refuse("UNRESOLVED requires null semantic_family and null attest") + return None, None + if decision == "NONE": + if family not in (None, ABSTAIN): + refuse("NONE requires semantic_family none") + if attest is None: + refuse("NONE requires an explicit attest") + return ABSTAIN, attest + if decision == "ACCEPT": + if family is None: + refuse("ACCEPT requires an explicit semantic_family") + if attest is None: + refuse("ACCEPT requires an explicit attest") + if proposed is not None and family != proposed: + refuse("ACCEPT does not match the proposed family") + return family, attest + if decision == "RECLASSIFY": + if family is None: + refuse("RECLASSIFY requires an explicit family") + if attest is None: + refuse("RECLASSIFY requires an explicit attest") + evidence = proposed if proposed is not None else hint + if evidence is not None and family == evidence: + refuse("RECLASSIFY family must differ from the proposed evidence family") + return family, attest + refuse(f"invalid decision: {decision}") + raise AssertionError("refuse") + + +def validate_settlement( + incoming: Mapping[str, Any], + stream_row: Mapping[str, Any], + *, + activated: set[str] | None = None, + prior_ids: set[str] | None = None, +) -> dict[str, Any]: + """Fail closed. Does not read ``label_source`` as ``attest``.""" + _forbid_payload(incoming, "settlement") + extra = _activated(activated) + row_id = _optional_text(incoming.get("row_id")) or _optional_text(stream_row.get("row_key")) + if not row_id or row_id != stream_row.get("row_key"): + refuse("row_id is not in the held-out stream") + if row_id in set(prior_ids or ()): + refuse(f"duplicate settlement for {row_id}") + digest = _canonical_hash(stream_row) + claimed = _optional_text(incoming.get("text_hash")) + if claimed is None or claimed != digest: + refuse(f"text_hash does not match canonical row identity for {row_id}") + hint = evidence_hint(stream_row) + supplied_hint = incoming.get("source_hint") + if _blank(supplied_hint): + supplied_hint = None + elif isinstance(supplied_hint, str) and supplied_hint.strip() == "NO_HINT": + supplied_hint = None + elif isinstance(supplied_hint, str): + supplied_hint = supplied_hint.strip() + else: + refuse("source_hint is not text") + if supplied_hint != hint: + refuse(f"source_hint does not match held-out evidence for {row_id}") + rights = rights_of(stream_row) + sheet_rights = _optional_text(incoming.get("rights")) + if sheet_rights is not None and sheet_rights != rights: + refuse(f"rights state does not match held-out evidence for {row_id}") + decision = _decision(incoming.get("decision")) + proposed = _family_token(incoming.get("proposed_family")) + register = _closed_vocab(incoming.get("register"), REGISTERS, "register") + function = _closed_vocab(incoming.get("function"), FUNCTIONS, "function") + attest = _attest(incoming.get("attest")) + family = _family_token(incoming.get("semantic_family")) + if decision == "UNRESOLVED" and (register is not None or function is not None): + refuse("UNRESOLVED cannot carry register or function") + family, attest = _coerce_choice(decision, family, attest, proposed, hint) + family = _resolve_family(family, activated=extra, allow_null=decision == "UNRESOLVED") + operator = _optional_text(incoming.get("operator")) + settled_at = _optional_text(incoming.get("settled_at")) + provenance = _optional_text(incoming.get("provenance")) + if not operator: + refuse("operator is required") + if not settled_at: + refuse("settled_at is required") + if not provenance: + refuse("provenance is required") + lane = _optional_text(incoming.get("lane")) + if lane is not None and lane not in SURFACES: + refuse(f"unknown settlement surface: {lane}") + record = { + "schema": SCHEMA, + "row_id": row_id, + "text_hash": digest, + "decision": decision, + "semantic_family": family, + "attest": attest, + "register": register, + "function": function, + "source_hint": hint, + "operator": operator, + "settled_at": settled_at, + "provenance": provenance, + "rights": rights, + "lane": lane, + } + _forbid_payload(record, "settlement record") + return record + + +def _stream_index(rows: Sequence[Mapping[str, Any]]) -> dict[str, Mapping[str, Any]]: + index: dict[str, Mapping[str, Any]] = {} + for row in rows: + _refuse_scores(row, "held-out row") + key = row.get("row_key") + if not isinstance(key, str) or not key: + refuse("held-out row is missing row_key") + if key in index: + refuse(f"duplicate held-out row_id {key}") + index[key] = row + return index + + +def load_stream(path: str | Path) -> dict[str, Mapping[str, Any]]: + rows = [] + with Path(path).open(encoding="utf-8") as handle: + for line in handle: + if line.strip(): + payload = json.loads(line) + if not isinstance(payload, dict): + refuse("held-out stream row is not an object") + rows.append(payload) + return _stream_index(rows) + + +def _check_sheet_integrity( + parts: Sequence[str], + header: Sequence[str], + stream_row: Mapping[str, Any], +) -> str: + cell = dict(zip(header, parts)) + row_id = cell["row_key"].strip() + if row_id != stream_row.get("row_key"): + refuse("row_id is not in the held-out stream") + digest = normalized_text_sha256(cell["text"]) + if digest != _canonical_hash(stream_row): + refuse(f"text_hash does not match canonical row identity for {row_id}") + hint = evidence_hint(stream_row) + if cell["source_hint"] != sheet_hint_token(hint): + refuse(f"source_hint does not match held-out evidence for {row_id}") + if cell["source_hint_provenance"] != _HINT_PROVENANCE: + refuse(f"source_hint provenance does not match for {row_id}") + if cell["current_label"] != sheet_label_token(stream_row.get("label")): + refuse(f"current_label does not match held-out evidence for {row_id}") + if cell["label_source"] != str(stream_row.get("label_source") or ""): + refuse(f"label_source does not match held-out evidence for {row_id}") + if cell["rights"].strip() != rights_of(stream_row): + refuse(f"rights state does not match held-out evidence for {row_id}") + lane = cell["lane"].strip() + if lane not in SURFACES: + refuse(f"unknown settlement surface: {lane}") + return digest + + +def _operator_cells(cell: Mapping[str, str]) -> dict[str, str]: + return { + "semantic_family": cell["semantic_family"], + "attest": cell["attest"], + "register": cell["register"], + "function": cell["function"], + "decision": cell["decision"], + } + + +def parse_sheet( + path: str | Path, + stream: Mapping[str, Mapping[str, Any]], + *, + operator: str, + provenance: str, + settled_at: str, + activated: set[str] | None = None, + prior_ids: set[str] | None = None, +) -> dict[str, Any]: + """Validate a private lane sheet. Does not write the sheet or fill decisions.""" + lines = Path(path).read_text(encoding="utf-8").splitlines() + if not lines: + refuse("settlement sheet is empty") + header = tuple(lines[0].split("\t")) + if header != SHEET_COLUMNS: + refuse("sheet header does not match the operator lane contract") + records = [] + unset = 0 + lane_rows: dict[str, int] = {} + seen: set[str] = set() + errors: list[str] = [] + for line in lines[1:]: + if not line.strip(): + continue + parts = line.split("\t") + if len(parts) != len(header): + errors.append("REFUSE: sheet row is not rectangular") + continue + row_id = parts[1].strip() + stream_row = stream.get(row_id) + if stream_row is None: + errors.append(f"REFUSE: row_id is not in the held-out stream: {row_id}") + continue + if row_id in seen or row_id in set(prior_ids or ()): + errors.append(f"REFUSE: duplicate settlement for {row_id}") + continue + seen.add(row_id) + try: + digest = _check_sheet_integrity(parts, header, stream_row) + cell = dict(zip(header, parts)) + lane = cell["lane"].strip() + lane_rows[lane] = lane_rows.get(lane, 0) + 1 + chosen = _operator_cells(cell) + if all(_blank(value) for value in chosen.values()): + unset += 1 + continue + if _blank(chosen["decision"]): + refuse(f"decision is empty for {row_id}; refusing to infer one") + record = validate_settlement( + { + "row_id": row_id, + "text_hash": digest, + "decision": chosen["decision"], + "semantic_family": chosen["semantic_family"], + "attest": chosen["attest"], + "register": chosen["register"], + "function": chosen["function"], + "source_hint": evidence_hint(stream_row), + "proposed_family": cell["proposed_family_evidence"], + "operator": operator, + "settled_at": settled_at, + "provenance": provenance, + "rights": cell["rights"], + "lane": lane, + }, + stream_row, + activated=activated, + prior_ids=prior_ids, + ) + records.append(record) + except SystemExit as exc: + errors.append(str(exc)) + if errors: + refuse("; ".join(message.removeprefix("REFUSE: ") for message in errors)) + return {"records": records, "unset_row_count": unset, "lane_rows": lane_rows, "row_ids": seen} + + +def load_record_file( + path: str | Path, + stream: Mapping[str, Mapping[str, Any]], + *, + operator: str, + provenance: str, + settled_at: str, + activated: set[str] | None = None, + prior_ids: set[str] | None = None, +) -> list[dict[str, Any]]: + records = [] + seen = set(prior_ids or ()) + with Path(path).open(encoding="utf-8") as handle: + for line in handle: + if not line.strip(): + continue + payload = json.loads(line) + if not isinstance(payload, dict): + refuse("settlement record is not an object") + row_id = payload.get("row_id") + if not isinstance(row_id, str) or row_id not in stream: + refuse(f"row_id is not in the held-out stream: {row_id}") + if row_id in seen: + refuse(f"duplicate settlement for {row_id}") + payload = dict(payload) + payload.setdefault("operator", operator) + payload.setdefault("provenance", provenance) + payload.setdefault("settled_at", settled_at) + records.append( + validate_settlement( + payload, + stream[row_id], + activated=activated, + prior_ids=prior_ids, + ) + ) + seen.add(row_id) + return records + + +def _decision_counts(records: Sequence[Mapping[str, Any]]) -> dict[str, int]: + counts = {name: 0 for name in DECISIONS} + for record in records: + counts[str(record["decision"])] += 1 + return counts + + +def _family_counts(records: Sequence[Mapping[str, Any]]) -> dict[str, int]: + counts = {name: 0 for name in (*ACTIVE_FAMILIES, ABSTAIN)} + for record in records: + if record["decision"] not in SETTLED_DECISIONS: + continue + family = record.get("semantic_family") + if family in counts: + counts[str(family)] += 1 + return counts + + +def build_receipt( + records: Sequence[Mapping[str, Any]], + *, + batch_id: str, + operator: str, + provenance: str, + unset_row_count: int, + lane_rows: Mapping[str, int], + stream_run_id: str = "", + settled_at: str = "", + input_sheets: Sequence[Mapping[str, str]] | None = None, +) -> dict[str, Any]: + if not str(batch_id or "").strip(): + refuse("batch_id is required") + if not str(operator or "").strip() or not str(provenance or "").strip(): + refuse("operator and provenance are required") + settled = [record for record in records if record["decision"] in SETTLED_DECISIONS] + unresolved = [record for record in records if record["decision"] == "UNRESOLVED"] + body = { + "schema": RECEIPT_SCHEMA, + "batch_id": batch_id, + "operator": operator, + "provenance": provenance, + "settled_row_count": len(settled), + "unresolved_row_count": len(unresolved), + "unset_row_count": unset_row_count, + "decision_counts": _decision_counts(records), + "family_counts": _family_counts(records), + "observed_count": sum(1 for record in settled if record.get("attest") == "OBSERVED"), + "inferred_count": sum(1 for record in settled if record.get("attest") == "INFERRED"), + "rights_blocked_settled_count": sum( + 1 for record in settled if record.get("rights") == "RIGHTS_UNRESOLVED" + ), + "reserve_eligible_count": sum(1 for record in records if settlement_allows_reserve(record)), + "reserve_added": 0, + "ledger_mutated": False, + "vendor_calls": VENDOR_CALLS, + "evaluation_enabled": False, + "lane_rows": dict(lane_rows), + } + run_id = str(stream_run_id or "").strip() + if run_id: + body["stream_run_id"] = run_id + stamp = str(settled_at or "").strip() + if stamp: + body["settled_at"] = stamp + if input_sheets: + sheets = [] + for item in input_sheets: + identity = str(item.get("identity") or "").strip() + digest = str(item.get("sha256") or "").strip() + if not identity or len(digest) != 64: + refuse("input sheet identity is incomplete") + sheets.append({"identity": identity, "sha256": digest}) + body["input_sheets"] = sheets + _walk_forbid(body) + return seal_receipt(body) + + +def seal_receipt(body: Mapping[str, Any]) -> dict[str, Any]: + payload = {key: value for key, value in body.items() if key != "receipt_sha256"} + raw = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + sealed = dict(payload) + sealed["receipt_sha256"] = hashlib.sha256(raw).hexdigest() + return sealed + + +def receipt_sha_matches(receipt: Mapping[str, Any]) -> bool: + sealed = seal_receipt(receipt) + return sealed["receipt_sha256"] == receipt.get("receipt_sha256") + + +def _walk_forbid(value: Any) -> None: + if isinstance(value, dict): + for key, item in value.items(): + if key in _FORBIDDEN_KEYS: + refuse(f"receipt must not carry {key}") + _walk_forbid(item) + elif isinstance(value, list): + for item in value: + _walk_forbid(item) + + +def load_logged_ids(path: str | Path) -> set[str]: + log = Path(path) + if not log.exists(): + return set() + found = set() + for line in log.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + payload = json.loads(line) + if not isinstance(payload, dict): + refuse("settlement log row is not an object") + _forbid_payload(payload, "settlement log") + row_id = payload.get("row_id") + if not isinstance(row_id, str) or not row_id: + refuse("settlement log row is missing row_id") + if row_id in found: + refuse(f"settlement log already contains a duplicate for {row_id}") + found.add(row_id) + return found + + +def _write_exclusive(path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(text) + + +def commit_settlement( + records: Sequence[Mapping[str, Any]], + receipt: Mapping[str, Any], + *, + log_path: str | Path, + receipt_path: str | Path, +) -> dict[str, Any]: + """Append settlement rows, then write a new receipt. Never rewrites either.""" + receipt_file = Path(receipt_path) + if receipt_file.exists(): + refuse(f"receipt already exists: {receipt_file}") + log = Path(log_path) + prior = load_logged_ids(log) + fresh_ids = [] + for record in records: + _forbid_payload(record, "settlement record") + row_id = str(record["row_id"]) + if row_id in prior or row_id in fresh_ids: + refuse(f"duplicate settlement for {row_id}") + fresh_ids.append(row_id) + if not log.exists(): + _write_exclusive(log, "") + if records: + with log.open("a", encoding="utf-8") as handle: + for record in records: + handle.write(json.dumps(record, sort_keys=True) + "\n") + sealed = dict(receipt) + _walk_forbid(sealed) + _write_exclusive(receipt_file, json.dumps(sealed, indent=2, sort_keys=True) + "\n") + os.chmod(log, 0o600) + return sealed + + +def settle( + stream_rows: Sequence[Mapping[str, Any]], + decisions: Sequence[Mapping[str, Any]], + *, + batch_id: str, + operator: str, + provenance: str, + settled_at: str, + activated: set[str] | None = None, + prior_ids: set[str] | None = None, + unset_row_count: int = 0, + lane_rows: Mapping[str, int] | None = None, +) -> dict[str, Any]: + """Validate explicit decisions. Blank callers pass no decisions and settle nothing.""" + stream = _stream_index(stream_rows) + prior = set(prior_ids or ()) + records = [] + seen: set[str] = set() + for incoming in decisions: + row_id = incoming.get("row_id") + stream_row = stream.get(row_id) if isinstance(row_id, str) else None + if stream_row is None: + refuse(f"row_id is not in the held-out stream: {row_id}") + if row_id in seen: + refuse(f"duplicate settlement for {row_id}") + seen.add(str(row_id)) + payload = dict(incoming) + payload.setdefault("operator", operator) + payload.setdefault("provenance", provenance) + payload.setdefault("settled_at", settled_at) + records.append( + validate_settlement(payload, stream_row, activated=activated, prior_ids=prior) + ) + receipt = build_receipt( + records, + batch_id=batch_id, + operator=operator, + provenance=provenance, + unset_row_count=unset_row_count, + lane_rows=dict(lane_rows or {}), + ) + return {"records": records, "receipt": receipt} + + +def run_settlement_apply(args: Any) -> int: + """CLI entry. Reads operator cells. Does not choose them or touch the ledger.""" + import sys + + stream = load_stream(args.stream_rows) + activated = set(args.activated_family or []) + sheets = list(args.sheet or []) + records_path = str(args.records or "").strip() + if not sheets and not records_path: + refuse("no operator surface supplied") + prior = load_logged_ids(args.settlement_log) + records: list[dict[str, Any]] = [] + unset = 0 + lane_rows: dict[str, int] = {} + decided: set[str] = set() + surfaced: set[str] = set() + for sheet in sheets: + parsed = parse_sheet( + sheet, + stream, + operator=args.operator, + provenance=args.provenance, + settled_at=args.settled_at, + activated=activated, + prior_ids=prior | decided, + ) + for row_id in parsed["row_ids"]: + if row_id in surfaced: + refuse(f"duplicate settlement for {row_id}") + surfaced.add(row_id) + for record in parsed["records"]: + decided.add(record["row_id"]) + records.append(record) + unset += int(parsed["unset_row_count"]) + for lane, count in parsed["lane_rows"].items(): + lane_rows[lane] = lane_rows.get(lane, 0) + count + if records_path: + loaded = load_record_file( + records_path, + stream, + operator=args.operator, + provenance=args.provenance, + settled_at=args.settled_at, + activated=activated, + prior_ids=prior | decided, + ) + for record in loaded: + if record["row_id"] in decided or record["row_id"] in surfaced: + refuse(f"duplicate settlement for {record['row_id']}") + decided.add(record["row_id"]) + records.append(record) + sheet_identities = [] + for sheet in sheets: + raw = Path(sheet).read_bytes() + sheet_identities.append( + {"identity": Path(sheet).name, "sha256": hashlib.sha256(raw).hexdigest()} + ) + receipt = build_receipt( + records, + batch_id=args.batch_id, + operator=args.operator, + provenance=args.provenance, + unset_row_count=unset, + lane_rows=lane_rows, + stream_run_id=str(getattr(args, "stream_run_id", "") or ""), + settled_at=str(args.settled_at or ""), + input_sheets=sheet_identities, + ) + sealed = commit_settlement( + records, + receipt, + log_path=args.settlement_log, + receipt_path=args.receipt, + ) + sys.stdout.write(json.dumps(sealed, indent=2, sort_keys=True) + "\n") + return 0 diff --git a/scripts/shadow/hyperlexical/eval_unbind.py b/scripts/shadow/hyperlexical/eval_unbind.py new file mode 100644 index 00000000..b573aa10 --- /dev/null +++ b/scripts/shadow/hyperlexical/eval_unbind.py @@ -0,0 +1,410 @@ +"""U3 eval harness. Stub/digest vs Spec 004 probe. Trunk-forward is opt-in. + +Stub/digest swap has no civilian filler lists, so unbind_token_f1 / +unbind_slot_f1 and the strict keys stay null. Trunk-forward scores the +same aligned filler lists as train val and fills those fields. Strict +keys use the raw lowercased filler; a ```` prediction is a miss. +``e2_train_overlap_count`` is filled on every report. name_gate is true +only for a trunk-forward eval of an approved pin (``name_gate.APPROVED_PINS``, A6). +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import os +import sys +from pathlib import Path + +from .name_gate import name_gate_for +from .unbind_metrics import null_unbind_secondary + +E2_SNAPSHOT_N = 48 +E2_LENGTH = 4 +E2_SEED = 7 +HARVEST_UNBIND_N = 24 + +HEAD_NAMES = ("heads.json", "model.safetensors", "heads.pt") +FORWARD_WEIGHT_NAMES = ("model.safetensors", "heads.pt") +DEFAULT_TRAIN_OUT = Path.home() / ".hyperlex" / "models" / "hyperlex-encoder-modernbert-base-seed" +DEFAULT_TRAIN_OUT_LIVE = Path.home() / ".hyperlex" / "models" / "hyperlex-encoder-modernbert-base-seed-live" +DEFAULT_TRUNK = Path.home() / ".hyperlex" / "models" / "trunks" / "ModernBERT-base" +TYPE_SLOT_TAGS = ("TOKEN", "SLOT", "MARKER") + + +class TrunkForwardError(RuntimeError): + """Fail-closed trunk-forward E2. Not a stub miss.""" + + +def _shadow() -> Path: + return Path(__file__).resolve().parents[1] + + +def stub_swap(spans: list[dict]) -> float: + """Chance-like filler guess from text hash. Not Hyperlexical.""" + hit = 0 + for sp in spans: + items = list(sp["item_ids"]) + guess = [] + for i, it in enumerate(items): + digest = hashlib.sha256(f"stub-unbind:{i}:{it}".encode()).digest() + guess.append(items[digest[0] % len(items)]) + if guess == items: + hit += 1 + return hit / max(1, len(spans)) + + +def model_swap(spans: list[dict], seed: bytes) -> float: + """Permutation guess seeded by loaded head bytes. Not a trunk forward.""" + hit = 0 + for sp in spans: + items = list(sp["item_ids"]) + guess = [] + for i, it in enumerate(items): + digest = hashlib.sha256(seed + f":unbind:{i}:{it}".encode()).digest() + guess.append(items[digest[0] % len(items)]) + if guess == items: + hit += 1 + return hit / max(1, len(spans)) + + +def spans_to_unbind_rows(spans: list[dict]) -> list[dict]: + """Export-shaped 004 rows: positional + type_slot only.""" + rows = [] + for sp in spans: + items = list(sp["item_ids"]) + tags = list(sp.get("type_tags") or []) + if len(tags) != len(items): + tags = [TYPE_SLOT_TAGS[k % len(TYPE_SLOT_TAGS)] for k in range(len(items))] + rows.append( + { + "text": " ".join(items), + "fillers": items, + "roles": [f"pos_{k}" for k in range(len(items))], + "role_scheme": "positional", + } + ) + rows.append( + { + "text": " ".join(f"{t}:{it}" for t, it in zip(tags, items)), + "fillers": items, + "roles": tags, + "role_scheme": "type_slot", + } + ) + return rows + + +def want_trunk_forward(cli_flag: bool = False) -> bool: + return bool(cli_flag) or os.environ.get("HYPERLEX_E2_TRUNK_FORWARD") == "1" + + +def resolve_trunk_dir() -> Path: + env = os.environ.get("HYPERLEX_TRUNK_DIR", "").strip() + return Path(env) if env else DEFAULT_TRUNK + + +def trunk_ready(trunk: Path) -> bool: + return trunk.is_dir() and (trunk / "config.json").is_file() + + +def torch_importable() -> bool: + try: + return importlib.util.find_spec("torch") is not None + except (ImportError, ValueError, ModuleNotFoundError): + return False + + +def resolve_model_dir(explicit: str | Path | None = None) -> Path | None: + if explicit not in (None, ""): + return Path(explicit) + env = os.environ.get("HYPERLEX_TRAIN_OUT", "").strip() + if env: + return Path(env) + for candidate in (DEFAULT_TRAIN_OUT_LIVE, DEFAULT_TRAIN_OUT): + if candidate.is_dir(): + return candidate + return None + + +def find_weight_file(model_dir: Path) -> Path | None: + for name in HEAD_NAMES: + path = model_dir / name + if path.is_file() and path.stat().st_size > 0: + return path + return None + + +def find_forward_weight_file(model_dir: Path) -> Path | None: + for name in FORWARD_WEIGHT_NAMES: + path = model_dir / name + if path.is_file() and path.stat().st_size > 0: + return path + return None + + +def load_heads_meta(weight_path: Path) -> dict: + """Torch-free load. JSON fixture or sibling layout/config + file digest.""" + model_dir = weight_path.parent + meta = { + "weight_file": weight_path.name, + "model_id": "hyperlex-encoder-modernbert-base-seed", + "digest": hashlib.sha256(weight_path.read_bytes()).hexdigest(), + } + if weight_path.suffix == ".json": + blob = json.loads(weight_path.read_text(encoding="utf-8")) + if not isinstance(blob, dict): + raise ValueError("heads.json must be an object") + if blob.get("model_id"): + meta["model_id"] = str(blob["model_id"]) + if blob.get("weight_digest"): + meta["digest"] = str(blob["weight_digest"]) + return meta + for name in ("config.json", "layout.json", "train-receipt.json"): + side = model_dir / name + if not side.is_file(): + continue + try: + side_blob = json.loads(side.read_text(encoding="utf-8")) + except json.JSONDecodeError: + continue + if isinstance(side_blob, dict) and side_blob.get("model_id"): + meta["model_id"] = str(side_blob["model_id"]) + break + return meta + + +def filler_tuple(span: dict) -> tuple[str, ...]: + """Lowercased filler identity. Spans use ``item_ids``; rows use ``fillers``.""" + raw = span.get("item_ids") + if raw is None: + raw = span.get("fillers") or () + return tuple(str(item).lower() for item in raw) + + +def e2_test_spans() -> list[dict]: + """12 Spec 004 TPR test spans. Same split as ``_probe_report`` (n=48, seed=7).""" + sys.path.insert(0, str(_shadow())) + from recoverable_structure.fit import _split + from recoverable_structure.fixtures import make_spans + + spans = make_spans(n=E2_SNAPSHOT_N, length=E2_LENGTH, seed=E2_SEED) + _train_idx, test_idx, _split_hash = _split(len(spans), seed=E2_SEED) + return [spans[i] for i in test_idx] + + +def e2_test_filler_tuples() -> set[tuple[str, ...]]: + return {filler_tuple(span) for span in e2_test_spans()} + + +def e2_train_overlap_count(n: int = HARVEST_UNBIND_N) -> int: + """How many E2 test spans share a filler tuple with ``harvest_unbind``. + + Respects ``HLX_E2_DISJOINT`` because the harvest drops those spans when + the flag is on. Computed on every E2 report, flag or not. + """ + from .export import harvest_unbind_spans + + train = {filler_tuple(span) for _index, span in harvest_unbind_spans(n)} + return sum(1 for span in e2_test_spans() if filler_tuple(span) in train) + + +def _probe_report() -> tuple[dict, list[dict], float, float, float, float]: + sys.path.insert(0, str(_shadow())) + from recoverable_structure.fit import run_probe + from recoverable_structure.fixtures import snapshot + + snap = snapshot("tpr", n=E2_SNAPSHOT_N, length=E2_LENGTH, dim=12, seed=E2_SEED) + probe = run_probe(snap, schemes=("positional", "type_slot"), seed=E2_SEED) + rec = probe["receipt"] + pos = next(b for b in rec["schemes"] if b["scheme"] == "positional") + typ = next(b for b in rec["schemes"] if b["scheme"] == "type_slot") + test_spans = [snap["spans"][i] for i in probe["test_idx"]] + stub_acc = stub_swap(test_spans) + probe_acc = min(float(pos["swap_accuracy"]), float(typ["swap_accuracy"])) + return rec, test_spans, stub_acc, probe_acc, float(pos["swap_accuracy"]), float(typ["swap_accuracy"]) + + +def require_trunk_forward(model_dir: str | Path | None = None) -> tuple[Path, Path, Path]: + if not torch_importable(): + raise TrunkForwardError("trunk-forward requested but torch is not importable") + trunk = resolve_trunk_dir() + if not trunk_ready(trunk): + raise TrunkForwardError( + f"trunk-forward requested but trunk is missing or has no config.json: {trunk}" + ) + directory = resolve_model_dir(model_dir) + if directory is None: + raise TrunkForwardError( + "trunk-forward requested but no train out dir " + "(--model-dir / HYPERLEX_TRAIN_OUT / " + "~/.hyperlex/models/hyperlex-encoder-modernbert-base-seed-live|seed)" + ) + weight = find_forward_weight_file(directory) + if weight is None: + raise TrunkForwardError( + f"trunk-forward requested but no model.safetensors or heads.pt in {directory}" + ) + return trunk, directory, weight + + +def _base_report(rec: dict, stub_acc: float, probe_acc: float, pos_acc: float, typ_acc: float, n_test: int) -> dict: + return { + "schema": "hyperlex.hyperlexical.eval_unbind.v0.1", + "probe_schema": rec.get("schema"), + "probe_positional_swap": pos_acc, + "probe_type_slot_swap": typ_acc, + "probe_swap_min": probe_acc, + "stub_swap": stub_acc, + "n_test": n_test, + "trunk": "answerdotai/ModernBERT-base", + "trunk_loaded": False, + "brier": None, + "forecast_eligible": False, + "e2_train_overlap_count": e2_train_overlap_count(), + **null_unbind_secondary(), + } + + +def _digest_or_stub(model_dir: str | Path | None, rec, test_spans, stub_acc, probe_acc, pos_acc, typ_acc) -> dict: + directory = resolve_model_dir(model_dir) + weight = find_weight_file(directory) if directory is not None else None + base = _base_report(rec, stub_acc, probe_acc, pos_acc, typ_acc, len(test_spans)) + if weight is None: + base.update( + { + "e2_pass": stub_acc > probe_acc, + "model_id": "stub", + "heads_loaded": False, + "weight_file": None, + "model_swap": None, + "note": ( + "E2 requires a trained T1 to beat the 004 probe. Stub is " + "expected to fail. unbind_token_f1/unbind_slot_f1 are null " + "(004 probe swap has no civilian filler lists)." + ), + } + ) + return base + meta = load_heads_meta(weight) + model_acc = model_swap(test_spans, meta["digest"].encode("utf-8")) + base.update( + { + "e2_pass": model_acc > probe_acc, + "model_id": meta["model_id"], + "heads_loaded": True, + "weight_file": meta["weight_file"], + "model_swap": model_acc, + "note": ( + f"Loaded heads from {weight}. E2 still requires beating the 004 probe. " + "unbind_token_f1/unbind_slot_f1 stay null (digest path has no " + "civilian filler lists)." + ), + } + ) + return base + + +def run_eval(model_dir: str | Path | None = None, trunk_forward: bool = False) -> dict: + if want_trunk_forward(trunk_forward): + trunk, directory, weight = require_trunk_forward(model_dir) + rec, test_spans, stub_acc, probe_acc, pos_acc, typ_acc = _probe_report() + from .eval_forward import run_unbind_exact + + scored = run_unbind_exact( + trunk_dir=trunk, + model_dir=directory, + weight_path=weight, + spans=test_spans, + ) + model_acc = float(scored["unbind_exact"]) + meta = load_heads_meta(weight) + report = _base_report(rec, stub_acc, probe_acc, pos_acc, typ_acc, len(test_spans)) + report.update( + { + "e2_pass": model_acc > probe_acc, + "model_id": meta["model_id"], + "heads_loaded": True, + "weight_file": meta["weight_file"], + "model_swap": model_acc, + "unbind_exact": model_acc, + "n_unbind_eval": scored["n_unbind_eval"], + "unbind_token_f1": scored.get("unbind_token_f1"), + "unbind_token_precision": scored.get("unbind_token_precision"), + "unbind_token_recall": scored.get("unbind_token_recall"), + "unbind_slot_f1": scored.get("unbind_slot_f1"), + "unbind_exact_strict": scored.get("unbind_exact_strict"), + "unbind_token_f1_strict": scored.get("unbind_token_f1_strict"), + "unbind_token_precision_strict": scored.get("unbind_token_precision_strict"), + "unbind_token_recall_strict": scored.get("unbind_token_recall_strict"), + "unbind_slot_f1_strict": scored.get("unbind_slot_f1_strict"), + "trunk_loaded": True, + "trunk_forward": True, + "trunk_dir": str(trunk), + "model_dir": str(directory), + "name_gate": name_gate_for(directory), + "device": scored.get("device"), + "encoder_trainable_loaded": scored.get("encoder_trainable_loaded", 0), + "encoder_trainable_present": scored.get("encoder_trainable_present", 0), + "note": ( + f"Trunk-forward unbind_exact + token/slot F1 from {weight} vs 004 " + "probe_swap_min. Strict keys (unbind_exact_strict, strict token/slot " + "F1) sit beside the legacy numbers; gold is the raw lowercased filler " + "and a prediction is a miss. Stub/digest leaves F1 null (no " + "civilian filler lists). name_gate is true only for an approved pin (A6). " + + scored.get( + "encoder_note", + "Encoder is the local trunk snapshot; heads from train out.", + ) + ), + } + ) + return report + rec, test_spans, stub_acc, probe_acc, pos_acc, typ_acc = _probe_report() + return _digest_or_stub(model_dir, rec, test_spans, stub_acc, probe_acc, pos_acc, typ_acc) + + +def main(argv=None) -> int: + p = argparse.ArgumentParser(prog="hyperlexical-eval-unbind") + p.add_argument("--out", default="") + p.add_argument( + "--model-dir", + default="", + help="train out dir with heads.json / heads.pt / model.safetensors", + ) + p.add_argument( + "--trunk-forward", + action="store_true", + help="load local ModernBERT + trained heads and score real unbind_exact (Spark)", + ) + args = p.parse_args(argv) + try: + report = run_eval(model_dir=args.model_dir or None, trunk_forward=args.trunk_forward) + except TrunkForwardError as exc: + print( + json.dumps( + { + "abort": True, + "error": str(exc), + "brier": None, + "e2_pass": False, + "trunk_loaded": False, + "name_gate": False, + }, + indent=2, + ), + file=sys.stderr, + ) + return 2 + text = json.dumps(report, indent=2, sort_keys=True) + if args.out: + Path(args.out).write_text(text + "\n", encoding="utf-8") + else: + print(text) + return 0 if report["e2_pass"] else 3 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/shadow/hyperlexical/export.py b/scripts/shadow/hyperlexical/export.py new file mode 100644 index 00000000..d9033399 --- /dev/null +++ b/scripts/shadow/hyperlexical/export.py @@ -0,0 +1,1558 @@ +"""U2 civilian exporter. No hyperlex import. No ~/.hyperlex copy.""" + +from __future__ import annotations + +import argparse +import ast +import hashlib +import json +import os +import sys +from pathlib import Path +from typing import Any, Mapping + +from .packet import RESTRICTED_MARKER, SCHEMES, sha256_hex +from ._negatives_data import NEGATIVES # ordinary prose; no slang +from .unbind_recipe import recipe_env_counts + +FAMILIES = ( + "betting-sharp", + "crypto-degen", + "ai-native", + "brainrot-aura", + "kinship-address", + "political-status", + "gaming-meta", + "workplace-corp", +) + +TYPOLOGY = { + "betting-sharp": ["status"], + "crypto-degen": ["status", "tribal"], + "ai-native": ["compression", "memory", "provenance", "context"], + "brainrot-aura": ["compression", "status"], + "kinship-address": ["tribal"], + "political-status": ["tribal", "irony_shield"], + "gaming-meta": ["status", "hook"], + "workplace-corp": ["camouflage"], +} + + +DIALECT = ( + "no cap fr", + "it's giving", + "locked in", + "crash out", + "left no crumbs", + "chat is this real", + "aura points", + "let him cook", +) + +ROW_KEYS = ( + "text", + "split", + "lineage", + "typology", + "stage", + "roles", + "fillers", + "role_scheme", + "task", + "provenance", + "class", + "license", +) + +COLLISION_HOLD = {"skill issue"} + +# Short slang / numeric codes that len≤2 or numeric filters would false-reject. +# Prefer explicit allowlist over blanket keep of all short/numeric tokens. +SHORT_SLANG_ALLOWLIST = frozenset( + { + "w", + "l", + "ez", + "gg", + "gm", + "gn", + "bs", + "a+", + "ai", + "ak", + "3p", + "ag", + "bf", + "bj", + "bk", + "bm", + "420", + "4/20", + "4:20", + "100", + "404", + "5150", + "10-4", + "304", + "143", + "007", + "411", + "730", + "10-20", + } +) + +# Spec 004 type_slot vocabulary (structural placeholders — not gloss-derived POS). +TYPE_SLOT_TAGS = ("TOKEN", "SLOT", "MARKER") + + +AI_NATIVE_TYPOLOGY = ("compression", "memory", "provenance", "context", "vernacular") + + +def ai_native_typology(typology: list[str]) -> list[str]: + """Order-stable dedupe. A set here made data_sha256 vary with PYTHONHASHSEED.""" + return list(dict.fromkeys([*typology, *AI_NATIVE_TYPOLOGY])) + + +def repo_root() -> Path: + return Path(__file__).resolve().parents[3] + + +def lexical_split(text: str) -> str: + """Frozen hash split. Do not change the hash, modulus, or bucket edges. + + Settle may add rows mid-experiment. A new text gets a bucket from *its* + hash only. Existing texts keep their split — val must not reshuffle. + """ + n = int(sha256_hex(text.lower())[:8], 16) % 10 + if n == 0: + return "test" + if n == 1: + return "val" + return "train" + + +def _norm_class(raw: str | None, default: str) -> str: + val = (raw or default).upper() + if val not in {"OBSERVED", "INFERRED", "SPECULATIVE"}: + return default + if val == "SPECULATIVE": + return "INFERRED" + return val + + +def _norm_role_scheme(raw: Any) -> str | None: + """Fail-closed: recoverable_structure allows positional|type_slot only. + + Dump / harvest leftovers such as ``civilian`` are not a third scheme. + Classify rows with an unknown label drop to None (no unbind gold). + """ + if raw in SCHEMES: + return str(raw) + return None + + +def _row(**kwargs: Any) -> dict[str, Any]: + text = kwargs["text"] + if RESTRICTED_MARKER in text: + raise ValueError("restricted text") + if text.lower() in COLLISION_HOLD and kwargs.get("task") == "classify": + kwargs = dict(kwargs) + kwargs["lineage"] = "none" + kwargs["class"] = "INFERRED" + kwargs["provenance"] = str(kwargs.get("provenance") or "") + ":collision-hold" + out = {k: kwargs.get(k) for k in ROW_KEYS} + # Spec 007 lexical split is train/val/test only. Reject store contamination + # (e.g. blanket-yes wrote split="live") so --include-live cannot bypass the hash split. + split = kwargs.get("split") + if split not in {"train", "val", "test"}: + split = lexical_split(text) + out["split"] = split + out["typology"] = list(out.get("typology") or []) + out["roles"] = list(out.get("roles") or []) + out["fillers"] = list(out.get("fillers") or []) + out["license"] = out.get("license") or "MIT-examples" + out["stage"] = out.get("stage") or "circulating" + out["role_scheme"] = _norm_role_scheme(out.get("role_scheme")) + return out + + +def load_registry(root: Path) -> list[dict[str, Any]]: + path = root / "src" / "hyperlex" / "analysis" / "__init__.py" + tree = ast.parse(path.read_text(encoding="utf-8")) + for node in tree.body: + if isinstance(node, ast.Assign): + targets = node.targets + elif isinstance(node, ast.AnnAssign): + targets = [node.target] + else: + continue + for target in targets: + if ( + isinstance(target, ast.Name) + and target.id == "LINEAGE_REGISTRY" + and node.value is not None + ): + return ast.literal_eval(node.value) + raise RuntimeError("LINEAGE_REGISTRY missing") + + +def harvest_registry(root: Path) -> list[dict[str, Any]]: + rows = [] + for entry in load_registry(root): + fam = entry["family_id"] + if fam not in FAMILIES: + continue + for term in entry.get("terms") or []: + rows.append( + _row( + text=term, + lineage=fam, + typology=TYPOLOGY.get(fam, []), + task="classify", + provenance=f"LINEAGE_REGISTRY:{fam}", + **{"class": "INFERRED"}, + role_scheme=None, + ) + ) + return rows + + +def harvest_receipts(root: Path) -> list[dict[str, Any]]: + rows = [] + gold = root / "examples" / "receipts" / "golden" + if not gold.is_dir(): + return rows + for path in sorted(gold.glob("*.json")): + if path.name == "MANIFEST.json": + continue + data = json.loads(path.read_text(encoding="utf-8")) + lineage = (data.get("analysis") or {}).get("lineage") or {} + fam = lineage.get("family_id") or "none" + if fam not in FAMILIES and fam != "none": + fam = "none" + terms = list(lineage.get("matched_terms") or []) + query = ((data.get("ingest") or {}).get("query") or "").strip() + if query: + terms.append(query) + seen = set() + for term in terms: + if not term or term in seen: + continue + seen.add(term) + rows.append( + _row( + text=term, + lineage=fam, + typology=TYPOLOGY.get(fam, []), + task="classify", + provenance=f"golden:{path.name}", + **{"class": "INFERRED"}, + role_scheme=None, + ) + ) + return rows + + +def harvest_backfill(root: Path) -> list[dict[str, Any]]: + rows = [] + pack_dir = root / "data" / "backfill" / "2026" + if not pack_dir.is_dir(): + return rows + for path in sorted(pack_dir.glob("2026-*.json")): + data = json.loads(path.read_text(encoding="utf-8")) + default = data.get("provenance_default") or "INFERRED" + for item in data.get("terms") or []: + term = (item.get("term") or "").strip() + if not term: + continue + fam = item.get("family_id") or "none" + if fam not in FAMILIES and fam != "none": + fam = "none" + rows.append( + _row( + text=term, + lineage=fam, + typology=TYPOLOGY.get(fam, []), + task="classify", + provenance=f"backfill:{path.name}", + **{"class": _norm_class(item.get("provenance"), default)}, + role_scheme=None, + ) + ) + return rows + + +def harvest_archive(root: Path) -> list[dict[str, Any]]: + rows = [] + archive = root / "docs" / "archive" + if not archive.is_dir(): + return rows + for path in sorted(archive.glob("**/receipts/*.json")): + try: + data = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + continue + lineage = (data.get("analysis") or {}).get("lineage") or {} + fam = lineage.get("family_id") or data.get("lineage_family") or "none" + if fam not in FAMILIES and fam != "none": + fam = "none" + terms = list(lineage.get("matched_terms") or []) + query = ((data.get("ingest") or {}).get("query") or "").strip() + if query: + terms.append(query) + seen = set() + for term in terms: + if not term or term in seen: + continue + seen.add(term) + rows.append( + _row( + text=term, + lineage=fam, + typology=TYPOLOGY.get(fam, []), + task="classify", + provenance=f"archive:{path.relative_to(root).as_posix()}", + **{"class": "INFERRED"}, + role_scheme=None, + ) + ) + return rows + + +def e2_disjoint_enabled() -> bool: + """Drop harvest spans whose filler tuple is in the E2 test set. + + Default off so existing exports stay byte-identical. Set ``HLX_E2_DISJOINT=1`` + to exclude them. Reporting of the leak (``e2_train_overlap_count``) is separate + and always on. + """ + return os.environ.get("HLX_E2_DISJOINT") == "1" + + +def harvest_unbind_spans(n: int = 24) -> list[tuple[int, dict[str, Any]]]: + """``(original index, span)`` from ``make_spans(n, length=4, seed=7)``. + + With ``HLX_E2_DISJOINT=1``, spans whose filler tuple appears in the E2 test + set are omitted. Original indices stay on the rows that remain. + """ + sys.path.insert(0, str(repo_root() / "scripts" / "shadow")) + from recoverable_structure.fixtures import make_spans + + indexed = list(enumerate(make_spans(n=n, length=4, seed=7))) + if e2_disjoint_enabled(): + from .eval_unbind import e2_test_filler_tuples, filler_tuple + + banned = e2_test_filler_tuples() + indexed = [(i, sp) for i, sp in indexed if filler_tuple(sp) not in banned] + return indexed + + +def harvest_unbind(n: int = 24) -> list[dict[str, Any]]: + """Spec 004 fixture gold under both schemes. Honest default n=24 (~45 unique). + + Fixture rows are provenance `004:tpr:*` only — not civilian name-gate gold. + ``HLX_E2_DISJOINT`` defaults off; set it to ``1`` to drop spans that share a + filler tuple with the E2 test set. + """ + rows = [] + for i, sp in harvest_unbind_spans(n): + items = list(sp["item_ids"]) + tags = list(sp["type_tags"]) + rows.append( + _row( + text=" ".join(items), + lineage="none", + typology=[], + stage="noise", + roles=[f"pos_{k}" for k in range(len(items))], + fillers=items, + role_scheme="positional", + task="unbind", + provenance=f"004:tpr:positional:{i}", + **{"class": "OBSERVED"}, + ) + ) + rows.append( + _row( + text=" ".join(f"{t}:{it}" for t, it in zip(tags, items)), + lineage="none", + typology=[], + stage="noise", + roles=tags, + fillers=items, + role_scheme="type_slot", + task="unbind", + provenance=f"004:tpr:type_slot:{i}", + **{"class": "OBSERVED"}, + ) + ) + return rows + + +def _structural_type_tags(n: int) -> list[str]: + """Assign Spec 004 TOKEN/SLOT/MARKER by index — not gloss/POS invention.""" + return [TYPE_SLOT_TAGS[i % len(TYPE_SLOT_TAGS)] for i in range(n)] + + +LIVE_UNBIND_MAX_LEN = 80 +LIVE_UNBIND_MAX_TOKENS = 6 +OBSERVED_MW_HARVEST_NAME = "harvest_unbind_observed_mw.jsonl" + + +def _unbind_dual_scheme_rows( + atom: str, + tokens: list[str], + *, + lineage: str, + stage: str, + epistemic: str, + pos_provenance: str, + type_provenance: str, + license: str | None = None, +) -> list[dict[str, Any]]: + """Emit positional + type_slot unbind rows. Fillers = real tokens only.""" + if lineage not in FAMILIES and lineage != "none": + lineage = "none" + extra: dict[str, Any] = {} + if license: + extra["license"] = license + pos = _row( + text=atom, + lineage=lineage, + typology=TYPOLOGY.get(lineage, []), + stage=stage, + roles=[f"pos_{k}" for k in range(len(tokens))], + fillers=tokens, + role_scheme="positional", + task="unbind", + provenance=pos_provenance, + **{"class": epistemic}, + **extra, + ) + tags = _structural_type_tags(len(tokens)) + typ = _row( + text=" ".join(f"{t}:{tok}" for t, tok in zip(tags, tokens)), + lineage=lineage, + typology=TYPOLOGY.get(lineage, []), + stage=stage, + roles=tags, + fillers=tokens, + role_scheme="type_slot", + task="unbind", + provenance=type_provenance, + **{"class": epistemic}, + **extra, + ) + return [pos, typ] + + +def _positional_unbind_atoms(rows: list[dict[str, Any]]) -> set[str]: + out: set[str] = set() + for row in rows: + if row.get("task") != "unbind" or row.get("role_scheme") != "positional": + continue + text = str(row.get("text") or "").strip() + if text: + out.add(text.lower()) + return out + + +def _phrase_like_atom(text: str) -> tuple[str, list[str]] | None: + """Return (stripped atom, tokens) for short multiword SoT phrases, else None.""" + atom = (text or "").strip() + if not atom or " " not in atom: + return None + if len(atom) > LIVE_UNBIND_MAX_LEN: + return None + if atom.lower() in COLLISION_HOLD: + return None + tokens = [t for t in atom.split() if t] + if len(tokens) < 2 or len(tokens) > LIVE_UNBIND_MAX_TOKENS: + return None + if reject_candidate_text(atom): + return None + return atom, tokens + + +def _live_unbind_epistemic(raw: dict[str, Any]) -> str: + """Copy store epistemic. Missing/None → INFERRED. Never invent OBSERVED.""" + for key in ("epistemic", "class"): + if key not in raw: + continue + val = raw.get(key) + if val is None or (isinstance(val, str) and not val.strip()): + continue + return _norm_class(str(val), "INFERRED") + return "INFERRED" + + +def _live_unbind_lineage(raw: dict[str, Any]) -> str: + for key in ("lineage", "family", "family_id", "lineage_family"): + val = raw.get(key) + if not val: + continue + fam = str(val).strip() + if fam in FAMILIES or fam == "none": + return fam + return "none" + + +def _default_held_unbind_atoms() -> set[str]: + """Civilian + fixture positional atoms. Fail-open if inventory cannot load.""" + try: + return _positional_unbind_atoms(harvest_unbind() + harvest_civilian_unbind(repo_root())) + except Exception: + return set() + + +def harvest_civilian_unbind(root: Path) -> list[dict[str, Any]]: + """Civilian unbind for multiword atoms under BOTH schemes. + + Fillers = real token atoms from golden/registry/dialect only. + type_slot roles = structural TOKEN/SLOT/MARKER (no gloss invent). + Collision-hold (`skill issue`) skipped on all paths (C37 / harvest card). + """ + rows: list[dict[str, Any]] = [] + seen_atom: set[str] = set() + + def _add(text: str, lineage: str, source_tag: str) -> None: + atom = text.strip() + if not atom or " " not in atom: + return + key = atom.lower() + if key in COLLISION_HOLD or key in seen_atom: + return + tokens = [t for t in atom.split() if t] + if len(tokens) < 2: + return + seen_atom.add(key) + rows.extend( + _unbind_dual_scheme_rows( + atom, + tokens, + lineage=lineage, + stage="circulating", + epistemic="OBSERVED", + pos_provenance=f"civilian-pos:{source_tag}", + type_provenance=f"civilian-type:{source_tag}", + ) + ) + + gold = root / "examples" / "receipts" / "golden" + if gold.is_dir(): + for path in sorted(gold.glob("*.json")): + if path.name == "MANIFEST.json": + continue + data = json.loads(path.read_text(encoding="utf-8")) + lineage = (data.get("analysis") or {}).get("lineage") or {} + fam = lineage.get("family_id") or "none" + for term in lineage.get("matched_terms") or []: + if isinstance(term, str) and " " in term.strip(): + _add(term.strip(), fam, f"golden:{path.name}") + + for entry in load_registry(root): + fam = entry.get("family_id") or "none" + for term in entry.get("terms") or []: + if isinstance(term, str) and " " in term.strip(): + _add(term.strip(), fam, f"registry:{fam}") + + for text in DIALECT: + if " " in text: + _add(text, "brainrot-aura", "seed:dialect-e6") + + return rows + + +def harvest_inferred_classify_pass(root: Path) -> list[dict[str, Any]]: + """Optional INFERRED classify rows from harvest classify-pass artifact. + + Never upgrades class to OBSERVED. Missing file → empty list. + """ + path = root / "specs" / "007-hyperlexical-model" / "harvest" / "inferred_classify_pass.jsonl" + if not path.is_file(): + return [] + rows: list[dict[str, Any]] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line: + continue + try: + raw = json.loads(line) + except json.JSONDecodeError: + continue + text = str(raw.get("text") or "").strip() + if not text: + continue + if reject_candidate_text(text): + continue + fam = raw.get("lineage") or "none" + if fam not in FAMILIES and fam != "none": + fam = "none" + if text.lower() in COLLISION_HOLD: + fam = "none" + try: + rows.append( + _row( + text=text, + lineage=fam, + typology=list(raw.get("typology") or TYPOLOGY.get(fam, [])), + stage=raw.get("stage") or "circulating", + task="classify", + provenance=str(raw.get("provenance") or "harvest:classify-pass"), + **{"class": "INFERRED"}, + role_scheme=None, + license=str(raw.get("license") or "operator-local; labels INFERRED"), + split=raw.get("split"), + ) + ) + except ValueError: + continue + return rows + + +def harvest_dialect() -> list[dict[str, Any]]: + return [ + _row( + text=text, + lineage="brainrot-aura", + typology=["compression"], + task="classify", + provenance="seed:dialect-e6", + **{"class": "OBSERVED"}, + role_scheme=None, + ) + for text in DIALECT + ] + + +def harvest_negatives() -> list[dict[str, Any]]: + return [ + _row( + text=text, + lineage="none", + typology=[], + stage="noise", + task="classify", + provenance="seed:negative-prose", + **{"class": "OBSERVED"}, + role_scheme=None, + ) + for text in NEGATIVES + ] + + +# Defaults the Moltbook adapters used to invent when the detector returned +# nothing. ``general`` is the unconditional fallback in +# moltbook_to_hyperlexical and create_high_signal_subset. The high-signal +# script's other fallback, ``KDR``, is not listed here: it is rejected by the +# surface-token check unless ``kdr`` is itself a token (see +# ``original_gold_unbind_verdict``). Do not add category names. A label that +# is a real surface token is original gold and may stay. +FALLBACK_PLACEHOLDER_LABELS = frozenset({"general"}) + + +def _gold_labels(fillers: Any) -> list[str]: + """Non-empty filler strings. A bare string is one label, not characters.""" + if fillers is None: + return [] + if isinstance(fillers, str): + fillers = [fillers] + labels: list[str] = [] + for item in fillers: + if item is None: + continue + text = str(item).strip() + if text: + labels.append(text) + return labels + + +def original_gold_unbind_verdict(text: str, fillers: Any) -> tuple[bool, str | None]: + """Decide whether original fillers may be unbind gold. + + Returns ``(True, None)`` only when every label is a real surface token + and none is a fallback placeholder. Otherwise ``(False, reason)``. + + Token membership reuses ``selection_surface.lenient_copy_hit`` (same rule + as ``score_holdout.copy_baseline``): whitespace split, then the substring + after the first colon, then ``.lower()``. It is not a substring search + and it does not strip punctuation. ``kdr`` inside ``kdrama`` misses. + ``KDR.`` is the token ``kdr.`` and does not match filler ``KDR``. + + Empty gold returns ``(False, None)``. ``harvest_moltbook`` records that + case as ``gold_demote_reason=no_gold`` and does not emit it as unbind. + The label string is never rewritten. + """ + labels = _gold_labels(fillers) + if not labels: + return False, None + if any(label.lower() in FALLBACK_PLACEHOLDER_LABELS for label in labels): + return False, "fallback_label" + # Lazy import: selection_surface imports TYPE_SLOT_TAGS from this module. + from .selection_surface import lenient_copy_hit + + if not lenient_copy_hit({"text": text, "fillers": labels}): + return False, "gold_not_in_text" + return True, None + + +def provenance_source_tag(row: Mapping[str, Any]) -> str: + prov = row.get("provenance") + if isinstance(prov, dict): + return str(prov.get("source") or "unknown") + return str(prov or "unknown") + + +def summarize_gold_demotions(rows: list[dict[str, Any]]) -> dict[str, dict[str, int]]: + """Pre-dedupe counts of demoted rows, keyed by provenance source then reason.""" + counts: dict[str, dict[str, int]] = {} + for row in rows: + reason = row.get("gold_demote_reason") + if not reason: + continue + source = provenance_source_tag(row) + bucket = counts.setdefault(source, {}) + bucket[str(reason)] = bucket.get(str(reason), 0) + 1 + return {src: dict(sorted(reasons.items())) for src, reasons in sorted(counts.items())} + + +def gold_demoted_after_dedupe( + rows: list[dict[str, Any]], +) -> tuple[dict[str, dict[str, int]], dict[str, int]]: + """Train/val demotion counts after dedupe. + + ``drop_test_rows`` runs before a demotion reason or provenance is read. + ``split=test`` rows are not copied and do not enter either count. + """ + from .selection_surface import drop_test_rows + + kept, _discarded = drop_test_rows(rows) + train_val = [ + row + for row in kept + if row.get("split") in {"train", "val"} and row.get("gold_demote_reason") + ] + by_split = { + "train": sum(row.get("split") == "train" for row in train_val), + "val": sum(row.get("split") == "val" for row in train_val), + } + return summarize_gold_demotions(train_val), by_split + + +def audit_seed_examples_gold(root: Path) -> dict[str, int]: + """Read-only verdict counts for ``seed_examples.jsonl``. Emits no rows. + + Original gold is ``labels.context_loss``. Rows with no such label are + ignored. Counts are ``fallback_label``, ``gold_not_in_text``, and ``kept``. + This does not relabel the file and does not enter the export. + """ + counts = {"fallback_label": 0, "gold_not_in_text": 0, "kept": 0} + path = root / "data" / "agent_memetics" / "seed_examples.jsonl" + if not path.is_file(): + return counts + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + try: + rec = json.loads(line) + except json.JSONDecodeError: + continue + labels = rec.get("labels") if isinstance(rec.get("labels"), dict) else {} + raw = labels.get("context_loss") + if not isinstance(raw, str) or not raw.strip(): + continue + keep, reason = original_gold_unbind_verdict(str(rec.get("text") or ""), [raw]) + if keep: + counts["kept"] += 1 + elif reason in counts: + counts[reason] += 1 + return counts + + +def harvest_moltbook(root: Path, *, stats: dict[str, int] | None = None) -> list[dict[str, Any]]: + """Moltbook agent discourse → ai-native rows for hyperlexical training. + + Uses pre-classified rows from scripts/moltbook_to_hyperlexical.py + (memory tiers, efficiency, provenance, context loss). + Also loads the high-signal subset when present, including + ``moltbook-curated-seed`` rows written by ``create_high_signal_subset``. + ``seed_examples.jsonl`` itself is not harvested; see + ``audit_seed_examples_gold``. + + classify+unbind is emitted only when every original filler is a surface + token and not a fallback placeholder. Otherwise the row is classify-only: + no fillers, no roles, no role scheme, ``class`` INFERRED, and + ``gold_demote_reason`` set. Empty gold (no filler strings) is tagged + ``no_gold``. ``original_gold_unbind_verdict`` still returns ``(False, None)`` + for that case; this harvest records the reason. Labels are not replaced + and nothing is promoted to OBSERVED. The text is unchanged. + + A line that is not JSON increments ``stats["moltbook_skipped_lines"]`` + when ``stats`` is passed. Any other error propagates. + """ + rows = [] + skipped = 0 + for p in [ + root / "data" / "moltbook_hyperlexical_rows.jsonl", + root / "data" / "moltbook_hyperlexical_high.jsonl", + root / "data" / "moltbook_hyperlexical_high_signal.jsonl", + ]: + if not p.exists(): + continue + for line in p.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + try: + r = json.loads(line) + except json.JSONDecodeError: + skipped += 1 + continue + if not isinstance(r, dict): + raise ValueError(f"{p}: line is JSON but not an object") + if r.get("lineage") != "ai-native": + continue + text = r.get("text", "") + fillers = r.get("fillers", []) + keep, reason = original_gold_unbind_verdict(text, fillers) + if keep: + row = _row( + text=text, + lineage="ai-native", + typology=r.get("typology", ["compression"]), + stage=r.get("stage", "circulating"), + roles=r.get("roles", []), + fillers=fillers, + role_scheme=r.get("role_scheme"), + task="classify+unbind", + provenance=r.get("provenance", {"source": "moltbook"}), + **{"class": r.get("class", "INFERRED")}, + license=r.get("license", "MIT (distilled)"), + ) + else: + row = _row( + text=text, + lineage="ai-native", + typology=r.get("typology", ["compression"]), + stage=r.get("stage", "circulating"), + roles=[], + fillers=[], + role_scheme=None, + task="classify", + provenance=r.get("provenance", {"source": "moltbook"}), + **{"class": "INFERRED"}, + license=r.get("license", "MIT (distilled)"), + ) + # None means empty gold. Do not invent a filler in its place. + row["gold_demote_reason"] = reason or "no_gold" + rows.append(row) + if stats is not None: + stats["moltbook_skipped_lines"] = skipped + return rows + + +def dedupe(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Dedupe by (task, text, role_scheme, lineage). + + First-seen order is preserved, but a later row with a stronger ``class`` + upgrades the kept row (OBSERVED > INFERRED > other). This lets + ``--include-live`` settled OBSERVED replace an earlier base INFERRED + duplicate without inventing new OBSERVED labels. + """ + rank = {"OBSERVED": 2, "INFERRED": 1, "SPECULATIVE": 0} + best: dict[tuple, dict[str, Any]] = {} + order: list[tuple] = [] + for row in rows: + key = (row["task"], row["text"], row.get("role_scheme"), row["lineage"]) + if key not in best: + best[key] = row + order.append(key) + continue + prev = best[key] + if rank.get(str(row.get("class")), 0) > rank.get(str(prev.get("class")), 0): + best[key] = row + return [best[k] for k in order] + + +def _strip_type_slot_tags(text: str) -> str: + """Recover underlying phrase from ``TOKEN:x SLOT:y`` type_slot display text.""" + parts: list[str] = [] + for tok in (text or "").split(): + if ":" in tok and tok.split(":", 1)[0] in TYPE_SLOT_TAGS: + parts.append(tok.split(":", 1)[1]) + else: + parts.append(tok) + return " ".join(parts) + + +def reject_wiki_scaffolding_text(text: str) -> str | None: + """Return reject reason for wiki/dictionary chrome, else None. + + Keeps civilian slang atoms; drops etymology / quotations / synonym-table / + language-gloss / Trends / declension scaffolding that walls unbind val. + Also rejects type_slot-tagged forms of the same chrome + (``TOKEN:Alternative SLOT:form …``), which otherwise miss contiguous + substring checks. Does not invent or settle OBSERVED. + """ + raw = (text or "").strip() + if not raw: + return None + candidates = [raw, _strip_type_slot_tags(raw)] + for cand in candidates: + low = cand.lower() + # Dictionary / wiktionary chrome (substring, case-insensitive). + for needle, reason in ( + ("etymology", "wiki_etymology"), + ("google trends", "wiki_trends"), + ("alternative form of", "wiki_alt_form"), + ("alternative letter-case form of", "wiki_alt_form"), + ("declension of", "wiki_declension"), + ("wiktionary", "wiki_wiktionary"), + ("quotations ▼", "wiki_quotations"), + ("▲quotations", "wiki_quotations"), + ("synonym ▲", "wiki_synonym_table"), + ("antonym ▲", "wiki_antonym_table"), + ("synonym:", "wiki_synonym_table"), + ("antonym:", "wiki_antonym_table"), + ("(neologism)", "wiki_neologism_gloss"), + ("armenian:", "wiki_lang_gloss"), + ("show ▼", "wiki_declension"), + ): + if needle in low: + return reason + # Language-label gloss dumps: "Armenian: …" already covered; bare ▼/▲ UI chrome + # only when paired with dictionary lemmata (handled above) or lone UI tokens. + if cand in {"▼", "▲", "quotations ▼", "▲quotations"}: + return "wiki_ui_chrome" + return None + + +def reject_candidate_text(text: str) -> str | None: + """Return reject reason for junk live candidates, else None. + + Filters: empty/punct-only, len≤2, pure numeric, Unsupported titles, + wiki/dictionary scaffolding chrome. + SHORT_SLANG_ALLOWLIST exempts known slang/codes from len/numeric kills. + Does not promote or settle labels. + """ + raw = (text or "").strip() + if not raw: + return "empty" + low = raw.lower() + if low in SHORT_SLANG_ALLOWLIST: + return None + alnum = "".join(ch for ch in raw if ch.isalnum()) + if not alnum: + return "punct_only" + if alnum.isdigit(): + return "numeric" + # numeric-ish codes with separators (4/20, 10-4) — still reject unless allowlisted + if all(ch.isdigit() or ch in "-/:." for ch in raw) and any(ch.isdigit() for ch in raw): + return "numeric" + if len(raw) <= 2: + return "len_le_2" + if "unsupported title" in low or low.startswith("unsupported"): + return "unsupported_title" + scaff = reject_wiki_scaffolding_text(raw) + if scaff: + return scaff + return None + + +class LiveStoreMissing(FileNotFoundError): + """include-live was requested but the candidate store is not on disk.""" + + +def default_live_store() -> Path: + override = os.environ.get("HYPERLEX_LIVE_STORE", "").strip() + if override: + return Path(override) + return Path.home() / ".hyperlex" / "hyperlexical" / "ingest_candidates.jsonl" + + +def resolve_live_store(live_store: Path | None = None, *, required: bool = False) -> Path: + store = Path(live_store) if live_store is not None else default_live_store() + if required and not store.is_file(): + raise LiveStoreMissing( + f"include-live requested but live store missing: {store}. " + "Write ingest_candidates.jsonl or unset --include-live / HYPERLEX_INCLUDE_LIVE." + ) + return store + + +def load_live_candidates(store: Path) -> list[dict[str, Any]]: + """Load SHADOW ingest candidates for --include-live. + + Copies ``class`` from the candidate store (OBSERVED stays OBSERVED). + Unset / unknown / SPECULATIVE → INFERRED. Never invents OBSERVED. + """ + if not store.is_file(): + return [] + out: list[dict[str, Any]] = [] + for line in store.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line: + continue + try: + raw_row = json.loads(line) + except json.JSONDecodeError: + continue + text = str(raw_row.get("text") or "") + if reject_candidate_text(text): + continue + prov = str(raw_row.get("provenance") or "ingest:store") + # Preserve settled OBSERVED; default unset/live crawl to INFERRED. + cls = _norm_class(raw_row.get("class"), "INFERRED") + label_tag = f"labels {cls}" + if prov.startswith("ingest:inbox") or "wiktionary" in prov.lower(): + license_ = f"CC-BY-SA-4.0+GFDL (Wiktionary text); {label_tag}" + elif prov.startswith("ingest:pipeline"): + license_ = f"operator-local-crawl; {label_tag}" + else: + license_ = str(raw_row.get("license") or "operator-local") + f"; {label_tag}" + fam = raw_row.get("lineage") or "none" + if fam not in FAMILIES and fam not in {"none", "ytd_leaf"}: + fam = "none" + try: + out.append( + _row( + text=text, + split=raw_row.get("split"), + lineage=fam, + typology=list(raw_row.get("typology") or TYPOLOGY.get(fam, [])), + stage=raw_row.get("stage") or "circulating", + roles=list(raw_row.get("roles") or []), + fillers=list(raw_row.get("fillers") or []), + role_scheme=raw_row.get("role_scheme"), + task=raw_row.get("task") or "classify", + provenance=prov if prov.endswith(":live") else f"{prov}:live", + **{"class": cls}, + license=license_, + ) + ) + except ValueError: + continue + return out + + +def _iter_jsonl_dicts(path: Path) -> list[dict[str, Any]]: + if not path.is_file(): + return [] + out: list[dict[str, Any]] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line: + continue + try: + raw = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(raw, dict): + out.append(raw) + return out + + +def resolve_observed_mw_harvest( + live_store: Path, + explicit: Path | None = None, +) -> Path | None: + """Wave A OBSERVED sidecar next to the live store (Spark path). + + Sibling ``harvest_unbind_observed_mw.jsonl``, or ``HYPERLEX_LIVE_UNBIND_OBSERVED``. + Missing file → None. Filename does not invent OBSERVED labels. + """ + if explicit is not None: + path = Path(explicit) + return path if path.is_file() else None + env = os.environ.get("HYPERLEX_LIVE_UNBIND_OBSERVED", "").strip() + if env: + path = Path(env) + return path if path.is_file() else None + sibling = Path(live_store).expanduser().resolve().parent / OBSERVED_MW_HARVEST_NAME + return sibling if sibling.is_file() else None + + +def _atom_key_from_unbind_row(row: dict[str, Any]) -> str: + if row.get("role_scheme") == "positional": + return str(row.get("text") or "").strip().lower() + fillers = [str(t).strip() for t in (row.get("fillers") or []) if str(t).strip()] + return " ".join(fillers).lower() + + +def _formed_unbind_row(raw: dict[str, Any]) -> dict[str, Any] | None: + """Adopt an already-built unbind row. Never upgrades missing class to OBSERVED.""" + task = raw.get("task") + if task not in (None, "unbind"): + return None + scheme = _norm_role_scheme(raw.get("role_scheme")) + if scheme not in SCHEMES: + return None + text = str(raw.get("text") or "").strip() + fillers = [str(t) for t in (raw.get("fillers") or []) if str(t).strip()] + roles = [str(t) for t in (raw.get("roles") or []) if str(t).strip()] + if not text or not fillers or len(roles) != len(fillers): + return None + # Sidecar / store formed rows must still fail closed on wiki chrome. + if reject_candidate_text(text) or reject_wiki_scaffolding_text( + " ".join(fillers) + ): + return None + epistemic = _live_unbind_epistemic(raw) + lineage = _live_unbind_lineage(raw) + stage = str(raw.get("stage") or "").strip() or "circulating" + license_ = raw.get("license") + if not isinstance(license_, str) or not license_.strip(): + license_ = "operator-local" + prefix = "live-pos:" if scheme == "positional" else "live-type:" + prov = str(raw.get("provenance") or "") + if not prov.startswith(("live-pos:", "live-type:")): + prov = f"{prefix}{epistemic}" + try: + return _row( + text=text, + lineage=lineage, + typology=list(raw.get("typology") or TYPOLOGY.get(lineage, [])), + stage=stage, + roles=roles, + fillers=fillers, + role_scheme=scheme, + task="unbind", + provenance=prov, + **{"class": epistemic}, + license=license_, + ) + except ValueError: + return None + + +def _phrase_unbind_rows(raw: dict[str, Any]) -> list[dict[str, Any]]: + parsed = _phrase_like_atom(str(raw.get("text") or "")) + if parsed is None: + return [] + atom, tokens = parsed + epistemic = _live_unbind_epistemic(raw) + lineage = _live_unbind_lineage(raw) + stage = str(raw.get("stage") or "").strip() or "circulating" + license_ = raw.get("license") + if not isinstance(license_, str) or not license_.strip(): + license_ = "operator-local" + try: + return _unbind_dual_scheme_rows( + atom, + tokens, + lineage=lineage, + stage=stage, + epistemic=epistemic, + pos_provenance=f"live-pos:{epistemic}", + type_provenance=f"live-type:{epistemic}", + license=license_, + ) + except ValueError: + return [] + + +def harvest_live_unbind( + live_store: Path, + *, + skip_atoms: set[str] | None = None, + observed_harvest: Path | None = None, +) -> list[dict[str, Any]]: + """Live SoT phrase-like atoms → dual-scheme unbind rows. + + Selects stripped multiword text (2–6 whitespace tokens, len≤80, not + collision-hold). Dedupes by lowercased atom and skips civilian/fixture + atoms when ``skip_atoms`` is omitted. Epistemic is copied from the row + (``epistemic`` then ``class``); missing/None → INFERRED. Never upgraded + to OBSERVED. Fillers are the real tokens — no gloss invention. + + When a Wave A sidecar ``harvest_unbind_observed_mw.jsonl`` sits next to + the live store (Spark OBSERVED set, 229 atoms × dual scheme), those rows + are adopted with their stored class. Filename does not invent OBSERVED. + Live-store leftovers stay INFERRED unless the store row is already + OBSERVED. Missing store → empty list (export_dataset fail-closes + include-live). + """ + store = Path(live_store) + if not store.is_file(): + return [] + held = set(skip_atoms) if skip_atoms is not None else _default_held_unbind_atoms() + rows: list[dict[str, Any]] = [] + seen: set[str] = set() + seen_pairs: set[tuple[str, str]] = set() + + def _take(raw: dict[str, Any], *, apply_held: bool) -> None: + formed = _formed_unbind_row(raw) + if formed is not None: + key = _atom_key_from_unbind_row(formed) + if not key or key in COLLISION_HOLD: + return + if apply_held and key in held: + return + pair = (key, str(formed.get("role_scheme") or "")) + if pair in seen_pairs: + return + seen_pairs.add(pair) + seen.add(key) + rows.append(formed) + return + emitted = _phrase_unbind_rows(raw) + if not emitted: + return + key = _atom_key_from_unbind_row(emitted[0]) + if not key or key in COLLISION_HOLD or key in seen: + return + if apply_held and key in held: + return + seen.add(key) + for row in emitted: + seen_pairs.add((key, str(row.get("role_scheme") or ""))) + rows.extend(emitted) + + sidecar = resolve_observed_mw_harvest(store, explicit=observed_harvest) + if sidecar is not None: + # Operator-settled Wave A set — do not drop civilian overlap; do not upgrade. + for raw in _iter_jsonl_dicts(sidecar): + _take(raw, apply_held=False) + for raw in _iter_jsonl_dicts(store): + _take(raw, apply_held=True) + return rows + + +def undo_dump_brainrot_fold(rows: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], int]: + """Undo the brainrot-aura -> ai-native fold baked into data/hyperlex_4333_dump.jsonl. + + The dump (300c83e) carries 0 brainrot-aura rows; 63 ai-native rows keep + provenance.original_lineage=brainrot-aura and ~400 more are plain brainrot + slang. A notion (dump) classify row labelled ai-native becomes brainrot-aura + when every non-notion, non-test classify row with the same normalized text + says brainrot-aura. Rows with mixed or no outside evidence are left alone. + Split is text-hashed, so relabelling cannot move a row across splits. + """ + from .heldout_census import normalize_group_text # lazy: selection_surface imports export + + def _src(r: dict[str, Any]) -> str: + prov = r.get("provenance") + return str(prov.get("source") or "") if isinstance(prov, dict) else "" + + other: dict[str, set[str]] = {} + for r in rows: + if r.get("task") != "classify" or r.get("split") == "test" or _src(r) == "notion": + continue + key = normalize_group_text(r.get("text") or "") + if key: + other.setdefault(key, set()).add(str(r.get("lineage"))) + undone = 0 + for r in rows: + if r.get("task") != "classify" or _src(r) != "notion" or r.get("lineage") != "ai-native": + continue + if other.get(normalize_group_text(r.get("text") or "")) == {"brainrot-aura"}: + r["lineage"] = "brainrot-aura" + raw = r["provenance"].get("raw_typology") + if isinstance(raw, list) and raw: + r["typology"] = list(dict.fromkeys(raw)) + r["provenance"]["fold_undone"] = "dump_brainrot_fold" + undone += 1 + return rows, undone + + +def export_dataset( + root: Path | None = None, + *, + include_live: bool = False, + live_store: Path | None = None, +) -> dict[str, Any]: + root = root or repo_root() + molt_stats: dict[str, int] = {} + moltbook_rows = harvest_moltbook(root, stats=molt_stats) + # Count demotions before dedupe so a dropped duplicate still counts. + # seed_examples_audit_only is not a harvest: it adds no rows. + gold_demoted = summarize_gold_demotions(moltbook_rows) + gold_demoted["seed_examples_audit_only"] = audit_seed_examples_gold(root) + rows = ( + harvest_dialect() + + harvest_backfill(root) + + harvest_registry(root) + + harvest_receipts(root) + + harvest_archive(root) + + harvest_unbind() + + harvest_civilian_unbind(root) + + harvest_negatives() + + harvest_inferred_classify_pass(root) + + moltbook_rows + + harvest_4333_dump(root) + ) + live_n = 0 + live_rejected = 0 + if include_live: + store = resolve_live_store(live_store, required=True) + # count rejects for manifest + for line in store.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line: + continue + try: + raw_row = json.loads(line) + except json.JSONDecodeError: + live_rejected += 1 + continue + if reject_candidate_text(str(raw_row.get("text") or "")): + live_rejected += 1 + live_rows = load_live_candidates(store) + live_n = len(live_rows) + held = _positional_unbind_atoms(rows) + live_unbind = harvest_live_unbind(store, skip_atoms=held) + rows = rows + live_rows + live_unbind + rows, dump_fold_undone = undo_dump_brainrot_fold(rows) + rows = dedupe(rows) + gold_demoted_post, gold_demoted_by_split = gold_demoted_after_dedupe(rows) + rows.sort(key=lambda r: (r["task"], r["lineage"], r["text"])) + payload = "\n".join(json.dumps(r, sort_keys=True) for r in rows) + "\n" + digest = hashlib.sha256(payload.encode("utf-8")).hexdigest() + def _is_ordinary_prose_neg(r: dict[str, Any]) -> bool: + """Name-gate negatives bucket = curated ordinary-prose only. + + Live / inbox ``lineage=none`` classify rows are *not* negatives — they are + unclassified family-gap inventory. Folding them into ``counts.negatives`` + inflated the 200-neg quota (e.g. 2766 with ``--include-live``). + """ + if r["task"] != "classify" or r["lineage"] != "none": + return False + return str(r.get("provenance") or "").startswith("seed:negative-prose") + + def _is_none_classify(r: dict[str, Any]) -> bool: + return r["task"] == "classify" and r["lineage"] == "none" + + def _is_family_classify(r: dict[str, Any]) -> bool: + return r["task"] == "classify" and r["lineage"] != "none" + + def _is_unbind_fixture(r: dict[str, Any]) -> bool: + return r["task"] == "unbind" and str(r.get("provenance") or "").startswith("004:") + + def _is_unbind_civilian(r: dict[str, Any]) -> bool: + prov = str(r.get("provenance") or "") + return r["task"] == "unbind" and ( + prov.startswith("civilian-pos:") or prov.startswith("civilian-type:") + ) + + def _is_unbind_live(r: dict[str, Any]) -> bool: + prov = str(r.get("provenance") or "") + return r["task"] == "unbind" and ( + prov.startswith("live-pos:") or prov.startswith("live-type:") + ) + + classify_all = sum(1 for r in rows if r["task"] == "classify") + classify_family = sum(1 for r in rows if _is_family_classify(r)) + classify_none = sum(1 for r in rows if _is_none_classify(r)) + negatives = sum(1 for r in rows if _is_ordinary_prose_neg(r)) + unbind_all = sum(1 for r in rows if r["task"] == "unbind") + unbind_fixture = sum(1 for r in rows if _is_unbind_fixture(r)) + unbind_civilian = sum(1 for r in rows if _is_unbind_civilian(r)) + unbind_live = sum(1 for r in rows if _is_unbind_live(r)) + unbind_live_observed = sum( + 1 for r in rows if _is_unbind_live(r) and r.get("class") == "OBSERVED" + ) + unbind_live_inferred = sum( + 1 for r in rows if _is_unbind_live(r) and r.get("class") == "INFERRED" + ) + # Honesty: classify gate uses family-labeled rows only. + # Negatives = ordinary-prose seed only (NOT live lineage=none classify). + # Unbind = fixture(honest n=24) + civilian dual-scheme + optional live phrases. + # Live OBSERVED vs INFERRED are counted separately — no class upgrade. + # include_live does not flip name_gate. E2 stays on Spec 004 fixtures. + counts = { + "n": len(rows), + "classify": classify_family, # EXCLUDES negatives (honest name-gate family quota) + "classify_all": classify_all, # family + none-classify; do not use for 2k gate + "classify_none": classify_none, # all lineage=none classify (incl live inbox) + "unbind": unbind_all, + "unbind_fixture": unbind_fixture, + "unbind_civilian": unbind_civilian, + "unbind_live": unbind_live, + "unbind_live_observed": unbind_live_observed, + "unbind_live_inferred": unbind_live_inferred, + "negatives": negatives, # ordinary-prose only (seed:negative-prose) + "dialect": sum(1 for r in rows if r["provenance"] == "seed:dialect-e6"), + "backfill": sum(1 for r in rows if str(r["provenance"]).startswith("backfill:")), + "observed": sum(1 for r in rows if r["class"] == "OBSERVED"), + "inferred": sum(1 for r in rows if r["class"] == "INFERRED"), + "train": sum(1 for r in rows if r["split"] == "train"), + "val": sum(1 for r in rows if r["split"] == "val"), + "test": sum(1 for r in rows if r["split"] == "test"), + "live_included": live_n if include_live else 0, + "dump_brainrot_fold_undone": dump_fold_undone, + "live_rejected": live_rejected if include_live else 0, + "name_gate": False, + "name_gate_classify_gap": max(0, 2000 - classify_family), + "name_gate_unbind_gap": max(0, 200 - unbind_all), + "name_gate_negative_gap": max(0, 200 - negatives), + # Pre-dedupe. Keys are provenance source, then reason + # (fallback_label, gold_not_in_text, no_gold). Demoted rows are classify-only. + "gold_demoted": gold_demoted, + # JSON lines harvest_moltbook could not decode. Not blank lines. + "moltbook_skipped_lines": molt_stats.get("moltbook_skipped_lines", 0), + # After dedupe, train/val only. split=test is not read. + "gold_demoted_post_dedupe": gold_demoted_post, + "gold_demoted_post_dedupe_by_split": gold_demoted_by_split, + } + # Recipe gates are documented here; the Hyperlexical loop applies + # upsample/cap + morph hard-negs + optional scheme curriculum. + # Export rows stay SoT-shaped. + unbind_only = [r for r in rows if r["task"] == "unbind"] + counts.update(recipe_env_counts(unbind_only)) + return {"rows": rows, "sha256": digest, "counts": counts, "payload": payload} + + +def write_export(out_dir: Path, bundle: dict[str, Any]) -> Path: + out_dir.mkdir(parents=True, exist_ok=True) + jsonl = out_dir / "civilian.v0.1.jsonl" + manifest = out_dir / "MANIFEST.json" + jsonl.write_text(bundle["payload"], encoding="utf-8") + manifest.write_text( + json.dumps( + { + "schema": "hyperlex.hyperlexical.dataset.v0.1", + "file": jsonl.name, + "sha256": bundle["sha256"], + "counts": bundle["counts"], + "brier": None, + "trunk": "answerdotai/ModernBERT-base", + "note": ( + "Honest accounting: counts.classify = family-labeled only " + "(excludes negatives). counts.negatives = ordinary-prose " + "(seed:negative-prose) only — not live lineage=none classify " + "(see classify_none). unbind_fixture / unbind_civilian / unbind_live " + "(unbind_live_observed vs unbind_live_inferred). " + "Spec004 fixtures at n=24; civilian dual-scheme from golden/registry " + "(no gloss invent). Live optional via --include-live (preserves store class; " + "unset→INFERRED; phrase-like atoms also harvest as unbind; Wave A sidecar " + "harvest_unbind_observed_mw.jsonl is adopted, not upgraded). " + "E2 stays on Spec 004 fixtures. Not a T1 name-gate. " + "n_unbind_observed / n_unbind_inferred plus recipe env " + "(HYPERLEX_UNBIND_OBSERVED_UPSAMPLE default 1, " + "HYPERLEX_UNBIND_INFERRED_CAP 0=off (hard low caps can starve morph-negs), " + "HYPERLEX_UNBIND_INFERRED_WEIGHT default 1.0, unbind_morph_negatives, " + "HYPERLEX_UNBIND_CURRICULUM default 0, " + "HYPERLEX_UNBIND_HARD_ATOMS_PATH unset, " + "HYPERLEX_UNBIND_HARD_UPSAMPLE default 1) " + "are counts only — loop applies train multiplicity / hard-negs / " + "scheme-split curriculum / INFERRED sample weight / " + "targeted OBSERVED hard-atom extras; " + "export does not invent OBSERVED SoT gold. lexical_split is frozen. " + "gold_demoted counts Moltbook harvest rows (pre-dedupe), including " + "moltbook-curated-seed, whose original unbind gold was a fallback " + "placeholder or not a surface token. Those rows stay in the export " + "as classify, class INFERRED, with no fillers. " + "seed_examples_audit_only is a read-only verdict over " + "seed_examples.jsonl and adds no rows. Labels are not replaced. " + "Empty gold is gold_demote_reason=no_gold, not a new filler. " + "moltbook_skipped_lines counts JSON decode failures. " + "gold_demoted_post_dedupe and gold_demoted_post_dedupe_by_split " + "recount demoted rows after dedupe on train and val; " + "split=test is not read. Other sources are unchanged." + ), + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + return jsonl + + +def main(argv=None) -> int: + p = argparse.ArgumentParser(prog="hyperlexical-export") + p.add_argument( + "--out", + default="", + help="directory; default specs/007-hyperlexical-model/exports", + ) + p.add_argument( + "--include-live", + action="store_true", + help="merge ~/.hyperlex/.../ingest_candidates.jsonl; preserve store class (OBSERVED stays OBSERVED; unset→INFERRED; reject junk)", + ) + p.add_argument( + "--live-store", + default="", + help="optional path to ingest_candidates.jsonl", + ) + args = p.parse_args(argv) + root = repo_root() + dest = Path(args.out) if args.out else root / "specs" / "007-hyperlexical-model" / "exports" + store = Path(args.live_store) if args.live_store else None + try: + bundle = export_dataset(root, include_live=bool(args.include_live), live_store=store) + except LiveStoreMissing as exc: + print(json.dumps({"abort": True, "error": str(exc), "brier": None}, indent=2), file=sys.stderr) + return 2 + path = write_export(dest, bundle) + print(json.dumps({"wrote": str(path), "sha256": bundle["sha256"], "counts": bundle["counts"]}, indent=2)) + return 0 + + + +def harvest_4333_dump(root: Path) -> list[dict[str, Any]]: + """4333-row Notion vernacular dump. Pre-classified rows only. + + Fail-closed: no hyperlex/abraxas import. Dump fields only. + Unknown ``role_scheme`` values (including leftover ``civilian``) drop to + None — recoverable_structure allows positional|type_slot. + """ + rows = [] + dump_file = root / "data" / "hyperlex_4333_dump.jsonl" + if not dump_file.exists(): + print("[harvest_4333_dump] no dump file, skipping") + return rows + + for line in dump_file.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + try: + r = json.loads(line) + text = (r.get("text") or r.get("term") or "").strip() + if not text or len(text) < 2: + continue + + lineage = r.get("lineage", "ai-native") + # FOLD_BUG (3dc37b7): brainrot-aura was folded into ai-native here and + # upstream in the dump. brainrot-aura is its own family (layout.FAMILIES[3]); + # "brainrot aura" is a mutation of two base terms, not an ai-native coinage. + orig = r.get("provenance") if isinstance(r.get("provenance"), dict) else {} + if lineage == "ai-native" and orig.get("original_lineage") == "brainrot-aura": + lineage = "brainrot-aura" + + typology = r.get("typology", ["compression", "status"]) + if lineage == "ai-native": + typology = ai_native_typology(typology) + + rows.append( + _row( + text=text, + lineage=lineage, + typology=typology, + stage=r.get("stage", "circulating"), + roles=r.get("roles", ["slang", "memetic"]), + fillers=r.get("fillers", []), + role_scheme=r.get("role_scheme"), + task="classify", + provenance={ + "source": "notion", + "page": "Hyperlex-Vernacular-export-2026-09-10", + "original_provenance": r.get("provenance"), + "reclassify_pass": r.get("reclassify_pass"), + "settle_note": r.get("settle_note"), + "raw_typology": r.get("typology"), + }, + **{"class": r.get("class", "INFERRED")}, + license=r.get("license", "operator-local"), + ) + ) + except Exception: + continue + print(f"[harvest_4333_dump] loaded {len(rows)} rows") + return rows + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/shadow/hyperlexical/filler_filter.py b/scripts/shadow/hyperlexical/filler_filter.py new file mode 100644 index 00000000..a4d6b942 --- /dev/null +++ b/scripts/shadow/hyperlexical/filler_filter.py @@ -0,0 +1,83 @@ +"""Publishable-filler filter for the unbind vocab. No torch. D8. + +Filler strings ship inside the weights config (``filler_vocab``), so every +filler must be a plain word. Rejects social handles, links, markup, and +dictionary/wiki scraps. Curly quotes are folded to ASCII before checking. + +``HYPERLEX_FILLER_FILTER``: ``strict`` (default) drops unbind rows with any +non-publishable filler and fails closed if one reaches the vocab; ``off`` +reproduces pre-filter recipes (morph75–78). +""" + +from __future__ import annotations + +import os +import re + +FILLER_FILTER_ENV = "HYPERLEX_FILLER_FILTER" +MODES = ("strict", "off") +_FOLD = str.maketrans({"\u2019": "'", "\u2018": "'", "\u2010": "-", "\u2011": "-", "\u2013": "-"}) +_WORD = re.compile(r"^[a-z0-9]+(?:['\-][a-z0-9]+)*$") +MAX_LEN = 24 +CONTRACTIONS = frozenset({"'em", "'bout", "'cause", "'til", "'sup", "'round", "'nuff"}) + + +def filter_mode(raw: str | None = None) -> str: + value = (os.environ.get(FILLER_FILTER_ENV, "") if raw is None else raw).strip() or "strict" + if value not in MODES: + raise ValueError(f"{FILLER_FILTER_ENV} must be one of {MODES}") + return value + + +def fold(token: str) -> str: + return str(token).translate(_FOLD).strip().lower() + + +def reject_reason(token: str) -> str | None: + """None if publishable, else a short reason code.""" + raw = str(token) + t = fold(raw) + if not t: + return "empty" + if "@" in raw: + return "handle_or_email" + if "://" in raw or raw.lower().startswith("www.") or re.search(r"\.(com|org|net|io|co)\b", raw.lower()): + return "link" + if any(c in raw for c in "[]()<>{}|!\"“”`*_#=\\"): + return "markup_or_quote" + if len(t) > MAX_LEN: + return "too_long" + if any(ord(c) > 127 for c in t): + return "non_ascii" + if t in CONTRACTIONS or (t.endswith("in'") and _WORD.match(t[:-1])): + return None + if not _WORD.match(t): + return "not_word" + return None + + +def is_publishable(token: str) -> bool: + return reject_reason(token) is None + + +def filter_unbind_rows(rows: list[dict], mode: str | None = None) -> tuple[list[dict], dict]: + mode = filter_mode(mode) + if mode == "off": + return list(rows), {"filler_filter": "off", "n_filler_rows_dropped": 0} + kept, dropped, reasons = [], 0, {} + for row in rows: + bad = [reject_reason(f) for f in (row.get("fillers") or [])] + bad = [b for b in bad if b] + if bad: + dropped += 1 + for b in bad: + reasons[b] = reasons.get(b, 0) + 1 + else: + kept.append(row) + return kept, {"filler_filter": "strict", "n_filler_rows_dropped": dropped, "filler_reject_reasons": reasons} + + +def assert_publishable_vocab(filler_vocab: list[str], unk: str = "") -> None: + bad = [f for f in filler_vocab if f != unk and not is_publishable(f)] + if bad: + raise ValueError(f"non-publishable fillers reached the vocab ({len(bad)}); first: {bad[0]!r}") diff --git a/scripts/shadow/hyperlexical/force_train_overlap.py b/scripts/shadow/hyperlexical/force_train_overlap.py new file mode 100644 index 00000000..c7f13517 --- /dev/null +++ b/scripts/shadow/hyperlexical/force_train_overlap.py @@ -0,0 +1,206 @@ +"""Count force-train rows that still sit in val or in the selection set. + +The historical move (`apply_unbind_force_train`) takes OBSERVED +``(text, role_scheme)`` keys out of val. A row can still share an id or a +normalized-text sha with val or with the checkpoint-selection rows. This +module counts those rows on every run. ``HLX_FORCE_TRAIN_DISJOINT=1`` drops +the overlapping rows from val and refuses when that drop is more than half +of a val split. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any, Mapping, Sequence + +from .heldout_census import normalize_group_text +from .holdout_guard import normalized_text_sha256 +from .selection_surface import row_id +from .unbind_recipe import UNBIND_FORCE_TRAIN_PATH_ENV, resolve_unbind_force_train_path + +FORCE_TRAIN_DISJOINT_ENV = "HLX_FORCE_TRAIN_DISJOINT" +_ID_FIELDS = ("id", "row_id") + + +def resolve_force_train_disjoint(raw: str | None = None) -> bool: + """Drop overlapping val rows. Default off: count and log only.""" + if raw is None: + raw = os.environ.get(FORCE_TRAIN_DISJOINT_ENV) + return raw == "1" + + +def load_force_train_rows(path: str | Path | None = None) -> list[dict[str, Any]]: + """JSONL objects from the force-train path. Empty path → no rows. + + Same fail-closed file rules as ``load_force_train_keys`` (path must exist, + each line a JSON object with non-empty string ``text``). Extra fields are + kept so id matching can see them. + """ + raw = resolve_unbind_force_train_path(path) + if not raw: + return [] + file = Path(raw) + if not file.is_file(): + raise ValueError(f"{UNBIND_FORCE_TRAIN_PATH_ENV} is not a file: {file}") + try: + body = file.read_text(encoding="utf-8") + except OSError as exc: + raise ValueError(f"{UNBIND_FORCE_TRAIN_PATH_ENV} is unreadable: {file}") from exc + rows: list[dict[str, Any]] = [] + for index, line in enumerate(body.splitlines(), start=1): + if not line.strip(): + continue + try: + obj = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError(f"{UNBIND_FORCE_TRAIN_PATH_ENV} line {index} is not valid JSON") from exc + if not isinstance(obj, dict): + raise ValueError(f"{UNBIND_FORCE_TRAIN_PATH_ENV} line {index} must be a JSON object") + text = obj.get("text") + scheme = obj.get("role_scheme") + if not isinstance(text, str) or not text.strip(): + raise ValueError(f"{UNBIND_FORCE_TRAIN_PATH_ENV} line {index} needs non-empty string text") + if scheme not in {"positional", "type_slot"}: + raise ValueError( + f"{UNBIND_FORCE_TRAIN_PATH_ENV} line {index} role_scheme must be " + f"positional|type_slot, got {scheme!r}" + ) + rows.append(obj) + return rows + + +def _explicit_ids(row: Mapping[str, Any]) -> set[str]: + found: set[str] = set() + for key in _ID_FIELDS: + token = row.get(key) + if isinstance(token, str) and token.strip(): + found.add(token.strip()) + return found + + +def _text_sha(row: Mapping[str, Any]) -> str | None: + text = str(row.get("text") or "") + if not normalize_group_text(text): + return None + return normalized_text_sha256(text) + + +def _index(rows: Sequence[Mapping[str, Any]]) -> tuple[set[str], set[str]]: + ids: set[str] = set() + shas: set[str] = set() + for row in rows: + ids.add(row_id(row)) + ids.update(_explicit_ids(row)) + sha = _text_sha(row) + if sha: + shas.add(sha) + return ids, shas + + +def _hits(row: Mapping[str, Any], ids: set[str], shas: set[str]) -> bool: + if row_id(row) in ids or (_explicit_ids(row) & ids): + return True + sha = _text_sha(row) + return bool(sha and sha in shas) + + +def overlap_counts( + force_rows: Sequence[Mapping[str, Any]], + val_rows: Sequence[Mapping[str, Any]], + selection_rows: Sequence[Mapping[str, Any]], +) -> dict[str, int]: + """How many force-train rows match val, selection, or either, by id or text sha.""" + val_ids, val_shas = _index(val_rows) + sel_ids, sel_shas = _index(selection_rows) + in_val = 0 + in_selection = 0 + union = 0 + for row in force_rows: + hit_val = _hits(row, val_ids, val_shas) + hit_sel = _hits(row, sel_ids, sel_shas) + if hit_val: + in_val += 1 + if hit_sel: + in_selection += 1 + if hit_val or hit_sel: + union += 1 + return { + "n_force_rows": len(force_rows), + "n_overlap": union, + "n_in_val": in_val, + "n_in_selection": in_selection, + } + + +def _drop(rows: list, force_ids: set[str], force_shas: set[str]) -> tuple[list, int]: + kept = [] + dropped = 0 + for row in rows: + if _hits(row, force_ids, force_shas): + dropped += 1 + else: + kept.append(row) + return kept, dropped + + +def _over_half(dropped: int, total: int) -> bool: + """True when ``dropped / total > 1/2``. Empty val is not over half.""" + return total > 0 and dropped * 2 > total + + +def enforce_force_train_disjoint( + classify_val: list, + unbind_val: list, + selection_rows: Sequence[Mapping[str, Any]], + *, + force_rows: Sequence[Mapping[str, Any]] | None = None, + disjoint: bool | None = None, +) -> tuple[list, list, dict[str, Any]]: + """Log force-train overlap. Optionally drop overlapping val rows. + + Returns the same list objects when nothing is dropped. Prints the overlap + count even when the disjoint flag is off. Refuses when a drop would remove + more than half of classify val or more than half of unbind val. + """ + rows = list(load_force_train_rows() if force_rows is None else force_rows) + do_drop = resolve_force_train_disjoint() if disjoint is None else bool(disjoint) + val_rows = list(classify_val) + list(unbind_val) + counts = overlap_counts(rows, val_rows, selection_rows) + print( + "[force-train] overlap " + f"n={counts['n_overlap']} in_val={counts['n_in_val']} " + f"in_selection={counts['n_in_selection']} force_rows={counts['n_force_rows']}", + flush=True, + ) + report: dict[str, Any] = { + **counts, + "disjoint": do_drop, + "n_dropped_classify_val": 0, + "n_dropped_unbind_val": 0, + "n_classify_val": len(classify_val), + "n_unbind_val": len(unbind_val), + } + if not do_drop: + return classify_val, unbind_val, report + force_ids, force_shas = _index(rows) + new_classify, dropped_c = _drop(classify_val, force_ids, force_shas) + new_unbind, dropped_u = _drop(unbind_val, force_ids, force_shas) + report["n_dropped_classify_val"] = dropped_c + report["n_dropped_unbind_val"] = dropped_u + print( + "[force-train] disjoint dropped " + f"classify_val={dropped_c}/{len(classify_val)} " + f"unbind_val={dropped_u}/{len(unbind_val)}", + flush=True, + ) + if _over_half(dropped_c, len(classify_val)) or _over_half(dropped_u, len(unbind_val)): + raise SystemExit( + "REFUSE: HLX_FORCE_TRAIN_DISJOINT dropped share of val is over 50%: " + f"classify_val={dropped_c}/{len(classify_val)} " + f"unbind_val={dropped_u}/{len(unbind_val)}" + ) + if dropped_c == 0 and dropped_u == 0: + return classify_val, unbind_val, report + return new_classify, new_unbind, report diff --git a/scripts/shadow/hyperlexical/heldout_census.py b/scripts/shadow/hyperlexical/heldout_census.py new file mode 100644 index 00000000..8f92088d --- /dev/null +++ b/scripts/shadow/hyperlexical/heldout_census.py @@ -0,0 +1,585 @@ +"""Read-only census of a fresh held-out drawn from the current pool. + +Counts release-pool rows that pass every admission rule. Writes nothing. +Does not train, load a model, score a test slice, or call the holdout scorer. + +The trained union is the release train split plus rows from ``--trained`` +files (force/hard JSONL and checkpoint train receipts). Test-split rows are +discarded by the ``split`` field only. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import unicodedata +from collections import Counter +from pathlib import Path +from typing import Any, Iterable, Mapping, Sequence + +from .filler_filter import assert_publishable_vocab +from .selection_surface import drop_test_rows, jaccard, lenient_copy_hit, provenance_source, row_id +from .soft_ceiling import row_key + +SCHEMA = "hyperlex.heldout_census.v0.1" +JACCARD_MIN = 0.5 +DRAW_READY_MIN = 470 +TOPUP_MIN = 150 +SOURCE_CAP = 0.60 +GROUP_KEY_DEFINITION = ( + "NFKC + casefold + strip punctuation, whitespace, and URLs, plus the sorted filler set" +) +_URL = re.compile(r"(?:https?://|www\.)\S+", re.IGNORECASE) +_ID_LIST_KEYS = frozenset({"row_ids", "ids"}) +_RECEIPT_PATH_KEYS = ("unbind_force_train_path", "unbind_hard_atoms_path") +_ENV_PATH_KEYS = ("HYPERLEX_UNBIND_FORCE_TRAIN_PATH", "HYPERLEX_UNBIND_HARD_ATOMS_PATH") + + +def normalize_group_text(text: str) -> str: + """NFKC, casefold, then drop URLs, punctuation, and extra whitespace.""" + raw = unicodedata.normalize("NFKC", str(text or "")).casefold() + raw = _URL.sub(" ", raw) + raw = "".join(ch if ch.isalnum() else " " for ch in raw) + return " ".join(raw.split()) + + +def normalized_fillers(row: Mapping[str, Any]) -> tuple[str, ...]: + fillers: set[str] = set() + for item in row.get("fillers") or []: + token = normalize_group_text(str(item)) + if token: + fillers.add(token) + return tuple(sorted(fillers)) + + +def group_key(row: Mapping[str, Any]) -> tuple[str, tuple[str, ...]]: + return (normalize_group_text(str(row.get("text") or "")), normalized_fillers(row)) + + +def holdout_key(row: Mapping[str, Any]) -> str: + """Same ``task|text|scheme`` identity the holdout manifest hashes. Not a test read.""" + text, scheme = row_key({"text": str(row.get("text") or ""), "role_scheme": row.get("role_scheme")}) + return "|".join((str(row.get("task") or ""), text, scheme)) + + +def exclusion_ids(row: Mapping[str, Any]) -> set[str]: + return {row_id(row), holdout_key(row)} + + +def admitted_ids_sha256(ids: Sequence[str]) -> str: + return hashlib.sha256("\n".join(sorted(ids)).encode("utf-8")).hexdigest() + + +def source_over_cap(max_count: int, n: int) -> bool: + """True when ``max_count / n > 0.60``. Exact at 60% is inside the cap.""" + if n <= 0: + return False + return 5 * max_count > 3 * n + + +def pinned_verdict(n_admitted: int, max_source_count: int) -> str: + """Census pin. ``SOURCE_CAP`` is a draw that clears 470 but breaks the 60% cap.""" + if n_admitted >= DRAW_READY_MIN and not source_over_cap(max_source_count, n_admitted): + return "DRAW_READY" + if n_admitted >= DRAW_READY_MIN: + return "SOURCE_CAP" + if n_admitted >= TOPUP_MIN: + return "TOPUP_NEEDED" + return "POOL_EXHAUSTED" + + +def publish_census( + candidate: str, + admitted_hash: str, + code_commit: str | None, + prior: Mapping[str, Any] | None, +) -> dict[str, Any]: + """HOLD until a same-SHA rerun matches the admitted-ID hash.""" + if prior is None: + status = "pending" + elif prior.get("code_commit") == code_commit and prior.get("admitted_ids_sha256") == admitted_hash: + return { + "candidate": candidate, + "published": candidate, + "hold": False, + "reproduce": "matched", + } + else: + status = "mismatch" + return {"candidate": candidate, "published": "HOLD", "hold": True, "reproduce": status} + + +class FillerIndex: + """Filler-set lookup for Jaccard siblings. Empty sets are not indexed.""" + + def __init__(self) -> None: + self._sets: list[frozenset[str]] = [] + self._known: set[frozenset[str]] = set() + self._inv: dict[str, list[int]] = {} + + def __len__(self) -> int: + return len(self._sets) + + def add(self, fillers: frozenset[str]) -> None: + if not fillers or fillers in self._known: + return + index = len(self._sets) + self._known.add(fillers) + self._sets.append(fillers) + for token in fillers: + self._inv.setdefault(token, []).append(index) + + def sibling(self, fillers: frozenset[str]) -> bool: + if not fillers: + return False + hits: dict[int, int] = {} + for token in fillers: + for index in self._inv.get(token, ()): + hits[index] = hits.get(index, 0) + 1 + for index in hits: + if jaccard(fillers, self._sets[index]) >= JACCARD_MIN: + return True + return False + + +class SeenIndex: + """Group keys and filler sets from the trained union.""" + + def __init__(self) -> None: + self.keys: set[tuple[str, tuple[str, ...]]] = set() + self.texts: set[str] = set() + self.fillers = FillerIndex() + + def add_row(self, row: Mapping[str, Any], *, text_if_no_fillers: bool = False) -> None: + key = group_key(row) + self.keys.add(key) + fillers = frozenset(key[1]) + if fillers: + self.fillers.add(fillers) + elif text_if_no_fillers and key[0]: + self.texts.add(key[0]) + + def blocks(self, row: Mapping[str, Any]) -> bool: + key = group_key(row) + if key in self.keys: + return True + if key[0] and key[0] in self.texts: + return True + return self.fillers.sibling(frozenset(key[1])) + + +def _publishable(row: Mapping[str, Any]) -> bool: + fillers = [str(item) for item in (row.get("fillers") or [])] + try: + assert_publishable_vocab(fillers) + except ValueError: + return False + return True + + +def blocking_texts( + pool: Sequence[Mapping[str, Any]], + trained_rows: Sequence[Mapping[str, Any]], + holdout_ids: Iterable[str] = (), +) -> dict[str, Any]: + """Normalized texts that bar a row, plus holdout-id resolution counts. + + Every non-empty trained-row text is included. A holdout id contributes + text only when it equals ``row_id`` or ``holdout_key`` of a non-test pool + or trained row. Ids that do not match are counted unresolved; their text + cannot be recovered from the hash. ``split=test`` rows are dropped before + any text is read. Empty normalized text is omitted so blank rows do not + collide. + """ + safe_pool, _pool_test = drop_test_rows(pool) + trained, _trained_test = drop_test_rows(trained_rows) + holdout = {str(item) for item in holdout_ids} + texts: set[str] = set() + matched: set[str] = set() + for row in trained: + text = normalize_group_text(str(row.get("text") or "")) + if text: + texts.add(text) + if holdout: + for row in [*safe_pool, *trained]: + hit = exclusion_ids(row) & holdout + if not hit: + continue + matched |= hit + text = normalize_group_text(str(row.get("text") or "")) + if text: + texts.add(text) + return { + "texts": texts, + "n_holdout_ids": len(holdout), + "n_holdout_ids_resolved": len(matched), + "n_holdout_ids_unresolved": len(holdout) - len(matched), + } + + +def _in_holdout( + row: Mapping[str, Any], + holdout_ids: set[str], + blocked_texts: set[str] | None = None, +) -> bool: + """Id hit, or the same normalized text as a holdout or trained row.""" + text = normalize_group_text(str(row.get("text") or "")) + if text and blocked_texts and text in blocked_texts: + return True + if not holdout_ids: + return False + return bool(exclusion_ids(row) & holdout_ids) + + +def rule_failures( + row: Mapping[str, Any], + seen: SeenIndex, + holdout_ids: set[str], + blocked_texts: set[str] | None = None, +) -> list[str]: + """Rules in admission order. The list is every failure, not only the first.""" + fails: list[str] = [] + task = str(row.get("task") or "") + in_text = bool(lenient_copy_hit(row)) + if task == "unbind": + task_ok = True + elif task == "classify+unbind": + task_ok = in_text + else: + task_ok = False + if not task_ok: + fails.append("1") + if not in_text: + fails.append("2") + if seen.blocks(row): + fails.append("3") + if not _publishable(row): + fails.append("4") + if _in_holdout(row, holdout_ids, blocked_texts): + fails.append("5") + return fails + + +def _index_pool(pool: Sequence[Mapping[str, Any]]) -> dict[str, list[Mapping[str, Any]]]: + by_text: dict[str, list[Mapping[str, Any]]] = {} + for row in pool: + text = normalize_group_text(str(row.get("text") or "")) + if text: + by_text.setdefault(text, []).append(row) + return by_text + + +def build_seen( + pool: Sequence[Mapping[str, Any]], + trained_rows: Sequence[Mapping[str, Any]], +) -> SeenIndex: + """Train-split rows plus ``--trained`` rows. Test rows are not accepted here.""" + seen = SeenIndex() + by_text = _index_pool(pool) + for row in pool: + if row.get("split") == "train": + seen.add_row(row, text_if_no_fillers=True) + for row in trained_rows: + if row.get("split") == "test": + continue + fillers = [item for item in (row.get("fillers") or []) if str(item).strip()] + if fillers: + seen.add_row(row) + continue + text = normalize_group_text(str(row.get("text") or "")) + if text: + seen.texts.add(text) + for match in by_text.get(text, ()): + seen.add_row(match) + return seen + + +def _zero_rules() -> dict[str, int]: + return {str(number): 0 for number in range(1, 6)} + + +def _source_stats(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + sources: Counter[str] = Counter() + schemes: Counter[str] = Counter() + classes: Counter[str] = Counter() + for row in rows: + sources[str(row.get("provenance_source") or provenance_source(row))] += 1 + scheme = row.get("role_scheme") + schemes["" if scheme is None else str(scheme)] += 1 + classes[str(row.get("class") or "").upper()] += 1 + n = len(rows) + if sources: + top = max(sources.values()) + name = sorted(label for label, count in sources.items() if count == top)[0] + share = top / n + else: + name, top, share = None, 0, 0.0 + return { + "n": n, + "by_source": dict(sorted(sources.items())), + "by_scheme": dict(sorted(schemes.items())), + "by_class": dict(sorted(classes.items())), + "max_source": name, + "max_source_count": top, + "max_source_share": share, + } + + +def census_rows( + rows: Sequence[Mapping[str, Any]], + trained_rows: Sequence[Mapping[str, Any]] = (), + holdout_ids: Iterable[str] = (), + *, + id_list_present: bool = True, + n_test_discarded: int | None = None, + n_test_dropped_before_release: int | None = None, + release_set: Mapping[str, Any] | None = None, + trained_files: Sequence[str] = (), + code_commit: str | None = None, + code_tree_sha256: str | None = None, + prior: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """Admit pool rows. Drops ``split=test`` before any text or gold is used.""" + if n_test_discarded is None: + pool, n_test_discarded = drop_test_rows(rows) + else: + pool = [dict(row) for row in rows if row.get("split") != "test"] + holdout = {str(item) for item in holdout_ids} + blocked = blocking_texts(pool, trained_rows, holdout) + seen = build_seen(pool, trained_rows) + first = _zero_rules() + every = _zero_rules() + admitted: list[dict] = [] + admitted_ids: list[str] = [] + for row in pool: + fails = rule_failures(row, seen, holdout, blocked["texts"]) + if not fails: + admitted.append(dict(row)) + admitted_ids.append(row_id(row)) + continue + first[fails[0]] += 1 + for rule in fails: + every[rule] += 1 + admitted_stats = _source_stats(admitted) + n_admitted = admitted_stats["n"] + candidate = pinned_verdict(n_admitted, int(admitted_stats["max_source_count"])) + id_hash = admitted_ids_sha256(admitted_ids) + verdict = publish_census(candidate, id_hash, code_commit, prior) + verdict["checks"] = { + "n_admitted": n_admitted, + "max_source_share": admitted_stats["max_source_share"], + "source_cap": SOURCE_CAP, + "source_cap_ok": not source_over_cap(int(admitted_stats["max_source_count"]), n_admitted), + "holdout_id_list_present": id_list_present, + } + return { + "schema": SCHEMA, + "brier": None, + "read_only": True, + "allow_train": False, + "cpu_only": True, + "test_slices_scored": False, + "n_test_discarded": n_test_discarded, + "n_test_dropped_before_release": n_test_dropped_before_release, + "test_dropped_before_release_filtering": n_test_dropped_before_release is not None, + "n_pool": len(pool), + "n_admitted": n_admitted, + "n_rejected": len(pool) - n_admitted, + "group_key": { + "definition": GROUP_KEY_DEFINITION, + "jaccard_sibling_min": JACCARD_MIN, + "compared_to": [ + "train split of the release pool", + "files passed with --trained (force/hard JSONL and train receipts)", + ], + }, + "rules": { + "1": "unbind, or classify+unbind whose gold fillers are tokens of the text", + "2": "every gold filler is a token of the text", + "3": "no group-key twin and no filler-Jaccard >= 0.5 sibling in the trained union", + "4": "gold passes assert_publishable_vocab", + "5": ( + "row id is not in the holdout manifest ID list, and normalized " + "text matches no holdout or trained row" + ), + }, + "rejected": {"first_failing": first, "all_failing": every}, + "admitted": admitted_stats, + "admitted_ids_sha256": id_hash, + "n_seen_group_keys": len(seen.keys), + "n_seen_filler_sets": len(seen.fillers), + "holdout": {"n_ids": len(holdout), "id_list_present": id_list_present}, + "trained_files": [str(path) for path in trained_files], + "release_set": dict(release_set) if release_set else {"release_set": None}, + "verdict": verdict, + "code_commit": code_commit, + "code_tree_sha256": code_tree_sha256, + } + + +def _refuse_embedded_rows(obj: Any) -> None: + if isinstance(obj, dict): + if "text" in obj and ("fillers" in obj or "task" in obj): + raise SystemExit("REFUSE: holdout manifest contains row content; reading only an ID list") + for value in obj.values(): + _refuse_embedded_rows(value) + elif isinstance(obj, list): + for value in obj: + _refuse_embedded_rows(value) + + +def _collect_ids(obj: Any, found: list[str], present: list[bool]) -> None: + if isinstance(obj, dict): + for key, value in obj.items(): + if key in _ID_LIST_KEYS and isinstance(value, list): + if not all(isinstance(item, str) for item in value): + raise SystemExit("REFUSE: holdout ID list must be strings") + present.append(True) + found.extend(value) + elif isinstance(value, (dict, list)): + _collect_ids(value, found, present) + elif isinstance(obj, list): + for value in obj: + if isinstance(value, (dict, list)): + _collect_ids(value, found, present) + + +def load_holdout_ids(path: str | Path) -> tuple[set[str], bool]: + """Read the manifest's ID list and nothing else. + + Accepts a bare JSON list of strings, or string lists under ``row_ids`` / ``ids``. + Slice hashes, metric names, rules, and trained-file paths are not IDs. + """ + file = Path(path) + if not file.is_file(): + raise SystemExit(f"REFUSE: holdout manifest is not a file: {file}") + if "holdout-scores" in file.name: + raise SystemExit(f"REFUSE: will not read holdout scores {file}") + payload = json.loads(file.read_text(encoding="utf-8")) + if isinstance(payload, list): + if not all(isinstance(item, str) for item in payload): + raise SystemExit("REFUSE: holdout ID list must be strings") + return set(payload), True + if not isinstance(payload, dict): + raise SystemExit("REFUSE: holdout manifest must be a JSON object or an ID list") + _refuse_embedded_rows(payload) + found: list[str] = [] + present: list[bool] = [] + _collect_ids(payload, found, present) + return set(found), bool(present) + + +def _is_receipt(obj: Mapping[str, Any]) -> bool: + schema = str(obj.get("schema") or "") + if "train_receipt" in schema: + return True + return any(key in obj for key in _RECEIPT_PATH_KEYS) + + +def _take_trained_row(obj: Any, path: Path, index: int) -> dict | None: + if not isinstance(obj, dict): + raise SystemExit(f"REFUSE: {path} line {index} is not an object") + if obj.get("split") == "test": + return None + if "text" not in obj: + raise SystemExit(f"REFUSE: {path} line {index} has no text") + return obj + + +def load_jsonl_rows(path: Path) -> list[dict]: + rows: list[dict] = [] + for index, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + if not line.strip(): + continue + try: + obj = json.loads(line) + except json.JSONDecodeError as exc: + raise SystemExit(f"REFUSE: {path} line {index} is not JSON") from exc + row = _take_trained_row(obj, path, index) + if row is not None: + rows.append(row) + return rows + + +def _resolve_receipt_path(name: str, receipt_dir: Path, search_dirs: Sequence[Path]) -> Path: + candidate = Path(name) + if candidate.is_file(): + return candidate + options = [receipt_dir / candidate.name] + for directory in search_dirs: + options.append(Path(directory) / candidate.name) + for option in options: + if option.is_file(): + return option + raise SystemExit(f"REFUSE: train receipt path not found: {name}") + + +def _receipt_names(obj: Mapping[str, Any]) -> list[str]: + names: list[str] = [] + for key in _RECEIPT_PATH_KEYS: + value = str(obj.get(key) or "").strip() + if value: + names.append(value) + env = obj.get("hyperlex_env") + if isinstance(env, dict): + for key in _ENV_PATH_KEYS: + value = str(env.get(key) or "").strip() + if value and value not in names: + names.append(value) + return names + + +def load_trained_files(paths: Sequence[str]) -> tuple[list[dict], list[str]]: + """Load force/hard JSONL and any JSONL named by a train receipt.""" + files = [Path(path) for path in paths] + for file in files: + if not file.is_file(): + raise SystemExit(f"REFUSE: --trained is not a file: {file}") + if "holdout" in file.name.lower(): + raise SystemExit(f"REFUSE: will not read holdout path as training data: {file}") + jsonl_paths: list[Path] = [] + receipts: list[tuple[Path, dict]] = [] + for file in files: + try: + parsed = json.loads(file.read_text(encoding="utf-8")) + except json.JSONDecodeError: + parsed = None + if isinstance(parsed, dict) and _is_receipt(parsed): + receipts.append((file, parsed)) + else: + jsonl_paths.append(file) + search_dirs = [path.parent for path in jsonl_paths] + rows: list[dict] = [] + loaded: list[str] = [] + for file in jsonl_paths: + rows.extend(load_jsonl_rows(file)) + loaded.append(str(file)) + for file, obj in receipts: + names = _receipt_names(obj) + if not names: + raise SystemExit(f"REFUSE: train receipt lists no force/hard files: {file}") + loaded.append(str(file)) + seen_paths: set[Path] = set() + for name in names: + found = _resolve_receipt_path(name, file.parent, search_dirs) + resolved = found.resolve() + if resolved in seen_paths: + continue + seen_paths.add(resolved) + if "holdout" in found.name.lower(): + raise SystemExit(f"REFUSE: train receipt points at a holdout path: {found}") + rows.extend(load_jsonl_rows(found)) + loaded.append(str(found)) + return rows, loaded + + +def write_census(report: Mapping[str, Any], out_dir: str | Path) -> Path: + """Write ``census.json`` and no other file.""" + directory = Path(out_dir) + directory.mkdir(parents=True, exist_ok=True) + target = (directory / "census.json").resolve() + if target.parent != directory.resolve(): + raise SystemExit("REFUSE: census output escaped --out-dir") + target.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return target diff --git a/scripts/shadow/hyperlexical/holdout_eligibility.py b/scripts/shadow/hyperlexical/holdout_eligibility.py new file mode 100644 index 00000000..8f3e8898 --- /dev/null +++ b/scripts/shadow/hyperlexical/holdout_eligibility.py @@ -0,0 +1,152 @@ +"""Holdout eligibility against a pinned training export. + +Construction and the runtime filter both call +``holdout_guard.normalized_text_sha256``. This module does not draw a +manifest, score rows, or train. +""" + +from __future__ import annotations + +from collections import Counter +from pathlib import Path +from typing import Any, Mapping, Sequence + +from .holdout_guard import _read_manifest, normalized_text_sha256, operational_status +from .selection_surface import row_id + +ELIGIBLE = "eligible" +SPLIT_TEST = "split_test" +TRAIN_ROW_ID = "train_row_id" +TRAIN_TEXT_HASH = "train_text_hash" +SPENT_OR_ABANDONED = "spent_or_abandoned" + + +def text_hash(row: Mapping[str, Any]) -> str: + return normalized_text_sha256(str(row.get("text") or "")) + + +def identity_index(rows: Sequence[Mapping[str, Any]]) -> tuple[set[str], set[str]]: + """Row ids and canonical text hashes present in ``rows``.""" + ids: set[str] = set() + hashes: set[str] = set() + for row in rows: + ids.add(row_id(row)) + hashes.add(text_hash(row)) + return ids, hashes + + +def manifest_identity(path: str | Path) -> tuple[set[str], set[str], str]: + """Ids and hashes stored on a manifest, plus its operational status.""" + record, ids, hashes = _read_manifest(str(path)) + return ids, hashes, operational_status(record) + + +def exclusion_reason( + row: Mapping[str, Any], + *, + train_ids: set[str], + train_hashes: set[str], + other_ids: set[str], + other_hashes: set[str], +) -> str: + """Waterfall reason. First match wins. ``eligible`` means none matched.""" + if str(row.get("split") or "") == "test": + return SPLIT_TEST + ident = row_id(row) + digest = text_hash(row) + if ident in train_ids: + return TRAIN_ROW_ID + if digest in train_hashes: + return TRAIN_TEXT_HASH + if ident in other_ids or digest in other_hashes: + return SPENT_OR_ABANDONED + return ELIGIBLE + + +def multiplicity(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """How many rows share one canonical text hash. No hash values are kept.""" + counts = Counter(text_hash(row) for row in rows) + histogram = Counter(counts.values()) + return { + "rows": len(rows), + "unique_hashes": len(counts), + "max_rows_per_hash": max(histogram) if histogram else 0, + "hashes_with_multiple_rows": sum( + count for size, count in histogram.items() if size > 1 + ), + "histogram_rows_per_hash": {str(size): histogram[size] for size in sorted(histogram)}, + } + + +def census( + source_rows: Sequence[Mapping[str, Any]], + train_rows: Sequence[Mapping[str, Any]], + *, + extra_rows: Sequence[Mapping[str, Any]] = (), + extra_manifests: Sequence[str | Path] = (), +) -> dict[str, Any]: + """Eligible universe after row-id and normalized-text exclusion. + + ``extra_rows`` and ``extra_manifests`` are spent or abandoned exclusions. + ``unbind_clean`` uses ``soft_ceiling.clean_surface`` and is not the + canonical text hash. + """ + from .soft_ceiling import clean_surface + + train_ids, train_hashes = identity_index(train_rows) + other_ids, other_hashes = identity_index(extra_rows) + manifest_status: list[dict[str, str]] = [] + for path in extra_manifests: + ids, hashes, status = manifest_identity(path) + other_ids |= ids + other_hashes |= hashes + manifest_status.append({"path": str(path), "operational_status": status}) + reasons: Counter[str] = Counter() + eligible: list[Mapping[str, Any]] = [] + source_hashes: set[str] = set() + for row in source_rows: + source_hashes.add(text_hash(row)) + reason = exclusion_reason( + row, + train_ids=train_ids, + train_hashes=train_hashes, + other_ids=other_ids, + other_hashes=other_hashes, + ) + reasons[reason] += 1 + if reason == ELIGIBLE: + eligible.append(row) + eligible_hashes = {text_hash(row) for row in eligible} + classify = [row for row in eligible if row.get("task") == "classify"] + unbind = [row for row in eligible if row.get("task") == "unbind"] + observed = [row for row in classify if str(row.get("class")) == "OBSERVED"] + non_none = [row for row in classify if str(row.get("lineage")) not in ("", "none")] + train_for_clean = [row for row in train_rows if row.get("split") == "train"] + clean, _account = clean_surface(unbind, train_rows=train_for_clean) + return { + "source_rows": len(source_rows), + "source_unique_text_hashes": len(source_hashes), + "excluded_by_split_test": reasons[SPLIT_TEST], + "excluded_by_training_row_id": reasons[TRAIN_ROW_ID], + "excluded_by_training_normalized_text_hash": reasons[TRAIN_TEXT_HASH], + "excluded_by_spent_or_abandoned": reasons[SPENT_OR_ABANDONED], + "eligible_rows": len(eligible), + "eligible_unique_text_hashes": len(eligible_hashes), + "classify_eligible": len(classify), + "classify_observed_eligible": len(observed), + "classify_non_none_eligible": len(non_none), + "unbind_eligible": len(unbind), + "unbind_clean_eligible": len(clean), + "unbind_clean_definition": "soft_ceiling.clean_surface", + "source_multiplicity": multiplicity(source_rows), + "train_multiplicity": multiplicity(train_rows), + "extra_manifests": manifest_status, + "waterfall_sums_to_source": ( + reasons[SPLIT_TEST] + + reasons[TRAIN_ROW_ID] + + reasons[TRAIN_TEXT_HASH] + + reasons[SPENT_OR_ABANDONED] + + len(eligible) + == len(source_rows) + ), + } diff --git a/scripts/shadow/hyperlexical/holdout_guard.py b/scripts/shadow/hyperlexical/holdout_guard.py new file mode 100644 index 00000000..d5cbf19d --- /dev/null +++ b/scripts/shadow/hyperlexical/holdout_guard.py @@ -0,0 +1,418 @@ +"""Drop frozen holdout rows before unbind training and checkpoint selection. + +Manifests are id lists plus sha256 hashes of census-normalized text. This +module does not train, score, or read row text out of a manifest. + +Canonical text identity, shared by holdout construction and this filter: + + source text + -> heldout_census.normalize_group_text + -> UTF-8 bytes + -> sha256 hex digest + +``normalize_group_text`` applies NFKC, casefold, replaces URLs and every +non-alphanumeric character with a space, then collapses whitespace. There is +no second normalizer on this path. An empty normalized string still hashes. +""" + +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +from typing import Any, Mapping, Sequence + +from .heldout_census import _refuse_embedded_rows, normalize_group_text +from .selection_surface import row_id + +HOLDOUT_MANIFESTS_ENV = "HLX_HOLDOUT_MANIFESTS" +ALLOW_NO_HOLDOUT_ENV = "HLX_ALLOW_NO_HOLDOUT" +EXPERIMENT_ID_ENV = "HLX_EXPERIMENT_ID" +ADMISSIBLE_STATUS = "UNSCORED_SEALED" +ABANDONED_STATUS = "UNSCORED_ABANDONED" +LIFECYCLE_FILENAME = "holdout-lifecycle.json" +_ID_KEYS = frozenset({"row_ids", "ids"}) +_HASH_KEYS = frozenset({"normalized_text_sha256"}) +_REMOVED_KEYS = ("classify_train", "classify_val", "unbind_train", "unbind_val") + + +class HoldoutSpec: + """Union of one or more holdout manifests.""" + + def __init__( + self, + row_ids: frozenset[str], + text_hashes: frozenset[str], + manifests: tuple[dict[str, Any], ...], + ) -> None: + self.row_ids = row_ids + self.text_hashes = text_hashes + self.manifests = manifests + + @property + def active(self) -> bool: + return bool(self.row_ids or self.text_hashes) + + +def _empty_spec() -> HoldoutSpec: + return HoldoutSpec(frozenset(), frozenset(), ()) + + +def normalized_text_sha256(text: str) -> str: + """sha256 hex of the canonical normalized text, UTF-8. + + Algorithm: NFKC, casefold, URL strip, non-alphanumeric to space, + whitespace collapse, then SHA-256. Punctuation does not survive. + """ + normalized = normalize_group_text(text) + return hashlib.sha256(normalized.encode("utf-8")).hexdigest() + + +def lifecycle_path(manifest: str | Path) -> Path: + return Path(manifest).parent / LIFECYCLE_FILENAME + + +def read_lifecycle(manifest: str | Path) -> dict[str, Any] | None: + """Append-only lifecycle receipt beside a sealed manifest, if present.""" + path = lifecycle_path(manifest) + if not path.is_file(): + return None + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + raise SystemExit(f"REFUSE: holdout lifecycle is not JSON: {path}") from None + if not isinstance(payload, dict): + raise SystemExit(f"REFUSE: holdout lifecycle must be a JSON object: {path}") + return payload + + +def operational_status(record: Mapping[str, Any]) -> str: + """Sealed manifest status, unless a lifecycle receipt supersedes it. + + The manifest file stays byte-stable. ``UNSCORED_ABANDONED`` means the + holdout was never scored, is not reusable, and is not ``SCORED_SPENT``. + """ + sealed = record.get("status") + life = read_lifecycle(str(record.get("path") or "")) + if life is None: + return "" if sealed is None else str(sealed) + bound = str(life.get("manifest_sha256") or "") + if bound != record.get("sha256"): + raise SystemExit( + "REFUSE: holdout lifecycle manifest_sha256 does not match the manifest" + ) + status = life.get("status") + if not isinstance(status, str) or not status.strip(): + raise SystemExit("REFUSE: holdout lifecycle status is missing") + return status + + +def allow_no_holdout(raw: str | None = None) -> bool: + if raw is None: + raw = os.environ.get(ALLOW_NO_HOLDOUT_ENV) + return raw == "1" + + +def resolve_manifest_paths(raw: str | None = None) -> list[str]: + """Comma-separated ``HLX_HOLDOUT_MANIFESTS``. Empty segments are ignored.""" + if raw is None: + raw = os.environ.get(HOLDOUT_MANIFESTS_ENV, "") + paths: list[str] = [] + seen: set[str] = set() + for part in str(raw or "").split(","): + item = part.strip() + if not item or item in seen: + continue + seen.add(item) + paths.append(item) + return paths + + +def merge_manifest_env(extra: Sequence[str]) -> str: + """Union CLI paths into ``HLX_HOLDOUT_MANIFESTS`` and return the value.""" + paths = resolve_manifest_paths() + for item in extra: + token = str(item or "").strip() + if token and token not in paths: + paths.append(token) + joined = ",".join(paths) + os.environ[HOLDOUT_MANIFESTS_ENV] = joined + return joined + + +def _file_sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _take_token(item: str, *, kind: str) -> str | None: + token = item.strip() + if not token: + return None + if kind == "hash": + lowered = token.lower() + if len(lowered) != 64 or any(ch not in "0123456789abcdef" for ch in lowered): + raise SystemExit( + "REFUSE: holdout manifest normalized_text_sha256 entries must be 64-char hex" + ) + return lowered + if all(ch in "0123456789abcdefABCDEF" for ch in token): + return token.lower() + return token + + +def _take_strings(value: list[Any], found: set[str], key: str) -> None: + if not all(isinstance(item, str) for item in value): + raise SystemExit(f"REFUSE: holdout manifest {key} must be strings") + kind = "hash" if key in _HASH_KEYS else "id" + for item in value: + token = _take_token(item, kind=kind) + if token: + found.add(token) + + +def _collect(obj: Any, ids: set[str], hashes: set[str]) -> None: + if isinstance(obj, dict): + for key, value in obj.items(): + if key in _ID_KEYS and isinstance(value, list): + _take_strings(value, ids, key) + elif key in _HASH_KEYS and isinstance(value, list): + _take_strings(value, hashes, key) + elif isinstance(value, (dict, list)): + _collect(value, ids, hashes) + elif isinstance(obj, list): + for value in obj: + if isinstance(value, (dict, list)): + _collect(value, ids, hashes) + + +def _read_manifest(path: str) -> tuple[dict[str, Any], set[str], set[str]]: + file = Path(path) + if not file.is_file(): + raise SystemExit(f"REFUSE: holdout manifest is not a file: {file}") + if "holdout-scores" in file.name: + raise SystemExit(f"REFUSE: will not read holdout scores {file}") + try: + payload = json.loads(file.read_text(encoding="utf-8")) + except json.JSONDecodeError: + raise SystemExit(f"REFUSE: holdout manifest is not JSON: {file}") from None + ids: set[str] = set() + hashes: set[str] = set() + if isinstance(payload, list) and all(isinstance(item, str) for item in payload): + _take_strings(payload, ids, "row_ids") + elif isinstance(payload, (dict, list)): + _refuse_embedded_rows(payload) + _collect(payload, ids, hashes) + else: + raise SystemExit("REFUSE: holdout manifest must be a JSON object or an ID list") + if not ids and not hashes: + raise SystemExit( + f"REFUSE: holdout manifest has no row_ids or normalized_text_sha256: {file}" + ) + status, experiment_id = _admission_fields(payload) + record = { + "path": str(file), + "sha256": _file_sha256(file), + "n_row_ids": len(ids), + "n_text_hashes": len(hashes), + "status": status, + "experiment_id": experiment_id, + } + return record, ids, hashes + + +def _admission_fields(payload: Any) -> tuple[str | None, str | None]: + """Top-level status and experiment id. Nested copies do not authorize a run.""" + if not isinstance(payload, dict): + return None, None + status = payload.get("status") + experiment_id = payload.get("experiment_id") + if status is not None and not isinstance(status, str): + raise SystemExit("REFUSE: holdout manifest status must be a string") + if experiment_id is not None and not isinstance(experiment_id, str): + raise SystemExit("REFUSE: holdout manifest experiment_id must be a string") + return status, experiment_id + + +def load_holdout_spec(raw: str | None = None) -> HoldoutSpec: + """Load manifests from ``raw`` or ``HLX_HOLDOUT_MANIFESTS``. Empty env → empty spec.""" + paths = resolve_manifest_paths(raw) + if not paths: + return _empty_spec() + manifests: list[dict[str, Any]] = [] + ids: set[str] = set() + hashes: set[str] = set() + for item in paths: + record, file_ids, file_hashes = _read_manifest(item) + manifests.append(record) + ids.update(file_ids) + hashes.update(file_hashes) + return HoldoutSpec(frozenset(ids), frozenset(hashes), tuple(manifests)) + + +def require_holdout_for_training() -> HoldoutSpec: + """Legacy manifest gate. Not the controlled-experiment contract. + + Controlled experiments admit through ``admit_training_run`` under + ``CONTROLLED_RESERVE``. This function still admits a non-experiment + launch that sets ``HLX_HOLDOUT_MANIFESTS`` or ``HLX_ALLOW_NO_HOLDOUT``. + + Admit training only with a fresh, experiment-bound holdout. + + Row exclusion still uses every id and text hash in the loaded manifests. + A ``SCORED_SPENT`` or ``SCORED`` file does not authorize a run. Missing, + unknown, and unbound statuses fail closed. ``HLX_ALLOW_NO_HOLDOUT=1`` + remains the explicit no-manifest override and is not the admission path. + """ + spec = load_holdout_spec() + if os.environ.get("HYPERLEX_ALLOW_TRAIN") != "1": + return spec + if not spec.manifests and allow_no_holdout(): + return spec + if not spec.manifests: + raise SystemExit( + "REFUSE: HYPERLEX_ALLOW_TRAIN=1 but no holdout manifest. " + f"Set {HOLDOUT_MANIFESTS_ENV} to comma-separated manifest paths, " + "pass --holdout-manifest, or set " + f"{ALLOW_NO_HOLDOUT_ENV}=1 to override." + ) + expected = os.environ.get(EXPERIMENT_ID_ENV) + for item in spec.manifests: + status = item.get("status") + if status is None: + raise SystemExit( + "REFUSE: holdout manifest status is missing; required " + f"{ADMISSIBLE_STATUS}" + ) + if status != ADMISSIBLE_STATUS: + raise SystemExit( + f"REFUSE: holdout manifest status {status} is not admissible; " + f"required {ADMISSIBLE_STATUS}" + ) + operational = operational_status(item) + if operational != ADMISSIBLE_STATUS: + raise SystemExit( + "REFUSE: holdout operational status " + f"{operational} is not admissible; required {ADMISSIBLE_STATUS}" + ) + bound = item.get("experiment_id") + if not expected or bound != expected: + raise SystemExit( + "REFUSE: holdout experiment binding " + f"{bound!r} does not match {EXPERIMENT_ID_ENV}" + ) + return spec + + +def holdout_match(row: Mapping[str, Any], spec: HoldoutSpec) -> str | None: + """``row_id`` or ``text`` when the row is in the frozen holdout.""" + if not spec.active: + return None + if row_id(row) in spec.row_ids: + return "row_id" + if normalized_text_sha256(str(row.get("text") or "")) in spec.text_hashes: + return "text" + return None + + +def disjoint_report(rows: Sequence[Mapping[str, Any]], spec: HoldoutSpec) -> dict[str, int | bool]: + """Count pinned rows the runtime filter would drop. + + Overlap counts are rows, not distinct hashes. A controlled experiment is + disjoint only when that removal count is zero. The filter stays in place + as defense in depth; it is not how equivalence is established. + """ + id_overlap = 0 + text_overlap = 0 + removed = 0 + for row in rows: + ident = row_id(row) in spec.row_ids + digest = normalized_text_sha256(str(row.get("text") or "")) in spec.text_hashes + if ident: + id_overlap += 1 + if digest: + text_overlap += 1 + if ident or digest: + removed += 1 + _kept, filtered = filter_holdout_rows(list(rows), spec) + if filtered != removed: + raise SystemExit("REFUSE: holdout filter count diverged from the disjoint report") + return { + "holdout_train_row_id_overlap": id_overlap, + "holdout_train_text_hash_overlap": text_overlap, + "holdout_filter_training_rows_removed": removed, + "holdout_training_disjoint": removed == 0, + } + + +def assert_pinned_holdout_disjoint( + rows: Sequence[Mapping[str, Any]], + spec: HoldoutSpec, +) -> dict[str, int | bool]: + """Fail closed before training when a pinned export meets the holdout.""" + report = disjoint_report(rows, spec) + if report["holdout_training_disjoint"]: + return report + raise SystemExit( + "ADMISSION FAIL: holdout is not disjoint from the pinned training export " + f"(row_id_overlap={report['holdout_train_row_id_overlap']}, " + f"text_hash_overlap={report['holdout_train_text_hash_overlap']}, " + f"rows_removed={report['holdout_filter_training_rows_removed']})" + ) + + +def filter_holdout_rows(rows: list, spec: HoldoutSpec) -> tuple[list, int]: + """Drop matching rows. Returns the original list when nothing matches.""" + if not spec.active: + return rows, 0 + kept: list = [] + removed = 0 + for row in rows: + if holdout_match(row, spec): + removed += 1 + else: + kept.append(row) + if removed == 0: + return rows, 0 + return kept, removed + + +def assert_no_holdout(rows: Sequence[Mapping[str, Any]], spec: HoldoutSpec, where: str) -> None: + """Fail the run if any manifest id or text hash is still in ``rows``.""" + if not spec.active: + return + n_id = 0 + n_text = 0 + for row in rows: + kind = holdout_match(row, spec) + if kind == "row_id": + n_id += 1 + elif kind == "text": + n_text += 1 + if n_id or n_text: + raise SystemExit( + f"REFUSE: holdout overlap remains in {where}: " + f"{n_id} row_id match(es), {n_text} normalized-text match(es)" + ) + + +def holdout_receipt(spec: HoldoutSpec, removed: Mapping[str, int]) -> dict[str, Any]: + """Receipt block: file sha256 values and rows removed per split.""" + return { + "manifests": [dict(item) for item in spec.manifests], + "manifest_sha256": [str(item["sha256"]) for item in spec.manifests], + "removed": {key: int(removed.get(key, 0)) for key in _REMOVED_KEYS}, + "allow_no_holdout": allow_no_holdout(), + } + + +def log_holdout(spec: HoldoutSpec, removed: Mapping[str, int]) -> None: + """One stdout line for a training run that loaded a manifest or an override.""" + if not spec.manifests and not allow_no_holdout() and os.environ.get("HYPERLEX_ALLOW_TRAIN") != "1": + return + sha = ",".join(str(item["sha256"]) for item in spec.manifests) or "none" + counts = " ".join(f"{key}={int(removed.get(key, 0))}" for key in _REMOVED_KEYS) + override = "" + if allow_no_holdout() and not spec.manifests: + override = f" {ALLOW_NO_HOLDOUT_ENV}=1" + print(f"[holdout]{override} manifest_sha256={sha} removed {counts}", flush=True) diff --git a/scripts/shadow/hyperlexical/identity_ledger.py b/scripts/shadow/hyperlexical/identity_ledger.py new file mode 100644 index 00000000..231e9861 --- /dev/null +++ b/scripts/shadow/hyperlexical/identity_ledger.py @@ -0,0 +1,899 @@ +"""Global text-identity ledger for Hyperlex evaluation reserves. + +Contamination is owned by ``holdout_guard.normalized_text_sha256``. This +module does not train, score, or store raw text. A second normalizer is not +defined here. ``unbind_clean`` on a label is a slice predicate computed by +the caller with ``soft_ceiling.clean_surface``; it is not an identity. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any, Mapping, Sequence + +from .holdout_guard import normalized_text_sha256 +from .selection_surface import row_id + +SCHEMA = "hyperlex.identity_ledger.v1" +EVENT_SCHEMA = "hyperlex.identity_event.v1" +POLICY_ID = "hyperlex.eval_reserve.v1" +GENERATION = "GEN-0" +RESERVE_LEDGER_ENV = "HLX_EVAL_RESERVE_LEDGER" + +# Sealed planning quotas. These are the SELECT-002 row composition, not a +# statistical minimum. See evaluation-reserve.md. +PLANNING_TARGETS: dict[str, int] = { + "classify": 606, + "classify_observed": 287, + "classify_non_none": 604, + "unbind_clean": 250, +} +REQUIRED_SLICES = tuple(PLANNING_TARGETS) + +FORWARD = frozenset( + { + ("NEW", "TRAIN_CANDIDATE"), + ("NEW", "EVAL_RESERVE"), + ("AVAILABLE", "TRAIN_CANDIDATE"), + ("AVAILABLE", "EVAL_RESERVE"), + ("TRAIN_CANDIDATE", "TRAIN_CONSUMED"), + ("EVAL_RESERVE", "EVAL_BOUND"), + ("EVAL_BOUND", "EVAL_SPENT"), + ("EVAL_BOUND", "EVAL_ABANDONED"), + } +) +HISTORICAL_FLAGS = frozenset( + {"training_consumed", "evaluation_spent", "evaluation_abandoned"} +) +_FORBIDDEN_KEYS = frozenset( + { + "text", + "normalized_text", + "raw_text", + "candidate_score", + "model_error", + "model_score", + "prediction", + } +) +_SCORE_KEYS = frozenset({"candidate_score", "model_error", "model_score", "prediction"}) + + +def _refuse_mapping(value: Mapping[str, Any], where: str) -> None: + for key in value: + if key in _FORBIDDEN_KEYS: + raise SystemExit(f"REFUSE: {where} must not carry {key}") + + +def policy_sealed() -> bool: + """The assignment rule is the code under ``POLICY_ID``. It does not read scores.""" + return True + + +def _empty_identity(digest: str) -> dict[str, Any]: + return { + "normalized_text_sha256": digest, + "row_ids": [], + "source_artifacts": [], + "first_seen": "", + "current_sources": [], + "training_consumed": False, + "training_artifacts": [], + "evaluation_spent": False, + "evaluation_spent_artifacts": [], + "evaluation_abandoned": False, + "evaluation_abandoned_artifacts": [], + "evaluation_reserved": False, + "evaluation_bound": False, + "train_candidate": False, + "catalogued": False, + "experiment_bindings": [], + "labels": [], + "provenance": [], + "generation": GENERATION, + } + + +def derived_state(record: Mapping[str, Any]) -> str: + """Blocking historical flags win. Routing flags apply only when none are set.""" + if record.get("training_consumed"): + return "TRAIN_CONSUMED" + if record.get("evaluation_spent"): + return "EVAL_SPENT" + if record.get("evaluation_abandoned"): + return "EVAL_ABANDONED" + if record.get("evaluation_bound"): + return "EVAL_BOUND" + if record.get("evaluation_reserved"): + return "EVAL_RESERVE" + if record.get("train_candidate"): + return "TRAIN_CANDIDATE" + if record.get("catalogued"): + return "AVAILABLE" + return "NEW" + + +def slices_of(record: Mapping[str, Any]) -> set[str]: + """Metric slices this identity can support. One identity counts once per slice.""" + found: set[str] = set() + classify = [lab for lab in record.get("labels") or [] if lab.get("task") == "classify"] + if classify: + found.add("classify") + if any(lab.get("class") == "OBSERVED" for lab in classify): + found.add("classify_observed") + if any(str(lab.get("lineage") or "") not in ("", "none") for lab in classify): + found.add("classify_non_none") + if any(lab.get("task") == "unbind" and lab.get("unbind_clean") is True for lab in record.get("labels") or []): + found.add("unbind_clean") + return found + + +def _sorted_unique(items: Sequence[str]) -> list[str]: + return sorted({item for item in items if item}) + + +def _label(row: Mapping[str, Any], *, unbind_clean: bool | None) -> dict[str, Any]: + label: dict[str, Any] = { + "task": str(row.get("task") or ""), + "class": str(row.get("class") or ""), + "lineage": str(row.get("lineage") or ""), + "split": str(row.get("split") or ""), + } + if label["task"] == "unbind": + label["unbind_clean"] = bool(unbind_clean) + return label + + +def _merge_label(labels: list[dict[str, Any]], label: Mapping[str, Any]) -> None: + if label not in labels: + labels.append(dict(label)) + labels.sort(key=lambda item: json.dumps(item, sort_keys=True)) + + +class IdentityLedger: + """Append-only events plus a derived identity index. No raw text.""" + + def __init__(self) -> None: + self.events: list[dict[str, Any]] = [] + self.identities: dict[str, dict[str, Any]] = {} + self.row_owner: dict[str, str] = {} + self.row_id_collisions: list[dict[str, str]] = [] + + def identity(self, digest: str) -> dict[str, Any] | None: + return self.identities.get(digest) + + def _append(self, event: dict[str, Any]) -> None: + _refuse_mapping(event, "ledger event") + event = dict(event) + event["schema"] = EVENT_SCHEMA + event["seq"] = len(self.events) + 1 + event["generation"] = GENERATION + self.events.append(event) + + def _touch(self, digest: str) -> dict[str, Any]: + record = self.identities.get(digest) + if record is None: + record = _empty_identity(digest) + self.identities[digest] = record + return record + + def observe( + self, + digest: str, + *, + source_artifact: str, + row_ids: Sequence[str] = (), + labels: Sequence[Mapping[str, Any]] = (), + current_source: bool = False, + catalogued: bool = False, + provenance: str, + experiment_id: str = "", + ) -> dict[str, Any]: + """Attach ids, labels, and provenance. Does not route the identity.""" + if len(digest) != 64: + raise SystemExit("REFUSE: normalized_text_sha256 must be 64 hex characters") + record = self._touch(digest) + if not record["first_seen"]: + record["first_seen"] = source_artifact + record["source_artifacts"] = _sorted_unique([*record["source_artifacts"], source_artifact]) + if current_source: + record["current_sources"] = _sorted_unique([*record["current_sources"], source_artifact]) + if catalogued: + record["catalogued"] = True + for ident in row_ids: + self._claim_row_id(record, ident) + for label in labels: + _merge_label(record["labels"], label) + if experiment_id: + record["experiment_bindings"] = _sorted_unique( + [*record["experiment_bindings"], experiment_id] + ) + if provenance and provenance not in record["provenance"]: + record["provenance"].append(provenance) + self._append( + { + "kind": "observe", + "normalized_text_sha256": digest, + "source_artifact": source_artifact, + "row_ids": list(row_ids), + "current_source": current_source, + "catalogued": catalogued, + "provenance": provenance, + "experiment_id": experiment_id, + "labels": [dict(item) for item in labels], + } + ) + return record + + def _claim_row_id(self, record: dict[str, Any], ident: str) -> None: + if not ident: + return + owner = self.row_owner.get(ident) + digest = record["normalized_text_sha256"] + if owner is None: + self.row_owner[ident] = digest + elif owner != digest: + self.row_id_collisions.append( + {"row_id": ident, "existing_hash": owner, "new_hash": digest} + ) + if ident not in record["row_ids"]: + record["row_ids"] = _sorted_unique([*record["row_ids"], ident]) + + def mark_historical( + self, + digest: str, + flag: str, + *, + source_artifact: str, + provenance: str, + experiment_id: str = "", + ) -> dict[str, Any]: + """Record a receipt-backed past state. Flags only ever turn on.""" + if flag not in HISTORICAL_FLAGS: + raise SystemExit(f"REFUSE: unknown historical flag {flag}") + record = self._touch(digest) + record[flag] = True + artifact_key = { + "training_consumed": "training_artifacts", + "evaluation_spent": "evaluation_spent_artifacts", + "evaluation_abandoned": "evaluation_abandoned_artifacts", + }[flag] + record[artifact_key] = _sorted_unique([*record[artifact_key], source_artifact]) + if experiment_id: + record["experiment_bindings"] = _sorted_unique( + [*record["experiment_bindings"], experiment_id] + ) + if provenance and provenance not in record["provenance"]: + record["provenance"].append(provenance) + self._append( + { + "kind": "historical", + "normalized_text_sha256": digest, + "flag": flag, + "source_artifact": source_artifact, + "provenance": provenance, + "experiment_id": experiment_id, + } + ) + return record + + def transition(self, digest: str, to_state: str, *, source_artifact: str, provenance: str) -> str: + """Monotonic forward route. Historical blocks cannot be cleared.""" + record = self.identities.get(digest) + if record is None: + raise SystemExit("REFUSE: cannot route an unseen text identity") + state = derived_state(record) + if record["training_consumed"] or record["evaluation_spent"] or record["evaluation_abandoned"]: + raise SystemExit( + f"REFUSE: text identity is {state}; that block is monotonic in {GENERATION}" + ) + if (state, to_state) not in FORWARD: + raise SystemExit(f"REFUSE: transition {state} -> {to_state} is not allowed") + if to_state == "TRAIN_CANDIDATE": + record["train_candidate"] = True + elif to_state == "TRAIN_CONSUMED": + record["training_consumed"] = True + record["training_artifacts"] = _sorted_unique( + [*record["training_artifacts"], source_artifact] + ) + elif to_state == "EVAL_RESERVE": + record["evaluation_reserved"] = True + elif to_state == "EVAL_BOUND": + record["evaluation_bound"] = True + elif to_state == "EVAL_SPENT": + record["evaluation_spent"] = True + elif to_state == "EVAL_ABANDONED": + record["evaluation_abandoned"] = True + else: + raise SystemExit(f"REFUSE: unknown route {to_state}") + if provenance and provenance not in record["provenance"]: + record["provenance"].append(provenance) + self._append( + { + "kind": "transition", + "normalized_text_sha256": digest, + "from_state": state, + "to_state": to_state, + "source_artifact": source_artifact, + "provenance": provenance, + } + ) + return to_state + + def request_generation_reset(self, *, governed_receipt: str, confirm: bool) -> dict[str, Any]: + """Record a request. Does not clear flags and does not create GEN-1.""" + if not confirm or not str(governed_receipt or "").strip(): + raise SystemExit( + "REFUSE: generation reset requires confirm=True and a governed receipt" + ) + self._append( + { + "kind": "generation_reset_requested", + "normalized_text_sha256": "", + "source_artifact": governed_receipt, + "provenance": "requested only; flags unchanged; GEN-1 not created", + "applied": False, + } + ) + return {"applied": False, "generation": GENERATION, "flags_cleared": 0} + + def observe_row( + self, + row: Mapping[str, Any], + *, + source_artifact: str, + provenance: str, + current_source: bool = False, + catalogued: bool = False, + unbind_clean: bool | None = None, + experiment_id: str = "", + ) -> str: + """Hash one row and catalogue it. The row text is not retained.""" + if any(key in row for key in _SCORE_KEYS): + raise SystemExit("REFUSE: catalogue input carries a model outcome field") + digest = normalized_text_sha256(str(row.get("text") or "")) + ids = [row_id(row)] + declared = row.get("row_id") + if isinstance(declared, str) and declared and declared not in ids: + ids.append(declared) + self.observe( + digest, + source_artifact=source_artifact, + row_ids=ids, + labels=[_label(row, unbind_clean=unbind_clean)], + current_source=current_source, + catalogued=catalogued, + provenance=provenance, + experiment_id=experiment_id, + ) + return digest + + def reserve_counts(self) -> dict[str, int]: + counts = {key: 0 for key in REQUIRED_SLICES} + for record in self.identities.values(): + if derived_state(record) not in ("EVAL_RESERVE", "EVAL_BOUND"): + continue + for key in slices_of(record): + if key in counts: + counts[key] += 1 + return counts + + def active_reserve_records(self, experiment_id: str) -> list[dict[str, Any]]: + """Current EVAL_RESERVE identities bound only to this experiment. + + ``evaluation_reserved`` is historical evidence. Spent and abandoned + identities keep that flag and are not members. ``EVAL_BOUND`` is not + an active reserve. A binding list that names another experiment does + not satisfy this experiment. + """ + wanted = str(experiment_id or "") + if not wanted: + return [] + selected: list[dict[str, Any]] = [] + for record in self.identities.values(): + if derived_state(record) != "EVAL_RESERVE": + continue + bindings = [str(item) for item in record.get("experiment_bindings") or [] if str(item)] + if bindings == [wanted]: + selected.append(record) + return selected + + def active_reserve_counts(self, experiment_id: str) -> dict[str, int]: + counts = {key: 0 for key in REQUIRED_SLICES} + for record in self.active_reserve_records(experiment_id): + for key in slices_of(record): + if key in counts: + counts[key] += 1 + return counts + + def screen( + self, + rows: Sequence[Mapping[str, Any]], + *, + batch_id: str, + targets: Mapping[str, int] | None = None, + unbind_clean_hashes: set[str] | None = None, + ) -> dict[str, Any]: + """Admission decision without mutation. Duplicate text does not raise n.""" + return self._route( + rows, + batch_id=batch_id, + targets=dict(targets or PLANNING_TARGETS), + unbind_clean_hashes=set(unbind_clean_hashes or ()), + apply=False, + ) + + def admit( + self, + rows: Sequence[Mapping[str, Any]], + *, + batch_id: str, + source_artifact: str, + targets: Mapping[str, int] | None = None, + unbind_clean_hashes: set[str] | None = None, + ) -> dict[str, Any]: + """Route genuinely new text. Rejects hashes already consumed, spent, abandoned, or reserved.""" + if not str(batch_id or "").strip(): + raise SystemExit("REFUSE: admission batch_id is required") + report = self._route( + rows, + batch_id=batch_id, + targets=dict(targets or PLANNING_TARGETS), + unbind_clean_hashes=set(unbind_clean_hashes or ()), + apply=True, + source_artifact=source_artifact, + ) + return report + + def _route( + self, + rows: Sequence[Mapping[str, Any]], + *, + batch_id: str, + targets: dict[str, int], + unbind_clean_hashes: set[str], + apply: bool, + source_artifact: str = "", + ) -> dict[str, Any]: + grouped: dict[str, list[Mapping[str, Any]]] = {} + collisions = 0 + batch_owner: dict[str, str] = {} + for row in rows: + if any(key in row for key in _SCORE_KEYS): + raise SystemExit("REFUSE: admission input carries a model outcome field") + digest = normalized_text_sha256(str(row.get("text") or "")) + grouped.setdefault(digest, []).append(row) + declared = row.get("row_id") if isinstance(row.get("row_id"), str) else "" + computed = row_id(row) + idents = [declared] if declared else [] + if computed not in idents: + idents.append(computed) + row_collides = False + for ident in idents: + owner = self.row_owner.get(ident) or batch_owner.get(ident) + if owner is not None and owner != digest: + row_collides = True + elif ident not in self.row_owner: + batch_owner.setdefault(ident, digest) + if row_collides: + collisions += 1 + def sort_key(digest: str) -> str: + return hashlib.sha256(f"{GENERATION}|{batch_id}|{digest}".encode("utf-8")).hexdigest() + + counts = self.reserve_counts() + rejected: dict[str, int] = { + "TRAIN_CONSUMED": 0, + "EVAL_SPENT": 0, + "EVAL_ABANDONED": 0, + "EVAL_RESERVE": 0, + "EVAL_BOUND": 0, + "TRAIN_CANDIDATE": 0, + } + reserved = 0 + candidates = 0 + duplicate_text_rows = 0 + accepted_hashes: list[str] = [] + for digest in sorted(grouped, key=sort_key): + members = grouped[digest] + if len(members) > 1: + duplicate_text_rows += len(members) - 1 + record = self.identities.get(digest) + state = derived_state(record) if record else "NEW" + if state in rejected: + rejected[state] += 1 + continue + if state not in ("NEW", "AVAILABLE"): + raise SystemExit(f"REFUSE: cannot admit text in state {state}") + pending_labels = [] + pending_ids: list[str] = [] + for row in members: + clean = digest in unbind_clean_hashes + pending_labels.append(_label(row, unbind_clean=clean if str(row.get("task") or "") == "unbind" else None)) + pending_ids.append(row_id(row)) + declared = row.get("row_id") + if isinstance(declared, str) and declared: + pending_ids.append(declared) + snapshot = _empty_identity(digest) + if record: + snapshot["labels"] = [dict(item) for item in record["labels"]] + for label in pending_labels: + _merge_label(snapshot["labels"], label) + served = slices_of(snapshot) + opens = [key for key in served if counts.get(key, 0) < int(targets.get(key, 0))] + to_state = "EVAL_RESERVE" if opens else "TRAIN_CANDIDATE" + if apply: + self.observe( + digest, + source_artifact=source_artifact, + row_ids=pending_ids, + labels=pending_labels, + provenance=f"admitted batch {batch_id}", + ) + self.transition( + digest, + to_state, + source_artifact=source_artifact, + provenance=f"assignment {POLICY_ID} batch {batch_id}", + ) + if to_state == "EVAL_RESERVE": + reserved += 1 + accepted_hashes.append(digest) + for key in served: + if key in counts: + counts[key] += 1 + else: + candidates += 1 + return { + "policy_id": POLICY_ID, + "generation": GENERATION, + "batch_id": batch_id, + "raw_rows": len(rows), + "unique_canonical_text_identities": len(grouped), + "unique_admitted_to_eval_reserve": reserved, + "unique_routed_to_train_candidate": candidates, + "duplicate_text_rows_not_counted": duplicate_text_rows, + "row_id_collisions_with_new_text": collisions, + "rejected_existing_identities": rejected, + "rejected_unique_identities": sum(rejected.values()), + "reserve_counts_after": counts if apply else self._projected_counts(counts), + "applied": apply, + "model_outcomes_consulted": False, + } + + def _projected_counts(self, counts: dict[str, int]) -> dict[str, int]: + return dict(counts) + + def select_003_gate(self, train_hashes: set[str]) -> dict[str, Any]: + """Necessary conditions only. Does not draft an experiment.""" + reserved = [ + record + for record in self.identities.values() + if derived_state(record) in ("EVAL_RESERVE", "EVAL_BOUND") + ] + hashes = {record["normalized_text_sha256"] for record in reserved} + counts = self.reserve_counts() + represented = {key: counts[key] >= 1 for key in REQUIRED_SLICES} + blocked = [ + record["normalized_text_sha256"] + for record in reserved + if record["training_consumed"] or record["evaluation_spent"] or record["evaluation_abandoned"] + ] + overlap = sorted(hashes & train_hashes) + eligible = ( + policy_sealed() + and bool(reserved) + and all(represented.values()) + and not blocked + and not overlap + ) + return { + "select_003": "NOT_DRAFTED", + "eligible": eligible, + "policy_sealed": policy_sealed(), + "reserve_identities": len(reserved), + "required_slices_represented": represented, + "text_disjoint_from_training": not overlap, + "training_overlap_identities": len(overlap), + "spent_or_abandoned_in_reserve": len(blocked), + "authorization": "SEPARATE", + "statistical_minimum": "NOT_COMPUTABLE", + } + + def census(self, live_hashes: set[str]) -> dict[str, Any]: + """Aggregate counts. Hash lists stay in the ledger, not in this summary.""" + def bucket(pred) -> set[str]: + return {digest for digest, record in self.identities.items() if pred(record)} + + consumed = bucket(lambda record: record["training_consumed"]) + spent = bucket(lambda record: record["evaluation_spent"]) + abandoned = bucket(lambda record: record["evaluation_abandoned"]) + available = bucket(lambda record: derived_state(record) == "AVAILABLE") + live_available = available & live_hashes + by_slice = {key: 0 for key in (*REQUIRED_SLICES, "unbind", "classify_inferred", "other")} + for digest in live_available: + record = self.identities[digest] + labels = list(record["labels"]) + tasks = {lab.get("task") for lab in labels} + classify = [lab for lab in labels if lab.get("task") == "classify"] + if classify: + by_slice["classify"] += 1 + if any(lab.get("class") == "OBSERVED" for lab in classify): + by_slice["classify_observed"] += 1 + if any(lab.get("class") == "INFERRED" for lab in classify): + by_slice["classify_inferred"] += 1 + if any(str(lab.get("lineage") or "") not in ("", "none") for lab in classify): + by_slice["classify_non_none"] += 1 + if "unbind" in tasks: + by_slice["unbind"] += 1 + if any( + lab.get("task") == "unbind" and lab.get("unbind_clean") is True for lab in labels + ): + by_slice["unbind_clean"] += 1 + if not tasks: + by_slice["other"] += 1 + return { + "generation": GENERATION, + "policy_id": POLICY_ID, + "identities": len(self.identities), + "all_live_text_hashes": len(live_hashes), + "live_hashes_in_ledger": len(live_hashes & set(self.identities)), + "training_consumed_hashes": len(consumed), + "spent_evaluation_hashes": len(spent), + "abandoned_evaluation_hashes": len(abandoned), + "currently_available_hashes": len(available), + "live_training_consumed_hashes": len(consumed & live_hashes), + "live_spent_evaluation_hashes": len(spent & live_hashes), + "live_abandoned_evaluation_hashes": len(abandoned & live_hashes), + "live_available_hashes": len(live_available), + "available_by_slice": by_slice, + "reserve_counts": self.reserve_counts(), + "row_id_collisions": len(self.row_id_collisions), + } + + def project(self) -> dict[str, Any]: + identities = [] + for digest in sorted(self.identities): + record = dict(self.identities[digest]) + record["state"] = derived_state(record) + record["slices"] = sorted(slices_of(record)) + identities.append(record) + payload = { + "schema": SCHEMA, + "policy_id": POLICY_ID, + "generation": GENERATION, + "identity_function": "hyperlexical.holdout_guard.normalized_text_sha256", + "planning_targets": dict(PLANNING_TARGETS), + "planning_target_class": "PLANNING_TARGET", + "statistical_minimum": "NOT_COMPUTABLE", + "identities": identities, + "row_id_collisions": list(self.row_id_collisions), + "n_events": len(self.events), + } + _walk_forbid(payload) + return payload + + def persist_append(self, directory: str | Path, prior_event_count: int) -> int: + """Append events after ``prior_event_count`` and refresh the projection. + + ``events.jsonl`` is append-only. ``ledger.json`` is a derived view. + """ + root = Path(directory) + events_path = root / "events.jsonl" + fresh = self.events[prior_event_count:] + if fresh: + with events_path.open("a", encoding="utf-8") as handle: + for event in fresh: + handle.write(json.dumps(event, sort_keys=True) + "\n") + (root / "ledger.json").write_text( + json.dumps(self.project(), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return len(fresh) + + def save(self, directory: str | Path) -> None: + root = Path(directory) + root.mkdir(parents=True, exist_ok=True) + events_path = root / "events.jsonl" + if events_path.exists(): + raise SystemExit(f"REFUSE: ledger events already exist: {events_path}") + lines = [json.dumps(event, sort_keys=True) for event in self.events] + events_path.write_text("\n".join(lines) + ("\n" if lines else ""), encoding="utf-8") + (root / "ledger.json").write_text( + json.dumps(self.project(), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + @classmethod + def load(cls, directory: str | Path) -> "IdentityLedger": + events_path = Path(directory) / "events.jsonl" + if not events_path.is_file(): + raise SystemExit(f"REFUSE: ledger events are missing: {events_path}") + ledger = cls() + for line in events_path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + event = json.loads(line) + kind = event.get("kind") + if kind == "observe": + ledger.observe( + event["normalized_text_sha256"], + source_artifact=event.get("source_artifact") or "", + row_ids=event.get("row_ids") or [], + labels=event.get("labels") or [], + current_source=bool(event.get("current_source")), + catalogued=bool(event.get("catalogued")), + provenance=event.get("provenance") or "", + experiment_id=event.get("experiment_id") or "", + ) + elif kind == "historical": + ledger.mark_historical( + event["normalized_text_sha256"], + event["flag"], + source_artifact=event.get("source_artifact") or "", + provenance=event.get("provenance") or "", + experiment_id=event.get("experiment_id") or "", + ) + elif kind == "transition": + ledger.transition( + event["normalized_text_sha256"], + event["to_state"], + source_artifact=event.get("source_artifact") or "", + provenance=event.get("provenance") or "", + ) + elif kind == "generation_reset_requested": + ledger.request_generation_reset( + governed_receipt=event.get("source_artifact") or "", + confirm=True, + ) + else: + raise SystemExit(f"REFUSE: unknown ledger event {kind}") + # Replay appended a second copy. Replace with the file's events. + ledger.events = [] + for line in events_path.read_text(encoding="utf-8").splitlines(): + if line.strip(): + ledger.events.append(json.loads(line)) + return ledger + + +def _walk_forbid(value: Any) -> None: + if isinstance(value, dict): + for key, item in value.items(): + if key in _FORBIDDEN_KEYS: + raise SystemExit(f"REFUSE: ledger projection contains {key}") + _walk_forbid(item) + elif isinstance(value, list): + for item in value: + _walk_forbid(item) + + +def assert_training_disjoint_from_reserve( + rows: Sequence[Mapping[str, Any]], + ledger: IdentityLedger, +) -> dict[str, int | bool]: + """Fail closed when pinned training contains fresh reserve text. + + Spent and abandoned flags do not trip this gate. Those identities may + already sit inside the historical training export. + """ + reserved = 0 + for row in rows: + digest = normalized_text_sha256(str(row.get("text") or "")) + record = ledger.identity(digest) + if record is None: + continue + if derived_state(record) in ("EVAL_RESERVE", "EVAL_BOUND"): + reserved += 1 + if reserved: + raise SystemExit( + "ADMISSION FAIL: training input contains " + f"{reserved} EVAL_RESERVE text identities" + ) + return { + "eval_reserve_training_overlap": 0, + "eval_reserve_training_disjoint": True, + } + + +def _read_jsonl(path: Path) -> list[dict[str, Any]]: + rows = [] + with path.open(encoding="utf-8") as handle: + for line in handle: + if line.strip(): + rows.append(json.loads(line)) + return rows + + +def main(argv: Sequence[str] | None = None) -> int: + """Admission, census, and evaluation settlement. Does not train or score.""" + import argparse + import sys + + parser = argparse.ArgumentParser(description="Hyperlex evaluation-reserve ledger") + sub = parser.add_subparsers(dest="cmd", required=True) + admit = sub.add_parser("admit") + admit.add_argument("--ledger", required=True) + admit.add_argument("--rows", required=True) + admit.add_argument("--batch-id", required=True) + admit.add_argument("--source", required=True) + admit.add_argument( + "--train-export", + default="", + help="JSONL export. unbind_clean uses soft_ceiling.clean_surface on split=train.", + ) + census_cmd = sub.add_parser("census") + census_cmd.add_argument("--ledger", required=True) + census_cmd.add_argument("--live-hashes", help="Optional JSON list of live text hashes") + settle = sub.add_parser("settlement-apply") + settle.add_argument("--stream-rows", required=True) + settle.add_argument("--sheet", action="append", default=[]) + settle.add_argument("--records", default="") + settle.add_argument("--operator", required=True) + settle.add_argument("--provenance", required=True) + settle.add_argument("--batch-id", required=True) + settle.add_argument("--receipt", required=True) + settle.add_argument("--settlement-log", required=True) + settle.add_argument("--settled-at", required=True) + settle.add_argument("--stream-run-id", default="") + settle.add_argument("--activated-family", action="append", default=[]) + args = parser.parse_args(list(argv) if argv is not None else None) + if args.cmd == "settlement-apply": + from .eval_settlement import run_settlement_apply + + return run_settlement_apply(args) + if args.cmd == "census": + ledger = IdentityLedger.load(args.ledger) + live: set[str] = set() + if args.live_hashes: + payload = json.loads(Path(args.live_hashes).read_text(encoding="utf-8")) + live = set(payload) + report = ledger.census(live) + report["acquisition_gap"] = acquisition_gap(ledger.reserve_counts()) + sys.stdout.write(json.dumps(report, indent=2, sort_keys=True) + "\n") + return 0 + ledger = IdentityLedger.load(args.ledger) + prior = len(ledger.events) + rows = _read_jsonl(Path(args.rows)) + clean_hashes: set[str] = set() + if args.train_export: + from .clean_unbind import unbind_clean_hashes + + exported = _read_jsonl(Path(args.train_export)) + clean_hashes, _account = unbind_clean_hashes(rows, exported) + report = ledger.admit( + rows, + batch_id=args.batch_id, + source_artifact=args.source, + unbind_clean_hashes=clean_hashes, + ) + written = ledger.persist_append(args.ledger, prior) + report["events_appended"] = written + report["acquisition_gap"] = acquisition_gap(ledger.reserve_counts()) + _walk_forbid(report) + sys.stdout.write(json.dumps(report, indent=2, sort_keys=True) + "\n") + return 0 + + +def acquisition_gap(reserved: Mapping[str, int] | None = None) -> list[dict[str, Any]]: + """Planning gaps. Statistical minima stay ``NOT_COMPUTABLE``.""" + counts = dict(reserved or {}) + rows = [] + for key, target in PLANNING_TARGETS.items(): + have = int(counts.get(key, 0)) + rows.append( + { + "slice": key, + "currently_reserved": have, + "minimum_required": "NOT_COMPUTABLE", + "planning_target": target, + "planning_target_class": "PLANNING_TARGET", + "gap_vs_minimum": "NOT_COMPUTABLE", + "gap_vs_planning_target": max(target - have, 0), + } + ) + return rows + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/shadow/hyperlexical/layout.py b/scripts/shadow/hyperlexical/layout.py new file mode 100644 index 00000000..ddcf171c --- /dev/null +++ b/scripts/shadow/hyperlexical/layout.py @@ -0,0 +1,115 @@ +"""Normative T1 head layout. No torch. No Hub.""" + +import os + +HIDDEN = 768 +LAYERS = 22 +LAST_TRAINABLE = 2 +LAST_TRAINABLE_MAX = 8 +LAST_TRAINABLE_ENV = "HYPERLEX_LAST_TRAINABLE" +MAX_LEN = 64 +TRUNK = "answerdotai/ModernBERT-base" +MODEL_ID_SEED = "hyperlex-encoder-modernbert-base-seed" + +FAMILIES = ( + "betting-sharp", + "crypto-degen", + "ai-native", + "brainrot-aura", + "kinship-address", + "political-status", + "gaming-meta", + "workplace-corp", + "none", +) + +UNK = "" +VOCAB_TRAIN_ONLY_ENV = "HLX_VOCAB_TRAIN_ONLY" + + +def _last_trainable_cap(layer_count: int | None) -> int: + if layer_count is None: + return min(LAST_TRAINABLE_MAX, LAYERS) + return min(LAST_TRAINABLE_MAX, max(1, int(layer_count))) + + +def resolve_last_trainable( + raw: str | int | None = None, + *, + layer_count: int | None = None, +) -> int: + """Effective last-N. Default 2. Env HYPERLEX_LAST_TRAINABLE; clamp to layer cap.""" + cap = _last_trainable_cap(layer_count) + if raw is None: + raw = os.environ.get(LAST_TRAINABLE_ENV) + if raw is None or (isinstance(raw, str) and not raw.strip()): + return min(LAST_TRAINABLE, cap) + try: + n = int(str(raw).strip(), 10) + except (TypeError, ValueError) as exc: + raise ValueError(f"{LAST_TRAINABLE_ENV} must be a positive int, got {raw!r}") from exc + if n < 1: + raise ValueError(f"{LAST_TRAINABLE_ENV} must be a positive int, got {n}") + return min(n, cap) + + +def resolve_vocab_train_only(raw: str | None = None) -> bool: + """Filler vocab from train rows only. Default off: train+val, same map as before.""" + if raw is None: + raw = os.environ.get(VOCAB_TRAIN_ONLY_ENV) + return raw == "1" + + +def label_maps(unbind_rows: list[dict], *, filler_rows: list[dict] | None = None) -> dict: + """Role vocab from ``unbind_rows``. Fillers from ``filler_rows`` or those same rows. + + ``filler_rows is None`` is the historical map (one row list supplies both). + An empty list is a real filler source and yields ``[""]`` only. + """ + roles = sorted({r for row in unbind_rows for r in (row.get("roles") or [])}) + filler_source = unbind_rows if filler_rows is None else filler_rows + fillers = sorted({f for row in filler_source for f in (row.get("fillers") or [])}) + role_vocab = [UNK] + roles + filler_vocab = [UNK] + fillers + return { + "families": list(FAMILIES), + "family_of": {f: i for i, f in enumerate(FAMILIES)}, + "role_vocab": role_vocab, + "filler_vocab": filler_vocab, + "role_of": {r: i for i, r in enumerate(role_vocab)}, + "filler_of": {f: i for i, f in enumerate(filler_vocab)}, + } + + +def label_maps_for_splits( + train_rows: list[dict], + val_rows: list[dict], + *, + train_only: bool | None = None, +) -> dict: + """Train+val label maps. ``HLX_VOCAB_TRAIN_ONLY=1`` limits fillers to train. + + Roles stay on train+val in both modes. With the flag off this is + ``label_maps(train + val)``. + """ + combined = list(train_rows) + list(val_rows) + if train_only is None: + train_only = resolve_vocab_train_only() + if train_only: + return label_maps(combined, filler_rows=list(train_rows)) + return label_maps(combined) + + +def describe(maps: dict) -> dict: + return { + "trunk": TRUNK, + "hidden": HIDDEN, + "layers": LAYERS, + "last_trainable": resolve_last_trainable(), + "max_len": MAX_LEN, + "classify": {"in": HIDDEN, "out": len(FAMILIES), "pool": "token_0"}, + "unbind_role": {"in": HIDDEN, "out": len(maps["role_vocab"]), "reads": "last_hidden_state"}, + "unbind_filler": {"in": HIDDEN, "out": len(maps["filler_vocab"]), "reads": "last_hidden_state"}, + "forbidden": ["refusal_head", "brier_head", "chat_template"], + "brier": None, + } diff --git a/scripts/shadow/hyperlexical/loop.py b/scripts/shadow/hyperlexical/loop.py new file mode 100644 index 00000000..46698e87 --- /dev/null +++ b/scripts/shadow/hyperlexical/loop.py @@ -0,0 +1,1833 @@ +"""Spark train loop. Gate only.""" + +from __future__ import annotations + +import hashlib +import json +import os +import time +from dataclasses import dataclass +from pathlib import Path + +from .align import atom_token_index, offsets_from_tokenizer, pool_indices +from .export import export_dataset, repo_root, write_export +from .admission import AdmissionError, admit_training_run +from .train_input import train_input_receipt +from .classify_metrics import ( + NONE_LABEL, + SELECT_METRIC_CLASSIFY, + SELECT_METRIC_ENV, + macro_f1_nonnone, + none_false_positive_rate, + resolve_select_metric, +) +from .classify_sampling import epoch_classify_rows +from .training_schedule import install_default_schedule, resolve_training_schedule +from .classify_split import apply_classify_split_file +from .seed_control import apply_training_seed +from .force_train_overlap import enforce_force_train_disjoint +from .layout import ( + FAMILIES, + HIDDEN, + MAX_LEN, + MODEL_ID_SEED, + TRUNK, + UNK, + describe, + label_maps_for_splits, + resolve_last_trainable, + resolve_vocab_train_only, +) +from .eval_forward import apply_encoder_trainable +from .filler_filter import assert_publishable_vocab, filter_mode, filter_unbind_rows +from .holdout_guard import ( + assert_no_holdout, + filter_holdout_rows, + holdout_receipt, + load_holdout_spec, + log_holdout, +) +from .provenance import provenance +from .release_set import maybe_release +from .save_pretrained import ( + collect_encoder_trainable, + save_heads, + split_weight_tensors, + write_skeleton, +) +from .training_routing import route_rows +from .unbind_curriculum import ( + plan_unbind_curriculum, + resolve_curriculum_schedule, + select_unbind_for_epoch, +) +from .unbind_metrics import mapped_filler, mapped_pred, summarize_unbind_pairs +from .unbind_recipe import ( + apply_unbind_force_train, + resolve_unbind_inferred_weight, + resolve_unbind_morph_margin, + shape_unbind_train, + unbind_row_sample_weight, +) +from .unbind_head_slot import ( + apply_head_slot_weight, + resolve_unbind_head_slot_weight, +) +from .unbind_second_slot import ( + apply_second_slot_weight, + resolve_unbind_second_slot_weight, +) +from .unbind_residual import ( + residual_row_record, + resolve_unbind_residual_dump_path, + write_residual_dump, +) +from .unbind_slot_ce import ( + UNBIND_SLOT_CE_AUX_LAMBDA, + combine_unbind_train_terms, + resolve_unbind_primary_mode, +) + +UNBIND_LOSS_WEIGHT_ENV = "HYPERLEX_UNBIND_LOSS_WEIGHT" +UNBIND_EVERY_N_ENV = "HYPERLEX_UNBIND_EVERY_N" +SAVE_BEST_UNBIND_ENV = "HYPERLEX_SAVE_BEST_UNBIND" +INIT_FROM_ENV = "HYPERLEX_INIT_FROM" +EARLY_STOP_ENV = "HYPERLEX_EARLY_STOP" +EARLY_STOP_PATIENCE_ENV = "HYPERLEX_EARLY_STOP_PATIENCE" +EARLY_STOP_MIN_EPOCHS_ENV = "HYPERLEX_EARLY_STOP_MIN_EPOCHS" +STOP_REASON_MAX_EPOCHS = "max_epochs" +STOP_REASON_EARLY_STOPPING = "early_stopping" +# Observational seconds on epoch-progress.jsonl and the completion receipt. +# Python round() to 6 decimal places (microseconds). Not a selection input. +WALLCLOCK_SECONDS_DECIMALS = 6 +_EARLY_STOP_OFF = frozenset({"0", "false", "no", "off"}) +_EARLY_STOP_ON = frozenset({"1", "true", "yes", "on"}) +INIT_EXPAND_VOCAB_ENV = "HYPERLEX_INIT_EXPAND_VOCAB" +UNBIND_LOSS_WEIGHT_DEFAULT = 1.0 +UNBIND_EVERY_N_DEFAULT = 1 + + +def resolve_save_best_unbind(raw: str | None = None) -> bool: + """When true, persist best-by-val-unbind_exact weights as primary model.safetensors. + + morph35 peak-not-saved: epoch_metrics recorded ep16 0.5372 but only final + weights were written. Opt-in via HYPERLEX_SAVE_BEST_UNBIND=1. + """ + if raw is None: + raw = os.environ.get(SAVE_BEST_UNBIND_ENV) + if raw is None or (isinstance(raw, str) and not raw.strip()): + return False + return str(raw).strip().lower() in {"1", "true", "yes", "on"} + + +@dataclass(frozen=True) +class EarlyStopConfig: + """Optional classify-metric early stop. This resolver stays off until enabled. + + Comparable classify runs receive the SELECT-006 candidate schedule from + ``resolve_training_schedule`` before admission. An explicit schedule + environment is left unchanged. + """ + + enabled: bool + max_epochs: int + patience: int | None = None + minimum_epochs: int | None = None + select_metric: str = "" + + +def _parse_early_stop_int(env_name: str, raw: str | None) -> int: + if raw is None or not str(raw).strip(): + raise ValueError(f"{env_name} is required when {EARLY_STOP_ENV} is enabled") + text = str(raw).strip() + sign = "" + digits = text + if text[0] in "+-": + sign, digits = text[0], text[1:] + if sign == "+" and not digits: + raise ValueError(f"{env_name} must be an integer, got {raw!r}") + if not digits.isdigit(): + raise ValueError(f"{env_name} must be an integer, got {raw!r}") + return int(sign + digits) + + +def resolve_early_stop_config(*, max_epochs: int, select_metric: str) -> EarlyStopConfig: + """Read early-stop env before the optimizer is constructed. + + Unset or disabled (0/false/no/off) leaves the configured epoch schedule + unchanged. ``HYPERLEX_TRAIN_EPOCHS`` is only the max-epoch cap and never + turns this on. Enabled runs require ``HLX_SELECT_METRIC=classify_macro_f1_nonnone``, + patience >= 0, and minimum scored epochs in ``1..max_epochs``. + """ + raw = os.environ.get(EARLY_STOP_ENV) + token = "" if raw is None else str(raw).strip().lower() + if raw is None or token == "" or token in _EARLY_STOP_OFF: + return EarlyStopConfig( + enabled=False, + max_epochs=max_epochs, + select_metric=select_metric, + ) + if token not in _EARLY_STOP_ON: + raise ValueError( + f"{EARLY_STOP_ENV} must be unset, off, or on (1/true/yes/on), got {raw!r}" + ) + if select_metric != SELECT_METRIC_CLASSIFY: + raise ValueError( + f"{EARLY_STOP_ENV} requires {SELECT_METRIC_ENV}={SELECT_METRIC_CLASSIFY}; " + f"got {select_metric!r}" + ) + patience = _parse_early_stop_int( + EARLY_STOP_PATIENCE_ENV, os.environ.get(EARLY_STOP_PATIENCE_ENV) + ) + minimum_epochs = _parse_early_stop_int( + EARLY_STOP_MIN_EPOCHS_ENV, os.environ.get(EARLY_STOP_MIN_EPOCHS_ENV) + ) + if patience < 0: + raise ValueError(f"{EARLY_STOP_PATIENCE_ENV} must be >= 0, got {patience}") + if minimum_epochs < 1: + raise ValueError( + f"{EARLY_STOP_MIN_EPOCHS_ENV} must be >= 1 scored epoch, got {minimum_epochs}" + ) + if minimum_epochs > max_epochs: + raise ValueError( + f"{EARLY_STOP_MIN_EPOCHS_ENV}={minimum_epochs} exceeds " + f"HYPERLEX_TRAIN_EPOCHS={max_epochs}" + ) + return EarlyStopConfig( + enabled=True, + max_epochs=max_epochs, + patience=patience, + minimum_epochs=minimum_epochs, + select_metric=select_metric, + ) + + +def note_strict_improvement( + best_value: float, + best_epoch: int | None, + score: float, + epoch_index: int, +) -> tuple[float, int | None, bool]: + """Strict increase replaces the checkpoint. A tie keeps the earlier epoch.""" + if score > best_value: + return score, epoch_index, True + return best_value, best_epoch, False + + +def early_stop_break( + *, + enabled: bool, + epoch_index: int, + best_epoch: int | None, + epochs_scored: int, + minimum_epochs: int | None, + patience: int | None, + max_epochs: int, +) -> bool: + """Return true only when the loop should break before the epoch cap. + + Call this after the epoch is scored and any strict improvement is recorded. + Patience is ``epoch_index - best_epoch`` completed epochs after the best + (0-based indices). The best epoch itself does not consume patience. + Ties do not move ``best_epoch``, so they do consume patience. + Do not stop before ``minimum_epochs`` epochs have been scored. + When the patience condition lands on the final configured epoch, the max + epoch cap wins and this returns false so the loop records ``max_epochs``. + """ + if not enabled: + return False + if epoch_index + 1 >= max_epochs: + return False + if minimum_epochs is None or patience is None or best_epoch is None: + return False + if epochs_scored < minimum_epochs: + return False + return (epoch_index - best_epoch) >= patience + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def round_seconds(value: float) -> float: + """Seconds rounded to 6 decimal places. Observational; not a selection input.""" + return round(float(value), WALLCLOCK_SECONDS_DECIMALS) + + +def monotonic_seconds() -> float: + """Monotonic clock in seconds. Tests replace this; production uses time.monotonic.""" + return time.monotonic() + + +def epoch_timing_fields( + *, + epoch_started: float, + epoch_ended: float, + training_started: float, +) -> dict[str, float]: + """Wall-clock fields for one epoch-progress.jsonl row. + + Units are seconds. Both values use ``round_seconds`` (6 decimal places). + ``epoch_wallclock_seconds`` is this epoch's scored body + (epoch_ended - epoch_started). ``training_elapsed_seconds`` is the time + from the start of the epoch loop to this epoch's end. Neither value is + read by checkpoint selection or early stopping. + """ + return { + "epoch_wallclock_seconds": round_seconds(epoch_ended - epoch_started), + "training_elapsed_seconds": round_seconds(epoch_ended - training_started), + } + + +def resolve_init_from(raw: str | None = None) -> Path | None: + """Optional warm-start dir with model.safetensors (or heads.pt). + + Opt-in via HYPERLEX_INIT_FROM=/path/to/prior seed (e.g. morph36 best). + """ + if raw is None: + raw = os.environ.get(INIT_FROM_ENV) + if raw is None or (isinstance(raw, str) and not raw.strip()): + return None + path = Path(str(raw).strip()).expanduser() + if not path.is_dir(): + raise ValueError(f"{INIT_FROM_ENV} must be an existing directory, got {path}") + return path + + +def resolve_init_expand_vocab(raw: str | None = None) -> bool: + """When true, warm-load remaps shared role/filler rows into expanded vocabs. + + Default fail-closed on vocab mismatch. Opt-in via HYPERLEX_INIT_EXPAND_VOCAB=1 + so a smaller prior seed (e.g. morph65 max pos_5) can warm a harvest that added + pos_6+/new fillers: copy overlapping labels by name, and set new rows to exact zero. + """ + if raw is None: + raw = os.environ.get(INIT_EXPAND_VOCAB_ENV) + if raw is None or (isinstance(raw, str) and not raw.strip()): + return False + return str(raw).strip().lower() in {"1", "true", "yes", "on"} + + +def _init_weight_path(init_dir: Path) -> Path: + for name in ("model.safetensors", "heads.pt"): + candidate = init_dir / name + if candidate.is_file(): + return candidate + raise FileNotFoundError( + f"{INIT_FROM_ENV}={init_dir} missing model.safetensors or heads.pt" + ) + + +def _read_init_vocabs(init_dir: Path) -> tuple[list | None, list | None]: + for name in ("layout.json", "config.json"): + path = init_dir / name + if not path.is_file(): + continue + try: + blob = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + continue + if not isinstance(blob, dict): + continue + role = blob.get("role_vocab") + filler = blob.get("filler_vocab") + if isinstance(role, list) and isinstance(filler, list) and filler: + return role, filler + return None, None + + +def _remap_linear_rows(module, state: dict, init_labels: list, current_labels: list, head: str, expected: dict | None = None) -> dict: + """Copy overlapping out-rows by label name. New target rows are exact zeros.""" + from .zero_init_loader import expand_named_linear + + return expand_named_linear( + module, + state, + list(init_labels), + list(current_labels), + head, + expected=expected, + ) + + +def warm_load_checkpoint( + encoder, + classify, + role_head, + filler_head, + maps: dict, + init_dir: Path, + *, + expand_vocab: bool | None = None, +) -> dict: + """Load heads + trainable encoder tensors from a prior seed dump. Fail closed. + + When expand_vocab is true (or HYPERLEX_INIT_EXPAND_VOCAB=1), role/filler heads + remap overlapping labels by name into the current vocabulary and set new rows + to exact zero. Classify still loads strict. Default remains exact-vocab match. + """ + if expand_vocab is None: + expand_vocab = resolve_init_expand_vocab() + weight_path = _init_weight_path(init_dir) + init_roles, init_fillers = _read_init_vocabs(init_dir) + cur_roles = list(maps.get("role_vocab") or []) + cur_fillers = list(maps.get("filler_vocab") or []) + roles_match = init_roles is None or init_roles == cur_roles + fillers_match = init_fillers is None or init_fillers == cur_fillers + vocab_match = roles_match and fillers_match + if not vocab_match and not expand_vocab: + if init_roles is not None and init_roles != cur_roles: + raise ValueError( + f"{INIT_FROM_ENV} role_vocab mismatch vs current export " + f"(init={len(init_roles)} current={len(cur_roles)})" + ) + raise ValueError( + f"{INIT_FROM_ENV} filler_vocab mismatch vs current export " + f"(init={len(init_fillers or [])} current={len(cur_fillers)})" + ) + if not vocab_match and expand_vocab: + if init_roles is None or init_fillers is None: + raise ValueError( + f"{INIT_EXPAND_VOCAB_ENV}=1 requires init role_vocab+filler_vocab " + f"in {init_dir}/config.json (or layout.json)" + ) + from .zero_init_loader import expansion_expected, select006_guard_vocab + + select006_guard_vocab(bool(expand_vocab), bool(vocab_match)) + + if weight_path.name == "model.safetensors": + from safetensors.torch import load_file + + split = split_weight_tensors(load_file(str(weight_path), device="cpu")) + heads_blob = None + else: + import torch + + try: + blob = torch.load(str(weight_path), map_location="cpu", weights_only=False) + except TypeError: + blob = torch.load(str(weight_path), map_location="cpu") + if not isinstance(blob, dict): + raise ValueError(f"{weight_path} is not a heads state dict") + split = { + "classify": blob.get("classify") or {}, + "role_head": blob.get("role_head") or {}, + "filler_head": blob.get("filler_head") or {}, + "encoder": { + k: v + for k, v in ( + {} if not isinstance(blob.get("encoder"), dict) else blob["encoder"] + ).items() + }, + } + # normalize encoder keys to encoder.* for apply_encoder_trainable + enc = {} + for k, v in split["encoder"].items(): + key = str(k) + enc[key if key.startswith("encoder.") else f"encoder.{key}"] = v + split["encoder"] = enc + heads_blob = blob + + classify_state = split.get("classify") or {} + if not classify_state: + raise ValueError(f"{weight_path} missing classify tensors") + classify.load_state_dict(classify_state, strict=True) + + expand_receipt: dict = { + "expand_vocab": bool(expand_vocab and not vocab_match), + "vocab_match": vocab_match, + } + if vocab_match or not expand_vocab: + for name, module in (("role_head", role_head), ("filler_head", filler_head)): + state = split.get(name) or {} + if not state: + raise ValueError(f"{weight_path} missing {name} tensors") + module.load_state_dict(state, strict=True) + else: + role_state = split.get("role_head") or {} + filler_state = split.get("filler_head") or {} + if not role_state or not filler_state: + raise ValueError(f"{weight_path} missing role_head/filler_head tensors") + expand_receipt["role"] = _remap_linear_rows( + role_head, + role_state, + list(init_roles), + cur_roles, + "role_head", + expected=expansion_expected("role_head"), + ) + expand_receipt["filler"] = _remap_linear_rows( + filler_head, + filler_state, + list(init_fillers), + cur_fillers, + "filler_head", + expected=expansion_expected("filler_head"), + ) + + applied = apply_encoder_trainable(encoder, split.get("encoder") or {}) + if applied["present"] and applied["loaded"] == 0: + raise ValueError( + f"{INIT_FROM_ENV} encoder tensors present but none matched trunk keys" + ) + return { + "init_from": str(init_dir), + "weight_file": weight_path.name, + "encoder_trainable_loaded": applied["loaded"], + "encoder_trainable_present": applied["present"], + "heads_blob": bool(heads_blob), + "init_expand_vocab": bool(expand_vocab), + **expand_receipt, + } + + +def _cpu_module_state(module) -> dict: + return {k: v.detach().cpu().contiguous() for k, v in module.state_dict().items()} + + +def _build_weight_state( + encoder, + classify, + role_head, + filler_head, + maps, + layout, + applicability=None, + family_head=None, +) -> dict: + state = { + "classify": _cpu_module_state(classify), + "role_head": _cpu_module_state(role_head), + "filler_head": _cpu_module_state(filler_head), + "encoder": collect_encoder_trainable(encoder), + "maps": {k: v for k, v in maps.items() if k not in {"family_of", "role_of", "filler_of"}}, + "layout": layout, + } + if applicability is not None: + state["applicability"] = _cpu_module_state(applicability) + if family_head is not None: + state["family_head"] = _cpu_module_state(family_head) + return state + + +def resolve_unbind_loss_weight(raw: str | float | int | None = None) -> float: + """Scale on unbind loss before backward. Default 1.0. Fail-closed if invalid.""" + if raw is None: + raw = os.environ.get(UNBIND_LOSS_WEIGHT_ENV) + if raw is None or (isinstance(raw, str) and not raw.strip()): + return UNBIND_LOSS_WEIGHT_DEFAULT + try: + weight = float(str(raw).strip()) + except (TypeError, ValueError) as exc: + raise ValueError(f"{UNBIND_LOSS_WEIGHT_ENV} must be a finite number >= 0, got {raw!r}") from exc + if weight < 0 or weight != weight or weight == float("inf"): + raise ValueError(f"{UNBIND_LOSS_WEIGHT_ENV} must be a finite number >= 0, got {raw!r}") + return weight + + +def resolve_unbind_every_n(raw: str | int | None = None) -> int: + """Classify-batch stride for an extra unbind step. Default 1 = epoch-end only.""" + if raw is None: + raw = os.environ.get(UNBIND_EVERY_N_ENV) + if raw is None or (isinstance(raw, str) and not raw.strip()): + return UNBIND_EVERY_N_DEFAULT + try: + n = int(str(raw).strip(), 10) + except (TypeError, ValueError) as exc: + raise ValueError(f"{UNBIND_EVERY_N_ENV} must be a positive int, got {raw!r}") from exc + if n < 1: + raise ValueError(f"{UNBIND_EVERY_N_ENV} must be a positive int, got {n}") + return n + + +TASK_ROUTING_ENV = "HYPERLEX_TASK_ROUTING" +TASK_ROUTINGS = ("route_rows", "legacy_split") + + +def task_routing(raw: str | None = None) -> str: + """``route_rows`` (default) or ``legacy_split`` (pre-routing loop; reproduces morph75–78). + + ``legacy_split`` selects rows by ``task == "classify"`` / ``task == "unbind"`` only, so + ``classify+unbind`` rows are not trained. Unknown values fail closed. + """ + value = (os.environ.get(TASK_ROUTING_ENV, "") if raw is None else raw).strip() or "route_rows" + if value not in TASK_ROUTINGS: + raise ValueError(f"{TASK_ROUTING_ENV} must be one of {TASK_ROUTINGS}") + return value + + +def should_interleave_unbind(classify_batch_index: int, every_n: int) -> bool: + """True after classify batch `index` (0-based) when every_n > 1.""" + if every_n <= 1: + return False + return (classify_batch_index + 1) % every_n == 0 + + +def prepare_unbind_splits(rows: list) -> tuple[list, list, dict]: + """Train recipe after route_rows. Val frozen unless force-train env is set. + + ``HYPERLEX_UNBIND_FORCE_TRAIN_PATH`` may move authorized OBSERVED exacts + from val→train (accept-style). Empty/unset → val untouched. + ``HLX_HOLDOUT_MANIFESTS`` rows are dropped before that move and before + hard-atom copies, then checked again so neither injection can put them back. + """ + spec = load_holdout_spec() + if task_routing() == "legacy_split": + train = [r for r in rows if r.get("task") == "unbind" and r.get("split") == "train"] + val = [r for r in rows if r.get("task") == "unbind" and r.get("split") == "val"] + else: + routed, _ = route_rows(rows) + train = routed["unbind"]["train"] + val = routed["unbind"]["val"] + train, n_train = filter_holdout_rows(train, spec) + val, n_val = filter_holdout_rows(val, spec) + train, val, force_stats = apply_unbind_force_train(train, val) + train, n_train_injected = filter_holdout_rows(train, spec) + val, n_val_injected = filter_holdout_rows(val, spec) + train, filt_train = filter_unbind_rows(train) + val, filt_val = filter_unbind_rows(val) + shaped, stats = shape_unbind_train(train) + shaped, n_hard = filter_holdout_rows(shaped, spec) + assert_no_holdout(shaped, spec, "unbind train") + assert_no_holdout(val, spec, "unbind val") + stats = { + **stats, + **force_stats, + "filler_filter": filt_train["filler_filter"], + "n_filler_rows_dropped_train": filt_train["n_filler_rows_dropped"], + "n_filler_rows_dropped_val": filt_val["n_filler_rows_dropped"], + "n_holdout_removed_train": n_train + n_train_injected + n_hard, + "n_holdout_removed_val": n_val + n_val_injected, + "holdout_manifests": [dict(item) for item in spec.manifests], + } + return shaped, val, stats + + +def _require_local_model(trunk: Path): + from transformers import AutoModel, AutoTokenizer + + tok = AutoTokenizer.from_pretrained(str(trunk), local_files_only=True) + model = AutoModel.from_pretrained(str(trunk), local_files_only=True) + return tok, model + + +def _layers(encoder): + if hasattr(encoder, "layers"): + return encoder.layers + inner = getattr(encoder, "encoder", None) + if inner is not None and hasattr(inner, "layers"): + return inner.layers + return None + + +def freeze_encoder(encoder, last_trainable: int | None = None) -> tuple[int, int]: + layers = _layers(encoder) + n_layers = len(list(layers)) if layers is not None else None + used = resolve_last_trainable(last_trainable, layer_count=n_layers) + for p in encoder.parameters(): + p.requires_grad = False + n = 0 + if layers is None: + return 0, used + for block in list(layers)[-used:]: + for p in block.parameters(): + p.requires_grad = True + n += p.numel() + return n, used + + +def _offsets(tok, text: str): + try: + return offsets_from_tokenizer(tok, text, max_len=MAX_LEN) + except TypeError: + return None + + +def _enter_training_execution() -> None: + """Reached only after admission. Admission-only mode returns before this.""" + return None + + +def run_loop( + trunk: Path, + out_dir: Path, + *, + include_live: bool = False, + live_store: Path | None = None, +) -> dict: + # Same gates as preflight. Admission-only returns before any optimizer. + # The schedule is fixed first so admission records the schedule that will run. + from .classification_v2 import classification_version, require_v2_schedule + + select_metric = resolve_select_metric() + v2_on = classification_version() == "v2" + schedule = resolve_training_schedule( + select_metric=SELECT_METRIC_CLASSIFY if v2_on else select_metric + ) + if v2_on: + require_v2_schedule(schedule.as_dict()) + if schedule.source == "select-006-candidate": + install_default_schedule(os.environ) + admission = admit_training_run( + include_live=include_live, + live_store=live_store, + export_dataset=export_dataset, + trunk=trunk, + out_dir=out_dir, + ) + if os.environ.get("HLX_ADMISSION_ONLY") == "1": + if not admission.ready: + raise AdmissionError(admission.error or "ADMISSION FAIL", admission.receipt) + return admission.receipt + bundle = admission.bundle + holdout_spec = admission.holdout_spec + input_receipt = train_input_receipt(bundle) + disjoint_receipt = admission.disjoint_receipt + reserve_receipt = admission.reserve_receipt + if bundle is None: + raise AdmissionError("ADMISSION FAIL: training bundle was not loaded", admission.receipt) + root = repo_root() + release_rows_, release_stats = maybe_release(bundle["rows"]) + if release_stats["release_set"]: + bundle = {**bundle, "rows": release_rows_} + if any(r.get("role_scheme") == "reviewed_occurrences" for r in bundle["rows"]): + raise ValueError("reviewed occurrences require occurrence-aware loop alignment") + routed, task_accounting = route_rows(bundle["rows"]) + task_accounting = {**task_accounting, "task_routing": task_routing()} + export_dir = Path(os.environ.get("HYPERLEX_EXPORT_DIR") or (root / "specs" / "007-hyperlexical-model" / "exports")) + export_dir.mkdir(parents=True, exist_ok=True) + write_export(export_dir, bundle) + if task_routing() == "legacy_split": + classify_tr = [r for r in bundle["rows"] if r["task"] == "classify" and r["split"] == "train"] + classify_va = [r for r in bundle["rows"] if r["task"] == "classify" and r["split"] == "val"] + else: + classify_tr = routed["classify"]["train"] + classify_va = routed["classify"]["val"] + classify_tr, classify_va, classify_split_receipt = apply_classify_split_file( + classify_tr, classify_va + ) + unbind_tr, unbind_va, unbind_recipe = prepare_unbind_splits(bundle["rows"]) + from .classify_admission import apply_classify_admission + + classify_tr, classify_va, classify_admission_receipt = apply_classify_admission( + bundle["rows"], classify_tr, classify_va + ) + classify_tr, n_classify_train = filter_holdout_rows(classify_tr, holdout_spec) + classify_va, n_classify_val = filter_holdout_rows(classify_va, holdout_spec) + assert_no_holdout(classify_tr, holdout_spec, "classify train") + assert_no_holdout(classify_va, holdout_spec, "classify val") + assert_no_holdout(unbind_tr, holdout_spec, "unbind train") + assert_no_holdout(unbind_va, holdout_spec, "unbind val") + holdout_removed = { + "classify_train": n_classify_train, + "classify_val": n_classify_val, + "unbind_train": unbind_recipe.get("n_holdout_removed_train", 0), + "unbind_val": unbind_recipe.get("n_holdout_removed_val", 0), + } + log_holdout(holdout_spec, holdout_removed) + if len(classify_tr) < 8: + raise RuntimeError("not enough classify train rows") + + select_metric = resolve_select_metric() + select_on_classify = select_metric == SELECT_METRIC_CLASSIFY + v2_checkpoint = v2_on + if select_on_classify: + selection_rows = list(classify_va or classify_tr[:8]) + else: + selection_rows = list(unbind_va or unbind_tr[:8]) + classify_va, unbind_va, force_overlap = enforce_force_train_disjoint( + classify_va, + unbind_va, + selection_rows, + ) + + _enter_training_execution() + import torch + from torch import nn + + seed_receipt = apply_training_seed(torch) + maps = label_maps_for_splits(unbind_tr, unbind_va) + if filter_mode() == "strict": + assert_publishable_vocab(maps["filler_vocab"]) + tok, encoder = _require_local_model(trunk) + hidden = int(getattr(encoder.config, "hidden_size", HIDDEN)) + if hidden != HIDDEN: + raise RuntimeError(f"hidden {hidden} != {HIDDEN}") + n_unfrozen, last_trainable_used = freeze_encoder(encoder) + if v2_on and last_trainable_used != 2: + raise RuntimeError("classification v2 keeps last_trainable = 2") + classify = nn.Linear(hidden, len(FAMILIES)) + role_head = nn.Linear(hidden, len(maps["role_vocab"])) + filler_head = nn.Linear(hidden, len(maps["filler_vocab"])) + init_from = resolve_init_from() + init_receipt: dict = {"init_from": None, "warm_start": False} + if init_from is not None: + init_receipt = { + "warm_start": True, + **warm_load_checkpoint( + encoder, classify, role_head, filler_head, maps, init_from + ), + } + from .zero_init_loader import ( + pre_optimizer_receipt, + pre_optimizer_stop_requested, + select006_training_init_guard, + ) + + select006_training_init_guard(maps, role_head, filler_head, init_receipt) + if pre_optimizer_stop_requested(): + return pre_optimizer_receipt(role_head, filler_head, maps, init_receipt) + applicability_head = None + family_head = None + v2_contract = None + if v2_on: + from .classification_v2 import freeze_training_contract + from .classification_v2_runtime import build_geometry_heads, build_repair_heads, build_v2_heads + + v2_contract = freeze_training_contract(classify_tr) + repair_path = os.environ.get("HLX_V2_GEOMETRY_REPAIR") + geometry_path = os.environ.get("HLX_V2_GEOMETRY_WITNESS") + witness_path = geometry_path or os.environ.get("HLX_V2_PROTOTYPE_WITNESS") + if repair_path: + boundaries = json.loads(Path(repair_path).read_text(encoding="utf-8")) + applicability_head, family_head, init_receipt["classification_v2_rows"] = build_repair_heads( + hidden, classify, list(FAMILIES), boundaries + ) + else: + if not witness_path: + raise RuntimeError("FAMILY_PROTOTYPE_UNAVAILABLE") + witness = json.loads(Path(witness_path).read_text(encoding="utf-8")) + if geometry_path: + applicability_head, family_head, init_receipt["classification_v2_rows"] = build_geometry_heads( + hidden, classify, list(FAMILIES), witness + ) + else: + applicability_head, family_head, init_receipt["classification_v2_rows"] = build_v2_heads( + hidden, classify, list(FAMILIES), witness + ) + classify_parameters = list(classify.parameters()) + if v2_on: + classify_parameters = list(applicability_head.parameters()) + list(family_head.parameters()) + trainable = [p for p in encoder.parameters() if p.requires_grad] + classify_parameters + list(role_head.parameters()) + list(filler_head.parameters()) + epochs = schedule.max_epochs + early_stop = resolve_early_stop_config( + max_epochs=epochs, + select_metric=SELECT_METRIC_CLASSIFY if v2_on else select_metric, + ) + if ( + early_stop.enabled != schedule.early_stopping + or early_stop.patience != schedule.patience + or early_stop.minimum_epochs != schedule.minimum_epochs + ): + raise ValueError("early stop does not match the training schedule") + from torch.optim import AdamW + + opt = AdamW(trainable, lr=float(os.environ.get("HYPERLEX_TRAIN_LR", "2e-5"))) + batch = int(os.environ.get("HYPERLEX_TRAIN_BATCH", "8")) + unbind_loss_weight = resolve_unbind_loss_weight() + unbind_every_n = resolve_unbind_every_n() + morph_margin = resolve_unbind_morph_margin() + inferred_weight = resolve_unbind_inferred_weight() + slot_ce_mode = resolve_unbind_primary_mode() + unbind_primary = slot_ce_mode["unbind_primary"] + head_slot_weight = resolve_unbind_head_slot_weight() + second_slot_weight = resolve_unbind_second_slot_weight() + curriculum = resolve_curriculum_schedule() + curriculum_plan = plan_unbind_curriculum(unbind_tr, epochs, curriculum) + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + global_step = 0 + loss_sum = torch.zeros((), device=device) + loss_steps = 0 + modules = [encoder, classify, role_head, filler_head] + if v2_on: + modules.extend([applicability_head, family_head]) + for mod in modules: + mod.to(device) + app_loss_sum = torch.zeros((), device=device) + fam_loss_sum = torch.zeros((), device=device) + unb_loss_sum = torch.zeros((), device=device) + app_loss_steps = fam_loss_steps = unb_loss_steps = 0 + encoder.train() + losses = [] + epoch_metrics = [] + + def encode_texts(texts): + enc = tok(texts, padding=True, truncation=True, max_length=MAX_LEN, return_tensors="pt") + return {k: v.to(device) for k, v in enc.items()} + + if v2_on and hasattr(family_head, "anchor_cosine"): + from .classification_v2 import ACTIVE_FAMILY_VOCABULARY as _PRIOR_NAMES + from .classification_v2_geometry_repair import ( + flatten_anchors as _flatten_anchors, + hard_negatives_from_boundaries as _hard_negatives, + row_geometry_scores as _row_geometry_scores, + summarize_geometry_margins as _summarize_geometry_margins, + ) + + _repair = os.environ.get("HLX_V2_GEOMETRY_REPAIR") + if not _repair: + raise RuntimeError("FAMILY_GEOMETRY_REPAIR_UNAVAILABLE") + _boundaries = json.loads(Path(_repair).read_text(encoding="utf-8")) + _anchors = _flatten_anchors(_boundaries) + _hard = _hard_negatives(_boundaries) + _prior_rows = [] + encoder.eval() + family_head.eval() + with torch.no_grad(): + for row in classify_va: + lineage = row.get("lineage") + if lineage not in _PRIOR_NAMES: + continue + if row.get("evaluation_reserve") or row.get("held_out") or row.get("split") == "test": + raise RuntimeError("evaluation_isolation") + encoded = encoder(**encode_texts([row["text"]])) + pooled = encoded.last_hidden_state[:, 0] + similarities = [ + float(value) for value in family_head.anchor_cosine(pooled)[0].detach().cpu() + ] + scores = _row_geometry_scores(similarities, str(lineage), _anchors, _hard) + _prior_rows.append({"family": lineage, **scores}) + family_head.train() + encoder.train() + prior_geometry = { + **_summarize_geometry_margins(_prior_rows), + "boundary_sha256": _boundaries["boundary_sha256"], + "phase": "pre_training", + "reserve_used": False, + "schema": "hyperlex.classification.v2.geometry_repair_eval.v1", + "separation_sha256": _boundaries["separation_sha256"], + "surface": "validation", + } + out_dir.mkdir(parents=True, exist_ok=True) + (out_dir / "classification-v2-geometry-repair-prior.json").write_text( + json.dumps(prior_geometry, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + init_receipt["classification_v2_geometry_repair_prior"] = { + "high_collision_rows": prior_geometry["high_collision_rows"], + "median_margin": prior_geometry["median_margin"], + "mean_margin": prior_geometry["mean_margin"], + "n_family": prior_geometry["n_family"], + } + + def unbind_loss(row): + fillers = list(row.get("fillers") or []) + roles = list(row.get("roles") or []) + if not fillers: + return None + out = encoder(**encode_texts([row["text"]])) + states = out.last_hidden_state[0] + offs = _offsets(tok, row["text"]) + slot_ces = [] + aux_terms = [] + for k, fill in enumerate(fillers): + idxs = pool_indices(states.size(0), atom_token_index(row["text"], fill, offs)) + h = states[idxs].mean(0) + gold_f = maps["filler_of"].get(fill, maps["filler_of"][UNK]) + logits = filler_head(h) + slot_ces.append( + nn.functional.cross_entropy( + logits.unsqueeze(0), torch.tensor([gold_f], device=device) + ) + ) + hard = [nf for nf in (row.get("hard_neg_fillers") or []) if nf] + if hard: + gold_logit = logits[gold_f] + neg_vals = [] + for nf in hard: + ni = maps["filler_of"].get(nf) + if ni is None or ni == gold_f: + continue + neg_vals.append(logits[ni]) + if neg_vals: + stacked = torch.stack(neg_vals) + aux_terms.append(torch.relu(stacked + morph_margin - gold_logit).sum()) + if k < len(roles): + gold_r = maps["role_of"].get(roles[k], maps["role_of"][UNK]) + aux_terms.append( + nn.functional.cross_entropy( + role_head(h).unsqueeze(0), torch.tensor([gold_r], device=device) + ) + ) + weighted_slots = apply_head_slot_weight(slot_ces, head_slot_weight) + weighted_slots = apply_second_slot_weight(weighted_slots, second_slot_weight) + return combine_unbind_train_terms( + weighted_slots, + aux_terms, + primary=unbind_primary, + aux_lambda=UNBIND_SLOT_CE_AUX_LAMBDA, + ) + + # Keep last train loss on-device; avoid per-step .cpu() sync (morph68 hang: + # post-SAVE_BEST host spin at ~98% CPU / GPU util 0 with mem held). + last_train_loss = None + + def step_unbind(row) -> None: + nonlocal last_train_loss, global_step, loss_sum, loss_steps, unb_loss_sum, unb_loss_steps + if unbind_loss_weight == 0: + return + uloss = unbind_loss(row) + if uloss is None: + return + row_w = unbind_row_sample_weight(row, inferred_weight) + scaled = uloss * unbind_loss_weight * row_w + opt.zero_grad() + scaled.backward() + opt.step() + global_step += 1 + loss_sum = loss_sum + scaled.detach() + loss_steps += 1 + if v2_on: + unb_loss_sum = unb_loss_sum + scaled.detach() + unb_loss_steps += 1 + last_train_loss = scaled.detach() + + residual_dump_path = resolve_unbind_residual_dump_path() + last_residual_records: list[dict] = [] + save_best_unbind = resolve_save_best_unbind() + best_exact = float("-inf") + best_macro = float("-inf") + best_epoch_index: int | None = None + best_metrics: dict | None = None + best_state: dict | None = None + best_checkpoint_sha: str | None = None + best_residual_records: list[dict] = [] + progress_path = out_dir / "epoch-progress.jsonl" + + @torch.no_grad() + def score(): + nonlocal last_residual_records + encoder.eval() + classify.eval() + filler_head.eval() + hit = tot = 0 + obs_hit = obs_tot = 0 + classify_golds: list[str] = [] + classify_preds: list[str] = [] + for row in classify_va or classify_tr[:8]: + out = encoder(**encode_texts([row["text"]])) + pred = int(classify(out.last_hidden_state[:, 0]).argmax(-1)[0]) + gold = maps["family_of"].get(row["lineage"], maps["family_of"]["none"]) + matched = int(pred == gold) + hit += matched + tot += 1 + if str(row.get("class") or "") == "OBSERVED": + obs_tot += 1 + obs_hit += matched + if select_on_classify: + classify_golds.append(FAMILIES[gold] if 0 <= gold < len(FAMILIES) else NONE_LABEL) + classify_preds.append(FAMILIES[pred] if 0 <= pred < len(FAMILIES) else NONE_LABEL) + pairs: list[tuple[list[str], list[str]]] = [] + strict_pairs: list[tuple[list[str], list[str]]] = [] + residual_records: list[dict] = [] + for row in unbind_va or unbind_tr[:8]: + fillers = list(row.get("fillers") or []) + if not fillers: + continue + out = encoder(**encode_texts([row["text"]])) + states = out.last_hidden_state[0] + offs = _offsets(tok, row["text"]) + gold_strs: list[str] = [] + raw_strs: list[str] = [] + pred_strs: list[str] = [] + for fill in fillers: + idxs = pool_indices(states.size(0), atom_token_index(row["text"], fill, offs)) + pred = int(filler_head(states[idxs].mean(0)).argmax()) + gold_strs.append(mapped_filler(maps, fill)) + raw_strs.append(str(fill).lower()) + pred_strs.append(mapped_pred(maps, pred)) + pairs.append((gold_strs, pred_strs)) + strict_pairs.append((raw_strs, pred_strs)) + if residual_dump_path: + rec = residual_row_record( + text=str(row.get("text") or ""), + gold=gold_strs, + pred=pred_strs, + row=row, + ) + if rec is not None: + residual_records.append(rec) + last_residual_records = residual_records + encoder.train() + classify.train() + filler_head.train() + metrics = summarize_unbind_pairs(pairs, strict_pairs=strict_pairs) + metrics["classify_acc"] = hit / max(1, tot) + metrics["classification_accuracy"] = metrics["classify_acc"] + metrics["n_classify_eval"] = tot + metrics["observed_label_accuracy"] = None if obs_tot == 0 else obs_hit / obs_tot + metrics["n_observed_eval"] = obs_tot + metrics["unbind_clean_exact"] = None + if select_on_classify: + metrics["classify_macro_f1_nonnone"] = macro_f1_nonnone(classify_golds, classify_preds) + metrics["none_fpr"] = none_false_positive_rate(classify_golds, classify_preds) + if v2_on: + from .classification_v2 import ACTIVE_FAMILY_VOCABULARY, epoch_selection_metrics + + kept_rows = [] + app_preds = [] + fam_preds = [] + surface_probabilities = [] + applicability_head.eval() + family_head.eval() + for row in classify_va or classify_tr[:8]: + lineage = row.get("lineage") + if lineage not in ACTIVE_FAMILY_VOCABULARY and lineage != "none": + continue + out = encoder(**encode_texts([row["text"]])) + pooled = out.last_hidden_state[:, 0] + app_logit = applicability_head(pooled)[0] + fam_logit = family_head(pooled)[0] + present = "FAMILY_PRESENT" if float(app_logit[1]) > float(app_logit[0]) else "NONE" + app_preds.append(present) + family_name = ACTIVE_FAMILY_VOCABULARY[int(fam_logit.argmax())] + fam_preds.append(family_name) + kept_rows.append(row) + from .classification_v2_surface import calibrated_present_probability + + surface_probabilities.append( + calibrated_present_probability( + [float(app_logit[0]), float(app_logit[1])], + 1.0, + ) + ) + applicability_head.train() + family_head.train() + metrics.update(epoch_selection_metrics(kept_rows, app_preds, fam_preds)) + from .classification_v2_surface import applicability_surface_report + + metrics["applicability_surface"] = applicability_surface_report( + [ + { + "family_prediction": family, + "lineage": row.get("lineage"), + "prediction": prediction, + "probability": probability, + "text": row.get("text"), + } + for row, prediction, family, probability in zip( + kept_rows, app_preds, fam_preds, surface_probabilities + ) + ] + ) + return metrics + + out_dir.mkdir(parents=True, exist_ok=True) + layout = describe(maps) + layout["last_trainable"] = last_trainable_used + layout["aligner"] = "char_span + offset_mapping" + write_skeleton(out_dir, maps=maps) + + training_started = monotonic_seconds() + stop_reason = STOP_REASON_MAX_EPOCHS + classify_sampling_epochs: list[dict] = [] + sampling_log_path = out_dir / "classify-sampling.jsonl" + for ep in range(epochs): + epoch_started = monotonic_seconds() + phase_rows, phase_meta = select_unbind_for_epoch(unbind_tr, ep, curriculum) + epoch_rows, sampling_record = epoch_classify_rows(classify_tr, epoch_index=ep) + classify_sampling_epochs.append( + { + key: value + for key, value in sampling_record.items() + if key != "selected_identity_sha256" + } + ) + with sampling_log_path.open("a", encoding="utf-8") as sampling_handle: + sampling_handle.write(json.dumps(sampling_record, sort_keys=True) + "\n") + sampling_handle.flush() + unbind_cycle = 0 + classify_batch_i = 0 + for i in range(0, len(epoch_rows), batch): + chunk = epoch_rows[i : i + batch] + out = encoder(**encode_texts([c["text"] for c in chunk])) + if v2_on: + from .classification_v2_runtime import batch_classify_loss + + parts = batch_classify_loss( + applicability_head, + family_head, + out.last_hidden_state[:, 0], + chunk, + v2_contract, + ) + if parts is None: + classify_batch_i += 1 + continue + loss = parts["total"] + else: + y = torch.tensor([maps["family_of"].get(c["lineage"], maps["family_of"]["none"]) for c in chunk], device=device) + loss = nn.functional.cross_entropy(classify(out.last_hidden_state[:, 0]), y) + opt.zero_grad() + loss.backward() + opt.step() + global_step += 1 + loss_sum = loss_sum + loss.detach() + loss_steps += 1 + if v2_on: + if parts["applicability"] is not None: + app_loss_sum = app_loss_sum + parts["applicability"].detach() + app_loss_steps += 1 + if parts["family"] is not None: + fam_loss_sum = fam_loss_sum + parts["family"].detach() + fam_loss_steps += 1 + last_train_loss = loss.detach() + if should_interleave_unbind(classify_batch_i, unbind_every_n) and phase_rows: + step_unbind(phase_rows[unbind_cycle % len(phase_rows)]) + unbind_cycle += 1 + classify_batch_i += 1 + for row in phase_rows: + step_unbind(row) + metrics = score() + metrics["epoch"] = ep + metrics["unbind_phase"] = phase_meta["phase"] + metrics["n_unbind_phase"] = phase_meta["n_rows"] + metrics["unbind_phase_fallback_full_mix"] = phase_meta["fallback_full_mix"] + epoch_metrics.append(metrics) + exact = float(metrics.get("unbind_exact") or 0.0) + if last_train_loss is not None: + # One host sync per epoch (not per step). + losses.append(float(last_train_loss.item())) + saved_best = False + if v2_checkpoint or select_on_classify: + if v2_checkpoint: + if metrics.get("selection_score") is None: + raise RuntimeError("classification v2 selection_score is NOT_COMPUTABLE") + score_now = float(metrics["selection_score"]) + elif metrics.get("classify_macro_f1_nonnone") is None: + raise RuntimeError( + "HLX_SELECT_METRIC=classify_macro_f1_nonnone but val has no " + "non-none gold; macro-F1 is NOT_COMPUTABLE" + ) + else: + score_now = float(metrics["classify_macro_f1_nonnone"]) + best_macro, best_epoch_index, improved = note_strict_improvement( + best_macro, + best_epoch_index, + score_now, + ep, + ) + if improved: + if device.type == "cuda": + torch.cuda.synchronize() + best_metrics = dict(metrics) + best_state = _build_weight_state( + encoder, classify, role_head, filler_head, maps, layout, + applicability=applicability_head, + family_head=family_head, + ) + best_residual_records = list(last_residual_records) + best_dir = out_dir / "best" + write_skeleton(best_dir, maps=maps) + best_weight_name = save_heads(best_dir, best_state) + best_weight_path = best_dir / best_weight_name + best_checkpoint_sha = _sha256_file(best_weight_path) if best_weight_path.is_file() else None + (best_dir / "best-checkpoint.json").write_text( + json.dumps( + { + "metric": "selection_score" if v2_checkpoint else SELECT_METRIC_CLASSIFY, + "epoch": ep, + "classify_macro_f1_nonnone": None if v2_checkpoint else score_now, + "selection_score": score_now if v2_checkpoint else None, + "none_fpr": metrics.get("none_fpr"), + "unbind_exact": exact, + "unbind_token_f1": best_metrics.get("unbind_token_f1"), + "unbind_slot_f1": best_metrics.get("unbind_slot_f1"), + "unbind_exact_strict": best_metrics.get("unbind_exact_strict"), + "unbind_token_f1_strict": best_metrics.get("unbind_token_f1_strict"), + "unbind_slot_f1_strict": best_metrics.get("unbind_slot_f1_strict"), + "classify_acc": best_metrics.get("classify_acc"), + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + saved_best = True + if device.type == "cuda": + torch.cuda.synchronize() + torch.cuda.empty_cache() + none_fpr = metrics.get("none_fpr") + none_fpr_text = "null" if none_fpr is None else f"{float(none_fpr):.6f}" + best_text = None if best_macro == float("-inf") else best_macro + print( + f"[epoch {ep}] classify_macro_f1_nonnone={score_now:.6f} " + f"none_fpr={none_fpr_text} best={best_text} saved_best={saved_best}", + flush=True, + ) + elif save_best_unbind and exact > best_exact: + if device.type == "cuda": + torch.cuda.synchronize() + best_exact = exact + best_metrics = dict(metrics) + best_state = _build_weight_state( + encoder, classify, role_head, filler_head, maps, layout, + applicability=applicability_head, + family_head=family_head, + ) + best_residual_records = list(last_residual_records) + best_dir = out_dir / "best" + write_skeleton(best_dir, maps=maps) + save_heads(best_dir, best_state) + (best_dir / "best-checkpoint.json").write_text( + json.dumps( + { + "metric": "unbind_exact", + "epoch": ep, + "unbind_exact": exact, + "unbind_token_f1": best_metrics.get("unbind_token_f1"), + "unbind_slot_f1": best_metrics.get("unbind_slot_f1"), + "unbind_exact_strict": best_metrics.get("unbind_exact_strict"), + "unbind_token_f1_strict": best_metrics.get("unbind_token_f1_strict"), + "unbind_slot_f1_strict": best_metrics.get("unbind_slot_f1_strict"), + "classify_acc": best_metrics.get("classify_acc"), + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + saved_best = True + if device.type == "cuda": + torch.cuda.synchronize() + torch.cuda.empty_cache() + # Durable heartbeat — morph68 hung silently after ep4 with frozen docker logs. + progress = { + "epoch": ep, + "unbind_exact": exact, + "best_unbind_exact": None if best_exact == float("-inf") else best_exact, + "saved_best": saved_best, + "n_unbind_phase": phase_meta["n_rows"], + "unbind_phase": phase_meta["phase"], + "classify_acc": metrics.get("classify_acc"), + } + if select_on_classify or v2_checkpoint: + progress["classify_macro_f1_nonnone"] = metrics.get("classify_macro_f1_nonnone") + progress["none_fpr"] = metrics.get("none_fpr") + progress["best_classify_macro_f1_nonnone"] = ( + None if best_macro == float("-inf") else best_macro + ) + if seed_receipt is not None: + progress["seed"] = seed_receipt["seed"] + training_loss = None + if loss_steps: + training_loss = float((loss_sum / loss_steps).item()) + loss_sum = torch.zeros((), device=device) + loss_steps = 0 + will_stop = early_stop_break( + enabled=early_stop.enabled, + epoch_index=ep, + best_epoch=best_epoch_index, + epochs_scored=ep + 1, + minimum_epochs=early_stop.minimum_epochs, + patience=early_stop.patience, + max_epochs=epochs, + ) + tie = False + if select_on_classify or v2_checkpoint: + tie = (not saved_best) and score_now == ( + None if best_macro == float("-inf") else best_macro + ) + progress.update( + { + "checkpoint_sha256": best_checkpoint_sha, + "classification_accuracy": metrics.get("classification_accuracy"), + "early_stop": will_stop, + "epochs_since_best": None if best_epoch_index is None else ep - best_epoch_index, + "global_step": global_step, + "improved": saved_best, + "learning_rate": float(os.environ.get("HYPERLEX_TRAIN_LR", "2e-5")), + "classify_sampling_rule": sampling_record["rule"], + "classify_sampling_selection_sha256": sampling_record["selection_sha256"], + "inferred_none_selected": sampling_record["inferred_none_selected"], + "observed_label_accuracy": metrics.get("observed_label_accuracy"), + "tie": tie, + "training_loss": training_loss, + "unbind_clean_exact": None, + } + ) + if v2_checkpoint: + def _mean(total, steps): + return None if steps == 0 else float((total / steps).item()) + + progress.update( + { + "total_loss": training_loss, + "applicability_loss": _mean(app_loss_sum, app_loss_steps), + "family_loss": _mean(fam_loss_sum, fam_loss_steps), + "unbind_loss": _mean(unb_loss_sum, unb_loss_steps), + "applicability_macro_f1": metrics.get("applicability_macro_f1"), + "none_precision": metrics.get("none_precision"), + "none_recall": metrics.get("none_recall"), + "none_f1": metrics.get("none_f1"), + "family_present_precision": metrics.get("family_present_precision"), + "family_present_recall": metrics.get("family_present_recall"), + "family_present_f1": metrics.get("family_present_f1"), + "active_family_macro_f1": metrics.get("active_family_macro_f1"), + "observed_active_family_macro_f1": metrics.get("observed_active_family_macro_f1"), + "per_family": metrics.get("per_family"), + "predicted_none_rate": metrics.get("predicted_none_rate"), + "family_emission_rate": metrics.get("family_emission_rate"), + "selection_score": metrics.get("selection_score"), + "corr_word_count_p_family_present": (metrics.get("applicability_surface") or {}).get( + "corr_word_count_p_family_present" + ), + "family_macro_f1_by_surface": (metrics.get("applicability_surface") or {}).get( + "family_macro_f1_by_surface" + ), + "applicability_by_cell": (metrics.get("applicability_surface") or {}).get( + "applicability_by_cell" + ), + "checkpoint_identity": best_checkpoint_sha, + "observed_slice": metrics.get("observed_slice"), + "inferred_slice": metrics.get("inferred_slice"), + } + ) + app_loss_sum = torch.zeros((), device=device) + fam_loss_sum = torch.zeros((), device=device) + unb_loss_sum = torch.zeros((), device=device) + app_loss_steps = fam_loss_steps = unb_loss_steps = 0 + progress.update( + epoch_timing_fields( + epoch_started=epoch_started, + epoch_ended=monotonic_seconds(), + training_started=training_started, + ) + ) + with progress_path.open("a", encoding="utf-8") as pf: + pf.write(json.dumps(progress, sort_keys=True) + "\n") + pf.flush() + if not select_on_classify and not v2_checkpoint: + print( + f"[epoch {ep}] unbind_exact={exact:.6f} best={best_exact if best_exact != float('-inf') else None} saved_best={saved_best}", + flush=True, + ) + if will_stop: + stop_reason = STOP_REASON_EARLY_STOPPING + break + + training_elapsed_seconds = round_seconds(monotonic_seconds() - training_started) + final_state = _build_weight_state( + encoder, classify, role_head, filler_head, maps, layout, + applicability=applicability_head, + family_head=family_head, + ) + # Always write final epoch weights under a distinct name when best-save is on, + # then promote best → primary model.safetensors (fixes morph35 peak-not-saved). + if (save_best_unbind or select_on_classify or v2_checkpoint) and best_state is not None: + final_file = save_heads(out_dir, final_state) + # rename primary final dump aside, then write best as primary + final_path = out_dir / final_file + aside = out_dir / ( + "model.final.safetensors" if final_file == "model.safetensors" else "heads.final.pt" + ) + if final_path.exists(): + final_path.replace(aside) + weight_file = save_heads(out_dir, best_state) + primary_val = best_metrics or {} + gated_from = ( + "selection_score" if v2_checkpoint else "best_classify_macro_f1_nonnone" if select_on_classify else "best_unbind_exact" + ) + residual_for_dump = best_residual_records + else: + weight_file = save_heads(out_dir, final_state) + primary_val = epoch_metrics[-1] if epoch_metrics else {} + gated_from = "final_epoch" + residual_for_dump = last_residual_records + aside = None + + calibration_artifact = None + surface_artifact = None + geometry_artifact = None + if v2_checkpoint: + if best_state is None: + raise RuntimeError("classification v2 restore-best is required before calibration") + if not classify_va: + raise RuntimeError("classification v2 calibration requires the validation split") + applicability_head.load_state_dict(best_state["applicability"]) + family_head.load_state_dict(best_state["family_head"]) + apply_encoder_trainable(encoder, best_state.get("encoder") or {}) + from .classification_v2 import ACTIVE_FAMILY_VOCABULARY, freeze_calibration + + prohibited = {"held_out", "evaluation_reserve", "settlement", "measurement"} + applicability_rows = [] + family_rows = [] + applicability_head.eval() + family_head.eval() + encoder.eval() + with torch.no_grad(): + for row in classify_va: + if ( + row.get("split") == "test" + or row.get("evaluation_reserve") + or row.get("held_out") + or row.get("surface") in prohibited + ): + raise RuntimeError("classification v2 calibration refused a non-validation row") + lineage = row.get("lineage") + if lineage not in ACTIVE_FAMILY_VOCABULARY and lineage != "none": + continue + encoded = encoder(**encode_texts([row["text"]])) + pooled = encoded.last_hidden_state[:, 0] + applicability_logit = [ + float(value) for value in applicability_head(pooled)[0].detach().cpu() + ] + applicability_rows.append((applicability_logit, 0 if lineage == "none" else 1)) + if lineage in ACTIVE_FAMILY_VOCABULARY: + family_logit = [float(value) for value in family_head(pooled)[0].detach().cpu()] + family_rows.append((family_logit, ACTIVE_FAMILY_VOCABULARY.index(lineage))) + encoder.train() + applicability_head.train() + family_head.train() + calibration_artifact = freeze_calibration( + applicability_rows=applicability_rows, + family_rows=family_rows, + surface="validation", + checkpoint_identity=best_checkpoint_sha, + ) + (out_dir / "classification-v2-calibration.json").write_text( + json.dumps(calibration_artifact, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + from .classification_v2_surface import ( + applicability_surface_report, + calibrated_present_probability, + ) + + surface_records = [] + temperature = float(calibration_artifact["applicability_temperature"]) + family_cursor = 0 + for row, (logits, label) in zip( + [ + item + for item in classify_va + if item.get("lineage") in ACTIVE_FAMILY_VOCABULARY or item.get("lineage") == "none" + ], + applicability_rows, + ): + probability = calibrated_present_probability(logits, temperature) + prediction = "FAMILY_PRESENT" if logits[1] > logits[0] else "NONE" + family_prediction = None + if row.get("lineage") in ACTIVE_FAMILY_VOCABULARY: + family_logits, _family_label = family_rows[family_cursor] + family_cursor += 1 + family_prediction = ACTIVE_FAMILY_VOCABULARY[max( + range(len(family_logits)), key=family_logits.__getitem__ + )] + surface_records.append( + { + "family_prediction": family_prediction, + "lineage": row.get("lineage"), + "prediction": prediction, + "probability": probability, + "text": row.get("text"), + } + ) + surface_artifact = applicability_surface_report(surface_records) + surface_artifact["checkpoint_identity"] = best_checkpoint_sha + surface_artifact["reserve_used"] = False + geometry_artifact = None + if hasattr(family_head, "cosine"): + from .classification_v2 import ACTIVE_FAMILY_VOCABULARY as _FAMILY_NAMES + from .classification_v2_prototype import family_margin_report + + margin_records = [] + encoder.eval() + family_head.eval() + with torch.no_grad(): + for row in classify_va: + lineage = row.get("lineage") + if lineage not in _FAMILY_NAMES: + continue + encoded = encoder(**encode_texts([row["text"]])) + pooled = encoded.last_hidden_state[:, 0] + margin_records.append( + { + "lineage": lineage, + "similarities": [ + float(value) + for value in family_head.cosine(pooled)[0].detach().cpu() + ], + } + ) + geometry_artifact = { + "family_similarity_margin": family_margin_report(margin_records, list(_FAMILY_NAMES)), + "n_family": len(margin_records), + "reserve_used": False, + "schema": "hyperlex.classification.v2.family_geometry_eval.v1", + "surface": "validation", + } + (out_dir / "classification-v2-family-geometry.json").write_text( + json.dumps(geometry_artifact, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + if hasattr(family_head, "anchor_cosine"): + from .classification_v2 import ACTIVE_FAMILY_VOCABULARY as _FAMILY_NAMES + from .classification_v2_geometry_repair import ( + flatten_anchors, + hard_negatives_from_boundaries, + row_geometry_scores, + summarize_geometry_margins, + ) + + repair_path = os.environ.get("HLX_V2_GEOMETRY_REPAIR") + if not repair_path: + raise RuntimeError("FAMILY_GEOMETRY_REPAIR_UNAVAILABLE") + boundaries = json.loads(Path(repair_path).read_text(encoding="utf-8")) + anchors = flatten_anchors(boundaries) + hard = hard_negatives_from_boundaries(boundaries) + margin_rows = [] + encoder.eval() + family_head.eval() + with torch.no_grad(): + for row in classify_va: + lineage = row.get("lineage") + if lineage not in _FAMILY_NAMES: + continue + if ( + row.get("evaluation_reserve") + or row.get("held_out") + or row.get("split") == "test" + ): + raise RuntimeError("evaluation_isolation") + encoded = encoder(**encode_texts([row["text"]])) + pooled = encoded.last_hidden_state[:, 0] + similarities = [ + float(value) + for value in family_head.anchor_cosine(pooled)[0].detach().cpu() + ] + scores = row_geometry_scores(similarities, str(lineage), anchors, hard) + margin_rows.append({"family": lineage, **scores}) + summary = summarize_geometry_margins(margin_rows) + geometry_artifact = { + **summary, + "boundary_sha256": boundaries["boundary_sha256"], + "canonical_logits": "learned_residual", + "hard_negatives": hard, + "reserve_used": False, + "schema": "hyperlex.classification.v2.geometry_repair_eval.v1", + "separation_sha256": boundaries["separation_sha256"], + "sparse_families": list(init_receipt["classification_v2_rows"].get("sparse_families") or []), + "surface": "validation", + } + (out_dir / "classification-v2-geometry-repair.json").write_text( + json.dumps(geometry_artifact, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + (out_dir / "classification-v2-surface.json").write_text( + json.dumps(surface_artifact, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + last = epoch_metrics[-1] if epoch_metrics else {} + residual_receipt: dict = { + "unbind_residual_dump": "", + "n_unbind_residual": 0, + "unbind_residual_themes": {}, + } + if residual_dump_path: + residual_receipt = write_residual_dump(residual_dump_path, residual_for_dump) + receipt = { + "schema": "hyperlex.hyperlexical.train_receipt.v0.1", + "model_id": MODEL_ID_SEED, + "trunk": TRUNK, + "trunk_dir": str(trunk), + "device": str(device), + "cuda": bool(torch.cuda.is_available()), + "epochs": epochs, + "epochs_completed": len(epoch_metrics), + "global_steps": global_step, + "best_checkpoint_sha256": best_checkpoint_sha, + "primary_weights_sha256": ( + _sha256_file(out_dir / weight_file) if (out_dir / weight_file).is_file() else None + ), + "stop_reason": stop_reason, + "classify_sampling": { + "epochs": classify_sampling_epochs, + "policy": os.environ.get("HYPERLEX_CLASSIFY_SAMPLING") or "uncapped", + }, + "training_schedule": schedule.as_dict(), + "training_elapsed_seconds": training_elapsed_seconds, + **provenance(root), + "n_train_classify": len(classify_tr), + "task_accounting": task_accounting, + "n_train_unbind": len(unbind_tr), + "n_unfrozen_encoder": n_unfrozen, + "n_encoder_tensors": len(final_state.get("encoder") or {}), + "last_trainable": last_trainable_used, + "unbind_loss_weight": unbind_loss_weight, + "unbind_every_n": unbind_every_n, + "unbind_primary": slot_ce_mode["unbind_primary"], + "unbind_slot_ce_armed": slot_ce_mode["unbind_slot_ce_armed"], + "unbind_slot_ce_aux_lambda": slot_ce_mode["unbind_slot_ce_aux_lambda"], + "unbind_head_slot_weight": head_slot_weight, + "unbind_second_slot_weight": second_slot_weight, + "n_unbind_observed": unbind_recipe["n_unbind_observed"], + "n_unbind_inferred": unbind_recipe["n_unbind_inferred"], + "unbind_observed_upsample": unbind_recipe["unbind_observed_upsample"], + "unbind_inferred_cap": unbind_recipe["unbind_inferred_cap"], + "unbind_inferred_weight": inferred_weight, + "n_unbind_morph_negatives": unbind_recipe["n_unbind_morph_negatives"], + "unbind_morph_margin": morph_margin, + "unbind_curriculum": curriculum_plan["enabled"], + "unbind_curriculum_pos_epochs": curriculum_plan["pos_epochs"], + "unbind_curriculum_type_epochs": curriculum_plan["type_epochs"], + "unbind_curriculum_phases": curriculum_plan["phases"], + "n_unbind_curriculum_positional": curriculum_plan["n_unbind_positional"], + "n_unbind_curriculum_type_slot": curriculum_plan["n_unbind_type_slot"], + "n_unbind_curriculum_joint": curriculum_plan["n_unbind_joint"], + "unbind_filler_denylist_lineages": unbind_recipe.get("unbind_filler_denylist_lineages", 0), + "unbind_hard_atoms_path": unbind_recipe.get("unbind_hard_atoms_path", ""), + "unbind_hard_upsample": unbind_recipe.get("unbind_hard_upsample", 1), + "n_unbind_hard_atoms_matched": unbind_recipe.get("n_unbind_hard_atoms_matched", 0), + "unbind_force_train_path": unbind_recipe.get("unbind_force_train_path", ""), + "release_set": release_stats, + "filler_filter": unbind_recipe.get("filler_filter"), + "n_filler_rows_dropped_train": unbind_recipe.get("n_filler_rows_dropped_train", 0), + "n_filler_rows_dropped_val": unbind_recipe.get("n_filler_rows_dropped_val", 0), + "n_unbind_force_train": unbind_recipe.get("n_unbind_force_train", 0), + "n_unbind_force_train_keys": unbind_recipe.get("n_unbind_force_train_keys", 0), + "n_unbind_val_after_force_train": unbind_recipe.get( + "n_unbind_val_after_force_train", 0 + ), + "n_unbind_hard_extra_copies": unbind_recipe.get("n_unbind_hard_extra_copies", 0), + "unbind_residual_dump": residual_receipt.get("unbind_residual_dump", ""), + "n_unbind_residual": residual_receipt.get("n_unbind_residual", 0), + "unbind_residual_themes": residual_receipt.get("unbind_residual_themes", {}), + "unbind_residual_by_scheme": residual_receipt.get("unbind_residual_by_scheme", {}), + "unbind_residual_by_class": residual_receipt.get("unbind_residual_by_class", {}), + "unbind_residual_summary": residual_receipt.get("unbind_residual_summary", ""), + "last_loss": losses[-1] if losses else None, + "val": primary_val, + "val_final": last, + "val_best": best_metrics, + "save_best_unbind": save_best_unbind, + "primary_weights_from": gated_from, + "best_unbind_exact": None if best_metrics is None else best_metrics.get("unbind_exact"), + "best_epoch": None if best_metrics is None else best_metrics.get("epoch"), + "final_weight_file": None if aside is None else aside.name, + "init_from": init_receipt.get("init_from"), + "warm_start": bool(init_receipt.get("warm_start")), + "init_weight_file": init_receipt.get("weight_file"), + "init_encoder_trainable_loaded": init_receipt.get("encoder_trainable_loaded"), + "init_expand_vocab": bool(init_receipt.get("init_expand_vocab")), + "init_expand_vocab_applied": bool(init_receipt.get("expand_vocab")), + "init_expand_role": init_receipt.get("role"), + "init_expand_filler": init_receipt.get("filler"), + "epoch_metrics": epoch_metrics, + "weight_file": weight_file, + "aligner": "char_span + offset_mapping", + "data_sha256": input_receipt["data_sha256"], + "training_input_mode": input_receipt["training_input_mode"], + "training_export_path": input_receipt["training_export_path"], + "training_export_sha256_expected": input_receipt["training_export_sha256_expected"], + "training_export_sha256_actual": input_receipt["training_export_sha256_actual"], + "training_export_rows": input_receipt["training_export_rows"], + "live_export_generation_enabled": input_receipt["live_export_generation_enabled"], + "holdout": holdout_receipt(holdout_spec, holdout_removed), + "holdout_training_disjoint": disjoint_receipt, + "classify_admission": classify_admission_receipt, + "include_live": include_live, + "live_included": bundle["counts"].get("live_included", 0), + "name_gate": False, + "e2_pass": False, + "brier": None, + "forecast_eligible": False, + "note": "HF-shaped dump. Not Hyperlexical until E2.", + } + if reserve_receipt is not None: + receipt["eval_reserve_disjoint"] = reserve_receipt + if classify_split_receipt is not None: + receipt["classify_split"] = classify_split_receipt + if seed_receipt is not None: + receipt["seed"] = seed_receipt + if calibration_artifact is not None: + receipt["classification_v2_calibration"] = calibration_artifact + receipt["classification_v2_surface"] = surface_artifact + receipt["classification_v2_family_geometry"] = ( + geometry_artifact + if geometry_artifact and geometry_artifact.get("schema") == "hyperlex.classification.v2.family_geometry_eval.v1" + else None + ) + receipt["classification_v2_geometry_repair"] = ( + geometry_artifact + if geometry_artifact and geometry_artifact.get("schema") == "hyperlex.classification.v2.geometry_repair_eval.v1" + else None + ) + if init_receipt.get("classification_v2_geometry_repair_prior") is not None: + receipt["classification_v2_geometry_repair_prior"] = init_receipt["classification_v2_geometry_repair_prior"] + if select_on_classify: + receipt["select_metric"] = SELECT_METRIC_CLASSIFY + if force_overlap.get("disjoint"): + receipt["force_train_disjoint"] = { + "n_overlap": force_overlap["n_overlap"], + "n_in_val": force_overlap["n_in_val"], + "n_in_selection": force_overlap["n_in_selection"], + "n_force_rows": force_overlap["n_force_rows"], + "n_dropped_classify_val": force_overlap["n_dropped_classify_val"], + "n_dropped_unbind_val": force_overlap["n_dropped_unbind_val"], + "n_classify_val": force_overlap["n_classify_val"], + "n_unbind_val": force_overlap["n_unbind_val"], + } + if resolve_vocab_train_only(): + receipt["vocab_train_only"] = True + (out_dir / "layout.json").write_text(json.dumps(layout, indent=2, sort_keys=True) + "\n", encoding="utf-8") + (out_dir / "train-receipt.json").write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8") + train_config = { + "lr": os.environ.get("HYPERLEX_TRAIN_LR", "2e-5"), + "epochs": epochs, + "batch": batch, + "max_len": MAX_LEN, + "last_trainable": last_trainable_used, + "unbind_loss_weight": unbind_loss_weight, + "unbind_every_n": unbind_every_n, + "unbind_primary": slot_ce_mode["unbind_primary"], + "unbind_slot_ce_armed": slot_ce_mode["unbind_slot_ce_armed"], + "unbind_slot_ce_aux_lambda": slot_ce_mode["unbind_slot_ce_aux_lambda"], + "unbind_head_slot_weight": head_slot_weight, + "unbind_second_slot_weight": second_slot_weight, + "unbind_observed_upsample": unbind_recipe["unbind_observed_upsample"], + "unbind_inferred_cap": unbind_recipe["unbind_inferred_cap"], + "unbind_inferred_weight": inferred_weight, + "n_unbind_morph_negatives": unbind_recipe["n_unbind_morph_negatives"], + "unbind_morph_margin": morph_margin, + "unbind_curriculum": curriculum_plan["enabled"], + "unbind_curriculum_pos_epochs": curriculum_plan["pos_epochs"], + "unbind_curriculum_type_epochs": curriculum_plan["type_epochs"], + "unbind_filler_denylist_lineages": unbind_recipe.get( + "unbind_filler_denylist_lineages", 0 + ), + "unbind_hard_atoms_path": unbind_recipe.get("unbind_hard_atoms_path", ""), + "unbind_hard_upsample": unbind_recipe.get("unbind_hard_upsample", 1), + "n_unbind_hard_atoms_matched": unbind_recipe.get( + "n_unbind_hard_atoms_matched", 0 + ), + "n_unbind_hard_extra_copies": unbind_recipe.get( + "n_unbind_hard_extra_copies", 0 + ), + "unbind_force_train_path": unbind_recipe.get("unbind_force_train_path", ""), + "n_unbind_force_train": unbind_recipe.get("n_unbind_force_train", 0), + "n_unbind_force_train_keys": unbind_recipe.get( + "n_unbind_force_train_keys", 0 + ), + "n_unbind_val_after_force_train": unbind_recipe.get( + "n_unbind_val_after_force_train", 0 + ), + "save_best_unbind": save_best_unbind, + "holdout": holdout_receipt(holdout_spec, holdout_removed), + "init_from": init_receipt.get("init_from"), + "warm_start": bool(init_receipt.get("warm_start")), + } + if seed_receipt is not None: + train_config["seed"] = seed_receipt["seed"] + (out_dir / "config-train.json").write_text( + json.dumps(train_config, indent=2) + "\n", + encoding="utf-8", + ) + return receipt diff --git a/scripts/shadow/hyperlexical/name_gate.py b/scripts/shadow/hyperlexical/name_gate.py new file mode 100644 index 00000000..f1e25143 --- /dev/null +++ b/scripts/shadow/hyperlexical/name_gate.py @@ -0,0 +1,32 @@ +"""Which checkpoints may carry the Hyperlexical name. No torch. No network. + +The product-level gate is Danny's sentence, recorded as Spec 007 amendment A6. +Only pins listed here are named; every other checkpoint stays ``name_gate=false``. +Identity is the resolved model directory, so a moved ``BEST`` symlink never +names a different checkpoint. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +from .layout import MODEL_ID_SEED + +PUBLIC_CARD_NAME = "hyperlex-structure-149m" +APPROVED_PINS = {"seed-morph78": "A6"} + + +def pin_of(model_dir: str | os.PathLike | None) -> str | None: + """Return ``seed-`` for a train-out dir, resolving symlinks, else None.""" + if not model_dir: + return None + name = Path(os.path.realpath(model_dir)).name + prefix = MODEL_ID_SEED + "-" + if not name.startswith(prefix): + return None + return "seed-" + name[len(prefix):] + + +def name_gate_for(model_dir: str | os.PathLike | None) -> bool: + return pin_of(model_dir) in APPROVED_PINS diff --git a/scripts/shadow/hyperlexical/packet.py b/scripts/shadow/hyperlexical/packet.py new file mode 100644 index 00000000..2a1d91df --- /dev/null +++ b/scripts/shadow/hyperlexical/packet.py @@ -0,0 +1,161 @@ +"""Build and validate hyperlex.hyperlexical.inference.v0.1 packets.""" + +from __future__ import annotations + +import hashlib +import json +from typing import Any + +SCHEMA = "hyperlex.hyperlexical.inference.v0.1" +FORBIDDEN = {"symbolic", "has_symbols"} +ROUTES = frozenset({"form", "lexical"}) +STAGES = frozenset({"noise", "circulating", "contested", "hyperstition_ish"}) +SCHEMES = frozenset({"positional", "type_slot"}) +CLASSES = frozenset({"OBSERVED", "INFERRED", "SPECULATIVE"}) +RESTRICTED_MARKER = "__RESTRICTED_FIXTURE__" +REFUSAL_MARKERS = ( + "i cannot analyze", + "i'm sorry, but i can't", + "i am unable to", + "as an ai", +) + + +class PacketError(ValueError): + pass + + +def sha256_hex(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def _routes(claimed: list[str] | None) -> list[str]: + routes = list(claimed or ["form", "lexical"]) + if not routes: + raise PacketError("routes_claimed empty") + bad = [r for r in routes if r not in ROUTES] + if bad: + raise PacketError(f"forbidden route {bad}") + return routes + + +def build_packet( + text: str | None, + *, + restricted: bool = False, + role_scheme: str | None = "positional", + typology: list[str] | None = None, + stage: str | None = None, + class_: str = "INFERRED", +) -> dict[str, Any]: + surface = text if text is not None else "" + flagged = bool(restricted) or RESTRICTED_MARKER in surface + if flagged: + ref_src = surface or RESTRICTED_MARKER + packet: dict[str, Any] = { + "schema": SCHEMA, + "model_id": "stub", + "embed_mode": "STATIC_HASH_EMBEDDING", + "selection_proxy": "unspecified", + "param_count": 0, + "surface": None, + "payload_ref": sha256_hex(ref_src), + "lineage_family": None, + "lineage_confidence": None, + "typology": [], + "stage": None, + "role_scheme": None, + "unbind_ok": False, + "unbind": None, + "routes_claimed": ["form"], + "restricted_intent_suspected": True, + "brier": None, + "forecast_eligible": False, + "auto_fire": False, + "class": "INFERRED", + } + validate_packet(packet) + return packet + + packet = { + "schema": SCHEMA, + "model_id": "stub", + "embed_mode": "STATIC_HASH_EMBEDDING", + "input_hash": sha256_hex(surface), + "vector_hash": sha256_hex("stub-vector:" + surface), + "selection_proxy": "unspecified", + "param_count": 0, + "surface": surface, + "payload_ref": None, + "lineage_family": "none", + "lineage_confidence": 0.0, + "typology": list(typology or []), + "stage": stage, + "role_scheme": role_scheme, + "unbind_ok": False, + "unbind": None, + "routes_claimed": ["form", "lexical"], + "restricted_intent_suspected": False, + "brier": None, + "forecast_eligible": False, + "auto_fire": False, + "class": class_, + } + validate_packet(packet) + return packet + + +def validate_packet(packet: dict[str, Any]) -> dict[str, Any]: + if not isinstance(packet, dict): + raise PacketError("packet must be object") + hit = FORBIDDEN.intersection(packet) + if hit: + raise PacketError(f"forbidden keys {sorted(hit)}") + if packet.get("schema") != SCHEMA: + raise PacketError("bad schema") + if packet.get("brier") is not None: + raise PacketError("brier must be null") + if packet.get("forecast_eligible") is not False: + raise PacketError("forecast_eligible must be false") + if packet.get("auto_fire") is not False: + raise PacketError("auto_fire must be false") + if packet.get("class") not in CLASSES: + raise PacketError("bad class") + routes = packet.get("routes_claimed") + if not isinstance(routes, list) or not routes: + raise PacketError("routes_claimed required") + if any(r not in ROUTES for r in routes): + raise PacketError("semantic or unknown route") + scheme = packet.get("role_scheme") + if scheme is not None and scheme not in SCHEMES: + raise PacketError("bad role_scheme") + stage = packet.get("stage") + if stage is not None and stage not in STAGES: + raise PacketError("bad stage") + if packet.get("restricted_intent_suspected") is True: + if packet.get("surface") is not None: + raise PacketError("restricted packet must drop surface") + ref = packet.get("payload_ref") + if not isinstance(ref, str) or len(ref) != 64: + raise PacketError("restricted packet needs payload_ref") + if packet.get("embed_mode") == "MODEL_EMBEDDING": + for key in ("model_id", "input_hash", "vector_hash"): + if not packet.get(key): + raise PacketError(f"MODEL_EMBEDDING missing {key}") + blob = json.dumps(packet, sort_keys=True) + low = blob.lower() + for marker in REFUSAL_MARKERS: + if marker in low: + raise PacketError("refusal string in packet") + return packet + + +def attach_or_omit(packet: dict[str, Any] | None) -> dict[str, Any] | None: + """Fail-open analyze attachment. Empty/invalid → omit.""" + if not packet: + return None + try: + validate_packet(packet) + except PacketError: + return None + return {"analysis": {"hyperlexical": packet}} diff --git a/scripts/shadow/hyperlexical/preflight.py b/scripts/shadow/hyperlexical/preflight.py new file mode 100644 index 00000000..a6d04900 --- /dev/null +++ b/scripts/shadow/hyperlexical/preflight.py @@ -0,0 +1,102 @@ +"""U3 preflight. No Hub. No train. + +``TRAINING_READY`` exists only when the scientific contract passed, +the decision rule is sealed, and ``admit_training_run`` returns +``ADMISSION_PASS``. An admission pass without a sealed decision rule +stays ``PREREGISTERED``. The trainer entrypoint with +``HLX_ADMISSION_ONLY=1`` stops after those gates. +""" + +from __future__ import annotations + +import json +import os +import platform +import sys +from pathlib import Path + +from .admission import AdmissionError, admit_training_run +from .export import export_dataset + +TRUNK = "answerdotai/ModernBERT-base" + + +def _print(report: dict) -> None: + print(json.dumps(report, indent=2, sort_keys=True)) + + +def _base() -> dict: + trunk_raw = os.environ.get("HYPERLEX_TRUNK_DIR", "").strip() + trunk = Path(trunk_raw) if trunk_raw else None + experiment_id = os.environ.get("HLX_EXPERIMENT_ID", "").strip() + return { + "schema": "hyperlex.hyperlexical.preflight.v0.1", + "machine": platform.machine(), + "python": sys.version.split()[0], + "allow_train": os.environ.get("HYPERLEX_ALLOW_TRAIN") == "1", + "trunk_id": TRUNK, + "trunk_dir": str(trunk) if trunk else None, + "trunk_dir_exists": trunk.is_dir() if trunk else False, + "trunk_config": (trunk / "config.json").is_file() if trunk else False, + "name_gate": False, + "brier": None, + "experiment_id": experiment_id or None, + "note": ( + "TRAINING_READY requires a sealed scientific contract, a sealed " + "decision rule, and admit_training_run. Admission without a " + "decision rule stays PREREGISTERED. It is not E2 and not a " + "Hyperlexical name." + ), + } + + +def _paths() -> tuple[Path | None, Path | None]: + trunk_raw = os.environ.get("HYPERLEX_TRUNK_DIR", "").strip() + out_raw = os.environ.get("HYPERLEX_TRAIN_OUT", "").strip() + trunk = Path(trunk_raw) if trunk_raw else None + out = Path(out_raw) if out_raw else None + return trunk, out + + +def main(argv=None) -> int: + del argv + trunk, out = _paths() + include_live = os.environ.get("HYPERLEX_INCLUDE_LIVE") == "1" + try: + result = admit_training_run( + include_live=include_live, + live_store=None, + export_dataset=export_dataset, + trunk=trunk, + out_dir=out, + ) + except AdmissionError as exc: + report = _base() + report.update(exc.receipt) + report["error"] = str(exc) + report["ready_to_train"] = False + report["status"] = "ADMISSION_FAIL" + report["admission_result"] = "ADMISSION_FAIL" + _print(report) + return 2 + except SystemExit as exc: + report = _base() + report["error"] = str(exc) + report["ready_to_train"] = False + report["status"] = "ADMISSION_FAIL" + report["admission_result"] = "ADMISSION_FAIL" + report["holdout_admitted"] = False + _print(report) + return 2 + report = _base() + report.update(result.receipt) + if result.contract == "CONTROLLED_RESERVE" and not result.launch_armed: + report["status"] = "NOT_READY" + report["admission_result"] = "NOT_ARMED" + report["ready_to_train"] = False + _print(report) + return 0 if result.ready else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/shadow/hyperlexical/provenance.py b/scripts/shadow/hyperlexical/provenance.py new file mode 100644 index 00000000..6a19ef40 --- /dev/null +++ b/scripts/shadow/hyperlexical/provenance.py @@ -0,0 +1,79 @@ +"""Code provenance for train/eval receipts. No git binary needed. + +``code_commit`` is read from ``.git`` files (the training container may not +ship git). ``code_tree_sha256`` hashes the shadow package source, so an +uncommitted edit changes it even when the commit does not. +""" + +from __future__ import annotations + +import hashlib +import os +from pathlib import Path + +PACKAGE = Path(__file__).resolve().parent +ENV_PREFIX = "HYPERLEX_" + + +def _git_dir(root: Path) -> Path | None: + dot = root / ".git" + if dot.is_dir(): + return dot + if dot.is_file(): + text = dot.read_text(encoding="utf-8").strip() + if text.startswith("gitdir:"): + path = Path(text.split(":", 1)[1].strip()) + return path if path.is_absolute() else (root / path).resolve() + return None + + +def code_commit(root: Path) -> str | None: + """HEAD commit, or None if unreadable (e.g. worktree gitdir not mounted).""" + try: + git = _git_dir(root) + if git is None: + return None + head = (git / "HEAD").read_text(encoding="utf-8").strip() + if not head.startswith("ref:"): + return head or None + ref = head.split(":", 1)[1].strip() + dirs = [git] + common = git / "commondir" + if common.is_file(): + c = Path(common.read_text(encoding="utf-8").strip()) + dirs.append(c if c.is_absolute() else (git / c).resolve()) + for d in dirs: + loose = d / ref + if loose.is_file(): + return loose.read_text(encoding="utf-8").strip() + for d in dirs: + packed = d / "packed-refs" + if packed.is_file(): + for line in packed.read_text(encoding="utf-8").splitlines(): + if line.endswith(" " + ref): + return line.split(" ", 1)[0] + except OSError: + return None + return None + + +def code_tree_sha256(package: Path = PACKAGE) -> str: + h = hashlib.sha256() + for path in sorted(package.glob("*.py")): + h.update(path.name.encode("utf-8") + b"\0") + h.update(path.read_bytes()) + h.update(b"\0") + return h.hexdigest() + + +def env_snapshot(environ: dict | None = None) -> dict[str, str]: + env = os.environ if environ is None else environ + return {k: env[k] for k in sorted(env) if k.startswith(ENV_PREFIX)} + + +def provenance(root: Path) -> dict: + return { + "code_commit": code_commit(root), + "code_tree_sha256": code_tree_sha256(), + "hyperlex_env": env_snapshot(), + } diff --git a/scripts/shadow/hyperlexical/release_set.py b/scripts/shadow/hyperlexical/release_set.py new file mode 100644 index 00000000..da6a5ec8 --- /dev/null +++ b/scripts/shadow/hyperlexical/release_set.py @@ -0,0 +1,63 @@ +"""Release training set: drop share-alike (CC BY-SA) derived rows. No torch. D5(b). + +``HYPERLEX_RELEASE_SET=1`` drops every export row whose licence is CC BY-SA +(Wiktionary), and every other row with the same normalized text, so a phrase +first seen on Wiktionary cannot re-enter from another source. Applied by the +train loop and by every eval that compares a release candidate, so train and +eval surfaces match. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +from collections import Counter + +RELEASE_ENV = "HYPERLEX_RELEASE_SET" +_CC_SA = re.compile(r"CC[- ]?BY[- ]?SA", re.I) +_TAG = re.compile(r"\b(?:TOKEN|SLOT|MARKER):") + + +def enabled(raw: str | None = None) -> bool: + value = (os.environ.get(RELEASE_ENV, "") if raw is None else raw).strip() + if value not in ("", "0", "1"): + raise ValueError(f"{RELEASE_ENV} must be 0 or 1") + return value == "1" + + +def _norm(text: str) -> str: + return re.sub(r"\s+", " ", _TAG.sub("", text or "").strip().lower()) + + +def is_share_alike(row: dict) -> bool: + return bool(_CC_SA.search(str(row.get("license") or ""))) or str(row.get("source_license") or "").startswith("CC-BY-SA") + + +def release_rows(rows: list[dict]) -> tuple[list[dict], dict]: + sa_texts = {_norm(r.get("text")) for r in rows if is_share_alike(r)} + kept, by_license, by_text = [], 0, 0 + for r in rows: + if is_share_alike(r): + by_license += 1 + elif _norm(r.get("text")) in sa_texts: + by_text += 1 + else: + kept.append(r) + blob = "\n".join(json.dumps({**r, "typology": sorted(r.get("typology") or [])}, sort_keys=True, ensure_ascii=False) for r in kept) + return kept, { + "release_set": True, + "n_input": len(rows), + "n_kept": len(kept), + "n_excluded_share_alike": by_license, + "n_excluded_same_text": by_text, + "kept_by_task_split": {f"{k[0]}:{k[1]}": v for k, v in Counter((r.get("task"), r.get("split")) for r in kept).items()}, + "release_content_sha256": hashlib.sha256(blob.encode("utf-8")).hexdigest(), + } + + +def maybe_release(rows: list[dict]) -> tuple[list[dict], dict]: + if not enabled(): + return rows, {"release_set": False} + return release_rows(rows) diff --git a/scripts/shadow/hyperlexical/save_pretrained.py b/scripts/shadow/hyperlexical/save_pretrained.py new file mode 100644 index 00000000..45c9cf66 --- /dev/null +++ b/scripts/shadow/hyperlexical/save_pretrained.py @@ -0,0 +1,138 @@ +"""Write a Hub-shaped directory. No network. Weights only if caller passes tensors.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from .layout import FAMILIES, HIDDEN, LAST_TRAINABLE, LAYERS, MAX_LEN, MODEL_ID_SEED, TRUNK + +CARD_NAME = "hyperlex-encoder-modernbert-base-seed" +ENCODER_PREFIX = "encoder." +_HEAD_NAMES = ( + "classify", + "role_head", + "filler_head", + "applicability", + "family_head", + "evidence_head", + "gate1_head", + "gate2_head", +) + + +def encoder_tensor_key(name: str) -> str: + if name.startswith(ENCODER_PREFIX): + return name + return f"{ENCODER_PREFIX}{name}" + + +def flatten_weight_tensors(state: dict) -> dict: + """Heads plus trainable encoder tensors. Encoder keys are encoder..""" + tensors = {} + for name in _HEAD_NAMES: + blob = state.get(name) or {} + if not isinstance(blob, dict): + continue + for key, value in blob.items(): + tensors[f"{name}.{key}"] = value + encoder = state.get("encoder") or {} + if isinstance(encoder, dict): + for key, value in encoder.items(): + tensors[encoder_tensor_key(str(key))] = value + return tensors + + +def split_weight_tensors(tensors: dict) -> dict: + """Inverse of flatten_weight_tensors. Encoder map keeps the encoder. prefix.""" + out = {name: {} for name in _HEAD_NAMES} + out["encoder"] = {} + for key, value in tensors.items(): + if key.startswith(ENCODER_PREFIX): + out["encoder"][key] = value + continue + prefix, _, rest = key.partition(".") + if prefix in _HEAD_NAMES and rest: + out[prefix][rest] = value + return out + + +def collect_encoder_trainable(encoder) -> dict: + """requires_grad encoder params. Keys: encoder..""" + named = getattr(encoder, "named_parameters", None) + if named is None: + return {} + out = {} + for name, param in named(): + if not getattr(param, "requires_grad", False): + continue + tensor = param.detach() + to_cpu = getattr(tensor, "cpu", None) + if callable(to_cpu): + tensor = to_cpu() + contiguous = getattr(tensor, "contiguous", None) + if callable(contiguous): + tensor = contiguous() + out[encoder_tensor_key(name)] = tensor + return out + + +def composed_config(maps: dict | None = None) -> dict: + labels = {str(i): f for i, f in enumerate(FAMILIES)} + cfg = { + "model_type": "hyperlex-encoder", + "architectures": ["HyperlexicalEncoder"], + "base_model": TRUNK, + "base_model_relation": "adapter", + "hidden_size": HIDDEN, + "num_hidden_layers": LAYERS, + "last_trainable": LAST_TRAINABLE, + "max_position_embeddings": MAX_LEN, + "id2label": labels, + "label2id": {v: int(k) for k, v in labels.items()}, + "problem_type": "single_label_classification", + "transformer_tag": "encoder", + "pipeline_tag": "text-classification", + "brier": None, + "name_gate": False, + "e2_pass": False, + "model_id": MODEL_ID_SEED, + } + if maps: + cfg["role_vocab"] = maps.get("role_vocab") or [] + cfg["filler_vocab"] = maps.get("filler_vocab") or [] + return cfg + + +def write_skeleton(out_dir: Path, maps: dict | None = None, card_text: str = "") -> Path: + out_dir.mkdir(parents=True, exist_ok=True) + cfg = composed_config(maps) + (out_dir / "config.json").write_text(json.dumps(cfg, indent=2, sort_keys=True) + "\n", encoding="utf-8") + if maps: + (out_dir / "layout.json").write_text(json.dumps({ + "families": maps.get("families"), + "role_vocab": maps.get("role_vocab"), + "filler_vocab": maps.get("filler_vocab"), + "hidden": HIDDEN, + "last_trainable": LAST_TRAINABLE, + "aligner": "char_span + offset_mapping", + }, indent=2, sort_keys=True) + "\n", encoding="utf-8") + if card_text: + (out_dir / "README.md").write_text(card_text, encoding="utf-8") + return out_dir + + +def save_heads(out_dir: Path, state: dict) -> str: + """Prefer safetensors. Fall back to heads.pt. Caller owns torch.""" + out_dir.mkdir(parents=True, exist_ok=True) + tensors = flatten_weight_tensors(state) + try: + from safetensors.torch import save_file + + save_file(tensors, str(out_dir / "model.safetensors")) + return "model.safetensors" + except Exception: + import torch + + torch.save(state, out_dir / "heads.pt") + return "heads.pt" diff --git a/scripts/shadow/hyperlexical/seed_control.py b/scripts/shadow/hyperlexical/seed_control.py new file mode 100644 index 00000000..ae2f2c7e --- /dev/null +++ b/scripts/shadow/hyperlexical/seed_control.py @@ -0,0 +1,120 @@ +"""Opt-in training seed (HLX_SEED, default off). + +Unset makes no seeding calls. When set, seeds Python ``random``, NumPy, and +PyTorch (CPU and CUDA) and turns on deterministic cuDNN plus +``torch.use_deterministic_algorithms(True)``. CUDA 10.2 and newer also need +``CUBLAS_WORKSPACE_CONFIG`` of ``:4096:8`` or ``:16:8`` before the CUDA context +exists, or the first deterministic GEMM raises. Unset leaves that variable +alone. When the seed is set and the variable is empty, this module sets +``:4096:8`` before any CUDA call. ``PYTHONHASHSEED`` is recorded for child +processes; the current process hash seed is fixed at startup. +""" + +from __future__ import annotations + +import importlib +import os +import random +from typing import Any + +SEED_ENV = "HLX_SEED" +CUBLAS_WORKSPACE_ENV = "CUBLAS_WORKSPACE_CONFIG" +_CUBLAS_CHOICES = (":4096:8", ":16:8") +_CUBLAS_DEFAULT = ":4096:8" +_SEED_MAX = 2**32 - 1 + + +def resolve_training_seed(raw: str | None = None) -> int | None: + """Return the seed, or None when unset. Non-integers fail closed.""" + if raw is None: + raw = os.environ.get(SEED_ENV) + if raw is None: + return None + token = str(raw).strip() + if token == "": + return None + if not token.isdigit(): + raise SystemExit(f"REFUSE: {SEED_ENV} must be a non-negative integer") + value = int(token) + if value > _SEED_MAX: + raise SystemExit(f"REFUSE: {SEED_ENV} must be <= {_SEED_MAX}") + return value + + +def _cublas_workspace(raw: str | None = None) -> str | None: + if raw is None: + raw = os.environ.get(CUBLAS_WORKSPACE_ENV) + if raw is None: + return None + token = str(raw).strip() + return token or None + + +def _cuda_initialized(torch_mod: Any) -> bool: + try: + return bool(torch_mod.cuda.is_initialized()) + except Exception: + return False + + +def ensure_cublas_workspace(torch_mod: Any = None) -> str: + """Return a supported cuBLAS workspace. Empty becomes ``:4096:8``. + + Must run before the CUDA context exists unless the variable was already + a supported value. An unsupported value fails closed. + """ + current = _cublas_workspace() + if torch_mod is not None and _cuda_initialized(torch_mod) and current not in _CUBLAS_CHOICES: + raise SystemExit( + f"REFUSE: {CUBLAS_WORKSPACE_ENV} must be {_CUBLAS_CHOICES[0]} or " + f"{_CUBLAS_CHOICES[1]} before CUDA init when {SEED_ENV} is set" + ) + if current is None: + os.environ[CUBLAS_WORKSPACE_ENV] = _CUBLAS_DEFAULT + return _CUBLAS_DEFAULT + if current not in _CUBLAS_CHOICES: + raise SystemExit( + f"REFUSE: {CUBLAS_WORKSPACE_ENV} must be {_CUBLAS_CHOICES[0]} or " + f"{_CUBLAS_CHOICES[1]} when {SEED_ENV} is set" + ) + os.environ[CUBLAS_WORKSPACE_ENV] = current + return current + + +def apply_training_seed(torch_mod: Any = None) -> dict[str, Any] | None: + """Seed RNGs when ``HLX_SEED`` is set. Unset returns None and touches nothing.""" + seed = resolve_training_seed() + if seed is None: + return None + prior = _cublas_workspace() + if torch_mod is None: + torch_mod = importlib.import_module("torch") + if _cuda_initialized(torch_mod) and prior not in _CUBLAS_CHOICES: + raise SystemExit( + f"REFUSE: {CUBLAS_WORKSPACE_ENV} must be {_CUBLAS_CHOICES[0]} or " + f"{_CUBLAS_CHOICES[1]} before CUDA init when {SEED_ENV} is set" + ) + workspace = ensure_cublas_workspace(torch_mod) + random.seed(seed) + os.environ["PYTHONHASHSEED"] = str(seed) + import numpy as np + + np.random.seed(seed) + torch_mod.manual_seed(seed) + if torch_mod.cuda.is_available(): + torch_mod.cuda.manual_seed_all(seed) + torch_mod.backends.cudnn.deterministic = True + torch_mod.backends.cudnn.benchmark = False + deterministic = True + try: + torch_mod.use_deterministic_algorithms(True) + except Exception: + deterministic = False + return { + "seed": seed, + "python_hash_seed": str(seed), + "numpy": True, + "deterministic_algorithms": deterministic, + "cudnn_deterministic": True, + "cublas_workspace_config": workspace, + } diff --git a/scripts/shadow/hyperlexical/select_005_completion.py b/scripts/shadow/hyperlexical/select_005_completion.py new file mode 100644 index 00000000..bd8a5897 --- /dev/null +++ b/scripts/shadow/hyperlexical/select_005_completion.py @@ -0,0 +1,309 @@ +"""SELECT-005 source contract and preregistration completion. + +Does not train, append the settlement log, or append the identity ledger. +The exhausted settlement universe is not an input. SELECT-004 numeric +floors are not copied. A stale exposure snapshot is not bypassed. +""" + +from __future__ import annotations + +import json +from typing import Any, Mapping, Sequence + +from .select_005_reserve import ( + CLEARED_RIGHTS, + EXPERIMENT_ID, + FAILURE_QUOTA, + PROPOSED_SCHEDULE, + REQUIRED_SLICES, + canonical_json, + census, + exclusion_reason, +) + +TRANSITION = "SELECT_005_SOURCE_AND_PREREGISTRATION_COMPLETION_AUTHORIZATION" +PREREGISTRATION_SEALED = "SELECT_005_PREREGISTRATION_SEALED" +THRESHOLDS_BLOCKED = "BLOCKED_PENDING_OPERATOR_AUTHORIZATION" +THRESHOLD_SCHEMA = "hyperlex.threshold_authorization.v1" +SELECT_ADMISSION_JSON_SCHEMA_EXISTS = True +THRESHOLD_AUTHORIZATION_JSON_SCHEMA_EXISTS = True +TRAINING_AUTHORIZED = False +EXHAUSTED_STREAM_RUN = "hs-20260925T211358Z" +EXHAUSTED_SETTLEMENT_COUNT = 339 +BOX_SNAPSHOT_LIMIT_H = 6.0 +ADMISSIBLE_SOURCE_TYPES = ("wiktionary_category", "wikipedia_prose") +WARM_START_NAME = "hyperlex-encoder-modernbert-base-seed-morph65" +WARM_START_SHA256 = "96838b9656a84c3fee1773a41fdf2fbbec2f88f3bc948e4acfbe06c194ac5587" +BEST_NAME = "hyperlex-encoder-modernbert-base-seed-select004" +BEST_SHA256 = "9fba0f66b1d5de6492470f53577d1447bfac1d29b9ac03869268abb70bbd97f6" +CONTROL_RECIPE_ENV_SHA256 = "edb7163f388de71c258db2d68d03ac4e6d6a011548236666250d5d0f5d05e0f1" +FRESH_BATCH_ID = "HLX-EVAL-RESERVE-SELECT-005-FRESH-001" +LABEL_BLIND_FORBIDDEN = frozenset( + { + "attest", + "class", + "decision", + "expected_slice", + "lineage", + "model_score", + "semantic_family", + "task", + "unbind_clean", + } +) +METRIC_NAMES = ( + "classify_macro_f1_nonnone", + "classification_accuracy", + "observed_label_accuracy", + "unbind_clean_exact", +) + + +def initialization_decision() -> dict[str, Any]: + """The written proposal warm-starts morph65 while BEST is already select004. + + That sentence is the experiment. The trainer does not choose between them. + """ + return { + "best_name": BEST_NAME, + "best_not_moved": True, + "best_sha256": BEST_SHA256, + "decision": "morph65_new_schedule", + "experiment_id": EXPERIMENT_ID, + "rejected_alternative": "select004_best_new_schedule", + "trainer_default_did_not_decide": True, + "warm_start_name": WARM_START_NAME, + "warm_start_sha256": WARM_START_SHA256, + } + + +def sealed_preregistration() -> dict[str, Any]: + """Freeze the control-arm recipe. Do not transfer SELECT-004's variable.""" + proposed = json_copy(PROPOSED_SCHEDULE) + return { + "experiment_id": EXPERIMENT_ID, + "held_constant": { + "batch_size": 8, + "gradient_accumulation": 1, + "learning_rate": "2e-5", + "seed": None, + "seed_policy": "HLX_SEED_UNSET_FROZEN", + }, + "held_constant_authority": { + "batch_size": "Recorded HYPERLEX_TRAIN_BATCH on the morph65 continuation control recipe.", + "control_recipe_env_sha256": CONTROL_RECIPE_ENV_SHA256, + "document": "specs/007-hyperlexical-model/evaluation-reserve.md", + "gradient_accumulation": ( + "The trainer takes one optimizer step per batch and has no " + "accumulation parameter. 1 freezes that behavior." + ), + "inherited_from_select_004": False, + "learning_rate": "Recorded HYPERLEX_TRAIN_LR on the morph65 continuation control recipe.", + "same_numeric_optimizer_values_as_select_004_candidate_env": True, + "seed": "The reconstructed recipe seals HLX_SEED unset. Both arms share that frozen policy.", + "select_004_scientific_variable_not_transferred": "HLX_SELECT_METRIC", + "trainer_default_did_not_decide": True, + }, + "initialization": initialization_decision(), + "pinned_export_rows": proposed["pinned_export_rows"], + "pinned_export_sha256": proposed["pinned_export_sha256"], + "schedule": { + "baseline": proposed["baseline"], + "candidate": proposed["candidate"], + "checkpoint_rule": proposed["checkpoint_rule"], + "selection_metric": proposed["selection_metric"], + }, + "schema": "hyperlex.preregistration.v1", + "scientific_variable": "train_schedule", + "sealed": True, + "state": PREREGISTRATION_SEALED, + "training_authorized": TRAINING_AUTHORIZED, + "training_launch_authorized": False, + "unset_fields": [], + } + + +def threshold_authorization_proposal() -> dict[str, Any]: + """Name the four metrics. Leave every number unset. + + The schedule design supports a strict improvement of the candidate arm + over the control arm. It does not assign a preservation floor. SELECT-004 + floors do not transfer, and no number is invented here. + """ + metrics = [ + { + "authorization_state": THRESHOLDS_BLOCKED, + "derivation": "OPERATOR_JUDGMENT_ABSENT", + "metric": name, + "numeric_threshold": None, + } + for name in METRIC_NAMES + ] + return { + "comparison_baseline": "SELECT-005 control arm", + "decision_thresholds": {}, + "experiment_id": EXPERIMENT_ID, + "inherited_from_select_004": False, + "metrics": metrics, + "not_seed_morph78": True, + "primary_comparison_supported_by_schedule_design": ( + "candidate classify_macro_f1_nonnone strictly greater than the " + "control arm; equality does not promote" + ), + "reason_unsealed": ( + "No SELECT-005 derivation assigns a number. Predecessor floors " + "do not transfer. A number was not invented." + ), + "schema": THRESHOLD_SCHEMA, + "sealed": False, + "state": THRESHOLDS_BLOCKED, + "threshold_authorization_json_schema_exists": THRESHOLD_AUTHORIZATION_JSON_SCHEMA_EXISTS, + "training_launch_authorized": False, + } + + +def source_contract() -> dict[str, Any]: + """Rights-cleared ingest rules for a reserve that is not the 339.""" + return { + "admissible_source_types": list(ADMISSIBLE_SOURCE_TYPES), + "cleared_rights": CLEARED_RIGHTS, + "exhausted_settlement_count": EXHAUSTED_SETTLEMENT_COUNT, + "exhausted_stream_is_a_source": False, + "exhausted_stream_run": EXHAUSTED_STREAM_RUN, + "experiment_id": EXPERIMENT_ID, + "harvest_fence": ( + "hs_run.py refuses when the box exposure snapshot is older than " + "6 hours. This pass does not bypass that fence and does not fetch." + ), + "label_blind": True, + "required_slices": list(REQUIRED_SLICES), + "selection_label_blind": True, + "settlement_slice_gap": ( + "The settlement record emits no task, class, lineage, or " + "unbind_clean. A decision-only row is SLICE_LABELS_ABSENT. " + "This pass does not attach those fields beside the contract." + ), + "transition": TRANSITION, + "unbind_clean": ( + "A new campaign must include compositional material that can " + "satisfy soft_ceiling.clean_surface. Which rows enter the review " + "surface is chosen before labels. Princeton WordNet is " + "compositional and is not the census rights token." + ), + "wordnet": { + "admitted_alone": False, + "census_rights_token": "not CC-BY-SA", + "license": "Princeton WordNet 3.0", + }, + } + + +def exposure_snapshot_refusal( + ages_hours: Mapping[str, float], + *, + limit_h: float = BOX_SNAPSHOT_LIMIT_H, +) -> str | None: + """Return the harvest refusal, or None when every snapshot is inside the fence.""" + if not ages_hours: + return "REFUSE: no box exposure snapshot" + stale = [age for age in ages_hours.values() if float(age) > limit_h] + if not stale: + return None + worst = max(stale) + return f"REFUSE: box exposure snapshot is {worst:.1f}h old (> {limit_h:g}h)" + + +def label_blind_surface(rows: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]: + """Identity, rights, and provenance only. Labels are a later operator act.""" + surface = [] + for row in rows: + carried = sorted(LABEL_BLIND_FORBIDDEN.intersection(row)) + if carried: + raise SystemExit("REFUSE: label-blind surface carries " + ",".join(carried)) + source_type = str(row.get("source_type") or "") + if source_type not in ADMISSIBLE_SOURCE_TYPES: + raise SystemExit( + "REFUSE: source_type is not rights-cleared for this contract: " + + (source_type or "missing") + ) + for key in ("row_id", "text_hash", "source_url", "license"): + if not str(row.get(key) or "").strip(): + raise SystemExit(f"REFUSE: provenance field missing: {key}") + license_text = str(row["license"]) + compact = license_text.upper().replace(" ", "").replace("-", "") + if "CCBYSA" not in compact: + raise SystemExit("REFUSE: license is not CC-BY-SA") + surface.append( + { + "license": license_text, + "row_id": str(row["row_id"]), + "source_type": source_type, + "source_url": str(row["source_url"]), + "text_hash": str(row["text_hash"]), + } + ) + return surface + + +def fresh_census( + events: Sequence[Mapping[str, Any]], + ledger_state: Mapping[str, str], + export_hashes: set[str], + *, + fences: Mapping[str, Mapping[str, set[str]]] | None = None, +) -> dict[str, Any]: + """Census a new source. The exhausted stream is refused before routing.""" + for event in events: + run = str(event.get("source_run") or "") + if run == EXHAUSTED_STREAM_RUN or event.get("exhausted_universe") is True: + raise SystemExit("REFUSE: exhausted settlement universe is not a SELECT-005 source") + result = census( + events, + ledger_state, + export_hashes, + fences=fences, + batch_id=FRESH_BATCH_ID, + ) + result["exhausted_universe_consulted"] = False + result["source"] = "fresh" + if result.get("failure") is None and result.get("reserve_row_count"): + result["reserve_status"] = "FROZEN" + else: + result["reserve_status"] = FAILURE_QUOTA if result.get("failure") == FAILURE_QUOTA else result.get("failure") + return result + + +def completion_report(snapshot_ages: Mapping[str, float]) -> dict[str, Any]: + """One reconstruction. Live ages are supplied by the caller.""" + refusal = exposure_snapshot_refusal(snapshot_ages) + body = fresh_census([], {}, set()) + return { + "acquisition_refusal": refusal, + "admission_receipt": None, + "census_failure": body["failure"], + "exhausted_universe_consulted": False, + "experiment_id": EXPERIMENT_ID, + "initialization": initialization_decision(), + "isolation": "NOT_RUN_NO_ROWS", + "label_blind_surface_count": 0, + "preregistration_state": PREREGISTRATION_SEALED, + "provenance": "NO_ADMITTED_ROWS", + "reserve_manifest_created": False, + "reserve_row_count": body["reserve_row_count"], + "rows_acquired": 0, + "settlements_appended": 0, + "snapshot_ages_hours": {key: snapshot_ages[key] for key in sorted(snapshot_ages)}, + "threshold_state": THRESHOLDS_BLOCKED, + "training_authorized": False, + "training_launch_authorized": False, + "transition": TRANSITION, + } + + +def json_copy(payload: Mapping[str, Any]) -> dict[str, Any]: + return json.loads(canonical_json(payload)) + + +def decision_only_exclusion(event: Mapping[str, Any]) -> str: + """Public alias so a settlement-shaped row is judged by the existing census.""" + return exclusion_reason(event, {}, set()) diff --git a/scripts/shadow/hyperlexical/select_005_harvest.py b/scripts/shadow/hyperlexical/select_005_harvest.py new file mode 100644 index 00000000..85a051e6 --- /dev/null +++ b/scripts/shadow/hyperlexical/select_005_harvest.py @@ -0,0 +1,75 @@ +"""SELECT-005 harvest authorization stops when the exposure snapshot is stale. + +This module does not fetch, harvest, settle, recensus, or train. +It does not modify the routing contract. A fresh fence is permission to +start a later harvest. It is not a harvest. +""" + +from __future__ import annotations + +from typing import Any, Mapping, Sequence + +from .select_005_completion import BOX_SNAPSHOT_LIMIT_H +from .select_005_reserve import EXPERIMENT_ID, canonical_json, sha256_text + +TRANSITION = "SELECT_005_FRESH_SOURCE_HARVEST_AND_ROUTING_AUTHORIZATION" +STALE = "EXPOSURE_SNAPSHOT_STALE" +FRESH = "EXPOSURE_SNAPSHOT_FRESH" +REQUIRED_KINDS = ("vern", "boxhash") + + +def exposure_fence( + snapshots: Sequence[Mapping[str, Any]], + *, + compared_at_unix: str, + limit_h: float = BOX_SNAPSHOT_LIMIT_H, +) -> dict[str, Any]: + """Return the exposure decision for one frozen snapshot pair. + + Ages are ``compared_at_unix`` minus each ``mtime_unix``. Both values are + decimal strings, so the record does not read a clock and does not fetch. + """ + if float(limit_h) > BOX_SNAPSHOT_LIMIT_H: + raise SystemExit("REFUSE: exposure fence cannot be widened") + rows = [] + for item in snapshots: + mtime = str(item.get("mtime_unix") or "") + age_h = (float(compared_at_unix) - float(mtime)) / 3600.0 + rows.append( + { + "age_hours": f"{age_h:.6f}", + "kind": str(item.get("kind") or ""), + "mtime_unix": mtime, + "path": str(item.get("path") or ""), + "sha256": str(item.get("sha256") or ""), + "snapshot_id": str(item.get("snapshot_id") or ""), + } + ) + rows.sort(key=lambda row: (row["kind"], row["snapshot_id"], row["path"])) + present = {row["kind"] for row in rows} + missing = [kind for kind in REQUIRED_KINDS if kind not in present] + worst = max((row["age_hours"] for row in rows), default=None) + stale = bool(missing) or worst is None or float(worst) > float(limit_h) + binding = None + if rows: + binding = max(rows, key=lambda row: (float(row["age_hours"]), row["kind"])) + record = { + "binding_snapshot": binding, + "compared_at_unix": str(compared_at_unix), + "experiment_id": EXPERIMENT_ID, + "exposure_fence_bypassed": False, + "fetch_authorized": not stale, + "harvest_executed": False, + "limit_hours": f"{float(limit_h):.1f}", + "missing_kinds": missing, + "network_requests": 0, + "rows_acquired": 0, + "snapshots": rows, + "state": STALE if stale else FRESH, + "stop_before": "source_fetch" if stale else None, + "transition": TRANSITION, + "worst_age_hours": worst, + } + body = {key: value for key, value in record.items() if key != "record_sha256"} + record["record_sha256"] = sha256_text(canonical_json(body)) + return record diff --git a/scripts/shadow/hyperlexical/select_005_reserve.py b/scripts/shadow/hyperlexical/select_005_reserve.py new file mode 100644 index 00000000..f1c0101e --- /dev/null +++ b/scripts/shadow/hyperlexical/select_005_reserve.py @@ -0,0 +1,531 @@ +"""SELECT-005 fresh-reserve census. Dry run only. + +This module does not train, score a candidate, append the identity ledger, +or seal a preregistration. ``admit`` remains the writer. A census that cannot +fill every required slice freezes no reserve rows. +""" + +from __future__ import annotations + +import hashlib +import json +from collections import Counter +from pathlib import Path +from typing import Any, Mapping, Sequence + +from .identity_ledger import ( + GENERATION, + PLANNING_TARGETS, + REQUIRED_SLICES, + derived_state, + slices_of, +) + +EXPERIMENT_ID = "HLX-EXP-2026-09-27-SELECT-005" +TRANSITION = "SELECT_005_RESERVE_AND_ADMISSION_PREPARATION_AUTHORIZATION" +BATCH_ID = "HLX-EVAL-RESERVE-SELECT-005-CENSUS-001" +ADMISSIBLE_DECISIONS = frozenset({"ACCEPT", "RECLASSIFY", "NONE"}) +CLEARED_RIGHTS = "CC-BY-SA" +REQUIRED_PROVENANCE = ("row_id", "text_hash", "rights", "decision", "provenance") +ADMISSION_FLOORS = {key: 1 for key in REQUIRED_SLICES} +FAILURE_QUOTA = "RESERVE_QUOTA_UNFILLED" +FAILURE_ISOLATION = "RESERVE_ISOLATION_FAILURE" +FAILURE_PROVENANCE = "RESERVE_PROVENANCE_FAILURE" +SELECT_ADMISSION_JSON_SCHEMA_EXISTS = True +TRAINING_AUTHORIZED = False + +# The written SELECT-005 proposal. Unset trainer fields stay unset. +PROPOSED_SCHEDULE: dict[str, Any] = { + "baseline": { + "HYPERLEX_EARLY_STOP": "0", + "HYPERLEX_EARLY_STOP_MIN_EPOCHS": None, + "HYPERLEX_EARLY_STOP_PATIENCE": None, + "HYPERLEX_TRAIN_EPOCHS": "40", + }, + "candidate": { + "HYPERLEX_EARLY_STOP": "1", + "HYPERLEX_EARLY_STOP_MIN_EPOCHS": "4", + "HYPERLEX_EARLY_STOP_PATIENCE": "4", + "HYPERLEX_TRAIN_EPOCHS": "12", + }, + "checkpoint_rule": { + "restore_best": True, + "strict_increase": True, + "ties": "keep_earlier", + }, + "experiment_id": EXPERIMENT_ID, + "pinned_export_rows": 9150, + "pinned_export_sha256": "64b7d3dede25047cb6dd2e5b663f7fa72946ec82ac1a8816ae34622d1aaac430", + "scientific_variable": "train_schedule", + "sealed": False, + "selection_metric": "classify_macro_f1_nonnone", + "unset_fields": [ + "batch_size", + "gradient_accumulation", + "learning_rate", + "seed", + ], + "warm_start_name": "hyperlex-encoder-modernbert-base-seed-morph65", +} + + +def canonical_json(payload: Mapping[str, Any]) -> str: + return json.dumps(payload, indent=2, sort_keys=True, ensure_ascii=True) + "\n" + + +def sha256_text(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def schedule_spec_sha256() -> str: + return sha256_text(canonical_json(PROPOSED_SCHEDULE)) + + +def _order_key(digest: str, batch_id: str = BATCH_ID) -> str: + return hashlib.sha256(f"{GENERATION}|{batch_id}|{digest}".encode("utf-8")).hexdigest() + + +def _slices(event: Mapping[str, Any]) -> set[str]: + """Slice labels only when the row already carries them. Attest is not a class.""" + task = str(event.get("task") or "") + if not task: + return set() + label: dict[str, Any] = { + "task": task, + "class": str(event.get("class") or ""), + "lineage": str(event.get("lineage") or ""), + "split": str(event.get("split") or ""), + } + if task == "unbind": + label["unbind_clean"] = event.get("unbind_clean") is True + return slices_of({"labels": [label]}) + + +def exclusion_reason( + event: Mapping[str, Any], + ledger_state: Mapping[str, str], + export_hashes: set[str], +) -> str: + """One reason. Provenance, then existing consumption, then decision and rights.""" + if any(not str(event.get(key) or "").strip() for key in REQUIRED_PROVENANCE): + return "PROVENANCE_INCOMPLETE" + digest = str(event["text_hash"]) + state = ledger_state.get(digest) + if state: + return state + if digest in export_hashes: + return "PINNED_EXPORT" + decision = str(event.get("decision") or "") + if decision == "UNRESOLVED": + return "DECISION_UNRESOLVED" + rights = str(event.get("rights") or "") + if rights != CLEARED_RIGHTS: + return "RIGHTS_UNRESOLVED" if rights == "RIGHTS_UNRESOLVED" else "RIGHTS_NOT_CLEARED" + if decision not in ADMISSIBLE_DECISIONS: + return "DECISION_NOT_ADMISSIBLE" + if not _slices(event): + return "SLICE_LABELS_ABSENT" + return "ELIGIBLE" + + +def _fence_counts( + rows: Sequence[Mapping[str, Any]], + fences: Mapping[str, Mapping[str, set[str]]], +) -> dict[str, dict[str, int]]: + counts: dict[str, dict[str, int]] = {} + for name, fence in fences.items(): + hashes = fence.get("hashes") or set() + row_ids = fence.get("row_ids") or set() + synsets = fence.get("synsets") or set() + sources = fence.get("source_ids") or set() + hash_hits = 0 + row_hits = 0 + synset_hits = 0 + source_hits = 0 + for row in rows: + if str(row.get("text_hash") or "") in hashes: + hash_hits += 1 + if str(row.get("row_id") or "") in row_ids: + row_hits += 1 + synset = str(row.get("pwn30_synset") or "") + if synset and synset in synsets: + synset_hits += 1 + source = str(row.get("source_identity") or "") + if source and source in sources: + source_hits += 1 + counts[name] = { + "normalized_identity": hash_hits, + "row_id": row_hits, + "source_identity": source_hits, + "synset": synset_hits, + } + return counts + + +def census( + events: Sequence[Mapping[str, Any]], + ledger_state: Mapping[str, str], + export_hashes: set[str], + *, + fences: Mapping[str, Mapping[str, set[str]]] | None = None, + batch_id: str = BATCH_ID, + routing_by_row: Mapping[str, Mapping[str, Any]] | None = None, +) -> dict[str, Any]: + """Route a settlement universe twice and freeze a reserve only when it qualifies. + + ``routing_by_row`` supplies derivative SELECT routing records. Omitting it + preserves the embedded-slice census. Routing is not a quota input. + """ + first = _once(events, ledger_state, export_hashes, fences or {}, batch_id, routing_by_row) + second = _once(events, ledger_state, export_hashes, fences or {}, batch_id, routing_by_row) + if first != second: + failed = dict(first) + failed["determinism"] = "NOT_DETERMINISTIC" + failed["failure"] = "NOT_DETERMINISTIC" + failed["reserve_rows"] = [] + failed["reserve_manifest_sha256"] = None + return failed + first["determinism"] = "IDENTICAL" + return first + + +def _census_reason( + event: Mapping[str, Any], + ledger_state: Mapping[str, str], + export_hashes: set[str], + routing_by_row: Mapping[str, Mapping[str, Any]] | None, +) -> tuple[str, list[str]]: + """Settlement exclusions stay first. Routing fills slices the settlement lacks.""" + reason = exclusion_reason(event, ledger_state, export_hashes) + embedded = sorted(_slices(event)) + if routing_by_row is None: + return reason, embedded + routing = routing_by_row.get(str(event.get("row_id") or "")) + if embedded and routing is not None and list(routing.get("slices") or []) != embedded: + return "ROUTING_SLICE_MISMATCH", [] + if reason != "SLICE_LABELS_ABSENT": + return reason, embedded + if routing is None: + return "SLICE_LABELS_ABSENT", [] + from .eval_routing import authorize_for_census + + routed = authorize_for_census(event, routing) + if routed != "ELIGIBLE": + return routed, [] + return "ELIGIBLE", list(routing.get("slices") or []) + + +def _once( + events: Sequence[Mapping[str, Any]], + ledger_state: Mapping[str, str], + export_hashes: set[str], + fences: Mapping[str, Mapping[str, set[str]]], + batch_id: str, + routing_by_row: Mapping[str, Mapping[str, Any]] | None = None, +) -> dict[str, Any]: + reasons: Counter[str] = Counter() + eligible: list[dict[str, Any]] = [] + for event in events: + reason, slices = _census_reason(event, ledger_state, export_hashes, routing_by_row) + reasons[reason] += 1 + if reason != "ELIGIBLE": + continue + digest = str(event["text_hash"]) + eligible.append( + { + "order_key": _order_key(digest, batch_id), + "pwn30_synset": str(event.get("pwn30_synset") or ""), + "rights": str(event.get("rights") or ""), + "row_id": str(event["row_id"]), + "slices": slices, + "source_identity": str(event.get("source_identity") or ""), + "text_hash": digest, + } + ) + eligible.sort(key=lambda row: (row["order_key"], row["text_hash"], row["row_id"])) + counts = {key: 0 for key in REQUIRED_SLICES} + routed: list[dict[str, Any]] = [] + for row in eligible: + served = set(row["slices"]) + if any(counts[key] < int(PLANNING_TARGETS[key]) for key in served): + routed.append(row) + for key in served: + counts[key] += 1 + fence_counts = _fence_counts(routed, fences) + isolation_hit = any( + value + for item in fence_counts.values() + for value in item.values() + ) + floors_met = all(counts[key] >= ADMISSION_FLOORS[key] for key in REQUIRED_SLICES) + if isolation_hit: + failure = FAILURE_ISOLATION + frozen: list[dict[str, Any]] = [] + frozen_counts = {key: 0 for key in REQUIRED_SLICES} + elif not floors_met: + failure = FAILURE_QUOTA + frozen = [] + frozen_counts = {key: 0 for key in REQUIRED_SLICES} + else: + failure = None + frozen = routed + frozen_counts = counts + manifest = None + if frozen: + manifest = { + "batch_id": batch_id, + "experiment_id": EXPERIMENT_ID, + "rows": [ + { + "rights": row["rights"], + "row_id": row["row_id"], + "slices": row["slices"], + "text_hash": row["text_hash"], + } + for row in frozen + ], + "schema": "hyperlex.select_reserve_manifest.v1", + } + return { + "admission_floors": dict(ADMISSION_FLOORS), + "batch_id": batch_id, + "eligible_count": len(eligible), + "eligible_order": [row["text_hash"] for row in eligible], + "exclusion_counts": dict(sorted(reasons.items())), + "experiment_id": EXPERIMENT_ID, + "failure": failure, + "frozen_slice_counts": frozen_counts, + "isolation": fence_counts, + "planning_targets": dict(PLANNING_TARGETS), + "reserve_manifest_sha256": sha256_text(canonical_json(manifest)) if manifest else None, + "reserve_row_count": len(frozen), + "reserve_rows": [ + {"row_id": row["row_id"], "slices": row["slices"], "text_hash": row["text_hash"]} + for row in frozen + ], + "routed_before_freeze_gate": len(routed), + "routed_slice_counts": counts, + "source_universe_count": len(events), + "training_authorized": TRAINING_AUTHORIZED, + } + + +def spent_reserve_hashes(records: Sequence[Mapping[str, Any]]) -> set[str]: + """EVAL_SPENT identities that still carry ``evaluation_reserved``. + + SELECT-004's 491 identities moved to EVAL_SPENT without a new binding + entry. The historical flag is the membership evidence. SELECT-001 and + SELECT-002 bindings are not this set. + """ + found: set[str] = set() + for record in records: + if derived_state(record) != "EVAL_SPENT" or not record.get("evaluation_reserved"): + continue + digest = str(record.get("normalized_text_sha256") or "") + if digest: + found.add(digest) + return found + + +def ledger_state_index(records: Sequence[Mapping[str, Any]]) -> dict[str, str]: + index: dict[str, str] = {} + for record in records: + digest = str(record.get("normalized_text_sha256") or "") + if digest: + index[digest] = derived_state(record) + return index + + +def unsettled_screen_counts(rows: Sequence[Mapping[str, Any]]) -> dict[str, int]: + """Operator-review packets are not a settlement universe.""" + counts: Counter[str] = Counter() + for row in rows: + if str(row.get("operator_decision") or "").strip(): + counts["OPERATOR_DECISION_PRESENT"] += 1 + else: + counts["OPERATOR_DECISION_ABSENT"] += 1 + bucket = str(row.get("bucket") or "UNBUCKETED") + counts[f"bucket:{bucket}"] += 1 + counts["rows"] = len(rows) + return dict(sorted(counts.items())) + + +def _read_jsonl(path: Path) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + with path.open(encoding="utf-8") as handle: + for line in handle: + line = line.strip() + if line: + payload = json.loads(line) + if isinstance(payload, dict): + rows.append(payload) + return rows + + +def _identity_fence(rows: Sequence[Mapping[str, Any]]) -> dict[str, set[str]]: + return { + "hashes": {str(row.get("normalized_text_sha256") or "") for row in rows if row.get("normalized_text_sha256")}, + "row_ids": {str(row.get("row_id") or "") for row in rows if row.get("row_id")}, + "source_ids": {str(row.get("source_identity") or "") for row in rows if row.get("source_identity")}, + "synsets": {str(row.get("pwn30_synset") or "") for row in rows if row.get("pwn30_synset")}, + } + + +def _write_private(path: Path, payload: Mapping[str, Any]) -> str: + if "hlx-private" not in path.parts: + raise SystemExit("REFUSE: SELECT-005 census evidence stays under hlx-private") + text = canonical_json(payload) + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + path.chmod(0o600) + return sha256_text(text) + + +def write_live_census( + *, + ledger_dir: Path, + settlement_events: Path, + export_path: Path, + review_candidates: Path, + calibration_manifest: Path, + measurement_manifest: Path, + calibration_v1_manifest: Path, + destination: Path, +) -> dict[str, str]: + """Re-census the live settlement stream and freeze the failure. No ledger append.""" + from .holdout_guard import normalized_text_sha256 + + ledger_payload = json.loads(ledger_dir.joinpath("ledger.json").read_text(encoding="utf-8")) + records = ledger_payload["identities"] + states = ledger_state_index(records) + state_totals: Counter[str] = Counter(states.values()) + bound_select = { + str(record.get("normalized_text_sha256") or "") + for record in records + if any("SELECT" in str(item) for item in record.get("experiment_bindings") or []) + and record.get("normalized_text_sha256") + } + spent_reserve = spent_reserve_hashes(records) + export_hashes: set[str] = set() + export_rows = 0 + for row in _read_jsonl(export_path): + export_rows += 1 + export_hashes.add(normalized_text_sha256(str(row.get("text") or ""))) + events = _read_jsonl(settlement_events) + fences = { + "prior_select_reserve": { + "hashes": set(spent_reserve), + "row_ids": set(), + "source_ids": set(), + "synsets": set(), + }, + "select_001_002_bindings": { + "hashes": set(bound_select), + "row_ids": set(), + "source_ids": set(), + "synsets": set(), + }, + "threshold_v1_calibration": _identity_fence(_read_jsonl(calibration_v1_manifest)), + "threshold_v2_calibration": _identity_fence(_read_jsonl(calibration_manifest)), + "threshold_v2_measurement": _identity_fence(_read_jsonl(measurement_manifest)), + "train_export": { + "hashes": set(export_hashes), + "row_ids": set(), + "source_ids": set(), + "synsets": set(), + }, + } + report = census(events, states, export_hashes, fences=fences) + review = unsettled_screen_counts(_read_jsonl(review_candidates)) + # Novel settlements are the rows the quota would have to come from. + novel = [ + row + for row in events + if str(row.get("text_hash") or "") not in states and str(row.get("text_hash") or "") not in export_hashes + ] + novel_fences = _fence_counts(novel, fences) + novel_decision_rights: Counter[str] = Counter( + f"{row.get('decision')}|{row.get('rights')}" for row in novel + ) + body = { + "active_reserve_identities": state_totals.get("EVAL_RESERVE", 0) + state_totals.get("AVAILABLE", 0), + "batch_id": report["batch_id"], + "determinism": report["determinism"], + "eligible_count": report["eligible_count"], + "exclusion_counts": report["exclusion_counts"], + "experiment_id": EXPERIMENT_ID, + "export_rows": export_rows, + "export_unique_hashes": len(export_hashes), + "failure": report["failure"], + "ledger_identity_count": len(states), + "ledger_state_counts": dict(sorted(state_totals.items())), + "novel_decision_rights": dict(sorted(novel_decision_rights.items())), + "novel_settlement_count": len(novel), + "novel_settlement_fence_overlaps": novel_fences, + "planning_targets_are_not_the_floor": True, + "reserve_manifest_sha256": report["reserve_manifest_sha256"], + "reserve_row_count": report["reserve_row_count"], + "review_packet_is_not_a_settlement": review, + "routed_before_freeze_gate": report["routed_before_freeze_gate"], + "routed_slice_counts": report["routed_slice_counts"], + "schema": "hyperlex.select_005_reserve_census.v1", + "select_001_002_bound_identities": len(bound_select), + "select_004_spent_reserve_identities": len(spent_reserve), + "source_universe": "settlement/events.jsonl", + "source_universe_count": report["source_universe_count"], + "training_authorized": False, + "transition": TRANSITION, + "unbind_source_disposition": "WORDNET_NOT_ADMITTED_ALONE", + "wordnet_rows_admitted": 0, + } + isolation = { + "experiment_id": EXPERIMENT_ID, + "failure": report["failure"], + "final_reserve_overlaps": report["isolation"], + "novel_settlement_fence_overlaps": novel_fences, + "reserve_row_count": report["reserve_row_count"], + "schema": "hyperlex.select_005_isolation_report.v1", + "train_overlap_count": 0, + } + provenance = { + "admitted_rows": 0, + "experiment_id": EXPERIMENT_ID, + "required_fields": list(REQUIRED_PROVENANCE), + "result": "NO_ADMITTED_ROWS", + "schema": "hyperlex.select_005_provenance_report.v1", + "source_universe_missing_required_provenance": report["exclusion_counts"].get("PROVENANCE_INCOMPLETE", 0), + } + destination.mkdir(mode=0o700, parents=True, exist_ok=True) + census_hash = _write_private(destination / "RESERVE_CENSUS.json", body) + isolation_hash = _write_private(destination / "ISOLATION_REPORT.json", isolation) + provenance_hash = _write_private(destination / "PROVENANCE_REPORT.json", provenance) + failure = { + "admission_receipt_created": False, + "admission_result": report["failure"], + "census_sha256": census_hash, + "decision_threshold_state": "BLOCKED_PENDING_OPERATOR_AUTHORIZATION", + "experiment_id": EXPERIMENT_ID, + "failure": report["failure"], + "isolation_sha256": isolation_hash, + "next_legal_transition": "NONE", + "preregistration_sealed": False, + "provenance_sha256": provenance_hash, + "reserve_manifest_created": False, + "schedule_sealed": False, + "schedule_spec_sha256": schedule_spec_sha256(), + "schema": "hyperlex.select_005_admission_failure.v1", + "select_admission_json_schema_exists": SELECT_ADMISSION_JSON_SCHEMA_EXISTS, + "status": report["failure"], + "training_authorized": False, + "training_executed": False, + "training_launch_authorized": False, + "transition": TRANSITION, + "weights_mutated": False, + } + failure_hash = _write_private(destination / "ADMISSION_FAILURE.json", failure) + return { + "admission_failure_sha256": failure_hash, + "census_sha256": census_hash, + "isolation_sha256": isolation_hash, + "provenance_sha256": provenance_hash, + } diff --git a/scripts/shadow/hyperlexical/select_006_admission.py b/scripts/shadow/hyperlexical/select_006_admission.py new file mode 100644 index 00000000..bfb700c8 --- /dev/null +++ b/scripts/shadow/hyperlexical/select_006_admission.py @@ -0,0 +1,279 @@ +"""SELECT-006 admission evidence. Read-only. Does not train or score the reserve.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from .eval_routing import ( + UNBIND_CLEAN_DERIVATION, + canonical_json, + contract_validation, + derive_batch, +) +from .identity_ledger import IdentityLedger +from .select_006_efficiency_preservation import ( + BEST_SHA256, + BEST_WEIGHTS, + EPSILON, + EXPERIMENT_ID, + TRUNK_SHA256, + WARM_START_DIR, + WARM_START_SHA256, + _schedules, + _shared_settings, +) +from .select_006_reserve_acquisition import SELECT_005_ID, metric_surface + +SELECT006_THRESHOLD_SCHEMA = "hyperlex.select_006_threshold_authorization.v1" +VERIFIED = "ZERO_INIT_LOADER_VERIFIED" +HISTORICAL_PIN_STATUS = "NOT_YET_IMPLEMENTED" + +PREFLIGHT_FAILURE = "ADMISSION_PREFLIGHT_FAILURE" +PIN_MISMATCH = "ADMISSION_PIN_MISMATCH" +RESERVE_FAILURE = "ADMISSION_RESERVE_FAILURE" +LOADER_NOT_VERIFIED = "ADMISSION_LOADER_NOT_VERIFIED" +WITNESS_MISMATCH = "ADMISSION_ZERO_INIT_WITNESS_MISMATCH" +IDENTITY_FAILURE = "ADMISSION_IDENTITY_FAILURE" + +PREREGISTRATION_SHA256 = "3692ac63425fcbd57e5fdc355a4e565d653ffc3c2a6703cfdda1fde7cc6404b8" +THRESHOLD_SHA256 = "91081de5f9b348102fa5d0359150f9aedca0d53e98c9f4275aeb127c9de3642c" +RESERVE_MANIFEST_SHA256 = "33ee588bdd13a322020e2a0105a71265899b856b44b6c3fcde40eb943b36cab6" +LEDGER_EVENTS_SHA256 = "4471e3339b3708f0f494f7fe60a0d30118609e7312d5d0334b946d2bbc4efbf1" +LEDGER_PROJECTION_SHA256 = "dad556c7f6bba58c7456a6b88b607c72ebe8149e36c176a602e0def2edfdc435" +ROLE_WEIGHT_SHA256 = "94aa92d4c6f17f56a6d7a68a5951ad98bb38764d029ed93916fb9b870c54c20d" +ROLE_BIAS_SHA256 = "ce7b6ecb2b276d4d5b46ae23f38a749d1939cadc336e82b76548e01065151b93" +FILLER_WEIGHT_SHA256 = "b3d11a8eb1fcfb4f616c36b9f5b32be84aab9c91d642e54294dccfea36a385a3" +FILLER_BIAS_SHA256 = "cce2ff11650d8b9734f84f1d05acd65003b68e31adb5e0c9e035beaa7f5a03bf" +COMBINED_WITNESS_SHA256 = "58a3e733087c33c5ffea5909b46098e666337e1b99fb83d87c3744db4aef91b2" + +SPEC = Path("/home/morpheus/hlx-private/exp-20260929-select-006/spec-001") +PREREGISTRATION = SPEC / "SELECT_006_PREREGISTRATION.json" +THRESHOLD = SPEC / "SELECT_006_THRESHOLD_AUTHORIZATION.json" +RESERVE_DIR = Path("/home/morpheus/hlx-private/exp-20260929-select-006/reserve-acquisition-002") +WITNESS = Path( + "/home/morpheus/hlx-private/exp-20260929-select-006/zero-init-loader-001/LOADER_WITNESS.json" +) +LEDGER = Path("/home/morpheus/hlx-private/eval-reserve-20260926") +SETTLEMENT_PROVENANCE = "Blind review of SELECT-006 reserve-acquisition-002." +EXPECTED_COUNTS = { + "classify": 32, + "classify_non_none": 32, + "classify_observed": 32, + "unbind_clean": 5, +} + + +class Select006AdmissionError(Exception): + def __init__(self, code: str, detail: str) -> None: + self.code = code + super().__init__(f"{code}: {detail}") + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _load(path: Path) -> dict[str, Any]: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise Select006AdmissionError(PREFLIGHT_FAILURE, f"{path.name} is unreadable") from exc + if not isinstance(payload, dict): + raise Select006AdmissionError(PREFLIGHT_FAILURE, f"{path.name} is not an object") + return payload + + +def _pin(path: Path, expected: str, label: str) -> str: + try: + actual = _sha256(path) + except OSError as exc: + raise Select006AdmissionError(PIN_MISMATCH, f"{label} is unreadable") from exc + if actual != expected: + raise Select006AdmissionError(PIN_MISMATCH, label) + return actual + + +def verify_zero_init_witness(witness: dict[str, Any]) -> None: + """Live loader state is the verified witness, not the historical pin text.""" + if witness.get("execution_loader_status") != VERIFIED: + raise Select006AdmissionError( + LOADER_NOT_VERIFIED, + "zero-init execution loader is not verified", + ) + expected = { + "expanded_role_weight_sha256": ROLE_WEIGHT_SHA256, + "expanded_role_bias_sha256": ROLE_BIAS_SHA256, + "expanded_filler_weight_sha256": FILLER_WEIGHT_SHA256, + "expanded_filler_bias_sha256": FILLER_BIAS_SHA256, + "expanded_loader_witness_sha256": COMBINED_WITNESS_SHA256, + } + for key, value in expected.items(): + if witness.get(key) != value: + raise Select006AdmissionError(WITNESS_MISMATCH, key) + if witness.get("optimizer_constructed") is not False or witness.get("training_started") is not False: + raise Select006AdmissionError(LOADER_NOT_VERIFIED, "zero-init witness records training") + + +def select006_threshold_is_sealed(payload: dict[str, Any], experiment_id: str) -> bool: + if payload.get("schema") != SELECT006_THRESHOLD_SCHEMA: + return False + if experiment_id != EXPERIMENT_ID or payload.get("experiment_id") != EXPERIMENT_ID: + raise Select006AdmissionError(PIN_MISMATCH, "threshold experiment_id") + if payload.get("sealed") is not True or payload.get("authorization_state") != "SEALED": + raise Select006AdmissionError(PIN_MISMATCH, "threshold authorization is not sealed") + if payload.get("epsilon") != 0 or payload.get("training_launch_authorized") is not False: + raise Select006AdmissionError(PIN_MISMATCH, "threshold epsilon or launch flag") + rule = payload.get("decision_rule") + if not isinstance(rule, dict) or rule.get("compensation") is not False or rule.get("primary_equality") != "PASS": + raise Select006AdmissionError(PIN_MISMATCH, "threshold decision rule") + return True + + +def _routing_metadata(raw: dict[str, Any]) -> dict[str, Any]: + task = "unbind" if raw.get("unbind_clean_derivation") == UNBIND_CLEAN_DERIVATION else "classify" + return { + "normalized_identity": raw["normalized_hash"], + "provenance_state": raw["provenance_status"], + "row_id": raw["candidate_id"], + "source_identity": raw["source_url"], + "task": task, + "unbind_clean": task == "unbind", + "unbind_clean_derivation": raw.get("unbind_clean_derivation"), + } + + +def _verify_routing(receipt: dict[str, Any]) -> str: + routing_path = RESERVE_DIR / "ROUTING_RECORDS.jsonl" + raw_path = RESERVE_DIR / "RAW_CANDIDATES.jsonl" + routing = [json.loads(line) for line in routing_path.read_text(encoding="utf-8").splitlines() if line.strip()] + raws = { + row["candidate_id"]: row + for row in ( + json.loads(line) for line in raw_path.read_text(encoding="utf-8").splitlines() if line.strip() + ) + } + events: dict[str, dict[str, Any]] = {} + for line in (LEDGER / "settlement" / "events.jsonl").read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + event = json.loads(line) + if event.get("provenance") == SETTLEMENT_PROVENANCE: + events[str(event["row_id"])] = event + if len(routing) != 37 or len(events) != 37: + raise Select006AdmissionError(RESERVE_FAILURE, "routing or settlement row count") + pairs = [] + for record in routing: + row_id = str(record["row_id"]) + if row_id not in events or row_id not in raws: + raise Select006AdmissionError(RESERVE_FAILURE, f"routing row {row_id} is unbound") + pairs.append((events[row_id], _routing_metadata(raws[row_id]))) + validation = contract_validation(pairs) + if validation["state"] != "ROUTING_VALIDATION_COMPLETE" or validation["determinism"] != "IDENTICAL": + raise Select006AdmissionError(RESERVE_FAILURE, "routing validation") + derived = derive_batch(pairs) + if [canonical_json(row) for row in derived] != [canonical_json(row) for row in routing]: + raise Select006AdmissionError(RESERVE_FAILURE, "routing replay differed") + routing_sha = _sha256(routing_path) + if routing_sha != receipt.get("routing_sha256"): + raise Select006AdmissionError(RESERVE_FAILURE, "routing sha256") + surface = metric_surface(derived) + if surface["slice_counts"] != EXPECTED_COUNTS or surface["head_mapped_non_none"] != 32: + raise Select006AdmissionError(RESERVE_FAILURE, "metric slice counts") + if surface["floors_met"] is not True or any(value is not True for value in surface["computable"].values()): + raise Select006AdmissionError(RESERVE_FAILURE, "metric computability") + return routing_sha + + +def select006_admission_evidence() -> dict[str, Any]: + """Verify sealed pins, the frozen reserve, and the zero-init witness.""" + prereg_sha = _pin(PREREGISTRATION, PREREGISTRATION_SHA256, "preregistration") + threshold_sha = _pin(THRESHOLD, THRESHOLD_SHA256, "threshold authorization") + manifest_sha = _pin(RESERVE_DIR / "RESERVE_MANIFEST.json", RESERVE_MANIFEST_SHA256, "reserve manifest") + warm_sha = _pin(Path(WARM_START_DIR) / "model.safetensors", WARM_START_SHA256, "warm start") + best_sha = _pin(Path(BEST_WEIGHTS), BEST_SHA256, "BEST") + events_sha = _pin(LEDGER / "events.jsonl", LEDGER_EVENTS_SHA256, "ledger events") + projection_sha = _pin(LEDGER / "ledger.json", LEDGER_PROJECTION_SHA256, "ledger projection") + if _sha256(Path("/home/morpheus/.hyperlex/models/trunks/ModernBERT-base/model.safetensors")) != TRUNK_SHA256: + raise Select006AdmissionError(PIN_MISMATCH, "trunk") + + prereg = _load(PREREGISTRATION) + if prereg.get("epsilon") != 0 or EPSILON != 0: + raise Select006AdmissionError(PIN_MISMATCH, "EPSILON") + control, candidate = _schedules() + if prereg.get("control_schedule") != control or prereg.get("candidate_schedule") != candidate: + raise Select006AdmissionError(PIN_MISMATCH, "schedules") + if prereg.get("shared_settings") != _shared_settings(): + raise Select006AdmissionError(PIN_MISMATCH, "shared settings") + historical = (prereg.get("vocabulary_expansion") or {}).get("execution_loader_status") + if historical != HISTORICAL_PIN_STATUS: + raise Select006AdmissionError(PIN_MISMATCH, "historical vocabulary pin") + if not select006_threshold_is_sealed(_load(THRESHOLD), EXPERIMENT_ID): + raise Select006AdmissionError(PIN_MISMATCH, "threshold authorization") + + witness = _load(WITNESS) + verify_zero_init_witness(witness) + + reserve_receipt = _load(RESERVE_DIR / "RESERVE_RECEIPT.json") + if reserve_receipt.get("state") != "RESERVE_FROZEN" or reserve_receipt.get("ledger_replay") != "PASS": + raise Select006AdmissionError(RESERVE_FAILURE, "reserve receipt") + isolation = _load(RESERVE_DIR / "ISOLATION_REPORT.json") + provenance = _load(RESERVE_DIR / "PROVENANCE_REPORT.json") + if isolation.get("result") != "PASS" or isolation.get("overlaps"): + raise Select006AdmissionError(RESERVE_FAILURE, "isolation") + if provenance.get("result") != "PASS": + raise Select006AdmissionError(RESERVE_FAILURE, "provenance") + if _sha256(RESERVE_DIR / "ISOLATION_REPORT.json") != reserve_receipt.get("isolation_sha256"): + raise Select006AdmissionError(RESERVE_FAILURE, "isolation sha256") + if _sha256(RESERVE_DIR / "PROVENANCE_REPORT.json") != reserve_receipt.get("provenance_sha256"): + raise Select006AdmissionError(RESERVE_FAILURE, "provenance sha256") + manifest = _load(RESERVE_DIR / "RESERVE_MANIFEST.json") + if manifest.get("slice_counts") != EXPECTED_COUNTS or manifest.get("head_mapped_non_none") != 32: + raise Select006AdmissionError(RESERVE_FAILURE, "manifest slices") + if len(manifest.get("rows") or []) != 37: + raise Select006AdmissionError(IDENTITY_FAILURE, "manifest identity count") + routing_sha = _verify_routing(reserve_receipt) + + ledger = IdentityLedger.load(LEDGER) + projected = json.dumps(ledger.project(), indent=2, sort_keys=True) + "\n" + if projected != (LEDGER / "ledger.json").read_text(encoding="utf-8"): + raise Select006AdmissionError(RESERVE_FAILURE, "ledger replay") + active = ledger.active_reserve_records(EXPERIMENT_ID) + counts = ledger.active_reserve_counts(EXPERIMENT_ID) + if len(active) != 37 or counts != EXPECTED_COUNTS: + raise Select006AdmissionError(IDENTITY_FAILURE, "active SELECT-006 reserve") + if len(ledger.active_reserve_records(SELECT_005_ID)) != 78: + raise Select006AdmissionError(RESERVE_FAILURE, "SELECT-005 reserve changed") + + return { + "admission_reason": "ADMISSION_PASS", + "best_sha256": best_sha, + "epsilon": 0, + "execution_loader_status": VERIFIED, + "expanded_filler_bias_sha256": FILLER_BIAS_SHA256, + "expanded_filler_weight_sha256": FILLER_WEIGHT_SHA256, + "expanded_loader_witness_sha256": COMBINED_WITNESS_SHA256, + "expanded_role_bias_sha256": ROLE_BIAS_SHA256, + "expanded_role_weight_sha256": ROLE_WEIGHT_SHA256, + "head_mapped_non_none": 32, + "historical_vocabulary_pin_status": HISTORICAL_PIN_STATUS, + "isolation": "PASS", + "ledger_events_sha256": events_sha, + "ledger_projection_sha256": projection_sha, + "ledger_replay": "PASS", + "metric_computability": "PASS", + "preregistration_sha256": prereg_sha, + "provenance": "PASS", + "reserve_manifest_sha256": manifest_sha, + "routing_validation": "PASS", + "routing_sha256": routing_sha, + "select_006_active_reserve": 37, + "threshold_authorization_sha256": threshold_sha, + "warm_start_sha256": warm_sha, + } diff --git a/scripts/shadow/hyperlexical/select_006_efficiency_preservation.py b/scripts/shadow/hyperlexical/select_006_efficiency_preservation.py new file mode 100644 index 00000000..c196e742 --- /dev/null +++ b/scripts/shadow/hyperlexical/select_006_efficiency_preservation.py @@ -0,0 +1,690 @@ +"""SELECT-006 efficiency-preservation specification. + +This module seals governance. It does not train, construct an optimizer, +create a checkpoint, score a reserve, or move BEST. +""" + +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +from typing import Any, Mapping + +EXPERIMENT_ID = "HLX-EXP-2026-09-29-SELECT-006" +SELECT_005_ID = "HLX-EXP-2026-09-27-SELECT-005" +TRANSITION = "SELECT_006_EFFICIENCY_PRESERVATION_SPEC" +NEXT_TRANSITION = "SELECT_006_RESERVE_ACQUISITION" +HYPOTHESIS = ( + "A reduced training schedule can preserve primary evaluation quality " + "and all preservation metrics while using substantially fewer optimizer " + "steps than the control schedule." +) +WARM_START_SHA256 = "96838b9656a84c3fee1773a41fdf2fbbec2f88f3bc948e4acfbe06c194ac5587" +BEST_SHA256 = "9fba0f66b1d5de6492470f53577d1447bfac1d29b9ac03869268abb70bbd97f6" +TRUNK_SHA256 = "340ac08b74eef0d7bdec2d7981a6a3d4249bf0e6aab60634b72ad02c2b8023a9" +EXPORT_SHA256 = "64b7d3dede25047cb6dd2e5b663f7fa72946ec82ac1a8816ae34622d1aaac430" +EXPORT_ROWS = 9150 +EXPORT_PATH = "/home/morpheus/hlx-private/d1-spark-tree-20260924T213846Z/morph78-train-export.jsonl" +WARM_START_DIR = "/home/morpheus/.hyperlex/models/hyperlex-encoder-modernbert-base-seed-morph65" +BEST_WEIGHTS = "/home/morpheus/.hyperlex/models/hyperlex-encoder-modernbert-base-seed-select004/model.safetensors" +TRUNK_DIR = "/home/morpheus/.hyperlex/models/trunks/ModernBERT-base" +IMAGE = "lmsysorg/sglang:dev-qwen38-27b-dflash2" +IMAGE_SHA256 = "616a3e97f45191af975896cfa644279096cb31bd408a071c2e99ca7209c3cafe" +LEDGER_EVENTS_SHA256 = "4ec441f545e37cd9e691ab322fa438269c79b6a6ed82f56b58ce740e94665763" +LEDGER_PROJECTION_SHA256 = "83d7dde24f723e750b487819aa1a5b6c7d847ca55fa5b1bf2dcd8b8157cc50d2" +SELECT_005_SETTLEMENT_SHA256 = "70f5ccddf41f357c75191d519427c2c1d5baac8f6a7b39575a325e8522d7ea27" +SELECT_005_METRIC_TABLE_SHA256 = "ba9e57a538c5e834321ace3cb6b7aa134e9c3c406cd05823853ebf540043fd39" +INVALID_LOADER_DIR = ( + "/home/morpheus/hlx-private/exp-20260927-select-005/launch-001/invalid-loader-001" +) +LEDGER_DIR = "/home/morpheus/hlx-private/eval-reserve-20260926" +WARM_START_WEIGHTS = WARM_START_DIR + "/model.safetensors" +CONTROL_CONFIG = ( + "/home/morpheus/.hyperlex/models/hyperlex-encoder-modernbert-base-seed-select005-control/config.json" +) +CANDIDATE_CONFIG = ( + "/home/morpheus/.hyperlex/models/hyperlex-encoder-modernbert-base-seed-select005/config.json" +) +SPEC_DIR = "/home/morpheus/hlx-private/exp-20260929-select-006/spec-001" +EPSILON = 0 +EFFICIENCY_MAX_STEP_RATIO = 0.25 +NEW_ROLES = ("pos_6", "pos_7", "unknown") +QUESTION = "non-inferiority plus compute efficiency" +SCHEMA_DIR = Path(__file__).resolve().parents[3] / "specs/007-hyperlexical-model/schemas/hyperlex/select" +WARM_START_PIN_MISMATCH = "WARM_START_PIN_MISMATCH" +VOCAB_EXPANSION_SPEC_INCOMPLETE = "VOCAB_EXPANSION_SPEC_INCOMPLETE" +EVALUATION_RESERVE_STATE_UNRESOLVED = "EVALUATION_RESERVE_STATE_UNRESOLVED" +THRESHOLD_UNDERSPECIFIED = "THRESHOLD_UNDERSPECIFIED" +EPSILON_UNJUSTIFIED = "EPSILON_UNJUSTIFIED" +SCHEDULE_UNDERSPECIFIED = "SCHEDULE_UNDERSPECIFIED" +ARTIFACT_MANIFEST_FAILURE = "ARTIFACT_MANIFEST_FAILURE" +SPEC_SEAL_FAILURE = "SPEC_SEAL_FAILURE" + + +class SpecSealError(SystemExit): + def __init__(self, code: str, detail: str) -> None: + self.code = code + super().__init__(f"{code}: {detail}") + + +def _fail(code: str, detail: str) -> None: + raise SpecSealError(code, detail) + + +def sha256_text(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def canonical_json(payload: Mapping[str, Any]) -> str: + return json.dumps(payload, indent=2, sort_keys=True, ensure_ascii=True) + "\n" + + +def _forbidden_training() -> dict[str, Any]: + return { + "best_changed": False, + "epochs": 0, + "gradient_steps": 0, + "optimizer_constructed": False, + "training_launch_authorized": False, + "training_started": False, + "weights_mutated": False, + } + + +def vocabulary_expansion_pin() -> dict[str, Any]: + """Explicit loader contract. New rows are zeros, not an unset torch draw.""" + return { + "filler_expansion": { + "mapped_existing_fillers": 1750, + "newly_initialized_fillers": 29, + "skipped_warm_start_only": 230, + "target_filler_count": 1779, + "warm_start_filler_count": 1980, + }, + "counts_include_unk_token": True, + "current_loader_behavior": ( + "warm_load_checkpoint with expand_vocab copies mapped rows by name " + "and leaves new rows at nn.Linear initialization. That draw is not " + "the sealed zero policy." + ), + "execution_loader_status": "NOT_YET_IMPLEMENTED", + "init_expand_vocab": True, + "initialization_policy": { + "mapped_rows": "copy warm-start weight and bias by label name", + "new_rows": "zeros", + "seed_used": False, + }, + "launch_blocked_until_loader_honors_zeros": True, + "mapped_existing_roles": 10, + "newly_initialized_roles": 3, + "new_role_names_in_sorted_order": list(NEW_ROLES), + "ordering_policy": "unk_then_sorted_labels", + "unk_token": "", + "unk_token_is_not_a_new_role": True, + "warm_start_only_roles": 0, + "schema": "hyperlex.select_006_vocabulary_expansion.v1", + "target_role_count": 13, + "warm_start_role_count": 10, + "warm_start_sha256": WARM_START_SHA256, + } + + +def _schedules() -> tuple[dict[str, Any], dict[str, Any]]: + control = { + "early_stopping": "disabled", + "max_epochs": 40, + "restore_best": True, + } + candidate = { + "early_stopping_patience": 4, + "improvement": "strict", + "max_epochs": 12, + "minimum_epochs": 4, + "restore_best": True, + "ties": "keep_earlier", + } + return control, candidate + + +def _shared_settings() -> dict[str, Any]: + return { + "batch_size": 8, + "environment_hash_omits": [ + "HLX_SEED", + "HYPERLEX_EARLY_STOP", + "HYPERLEX_EARLY_STOP_MIN_EPOCHS", + "HYPERLEX_EARLY_STOP_PATIENCE", + "HYPERLEX_FILLER_FILTER", + "HYPERLEX_INIT_EXPAND_VOCAB", + "HYPERLEX_INIT_FROM", + "HYPERLEX_LAST_TRAINABLE", + "HYPERLEX_TRAIN_BATCH", + "HYPERLEX_TRAIN_EPOCHS", + "HYPERLEX_TRAIN_LR", + "HYPERLEX_UNBIND_CURRICULUM", + "HYPERLEX_UNBIND_EVERY_N", + "HYPERLEX_UNBIND_LOSS_WEIGHT", + "HYPERLEX_UNBIND_PRIMARY", + ], + "filler_filter": "strict", + "gradient_accumulation": 1, + "gradient_accumulation_authority": ( + "The trainer takes one optimizer step per batch and has no " + "accumulation parameter. 1 freezes that behavior." + ), + "image": IMAGE, + "image_sha256": IMAGE_SHA256, + "last_trainable": 2, + "learning_rate": "2e-5", + "max_len": 64, + "must_remain_unset": ["HLX_SEED", "HYPERLEX_INCLUDE_LIVE"], + "not_in_sealed_env": ["HYPERLEX_TRAIN_OUT"], + "output_directories_not_reused": [ + "/home/morpheus/.hyperlex/models/hyperlex-encoder-modernbert-base-seed-select005", + "/home/morpheus/.hyperlex/models/hyperlex-encoder-modernbert-base-seed-select005-control", + ], + "seed_policy": "HLX_SEED_UNSET_FROZEN", + "selection_metric": "classify_macro_f1_nonnone", + "training_export_path": EXPORT_PATH, + "training_export_rows": EXPORT_ROWS, + "training_export_sha256": EXPORT_SHA256, + "trunk_dir": TRUNK_DIR, + "trunk_sha256": TRUNK_SHA256, + "unbind_curriculum": False, + "unbind_every_n": 1, + "unbind_loss_weight": 1.0, + "unbind_primary": "mixed", + } + + +def invalid_loader_exclusion() -> dict[str, Any]: + return { + "allowed_use": "debugging provenance", + "forbidden_uses": [ + "aggregate reporting", + "baseline comparison", + "BEST selection", + "SELECT-006 initialization", + "threshold calculation", + ], + "path": INVALID_LOADER_DIR, + "status": "PERMANENTLY_NON_RESULT", + } + + +def evaluation_reserve_decision() -> dict[str, Any]: + """SELECT-005's 78 rows stay bound to SELECT-005 and must not be scored again.""" + return { + "decision": "REUSE_FORBIDDEN", + "ledger_events_sha256": LEDGER_EVENTS_SHA256, + "ledger_projection_sha256": LEDGER_PROJECTION_SHA256, + "reason": ( + "The 78 identities are still EVAL_RESERVE and are bound only to " + "HLX-EXP-2026-09-27-SELECT-005. A reserve bound to another experiment " + "does not satisfy SELECT-006. The sealed reserve policy also forbids " + "repeated scoring. evaluation_spent was not flipped; the completed " + "SELECT-005 score is the scientific spend. SELECT-006 has no reserve." + ), + "repeated_scoring_allowed": False, + "satisfies_select_006": False, + "select_005_active_count": 78, + "select_005_binding": [SELECT_005_ID], + "select_005_ledger_lifecycle": "EVAL_RESERVE", + "select_005_slice_counts": { + "classify": 51, + "classify_non_none": 11, + "classify_observed": 11, + "unbind_clean": 27, + }, + "select_006_reserve": None, + } + + +def preregistration() -> dict[str, Any]: + control, candidate = _schedules() + return { + "candidate_schedule": candidate, + "control_schedule": control, + "efficiency_gate": { + "candidate_optimizer_steps_le_ratio_of_control": EFFICIENCY_MAX_STEP_RATIO, + "comparison": "candidate_optimizer_steps * 4 <= control_optimizer_steps", + "descriptive_only": ["epochs_ratio", "step_ratio", "step_reduction_percent", "wall_clock_ratio"], + "wall_clock_is_a_gate": False, + }, + "epsilon": EPSILON, + "epsilon_justification": ( + "SELECT-006 has no frozen reserve. The only scored surface is the " + "SELECT-005 reserve, which this experiment must not reuse. That surface " + "has 51 classify rows and 4 head-mapped non-none golds, and both arms " + "landed on exactly 1/3. A nonzero band would be finer than that surface " + "can support. EPSILON is 0: exact non-inferiority. A later reserve must " + "not widen it." + ), + "evaluation_reserve": evaluation_reserve_decision(), + "experiment_id": EXPERIMENT_ID, + "hypothesis": HYPOTHESIS, + "question": QUESTION, + "inherited_from_select_005_settlement": False, + "invalid_loader_exclusion": invalid_loader_exclusion(), + "schema": "hyperlex.select_006_preregistration.v1", + "scientific_variable": "train_schedule", + "sealed": True, + "select_005_citation": { + "metric_table_sha256": SELECT_005_METRIC_TABLE_SHA256, + "role": "prior empirical motivation only", + "settlement": "REJECT", + "settlement_sha256": SELECT_005_SETTLEMENT_SHA256, + "threshold": "FAIL", + "use_as_authorization": False, + }, + "shared_settings": _shared_settings(), + "state": "PREREGISTERED", + "telemetry": { + "incomplete": "EXPERIMENT_INVALID", + "per_epoch": [ + "checkpoint identity", + "classification_accuracy", + "classify_macro_f1_nonnone", + "epoch", + "global_step", + "learning_rate", + "observed_label_accuracy", + "training_loss", + "unbind_clean_exact", + ], + "required_reconstructions": [ + "best checkpoint", + "best epoch", + "final optimizer steps", + "patience trace", + "restore-best proof", + "stop reason", + ], + "unbind_clean_exact_on_train_val": ( + "null is required when the val split has no unbind_clean flag; " + "do not infer it. The reserve score is the gate value and must be computable." + ), + }, + "training_launch_authorized": False, + "training_started": False, + "vocabulary_expansion": vocabulary_expansion_pin(), + "warm_start_sha256": WARM_START_SHA256, + } + + +def threshold_authorization(preregistration_sha256: str) -> dict[str, Any]: + return { + "authorization_state": "SEALED", + "decision_rule": { + "compensation": False, + "efficiency": "candidate_optimizer_steps * 4 <= control_optimizer_steps", + "missing_required_metric": "FAIL", + "noncomputable_required_metric": "FAIL", + "primary_equality": "PASS", + "primary_strict_increase_required": False, + "select_005_rule_not_reused": "primary delta > 0", + "pass_requires": [ + "classification_accuracy", + "efficiency", + "observed_label_accuracy", + "primary_preservation", + "unbind_clean_exact", + ], + "preservation": "candidate >= control on each preservation metric", + "primary_preservation": "candidate_classify_macro_f1_nonnone >= control_classify_macro_f1_nonnone - EPSILON", + }, + "efficiency_max_step_ratio": EFFICIENCY_MAX_STEP_RATIO, + "epsilon": EPSILON, + "experiment_id": EXPERIMENT_ID, + "inherited_from_select_005_settlement": False, + "preregistration_sha256": preregistration_sha256, + "schema": "hyperlex.select_006_threshold_authorization.v1", + "sealed": True, + "training_launch_authorized": False, + } + + +def dependency_statement() -> dict[str, Any]: + return { + "best_not_moved": True, + "best_sha256": BEST_SHA256, + "best_weights": BEST_WEIGHTS, + "evaluation_reserve": evaluation_reserve_decision(), + "experiment_id": EXPERIMENT_ID, + "image_sha256": IMAGE_SHA256, + "invalid_loader_exclusion": invalid_loader_exclusion(), + "schema": "hyperlex.select_006_dependency_statement.v1", + "training_export_path": EXPORT_PATH, + "training_export_rows": EXPORT_ROWS, + "training_export_sha256": EXPORT_SHA256, + "trunk_dir": TRUNK_DIR, + "trunk_sha256": TRUNK_SHA256, + "warm_start_dir": WARM_START_DIR, + "warm_start_sha256": WARM_START_SHA256, + **_forbidden_training(), + } + + +def schema_errors(payload: Mapping[str, Any], schema_name: str) -> list[str]: + import jsonschema + + schema = json.loads((SCHEMA_DIR / schema_name).read_text(encoding="utf-8")) + validator = jsonschema.Draft202012Validator(schema) + return [error.message for error in validator.iter_errors(payload)] + + +def apply_decision( + *, + control: Mapping[str, float], + candidate: Mapping[str, float], + control_steps: int, + candidate_steps: int, +) -> dict[str, Any]: + """Apply the sealed rule to caller-supplied numbers. This does not score.""" + gates = [] + primary = "classify_macro_f1_nonnone" + for name in ( + primary, + "classification_accuracy", + "observed_label_accuracy", + "unbind_clean_exact", + ): + if name not in control or name not in candidate: + gates.append({"metric": name, "pass": False, "reason": "MISSING"}) + continue + left = control[name] + right = candidate[name] + if isinstance(left, bool) or isinstance(right, bool): + gates.append({"metric": name, "pass": False, "reason": "NON_COMPUTABLE"}) + continue + try: + delta = float(right) - float(left) + except (TypeError, ValueError): + gates.append({"metric": name, "pass": False, "reason": "NON_COMPUTABLE"}) + continue + limit = -EPSILON if name == primary else 0.0 + ok = delta >= limit + gates.append( + { + "delta": delta, + "metric": name, + "pass": ok, + "reason": "PASS" if ok else "PRESERVATION_DEGRADED", + } + ) + if ( + isinstance(control_steps, bool) + or isinstance(candidate_steps, bool) + or not isinstance(control_steps, int) + or not isinstance(candidate_steps, int) + or control_steps <= 0 + or candidate_steps < 0 + ): + efficiency = {"metric": "optimizer_steps", "pass": False, "reason": "NON_COMPUTABLE"} + else: + ratio = candidate_steps / control_steps + ok = candidate_steps * 4 <= control_steps + efficiency = { + "metric": "optimizer_steps", + "pass": ok, + "reason": "PASS" if ok else "EFFICIENCY_FAIL", + "step_ratio": ratio, + "step_reduction_percent": (1.0 - ratio) * 100.0, + } + gates.append(efficiency) + passed = all(gate["pass"] for gate in gates) + return {"compensation": False, "epsilon": EPSILON, "gates": gates, "outcome": "PASS" if passed else "FAIL", "pass": passed} + + +def _common_env() -> dict[str, str]: + return { + "HLX_EXPERIMENT_ID": EXPERIMENT_ID, + "HLX_SCIENTIFIC_VARIABLE": "train_schedule", + "HLX_SELECT_METRIC": "classify_macro_f1_nonnone", + "HLX_TRAIN_EXPORT_PATH": EXPORT_PATH, + "HLX_TRAIN_EXPORT_ROWS": str(EXPORT_ROWS), + "HLX_TRAIN_EXPORT_SHA256": EXPORT_SHA256, + "HYPERLEX_FILLER_FILTER": "strict", + "HYPERLEX_INIT_EXPAND_VOCAB": "1", + "HYPERLEX_INIT_FROM": WARM_START_DIR, + "HYPERLEX_LAST_TRAINABLE": "2", + "HYPERLEX_TRAIN_BATCH": "8", + "HYPERLEX_TRAIN_LR": "2e-5", + "HYPERLEX_TRUNK_DIR": TRUNK_DIR, + "HYPERLEX_UNBIND_CURRICULUM": "0", + "HYPERLEX_UNBIND_EVERY_N": "1", + "HYPERLEX_UNBIND_LOSS_WEIGHT": "1.0", + "HYPERLEX_UNBIND_PRIMARY": "mixed", + } + + +def baseline_env() -> dict[str, str]: + env = _common_env() + env["HYPERLEX_EARLY_STOP"] = "0" + env["HYPERLEX_TRAIN_EPOCHS"] = "40" + return env + + +def candidate_env() -> dict[str, str]: + env = _common_env() + env["HYPERLEX_EARLY_STOP"] = "1" + env["HYPERLEX_EARLY_STOP_MIN_EPOCHS"] = "4" + env["HYPERLEX_EARLY_STOP_PATIENCE"] = "4" + env["HYPERLEX_TRAIN_EPOCHS"] = "12" + return env + + +def _expected_observations() -> dict[str, Any]: + pin = vocabulary_expansion_pin() + reserve = evaluation_reserve_decision() + return { + "best_sha256": BEST_SHA256, + "candidate_config": CANDIDATE_CONFIG, + "configs_agree": True, + "control_config": CONTROL_CONFIG, + "export_sha256": EXPORT_SHA256, + "filler_expansion": pin["filler_expansion"], + "ledger_events_sha256": LEDGER_EVENTS_SHA256, + "ledger_projection_sha256": LEDGER_PROJECTION_SHA256, + "mapped_existing_roles": 10, + "new_role_names_in_sorted_order": list(NEW_ROLES), + "newly_initialized_roles": 3, + "role_order_ok": True, + "select_005_active_count": 78, + "select_005_slice_counts": reserve["select_005_slice_counts"], + "select_006_active_count": 0, + "target_role_count": 13, + "trunk_sha256": TRUNK_SHA256, + "warm_start_only_roles": 0, + "warm_start_role_count": 10, + "warm_start_sha256": WARM_START_SHA256, + } + + +def _check_observations(observations: Mapping[str, Any]) -> None: + expected = _expected_observations() + if observations.get("warm_start_sha256") != WARM_START_SHA256: + _fail(WARM_START_PIN_MISMATCH, "warm-start weights") + if observations.get("best_sha256") != BEST_SHA256: + _fail(SPEC_SEAL_FAILURE, "BEST pin") + if observations.get("export_sha256") != EXPORT_SHA256 or observations.get("trunk_sha256") != TRUNK_SHA256: + _fail(SPEC_SEAL_FAILURE, "export or trunk pin") + vocab_keys = ( + "configs_agree", + "filler_expansion", + "mapped_existing_roles", + "new_role_names_in_sorted_order", + "newly_initialized_roles", + "role_order_ok", + "target_role_count", + "warm_start_only_roles", + "warm_start_role_count", + ) + if any(observations.get(key) != expected[key] for key in vocab_keys): + _fail(VOCAB_EXPANSION_SPEC_INCOMPLETE, "live role or filler expansion") + reserve_keys = ( + "ledger_events_sha256", + "ledger_projection_sha256", + "select_005_active_count", + "select_005_slice_counts", + "select_006_active_count", + ) + if any(observations.get(key) != expected[key] for key in reserve_keys): + _fail(EVALUATION_RESERVE_STATE_UNRESOLVED, "ledger lifecycle") + + +def _check_documents(pre: Mapping[str, Any], threshold: Mapping[str, Any]) -> None: + if pre.get("epsilon") != 0 or "exact non-inferiority" not in str(pre.get("epsilon_justification") or ""): + _fail(EPSILON_UNJUSTIFIED, "EPSILON") + if pre.get("evaluation_reserve", {}).get("decision") != "REUSE_FORBIDDEN": + _fail(EVALUATION_RESERVE_STATE_UNRESOLVED, "reserve decision") + if pre.get("evaluation_reserve", {}).get("select_006_reserve") is not None: + _fail(EVALUATION_RESERVE_STATE_UNRESOLVED, "SELECT-006 has no reserve") + control, candidate = _schedules() + if pre.get("control_schedule") != control or pre.get("candidate_schedule") != candidate: + _fail(SCHEDULE_UNDERSPECIFIED, "schedule") + if baseline_env().get("HYPERLEX_EARLY_STOP_PATIENCE") or baseline_env().get("HYPERLEX_EARLY_STOP_MIN_EPOCHS"): + _fail(SCHEDULE_UNDERSPECIFIED, "control early-stop keys must stay absent") + vocab = pre.get("vocabulary_expansion") or {} + if ( + vocab.get("execution_loader_status") != "NOT_YET_IMPLEMENTED" + or vocab.get("new_role_names_in_sorted_order") != list(NEW_ROLES) + or vocab.get("initialization_policy", {}).get("new_rows") != "zeros" + or vocab.get("init_expand_vocab") is not True + ): + _fail(VOCAB_EXPANSION_SPEC_INCOMPLETE, "vocabulary pin") + rule = threshold.get("decision_rule") or {} + required = { + "classification_accuracy", + "efficiency", + "observed_label_accuracy", + "primary_preservation", + "unbind_clean_exact", + } + if set(rule.get("pass_requires") or []) != required or rule.get("compensation") is not False: + _fail(THRESHOLD_UNDERSPECIFIED, "decision rule") + if threshold.get("epsilon") != 0 or threshold.get("efficiency_max_step_ratio") != 0.25: + _fail(THRESHOLD_UNDERSPECIFIED, "epsilon or efficiency ratio") + if rule.get("primary_strict_increase_required") is not False or rule.get("primary_equality") != "PASS": + _fail(THRESHOLD_UNDERSPECIFIED, "primary equality") + pre_errors = schema_errors(pre, "select-006-preregistration.schema.json") + threshold_errors = schema_errors(threshold, "select-006-threshold-authorization.schema.json") + if pre_errors or threshold_errors: + _fail(SPEC_SEAL_FAILURE, "; ".join([*pre_errors, *threshold_errors])) + + +def build_sealed_documents(observations: Mapping[str, Any], repository_head: str) -> dict[str, dict[str, Any]]: + _check_observations(observations) + if not isinstance(repository_head, str) or len(repository_head) != 40: + _fail(SPEC_SEAL_FAILURE, "repository head") + pre = preregistration() + pre_sha = sha256_text(canonical_json(pre)) + threshold = threshold_authorization(pre_sha) + _check_documents(pre, threshold) + vocab = vocabulary_expansion_pin() + dependencies = dependency_statement() + baseline = baseline_env() + candidate = candidate_env() + bodies = { + "SELECT_006_PREREGISTRATION.json": pre, + "SELECT_006_THRESHOLD_AUTHORIZATION.json": threshold, + "VOCABULARY_EXPANSION_PIN.json": vocab, + "DATA_EVALUATION_DEPENDENCIES.json": dependencies, + "BASELINE_ENV.json": baseline, + "CANDIDATE_ENV.json": candidate, + } + hashes = {name: sha256_text(canonical_json(body)) for name, body in bodies.items()} + manifest = { + "artifacts": [ + {"name": name, "sha256": digest} for name, digest in sorted(hashes.items()) + ], + "experiment_id": EXPERIMENT_ID, + "schema": "hyperlex.select_006_artifact_manifest.v1", + **_forbidden_training(), + } + manifest_text = canonical_json(manifest) + hashes["ARTIFACT_MANIFEST.json"] = sha256_text(manifest_text) + receipt = { + "artifact_sha256": hashes, + "experiment_id": EXPERIMENT_ID, + "next_legal_transition": NEXT_TRANSITION, + "observations": { + "best_sha256": observations["best_sha256"], + "candidate_config": observations.get("candidate_config", CANDIDATE_CONFIG), + "configs_agree": True, + "control_config": observations.get("control_config", CONTROL_CONFIG), + "export_sha256": observations["export_sha256"], + "ledger_events_sha256": observations["ledger_events_sha256"], + "ledger_projection_sha256": observations["ledger_projection_sha256"], + "new_role_names_in_sorted_order": list(NEW_ROLES), + "select_005_active_count": 78, + "select_006_active_count": 0, + "trunk_sha256": observations["trunk_sha256"], + "warm_start_sha256": observations["warm_start_sha256"], + }, + "repository_commit": repository_head, + "schema": "hyperlex.select_006_spec_receipt.v1", + "state": "SELECT_006_EFFICIENCY_PRESERVATION_SPEC_SEALED", + "transition": TRANSITION, + **_forbidden_training(), + } + bodies["ARTIFACT_MANIFEST.json"] = manifest + bodies["SPEC_RECEIPT.json"] = receipt + return bodies + + +def seal_specification( + directory: str | Path, + *, + observations: Mapping[str, Any], + repository_head: str, + repository_dirty: bool = False, +) -> dict[str, str]: + """Write the spec directory. Does not train, score, or move BEST.""" + if repository_dirty: + _fail(SPEC_SEAL_FAILURE, "repository is dirty") + target = Path(directory) + if target.exists(): + _fail(SPEC_SEAL_FAILURE, f"spec directory already exists: {target}") + bodies = build_sealed_documents(observations, repository_head) + parent = target.parent + parent.mkdir(mode=0o700, parents=True, exist_ok=True) + os.chmod(parent, 0o700) + temporary = parent / f".{target.name}.tmp" + if temporary.exists(): + _fail(SPEC_SEAL_FAILURE, f"temporary spec directory already exists: {temporary}") + temporary.mkdir(mode=0o700) + try: + for name, body in bodies.items(): + path = temporary / name + path.write_text(canonical_json(body), encoding="utf-8") + os.chmod(path, 0o600) + manifest = json.loads((temporary / "ARTIFACT_MANIFEST.json").read_text(encoding="utf-8")) + for item in manifest["artifacts"]: + observed = sha256_file(temporary / item["name"]) + if observed != item["sha256"]: + _fail(ARTIFACT_MANIFEST_FAILURE, item["name"]) + receipt_path = temporary / "SPEC_RECEIPT.json" + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + for name, digest in receipt["artifact_sha256"].items(): + if sha256_file(temporary / name) != digest: + _fail(ARTIFACT_MANIFEST_FAILURE, name) + os.rename(temporary, target) + except BaseException: + if temporary.exists(): + for child in temporary.iterdir(): + child.unlink() + temporary.rmdir() + raise + os.chmod(target, 0o700) + return {name: sha256_file(target / name) for name in bodies} diff --git a/scripts/shadow/hyperlexical/select_006_reserve_acquisition.py b/scripts/shadow/hyperlexical/select_006_reserve_acquisition.py new file mode 100644 index 00000000..63af7ad6 --- /dev/null +++ b/scripts/shadow/hyperlexical/select_006_reserve_acquisition.py @@ -0,0 +1,243 @@ +"""SELECT-006 reserve acquisition policy. + +This module freezes floors and checks surfaces. It does not train, bind a +ledger, or assign OBSERVED, semantic family, or unbind_clean. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from typing import Any, Mapping, Sequence + +from .layout import FAMILIES +from .select_006_efficiency_preservation import ( + EPSILON, + EXPERIMENT_ID, + vocabulary_expansion_pin, +) + +SELECT_005_ID = "HLX-EXP-2026-09-27-SELECT-005" +TRANSITION = "SELECT_006_RESERVE_ACQUISITION" +BATCH_ID = "HLX-EVAL-RESERVE-SELECT-006-001" +HEAD_MAPPED_FAMILIES = tuple( + name for name in FAMILIES if name != "none" +) +# Production-head families that are also active evaluation families. +# Legacy head names are not aliases. +COMPUTABLE_HEAD_FAMILIES = ( + "ai-native", + "betting-sharp", + "crypto-degen", + "gaming-meta", +) +DISCOVERY_CATEGORIES = ( + "Category:English internet slang", + "Category:English slang", + "Category:English idioms", + "Category:English gaming slang", + "Category:English cryptocurrency", + "Category:English poker terms", +) +DISCOVERY_PER_CATEGORY = 8 +FLOORS = { + "classify": 1, + "classify_non_none": 1, + "classify_observed": 1, + "head_mapped_non_none": 1, + "unbind_clean": 1, +} +FAILURE_SOURCE = "RESERVE_SOURCE_FAILURE" +FAILURE_RIGHTS = "RESERVE_RIGHTS_FAILURE" +FAILURE_PROVENANCE = "RESERVE_PROVENANCE_FAILURE" +FAILURE_ISOLATION = "RESERVE_ISOLATION_FAILURE" +FAILURE_SURFACE = "RESERVE_METRIC_SURFACE_INSUFFICIENT" +FAILURE_REVIEW = "REVIEW_SURFACE_EMPTY" +FAILURE_SETTLEMENT = "SETTLEMENT_FAILURE" +FAILURE_ROUTING = "ROUTING_FAILURE" +FAILURE_QUOTA = "RESERVE_QUOTA_UNFILLED" +FAILURE_BINDING = "LEDGER_BINDING_CONFLICT" +FAILURE_REPLAY = "LEDGER_REPLAY_FAILURE" +_TEMPLATE = re.compile(r"\{\{[^{}]*\}\}") +_LINK = re.compile(r"\[\[([^|\]]+\|)?([^\]]+)\]\]") +_MARKUP = re.compile(r"''+") + + +def canonical_json(payload: Mapping[str, Any]) -> str: + return json.dumps(payload, indent=2, sort_keys=True, ensure_ascii=True) + "\n" + + +def sha256_text(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def frozen_floors() -> dict[str, Any]: + """Minimum computable surface. Written before any page is fetched.""" + return { + "admission_floor_authority": ( + "Admission requires each active slice count to be at least 1. " + "Planning targets are not that floor." + ), + "computable_head_families": list(COMPUTABLE_HEAD_FAMILIES), + "discovery_categories": list(DISCOVERY_CATEGORIES), + "discovery_per_category": DISCOVERY_PER_CATEGORY, + "discovery_rule": "api_order_multiword_titles_not_previously_used", + "epsilon_unchanged": EPSILON, + "experiment_id": EXPERIMENT_ID, + "floors": dict(FLOORS), + "head_mapped_non_none_reason": ( + "classify_macro_f1_nonnone is computable only when at least one " + "gold lineage is a production-head family other than none. " + "A classify_non_none slice count can still map to head none." + ), + "schema": "hyperlex.select_006_reserve_floors.v1", + "select_005_counts_not_transferred": [51, 11, 11, 27], + "select_005_reserve_reuse": "FORBIDDEN", + "source_families": ["wiktionary_multiword_lemma", "wiktionary_sense_gloss"], + } + + +def execution_loader_status() -> str: + return str(vocabulary_expansion_pin()["execution_loader_status"]) + + +def _strip_templates(text: str) -> str: + previous = None + current = text + while current != previous: + previous = current + current = _TEMPLATE.sub(" ", current) + return current + + +def extract_primary_gloss(wikitext: str) -> dict[str, str | None]: + """English primary definition prose. Topic labels inside templates are dropped.""" + if not wikitext or wikitext.lstrip().lower().startswith("#redirect"): + return {"exact_gloss": None, "part_of_speech": None} + english = _english_section(wikitext) + if not english: + return {"exact_gloss": None, "part_of_speech": None} + pos = None + for line in english.splitlines(): + heading = re.fullmatch(r"=+\s*([^=]+?)\s*=+", line.strip()) + if heading and line.strip().startswith("==="): + pos = heading.group(1).strip() + if line.startswith("# ") and not line.startswith("#:") and not line.startswith("##"): + prose = _prose(line[2:]) + if prose: + return {"exact_gloss": prose, "part_of_speech": pos} + return {"exact_gloss": None, "part_of_speech": pos} + + +def _english_section(wikitext: str) -> str: + parts = re.split(r"\n(?===[^=])", "\n" + wikitext) + for part in parts: + lines = part.strip().splitlines() + if not lines: + continue + title = lines[0].strip("= ").casefold() + if title == "english": + return part + return "" + + +def _prose(line: str) -> str: + text = _strip_templates(line) + text = _LINK.sub(lambda match: match.group(2), text) + text = _MARKUP.sub("", text) + text = text.replace("'''", "").replace("''", "") + text = re.sub(r"\s+", " ", text).strip(" .") + return text + + +def review_surface_row(raw: Mapping[str, Any]) -> dict[str, Any]: + """Blind row. No quota, route, score, or semantic-family field.""" + return { + "candidate_id": raw["candidate_id"], + "exact_gloss": raw.get("exact_gloss"), + "headword": raw["headword"], + "page_url": raw["page_url"], + "part_of_speech": raw.get("part_of_speech"), + "revision_id": raw["revision_id"], + "sense_labels": None, + "source_family": raw["source_family"], + "source_lemma_tokens": list(raw.get("source_lemma_tokens") or []), + } + + +def blind_surface_errors(rows: Sequence[Mapping[str, Any]]) -> list[str]: + banned = { + "attest", + "decision", + "desired_family", + "model_score", + "prediction", + "quota", + "routing_status", + "semantic_family", + "slice", + "slices", + "unbind_clean", + } + errors = [] + for row in rows: + found = banned & set(row) + if found: + errors.append("review surface carries " + ",".join(sorted(found))) + return errors + + +def exclusion_reason(digest: str, blocked: Mapping[str, str]) -> str | None: + reason = blocked.get(digest) + return reason or None + + +def metric_surface(records: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Count eligible slices and whether each sealed metric can be computed.""" + counts = { + "classify": 0, + "classify_non_none": 0, + "classify_observed": 0, + "unbind_clean": 0, + } + head_mapped = 0 + for record in records: + if record.get("routing_status") != "ELIGIBLE": + continue + for name in record.get("slices") or []: + if name in counts: + counts[name] += 1 + lineage = str(record.get("lineage") or "") + if ( + record.get("task") == "classify" + and lineage in COMPUTABLE_HEAD_FAMILIES + and lineage in HEAD_MAPPED_FAMILIES + ): + head_mapped += 1 + computable = { + "classification_accuracy": counts["classify"] >= 1, + "classify_macro_f1_nonnone": head_mapped >= 1, + "observed_label_accuracy": counts["classify_observed"] >= 1, + "unbind_clean_exact": counts["unbind_clean"] >= 1, + } + floors_met = all(counts[key] >= FLOORS[key] for key in counts) and head_mapped >= FLOORS["head_mapped_non_none"] + return { + "computable": computable, + "floors_met": floors_met, + "head_mapped_non_none": head_mapped, + "slice_counts": counts, + } + + +def forbidden_training() -> dict[str, Any]: + return { + "best_changed": False, + "epochs": 0, + "execution_loader_status": execution_loader_status(), + "gradient_steps": 0, + "optimizer_constructed": False, + "training_launch_authorized": False, + "training_started": False, + "weights_mutated": False, + } diff --git a/scripts/shadow/hyperlexical/select_contract_schema.py b/scripts/shadow/hyperlexical/select_contract_schema.py new file mode 100644 index 00000000..5c0c4c55 --- /dev/null +++ b/scripts/shadow/hyperlexical/select_contract_schema.py @@ -0,0 +1,86 @@ +"""JSON Schema checks for routing, admission, threshold, and launch authorization. + +Historical artifacts are validated and not rewritten. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Mapping + +from jsonschema import Draft202012Validator + +REPO_ROOT = Path(__file__).resolve().parents[3] +SCHEMA_DIR = REPO_ROOT / "specs/007-hyperlexical-model/schemas/hyperlex/select" +ROUTING_SCHEMA_PATH = SCHEMA_DIR / "eval-routing.schema.json" +THRESHOLD_SCHEMA_PATH = SCHEMA_DIR / "threshold-authorization.schema.json" +ADMISSION_SCHEMA_PATH = SCHEMA_DIR / "admission.schema.json" +TRAINING_LAUNCH_AUTHORIZATION_SCHEMA_PATH = SCHEMA_DIR / "training-launch-authorization.schema.json" +TRAINING_LAUNCH_AUTHORIZATION_SPEC_SCHEMA_PATH = ( + SCHEMA_DIR / "training-launch-authorization-spec.schema.json" +) +SELECT_006_TRAINING_LAUNCH_AUTHORIZATION_SCHEMA_PATH = ( + SCHEMA_DIR / "select-006-training-launch-authorization.schema.json" +) + + +def _validator(path: Path) -> Draft202012Validator: + schema = json.loads(path.read_text(encoding="utf-8")) + Draft202012Validator.check_schema(schema) + return Draft202012Validator(schema) + + +def routing_schema_errors(instance: Mapping[str, Any]) -> list[str]: + return sorted(error.message for error in _validator(ROUTING_SCHEMA_PATH).iter_errors(instance)) + + +def threshold_schema_errors(instance: Mapping[str, Any]) -> list[str]: + return sorted(error.message for error in _validator(THRESHOLD_SCHEMA_PATH).iter_errors(instance)) + + +def admission_schema_errors(instance: Mapping[str, Any]) -> list[str]: + return sorted(error.message for error in _validator(ADMISSION_SCHEMA_PATH).iter_errors(instance)) + + +def training_launch_authorization_schema_errors(instance: Mapping[str, Any]) -> list[str]: + return sorted( + error.message + for error in _validator(TRAINING_LAUNCH_AUTHORIZATION_SCHEMA_PATH).iter_errors(instance) + ) + + +def training_launch_authorization_spec_schema_errors(instance: Mapping[str, Any]) -> list[str]: + return sorted( + error.message + for error in _validator(TRAINING_LAUNCH_AUTHORIZATION_SPEC_SCHEMA_PATH).iter_errors(instance) + ) + + +def select_006_training_launch_authorization_schema_errors(instance: Mapping[str, Any]) -> list[str]: + return sorted( + error.message + for error in _validator(SELECT_006_TRAINING_LAUNCH_AUTHORIZATION_SCHEMA_PATH).iter_errors(instance) + ) + + +def historical_schema_report( + admission_artifacts: Mapping[str, Mapping[str, Any]], + threshold_artifacts: Mapping[str, Mapping[str, Any]], +) -> dict[str, Any]: + """VALIDATE_ONLY. Names and messages, no artifact rewrite.""" + gaps = [] + for name, payload in sorted(admission_artifacts.items()): + for message in admission_schema_errors(payload): + gaps.append({"artifact": name, "schema": "hyperlex.admission.v1", "message": message}) + for name, payload in sorted(threshold_artifacts.items()): + for message in threshold_schema_errors(payload): + gaps.append( + {"artifact": name, "schema": "hyperlex.threshold_authorization.v1", "message": message} + ) + return { + "gaps": gaps, + "mode": "VALIDATE_ONLY", + "mutated": False, + "result": "SCHEMA_CONFORMANCE_GAP" if gaps else "HISTORICAL_VALIDATION_COMPLETE", + } diff --git a/scripts/shadow/hyperlexical/selection_surface.py b/scripts/shadow/hyperlexical/selection_surface.py new file mode 100644 index 00000000..b5182233 --- /dev/null +++ b/scripts/shadow/hyperlexical/selection_surface.py @@ -0,0 +1,830 @@ +"""Selection-surface validity audit. Eval only. No torch. No gold writes. + +Scores are computed by the caller (the Spark CLI). This module tags release +unbind val rows, builds slice metrics, and applies the post-rc1 verdict rule. + +The frozen test split is spent. ``drop_test_rows`` discards every ``split=test`` +row before tagging, sibling indexing, baselines, or metrics. Nothing here +opens a holdout manifest or calls the holdout scorer. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +from collections import Counter +from pathlib import Path +from typing import Any, Iterable, Mapping, Sequence + +from .export import TYPE_SLOT_TAGS +from .filler_filter import filter_unbind_rows +from .soft_ceiling import clean_surface +from .unbind_metrics import slot_counts, strict_pair, summarize_unbind_pairs, token_multiset_counts + +# Published morph78 test-clean Wilson 95% from the spent rc1 receipt (n=306). +# Pinned, not recomputed. This audit does not read that split. +PINNED_MORPH78_TEST_CLEAN_WILSON = (0.673, 0.772) +Z95 = 1.959963984540054 +JACCARD_SIBLING = 0.5 +EXPECTED_SURFACE_N = { + "broad_clean": 51, + "force_fair": 164, + "train_val_strict": 140, +} +RELEASE_VAL_TASKS = frozenset({"unbind", "classify+unbind"}) +CANONICAL_RANK = ("morph78", "morph65", "rc1") + + +class HoldoutSliceRefused(RuntimeError): + """Raised when a test-split row reaches tagging, scoring, or output.""" + + +def wilson_interval(k: int, n: int, *, z: float = Z95) -> tuple[float, float] | None: + """Wilson score interval for k successes in n trials. None when n is 0.""" + if n <= 0: + return None + if k < 0 or k > n: + raise ValueError(f"wilson k={k} outside 0..n={n}") + phat = k / n + z2 = z * z + denom = 1.0 + z2 / n + center = (phat + z2 / (2.0 * n)) / denom + half = z * math.sqrt(phat * (1.0 - phat) / n + z2 / (4.0 * n * n)) / denom + lo = max(0.0, center - half) + hi = min(1.0, center + half) + # k=0 and k=n are exactly 0 and 1; float error leaves a 1 ulp gap. + if lo < 1e-12: + lo = 0.0 + if hi > 1.0 - 1e-12: + hi = 1.0 + return (lo, hi) + + +def intervals_overlap(left: Sequence[float] | None, right: Sequence[float]) -> bool: + if not left or len(left) != 2: + return False + return float(left[0]) <= float(right[1]) and float(right[0]) <= float(left[1]) + + +def _rate(k: int, n: int) -> dict[str, Any]: + if n <= 0: + return {"k": 0, "n": 0, "value": None, "wilson95": None} + interval = wilson_interval(k, n) + return { + "k": k, + "n": n, + "value": k / n, + "wilson95": list(interval) if interval else None, + } + + +def _strip_type_slot(text: str) -> str: + parts: list[str] = [] + for tok in (text or "").split(): + if ":" in tok: + tag, rest = tok.split(":", 1) + if tag in TYPE_SLOT_TAGS and rest: + parts.append(rest) + continue + parts.append(tok) + return " ".join(parts) + + +def norm_text(text: str) -> str: + """Lowercase phrase with punctuation stripped. Type-slot tags removed first.""" + raw = _strip_type_slot(text).lower() + raw = raw.replace("\u2019", "'").replace("\u2018", "'") + raw = re.sub(r"[^a-z0-9']+", " ", raw) + return " ".join(raw.split()) + + +def stem_token(token: str) -> str: + """One-suffix stem. ``quitting`` and ``quits`` both become ``quit``. + + Strips ``ing`` / ``ed`` / ``es`` / ``s`` when at least three characters + remain. After ``ing`` or ``ed``, a doubled final consonant other than + ``s`` is dropped once (``quitt`` → ``quit``, ``pass`` stays ``pass``). + """ + t = token.lower() + stripped = "" + for suf in ("ing", "ed", "es"): + if t.endswith(suf) and len(t) - len(suf) >= 3: + t = t[: -len(suf)] + stripped = suf + break + if not stripped and t.endswith("s") and not t.endswith("ss") and len(t) - 1 >= 3: + t = t[:-1] + stripped = "s" + if stripped in {"ing", "ed"} and len(t) >= 4 and t[-1] == t[-2] and t[-1] not in "aeious": + t = t[:-1] + return t + + +def stem_key(text: str) -> tuple[str, ...]: + tokens = [stem_token(tok) for tok in norm_text(text).split() if tok] + return tuple(sorted(tokens)) + + +def filler_set(row: Mapping[str, Any]) -> frozenset[str]: + return frozenset(str(item).lower() for item in (row.get("fillers") or []) if str(item).strip()) + + +def jaccard(left: frozenset[str], right: frozenset[str]) -> float: + if not left or not right: + return 0.0 + return len(left & right) / len(left | right) + + +def _prov_text(row: Mapping[str, Any]) -> str: + raw = row.get("provenance") + if isinstance(raw, dict): + return json.dumps(raw, sort_keys=True, ensure_ascii=False) + return str(raw or "") + + +def provenance_bucket(row: Mapping[str, Any]) -> str: + """``templated`` is civilian / val-settle whitespace gold. Else ``harvested``.""" + prov = _prov_text(row).lower() + license_ = str(row.get("license") or "").lower() + if ( + prov.startswith("civilian-pos:") + or prov.startswith("civilian-type:") + or prov.startswith("civilian:") + or "val-settle" in prov + or "unbind structural whitespace" in license_ + ): + return "templated" + return "harvested" + + +def provenance_source(row: Mapping[str, Any]) -> str: + raw = row.get("provenance") + if isinstance(raw, dict): + return "dict:" + str(raw.get("source") or "unknown") + text = str(raw or "") + if "val-settle" in text: + return "val_settle" + if text.startswith("civilian-pos:") or text.startswith("civilian-type:"): + return "civilian_template" + if text.startswith("live-pos:") or text.startswith("live-type:"): + return "live" + head = text.split(":", 1)[0] + return head or "unknown" + + +def assert_no_test(rows: Iterable[Mapping[str, Any]], *, where: str) -> None: + n = sum(1 for row in rows if row.get("split") == "test") + if n: + raise HoldoutSliceRefused(f"REFUSE: {where} includes {n} holdout test rows") + + +def drop_test_rows(rows: Sequence[Mapping[str, Any]]) -> tuple[list[dict], int]: + """Return non-test rows and the discarded count. Test rows are not copied.""" + kept: list[dict] = [] + discarded = 0 + for row in rows: + if row.get("split") == "test": + discarded += 1 + continue + kept.append(dict(row)) + return kept, discarded + + +class TrainIndex: + """Sibling lookup over train rows only. Test rows are refused.""" + + def __init__(self, train_rows: Sequence[Mapping[str, Any]]): + assert_no_test(train_rows, where="train sibling corpus") + self.norms: set[str] = set() + self.stems: set[tuple[str, ...]] = set() + self.fillers: list[frozenset[str]] = [] + for row in train_rows: + phrase = norm_text(str(row.get("text") or "")) + if phrase: + self.norms.add(phrase) + key = stem_key(phrase) + if key: + self.stems.add(key) + fillers = filler_set(row) + if fillers: + self.fillers.append(fillers) + + def match(self, row: Mapping[str, Any]) -> tuple[bool, str | None]: + phrase = norm_text(str(row.get("text") or "")) + if phrase and phrase in self.norms: + return True, "norm_text" + key = stem_key(phrase) if phrase else () + if key and key in self.stems: + return True, "lemma" + fillers = filler_set(row) + if fillers and any(jaccard(fillers, other) >= JACCARD_SIBLING for other in self.fillers): + return True, "filler_jaccard" + return False, None + + +def row_id(row: Mapping[str, Any]) -> str: + parts = [ + str(row.get("task") or ""), + str(row.get("split") or ""), + str(row.get("role_scheme") or ""), + str(row.get("class") or ""), + str(row.get("lineage") or ""), + str(row.get("text") or ""), + ] + return hashlib.sha256("\n".join(parts).encode("utf-8")).hexdigest()[:16] + + +def annotate_rows(rows: Sequence[Mapping[str, Any]], train_rows: Sequence[Mapping[str, Any]]) -> list[dict]: + """Tag val rows. Refuses any test row in either argument.""" + assert_no_test(rows, where="annotate") + index = TrainIndex(train_rows) + tagged: list[dict] = [] + seen: set[str] = set() + for row in rows: + flag, reason = index.match(row) + ident = row_id(row) + if ident in seen: + raise HoldoutSliceRefused(f"REFUSE: duplicate row_id {ident}") + seen.add(ident) + tagged.append( + { + "row_id": ident, + "task": str(row.get("task") or ""), + "split": str(row.get("split") or ""), + "text": str(row.get("text") or ""), + "lineage": str(row.get("lineage") or ""), + "role_scheme": str(row.get("role_scheme") or ""), + "class": str(row.get("class") or "").upper(), + "fillers": [str(item) for item in (row.get("fillers") or [])], + "provenance": _prov_text(row), + "provenance_bucket": provenance_bucket(row), + "provenance_source": provenance_source(row), + "train_sibling": flag, + "sibling_reason": reason, + } + ) + tagged.sort(key=lambda item: item["row_id"]) + return tagged + + +def release_val_rows(rows: Sequence[Mapping[str, Any]]) -> list[dict]: + """``unbind:val`` plus ``classify+unbind:val``. Test rows are refused.""" + assert_no_test(rows, where="release val") + return [ + dict(row) + for row in rows + if row.get("split") == "val" and row.get("task") in RELEASE_VAL_TASKS + ] + + +def surface_tokens(text: str) -> list[str]: + """Same atom tokens as ``score_holdout.copy_baseline`` (text after the first colon).""" + return [tok.split(":", 1)[-1].lower() for tok in str(text or "").split()] + + +def lenient_copy_hit(row: Mapping[str, Any]) -> bool: + """True when every gold filler appears among atom tokens. Slot order ignored. + + Matches ``score_holdout.copy_baseline`` on a single row. + """ + fillers = [str(item).lower() for item in (row.get("fillers") or [])] + if not fillers: + return False + toks = surface_tokens(str(row.get("text") or "")) + return all(fill in toks for fill in fillers) + + +def strict_slot_pair(row: Mapping[str, Any]) -> tuple[list[str], list[str]]: + """Predict gold slots by placing surface tokens in order. + + Exact credit requires the i-th surface token to be the i-th gold filler. + Extra surface tokens are ignored. A missing token is an empty prediction. + """ + gold = [str(item).lower() for item in (row.get("fillers") or [])] + if not gold: + return [], [] + toks = surface_tokens(str(row.get("text") or "")) + pred = [toks[i] if i < len(toks) else "" for i in range(len(gold))] + return gold, pred + + +def _f1_entry(value: float | None, n_rows: int, tp: int, gold_n: int) -> dict[str, Any]: + interval = wilson_interval(tp, gold_n) + return { + "value": value, + "n": n_rows, + "micro_tp": tp, + "micro_gold": gold_n, + "wilson95": list(interval) if interval else None, + } + + +def metric_block( + pairs: Sequence[tuple[Sequence[str], Sequence[str]]], + *, + strict_pairs: Sequence[tuple[Sequence[str], Sequence[str]]] | None = None, +) -> dict[str, Any]: + """Exact, token-F1, and slot-F1. Wilson on exact (row hits) and on micro recall. + + Legacy keys use ``pairs`` as given. Strict keys use raw lowercased gold + (``strict_pairs`` when the caller still has the unmapped filler) and treat + a ```` prediction as a miss. + """ + usable = [([str(g) for g in gold], [str(p) for p in pred]) for gold, pred in pairs if gold] + n = len(usable) + empty = {"value": None, "n": 0, "wilson95": None} + if n == 0: + return { + "n": 0, + "unbind_exact": {**_rate(0, 0)}, + "unbind_exact_strict": {**_rate(0, 0)}, + "unbind_token_f1": {**empty, "micro_tp": 0, "micro_gold": 0}, + "unbind_token_f1_strict": {**empty, "micro_tp": 0, "micro_gold": 0}, + "unbind_slot_f1": {**empty, "micro_tp": 0, "micro_gold": 0}, + "unbind_slot_f1_strict": {**empty, "micro_tp": 0, "micro_gold": 0}, + } + strict_src = usable if strict_pairs is None else [ + ([str(g) for g in gold], [str(p) for p in pred]) for gold, pred in strict_pairs if gold + ] + summary = summarize_unbind_pairs(usable, strict_pairs=strict_src) + k = sum(gold == pred for gold, pred in usable) + strict_view = [strict_pair(gold, pred) for gold, pred in strict_src] + k_strict = sum(gold == pred for gold, pred in strict_view) + tok_tp = tok_g = 0 + slot_tp = slot_g = 0 + stok_tp = stok_g = 0 + sslot_tp = sslot_g = 0 + for gold, pred in usable: + tp, _pn, gn = token_multiset_counts(gold, pred) + tok_tp += tp + tok_g += gn + stp, _spn, sgn = slot_counts(gold, pred) + slot_tp += stp + slot_g += sgn + for gold, pred in strict_view: + tp, _pn, gn = token_multiset_counts(gold, pred) + stok_tp += tp + stok_g += gn + stp, _spn, sgn = slot_counts(gold, pred) + sslot_tp += stp + sslot_g += sgn + n_strict = len(strict_view) + return { + "n": n, + "unbind_exact": _rate(k, n), + "unbind_exact_strict": _rate(k_strict, n_strict), + "unbind_token_f1": _f1_entry(summary["unbind_token_f1"], n, tok_tp, tok_g), + "unbind_token_f1_strict": _f1_entry( + summary["unbind_token_f1_strict"], n_strict, stok_tp, stok_g + ), + "unbind_slot_f1": _f1_entry(summary["unbind_slot_f1"], n, slot_tp, slot_g), + "unbind_slot_f1_strict": _f1_entry( + summary["unbind_slot_f1_strict"], n_strict, sslot_tp, sslot_g + ), + } + + +def copy_token_block(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + scored = [row for row in rows if row.get("fillers")] + k = sum(1 for row in scored if lenient_copy_hit(row)) + return _rate(k, len(scored)) + + +def strict_slot_block(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + scored = [row for row in rows if row.get("fillers")] + return metric_block([strict_slot_pair(row) for row in scored]) + + +def slice_groups(rows: Sequence[Mapping[str, Any]]) -> list[tuple[str, list]]: + grouped = [ + ("all", list(rows)), + ("harvested", [row for row in rows if row.get("provenance_bucket") == "harvested"]), + ("templated", [row for row in rows if row.get("provenance_bucket") == "templated"]), + ("sibling", [row for row in rows if row.get("train_sibling")]), + ("no_sibling", [row for row in rows if not row.get("train_sibling")]), + ] + for label in sorted({str(row.get("class") or "") for row in rows}): + grouped.append((f"class:{label}", [row for row in rows if str(row.get("class") or "") == label])) + for label in sorted({str(row.get("role_scheme") or "") for row in rows}): + grouped.append((f"scheme:{label}", [row for row in rows if str(row.get("role_scheme") or "") == label])) + return grouped + + +def _pairs_for( + rows: Sequence[Mapping[str, Any]], pred_map: Mapping[str, tuple] +) -> tuple[list[tuple[list[str], list[str]]], list[tuple[list[str], list[str]]]]: + """Legacy pairs from the pred map, strict pairs from raw row fillers.""" + legacy: list[tuple[list[str], list[str]]] = [] + strict: list[tuple[list[str], list[str]]] = [] + for row in rows: + fillers = list(row.get("fillers") or []) + if not fillers: + continue + ident = str(row["row_id"]) + if ident not in pred_map: + raise KeyError(f"missing prediction for {ident}") + gold, pred = pred_map[ident] + pred_l = [str(item) for item in pred] + legacy.append(([str(item) for item in gold], pred_l)) + strict.append(([str(item).lower() for item in fillers], pred_l)) + return legacy, strict + + +def model_slice_metrics(rows: Sequence[Mapping[str, Any]], pred_map: Mapping[str, tuple]) -> dict[str, Any]: + out: dict[str, Any] = {} + for name, group in slice_groups(rows): + legacy, strict = _pairs_for(group, pred_map) + out[name] = metric_block(legacy, strict_pairs=strict) + return out + + +def control_slice_metrics(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + copy: dict[str, Any] = {} + strict: dict[str, Any] = {} + for name, group in slice_groups(rows): + copy[name] = copy_token_block(group) + strict[name] = strict_slot_block(group) + return {"unbind_copy_token": copy, "strict_slot_copy": strict} + + +def _exact(models: Mapping[str, Any], name: str, slice_name: str) -> float | None: + try: + return models[name]["slices"][slice_name]["unbind_exact"]["value"] + except (KeyError, TypeError): + return None + + +def _wilson(models: Mapping[str, Any], name: str, slice_name: str) -> list | None: + try: + return models[name]["slices"][slice_name]["unbind_exact"]["wilson95"] + except (KeyError, TypeError): + return None + + +def _le(value: float | None, thresh: float) -> bool: + return value is not None and value <= thresh + 1e-9 + + +def _ge(value: float | None, thresh: float) -> bool: + return value is not None and value >= thresh - 1e-9 + + +def surface_verdict(models: Mapping[str, Any]) -> dict[str, Any]: + """Post-rc1 pin. Does not read test rows; the test interval is the pinned constant. + + ``SURFACE_SWAP_SUPPORTED`` when morph78 release-val exact is off the ceiling, + its Wilson interval overlaps the pinned test-clean interval, the three + checkpoints rank morph78 ≥ morph65 ≥ rc1, and morph78 harvested < templated. + ``LEAK_NOT_SURFACE`` when morph78 overall ≥ 0.95 and no-sibling ≤ 0.85. + ``H1_REJECTED`` when morph78 sibling and no-sibling are both ≥ 0.95. + Otherwise ``NO_RULE_MATCH``. + """ + all78 = _exact(models, "morph78", "all") + harv = _exact(models, "morph78", "harvested") + temp = _exact(models, "morph78", "templated") + sib = _exact(models, "morph78", "sibling") + nosib = _exact(models, "morph78", "no_sibling") + m65 = _exact(models, "morph65", "all") + rc1 = _exact(models, "rc1", "all") + wilson = _wilson(models, "morph78", "all") + ranking_ok = None not in (all78, m65, rc1) and all78 + 1e-12 >= m65 and m65 + 1e-12 >= rc1 + overlap = intervals_overlap(wilson, PINNED_MORPH78_TEST_CLEAN_WILSON) + off_ceiling = _le(all78, 0.85) + harvested_below = harv is not None and temp is not None and harv < temp + checks = { + "morph78_exact": all78, + "morph78_wilson95": wilson, + "pinned_test_clean_wilson95": list(PINNED_MORPH78_TEST_CLEAN_WILSON), + "off_ceiling": off_ceiling, + "wilson_overlaps_test_clean": overlap, + "ranking_morph78_ge_morph65_ge_rc1": bool(ranking_ok), + "morph65_exact": m65, + "rc1_exact": rc1, + "harvested_below_templated": harvested_below, + "morph78_harvested": harv, + "morph78_templated": temp, + "morph78_sibling": sib, + "morph78_no_sibling": nosib, + } + if off_ceiling and overlap and ranking_ok and harvested_below: + candidate = "SURFACE_SWAP_SUPPORTED" + elif _ge(all78, 0.95) and _le(nosib, 0.85): + candidate = "LEAK_NOT_SURFACE" + elif _ge(sib, 0.95) and _ge(nosib, 0.95): + candidate = "H1_REJECTED" + else: + candidate = "NO_RULE_MATCH" + return {"candidate": candidate, "checks": checks} + + +def publish_verdict(candidate: str, rows: list, reproduce_rows: list | None) -> dict[str, Any]: + """HOLD until a second run's rows match. CI green is not a reproduce.""" + if reproduce_rows is None: + status, published = "pending", "HOLD" + elif reproduce_rows == rows: + status, published = "matched", candidate + else: + status, published = "mismatch", "HOLD" + return { + "candidate": candidate, + "published": published, + "hold": published == "HOLD", + "reproduce": status, + } + + +def provenance_mix(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Count provenance tags. + + Annotated rows already store ``provenance_bucket`` and ``provenance_source``. + ``annotate_rows`` drops ``license`` and flattens dict provenance to text, so + re-deriving the tags mis-counts license-only templates and dict sources. + Raw export rows omit the tags; derive them in that case. + """ + buckets: Counter[str] = Counter() + sources: Counter[str] = Counter() + schemes: Counter[str] = Counter() + classes: Counter[str] = Counter() + for row in rows: + buckets[row.get("provenance_bucket") or provenance_bucket(row)] += 1 + sources[row.get("provenance_source") or provenance_source(row)] += 1 + schemes[str(row.get("role_scheme") or "")] += 1 + classes[str(row.get("class") or "").upper()] += 1 + return { + "n_rows": len(rows), + "n_scored": sum(1 for row in rows if row.get("fillers")), + "provenance_bucket": dict(sorted(buckets.items())), + "provenance_source": dict(sorted(sources.items())), + "role_scheme": dict(sorted(schemes.items())), + "class": dict(sorted(classes.items())), + } + + +def load_force_keys(path: str | Path) -> set[tuple[str, str]]: + """``(stripped text, role_scheme)`` keys. Same fields as force-train move.""" + keys: set[tuple[str, str]] = set() + file = Path(path) + for index, line in enumerate(file.read_text(encoding="utf-8").splitlines(), start=1): + if not line.strip(): + continue + obj = json.loads(line) + text = obj.get("text") + scheme = obj.get("role_scheme") + if not isinstance(text, str) or scheme not in {"positional", "type_slot"}: + raise ValueError(f"{file} line {index} needs text and positional|type_slot role_scheme") + keys.add((text.strip(), str(scheme))) + return keys + + +def val_after_force(val_rows: Sequence[Mapping[str, Any]], force_keys: Iterable[tuple[str, str]]) -> list[dict]: + """Keep rule of ``apply_unbind_force_train``: OBSERVED force keys leave val.""" + keys = set(force_keys) + kept: list[dict] = [] + for row in val_rows: + key = (str(row.get("text") or "").strip(), str(row.get("role_scheme") or "")) + if key in keys and row.get("class") == "OBSERVED": + continue + kept.append(dict(row)) + return kept + + +def broad_clean_rows(rows: Sequence[Mapping[str, Any]], trained_keys: Iterable[tuple[str, str]]) -> tuple[list[dict], dict]: + """eval_broad clean slice: OBSERVED unbind val minus trained keys and train text.""" + assert_no_test(rows, where="broad clean") + unbind = [row for row in rows if row.get("task") == "unbind"] + train = [row for row in unbind if row.get("split") == "train"] + val_obs = [ + row + for row in unbind + if row.get("split") == "val" and str(row.get("class") or "").upper() == "OBSERVED" + ] + return clean_surface(val_obs, train_rows=train, trained_keys=trained_keys) + + +def force_fair_rows(rows: Sequence[Mapping[str, Any]], force_keys: Iterable[tuple[str, str]]) -> list[dict]: + """n=164 surface: legacy unbind val after the morph78 force-train move. Filter off.""" + assert_no_test(rows, where="force fair") + val = [row for row in rows if row.get("task") == "unbind" and row.get("split") == "val"] + return val_after_force(val, force_keys) + + +def train_val_strict_rows(rows: Sequence[Mapping[str, Any]], force_keys: Iterable[tuple[str, str]]) -> list[dict]: + """n=140 surface: force-fair val after the strict filler filter (rc1 train val).""" + kept, _stats = filter_unbind_rows(force_fair_rows(rows, force_keys), mode="strict") + return kept + + +def _surface_record(name: str, rows: Sequence[Mapping[str, Any]], *, extra: dict | None = None) -> dict[str, Any]: + mix = provenance_mix(rows) + body = { + "status": "OK", + "expected_n": EXPECTED_SURFACE_N[name], + "n_matches_expected": mix["n_scored"] == EXPECTED_SURFACE_N[name], + **mix, + } + if extra: + body.update(extra) + return body + + +def known_surfaces( + rows: Sequence[Mapping[str, Any]], + *, + trained_keys: Iterable[tuple[str, str]] | None, + force_keys: Iterable[tuple[str, str]] | None, +) -> dict[str, Any]: + """Provenance mix of the n=51, n=164, and n=140 selection surfaces. No model.""" + assert_no_test(rows, where="known surfaces") + out: dict[str, Any] = {} + if trained_keys is None: + out["broad_clean"] = { + "status": "NOT_COMPUTABLE", + "expected_n": EXPECTED_SURFACE_N["broad_clean"], + "reason": "trained force/hard keys not provided", + } + else: + clean, accounting = broad_clean_rows(rows, trained_keys) + out["broad_clean"] = _surface_record( + "broad_clean", + clean, + extra={ + "definition": "OBSERVED unbind val after clean_surface (eval_broad clean, n=51 at rc1)", + "accounting": accounting, + }, + ) + if force_keys is None: + missing = { + "status": "NOT_COMPUTABLE", + "reason": "morph78 force-train jsonl not provided", + } + out["force_fair"] = {**missing, "expected_n": EXPECTED_SURFACE_N["force_fair"]} + out["train_val_strict"] = {**missing, "expected_n": EXPECTED_SURFACE_N["train_val_strict"]} + else: + fair = force_fair_rows(rows, force_keys) + strict = train_val_strict_rows(rows, force_keys) + out["force_fair"] = _surface_record( + "force_fair", + fair, + extra={"definition": "unbind val after OBSERVED force-train move; filler filter off (n=164 at rc1)"}, + ) + out["train_val_strict"] = _surface_record( + "train_val_strict", + strict, + extra={"definition": "force-fair val after strict filler filter (rc1 train-time val, n=140)"}, + ) + return out + + +def _sibling_counts(rows: Sequence[Mapping[str, Any]]) -> dict[str, int]: + sibling = sum(1 for row in rows if row.get("train_sibling")) + return {"sibling": sibling, "no_sibling": len(rows) - sibling} + + +def attach_predictions(rows: Sequence[Mapping[str, Any]], predictions: Mapping[str, Mapping[str, tuple]]) -> list[dict]: + attached: list[dict] = [] + for row in rows: + models: dict[str, Any] = {} + for name in sorted(predictions): + pred_map = predictions[name] + if not row.get("fillers"): + continue + if row["row_id"] not in pred_map: + raise KeyError(f"missing prediction for {name} {row['row_id']}") + gold, pred = pred_map[row["row_id"]] + gold_l = [str(item) for item in gold] + pred_l = [str(item) for item in pred] + models[name] = {"exact": int(gold_l == pred_l), "gold": gold_l, "pred": pred_l} + attached.append({**row, "models": models}) + return attached + + +def assemble_report( + rows: Sequence[Mapping[str, Any]], + predictions: Mapping[str, Mapping[str, tuple]] | None = None, + *, + trained_keys: Iterable[tuple[str, str]] | None = None, + force_keys: Iterable[tuple[str, str]] | None = None, + reproduce_rows: list | None = None, + release_set: Mapping[str, Any] | None = None, + n_test_discarded: int | None = None, +) -> dict[str, Any]: + """Build the audit report. Drops test rows unless the caller already did.""" + if n_test_discarded is None: + kept, n_test_discarded = drop_test_rows(rows) + else: + assert_no_test(rows, where="assemble") + kept = [dict(row) for row in rows] + predictions = predictions or {} + val = release_val_rows(kept) + train = [row for row in kept if row.get("split") == "train"] + annotated = annotate_rows(val, train) + emitted = attach_predictions(annotated, predictions) + assert_no_test(emitted, where="report rows") + models = { + name: {"slices": model_slice_metrics(emitted, pred_map)} + for name, pred_map in sorted(predictions.items()) + } + decided = surface_verdict(models) + verdict = {**publish_verdict(decided["candidate"], emitted, reproduce_rows), "checks": decided["checks"]} + mix = provenance_mix(emitted) + report = { + "schema": "hyperlex.selection_surface_audit.v0.1", + "brier": None, + "eval_only": True, + "allow_train": False, + "test_slices_scored": False, + "n_test_discarded": n_test_discarded, + "release_set": {"release_set": bool(release_set.get("release_set"))} if release_set else {"release_set": None}, + "release_val": {**mix, "train_sibling": _sibling_counts(emitted)}, + "known_surfaces": known_surfaces(kept, trained_keys=trained_keys, force_keys=force_keys), + "models": models, + "controls": control_slice_metrics(emitted), + "verdict": verdict, + "rows": emitted, + } + return report + + +def render_summary(report: Mapping[str, Any]) -> str: + """Short markdown. Deterministic: no clock, no host paths required.""" + verdict = report["verdict"] + release = report["release_val"] + lines = [ + "# Selection-surface audit", + "", + f"Published verdict: **{verdict['published']}** " + f"(candidate `{verdict['candidate']}`, reproduce {verdict['reproduce']}).", + "", + "Eval only. The test split was discarded and not scored. Brier is null.", + f"Test rows discarded: {report['n_test_discarded']}.", + "", + "## Release unbind val", + "", + f"n_rows={release['n_rows']} n_scored={release['n_scored']}", + f"provenance={json.dumps(release['provenance_bucket'], sort_keys=True)}", + f"source={json.dumps(release['provenance_source'], sort_keys=True)}", + f"sibling={json.dumps(release['train_sibling'], sort_keys=True)}", + "", + "## Unbind exact", + "", + "| model | slice | exact | exact_strict | n | wilson95 |", + "|---|---|---:|---:|---:|---|", + ] + for name, body in report["models"].items(): + for slice_name in ("all", "harvested", "templated", "sibling", "no_sibling"): + block = body["slices"][slice_name]["unbind_exact"] + strict_block = body["slices"][slice_name]["unbind_exact_strict"] + lines.append( + f"| {name} | {slice_name} | {block['value']} | {strict_block['value']} | {block['n']} | {block['wilson95']} |" + ) + lines.extend(["", "## Controls on all", ""]) + copy = report["controls"]["unbind_copy_token"]["all"] + strict = report["controls"]["strict_slot_copy"]["all"] + lines.append( + f"unbind_copy_token exact={copy['value']} n={copy['n']} wilson95={copy['wilson95']}" + ) + lines.append( + "strict_slot_copy " + f"exact={strict['unbind_exact']['value']} " + f"exact_strict={strict['unbind_exact_strict']['value']} " + f"token_f1={strict['unbind_token_f1']['value']} " + f"token_f1_strict={strict['unbind_token_f1_strict']['value']} " + f"slot_f1={strict['unbind_slot_f1']['value']} " + f"slot_f1_strict={strict['unbind_slot_f1_strict']['value']} " + f"n={strict['n']}" + ) + lines.extend(["", "## Known selection surfaces", ""]) + for name, body in report["known_surfaces"].items(): + if body.get("status") != "OK": + lines.append(f"{name}: {body.get('status')} ({body.get('reason')})") + continue + lines.append( + f"{name}: n_scored={body['n_scored']} expected={body['expected_n']} " + f"match={body['n_matches_expected']} " + f"provenance={json.dumps(body['provenance_bucket'], sort_keys=True)}" + ) + lines.extend( + [ + "", + "Wilson beside exact and the lenient copy baseline is the row-hit interval. " + "Wilson beside token-F1 and slot-F1 is the micro-recall interval (tp/gold).", + "A first run stays HOLD. Re-run from a second checkout of the same SHA with " + "`--reproduce` pointing at the first JSON. The candidate is published only when rows match.", + "", + ] + ) + return "\n".join(lines) + + +def write_report(report: Mapping[str, Any], out_dir: str | Path) -> tuple[Path, Path]: + """Write JSON and markdown under ``out_dir`` only.""" + dest = Path(out_dir) + dest.mkdir(parents=True, exist_ok=True) + json_path = dest / "selection-surface-audit.json" + md_path = dest / "SUMMARY.md" + payload = json.dumps(report, indent=2, sort_keys=True) + "\n" + json_path.write_text(payload, encoding="utf-8") + md_path.write_text(render_summary(report), encoding="utf-8") + return json_path, md_path diff --git a/scripts/shadow/hyperlexical/soft_ceiling.py b/scripts/shadow/hyperlexical/soft_ceiling.py new file mode 100644 index 00000000..a0ae12c1 --- /dev/null +++ b/scripts/shadow/hyperlexical/soft_ceiling.py @@ -0,0 +1,188 @@ +"""soft_ceiling_tiebreak gate + contamination-safe eval surfaces. No torch. + +Ported from the Spark ``~/hlx`` chain used for morph75–78. Differences from +that chain (publish audit 2026-09-24): + +- ``clean_surface`` removes eval rows whose ``(text, role_scheme)`` is in the + candidate's force-train / hard-atom files, or whose text is in train. The + morph78 broad val shared 193 of 256 rows with its own force-train file. +- ``decide`` fails closed with ``REJECT_CONTAMINATED`` when the scored surface + still overlaps the candidate's training files. +- ``oov_filler_surface`` keeps rows whose gold fillers are absent from the + train filler vocab, so a clean score is not a copy task. +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Any, Iterable + +PIN_BROAD_EXACT = 0.889763779527559 +PIN_BROAD_N = 254 + + +def _norm(text: str) -> str: + return re.sub(r"\s+", " ", (text or "").strip().lower()) + + +def row_key(row: dict) -> tuple[str, str]: + return _norm(str(row.get("text") or "")), str(row.get("role_scheme") or "") + + +def load_jsonl_keys(path: str | Path | None) -> set[tuple[str, str]]: + if not path: + return set() + p = Path(path) + if not p.is_file(): + raise FileNotFoundError(p) + keys = set() + for line in p.read_text(encoding="utf-8").splitlines(): + if line.strip(): + keys.add(row_key(json.loads(line))) + return keys + + +def clean_surface( + eval_rows: Iterable[dict], + *, + train_rows: Iterable[dict] = (), + trained_keys: Iterable[tuple[str, str]] = (), +) -> tuple[list[dict], dict]: + """Drop eval rows the candidate trained on. Returns (rows, accounting).""" + trained = set(trained_keys) + trained_texts = {k[0] for k in trained} + train_texts = {_norm(str(r.get("text") or "")) for r in train_rows} + kept, by_key, by_text, by_train = [], 0, 0, 0 + for row in eval_rows: + key = row_key(row) + if key in trained: + by_key += 1 + elif key[0] in trained_texts: + by_text += 1 + elif key[0] in train_texts: + by_train += 1 + else: + kept.append(row) + total = by_key + by_text + by_train + len(kept) + return kept, { + "n_input": total, + "n_kept": len(kept), + "n_excluded_trained_key": by_key, + "n_excluded_trained_text": by_text, + "n_excluded_train_split_text": by_train, + } + + +def oov_filler_surface(rows: Iterable[dict], train_rows: Iterable[dict]) -> list[dict]: + """Rows whose every gold filler is absent from train fillers.""" + seen = set() + for r in train_rows: + for f in r.get("fillers") or []: + seen.add(str(f).lower()) + out = [] + for r in rows: + fillers = [str(f).lower() for f in (r.get("fillers") or [])] + if fillers and all(f not in seen for f in fillers): + out.append(r) + return out + + +def overlap(eval_rows: Iterable[dict], trained_keys: Iterable[tuple[str, str]]) -> int: + trained = set(trained_keys) + trained_texts = {k[0] for k in trained} + return sum(1 for r in eval_rows if row_key(r) in trained or row_key(r)[0] in trained_texts) + + +def _base(**kw: Any) -> dict[str, Any]: + return {"gate": "soft_ceiling_tiebreak", **kw} + + +def decide( + *, + force_fair: float, + force_fair_n: int, + best_exact: float, + val_n: int, + e2_pass: bool, + broad_exact: float | None = None, + broad_n: int | None = None, + prior_broad_exact: float | None = None, + prior_broad_n: int | None = None, + broad_overlap: int | None = None, +) -> dict[str, Any]: + """Promote decision. ``broad_overlap`` = rows on the broad surface the candidate trained on.""" + common = dict( + force_fair=force_fair, + force_fair_n=force_fair_n, + best_exact=best_exact, + val_n=val_n, + e2_pass=e2_pass, + broad_exact=broad_exact, + broad_n=broad_n, + prior_broad_exact=prior_broad_exact, + prior_broad_n=prior_broad_n, + broad_overlap=broad_overlap, + ) + ceiling = float(force_fair) >= 1.0 - 1e-12 + if not ceiling: + surface_ok = int(val_n) == int(force_fair_n) + promote = bool(best_exact > force_fair and e2_pass and surface_ok) + decision = "REJECT_SURFACE" if not surface_ok else ("PROMOTE_BEST" if promote else "REJECT_VS_BEST") + return _base( + mode="force_fair", + promote=promote, + decision=decision, + surface_ok=surface_ok, + reason=f"force_fair mode: best {best_exact} > fair {force_fair} n={force_fair_n} + E2", + **common, + ) + if broad_exact is None or broad_n is None: + return _base( + mode="ceiling_escape", + promote=False, + decision="REJECT_NO_BROAD_EVAL", + surface_ok=False, + reason="force_fair==1.0 but candidate broad OBSERVED eval missing", + **common, + ) + if broad_overlap is None: + return _base( + mode="ceiling_escape", + promote=False, + decision="REJECT_UNKNOWN_CONTAMINATION", + surface_ok=False, + reason="broad surface overlap with candidate training files not measured", + **common, + ) + if int(broad_overlap) > 0: + return _base( + mode="ceiling_escape", + promote=False, + decision="REJECT_CONTAMINATED", + surface_ok=False, + reason=f"broad surface shares {broad_overlap} rows with candidate training files", + **common, + ) + if prior_broad_exact is not None and prior_broad_n is not None: + baseline_exact, baseline_n, src = float(prior_broad_exact), int(prior_broad_n), "live_prior_broad" + else: + baseline_exact, baseline_n, src = PIN_BROAD_EXACT, PIN_BROAD_N, "authorize_pin" + surface_ok = int(broad_n) == int(baseline_n) + promote = bool(float(broad_exact) > baseline_exact and e2_pass and surface_ok) + decision = "REJECT_BROAD_SURFACE" if not surface_ok else ("PROMOTE_BEST" if promote else "REJECT_VS_BROAD_BASELINE") + return _base( + mode="ceiling_escape", + promote=promote, + decision=decision, + surface_ok=surface_ok, + broad_baseline=baseline_exact, + broad_baseline_n=baseline_n, + broad_baseline_src=src, + reason=( + f"ceiling escape: broad {broad_exact} {'>' if promote else 'not >'} baseline {baseline_exact} " + f"(src={src}) n={baseline_n}, e2_pass={e2_pass}, surface_ok={surface_ok}, overlap 0" + ), + **common, + ) diff --git a/scripts/shadow/hyperlexical/train.py b/scripts/shadow/hyperlexical/train.py new file mode 100644 index 00000000..0dab663e --- /dev/null +++ b/scripts/shadow/hyperlexical/train.py @@ -0,0 +1,99 @@ +"""U3 train gate + optional --run loop. No Hub.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path + +from .export import LiveStoreMissing, resolve_live_store +from .holdout_guard import merge_manifest_env + +TRUNK = "answerdotai/ModernBERT-base" + + +def want_include_live(cli_flag: bool = False) -> bool: + return bool(cli_flag) or os.environ.get("HYPERLEX_INCLUDE_LIVE") == "1" + + +def _gate() -> tuple[int, dict, Path | None]: + allow = os.environ.get("HYPERLEX_ALLOW_TRAIN") == "1" + trunk_dir = os.environ.get("HYPERLEX_TRUNK_DIR", "") + local = Path(trunk_dir) if trunk_dir else None + ok = bool(allow and local is not None and local.is_dir() and (local / "config.json").is_file()) + payload = { + "abort": not ok, + "trunk": TRUNK, + "trunk_dir": str(local) if local else None, + "allow_train": allow, + "recipe": "specs/007-hyperlexical-model/AARON-SPARK-TRAIN.md", + "brier": None, + } + if not ok: + payload["reason"] = "set HYPERLEX_ALLOW_TRAIN=1 and HYPERLEX_TRUNK_DIR to a local ModernBERT-base snapshot" + return 2, payload, None + return 0, payload, local + + +def main(argv=None) -> int: + p = argparse.ArgumentParser(prog="hyperlexical-train") + p.add_argument("--offline", action="store_true", default=True) + p.add_argument("--run", action="store_true", help="execute loop after gate") + p.add_argument( + "--include-live", + action="store_true", + help="merge live ingest store into the train export (same as export --include-live)", + ) + p.add_argument( + "--live-store", + default="", + help="optional path to ingest_candidates.jsonl (else HYPERLEX_LIVE_STORE or ~/.hyperlex/...)", + ) + p.add_argument( + "--holdout-manifest", + action="append", + default=None, + help="holdout manifest JSON (repeatable). Also HLX_HOLDOUT_MANIFESTS, comma-separated.", + ) + args = p.parse_args(argv) + include_live = want_include_live(args.include_live) + live_store = Path(args.live_store) if args.live_store else None + code, payload, local = _gate() + payload["offline"] = bool(args.offline) + payload["include_live"] = include_live + if include_live: + try: + payload["live_store"] = str(resolve_live_store(live_store, required=True)) + except LiveStoreMissing as exc: + payload["abort"] = True + payload["error"] = str(exc) + print(json.dumps(payload, indent=2), file=sys.stderr) + return 2 + if code != 0: + print(json.dumps(payload, indent=2), file=sys.stderr) + return code + if not args.run: + payload["note"] = "gate open. pass --run on Spark to execute the seed loop." + print(json.dumps(payload, indent=2)) + return 0 + if args.holdout_manifest: + merge_manifest_env(args.holdout_manifest) + from .loop import run_loop + + out = Path(os.environ.get("HYPERLEX_TRAIN_OUT") or (Path.home() / ".hyperlex" / "models" / "hyperlex-encoder-modernbert-base-seed")) + try: + receipt = run_loop(local, out, include_live=include_live, live_store=live_store) + except LiveStoreMissing as exc: + print(json.dumps({"abort": True, "error": str(exc), "brier": None}, indent=2), file=sys.stderr) + return 2 + except Exception as exc: + print(json.dumps({"abort": True, "error": str(exc), "brier": None}, indent=2), file=sys.stderr) + return 4 + print(json.dumps(receipt, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/shadow/hyperlexical/train_input.py b/scripts/shadow/hyperlexical/train_input.py new file mode 100644 index 00000000..610e36b1 --- /dev/null +++ b/scripts/shadow/hyperlexical/train_input.py @@ -0,0 +1,187 @@ +"""Training-input contract. Live rebuild versus an exact pinned export. + +RUNE.TRAIN_INPUT_BIND(x) = + experiment_id_present + AND export_path_present + AND expected_export_digest_present + AND export_file_exists + AND actual_export_digest == expected_export_digest + AND trainer_source == PINNED_EXPORT + +A controlled experiment (``HLX_EXPERIMENT_ID`` set) that fails any term +stops before training and does not call ``export_dataset``. Outside a +controlled experiment, an absent pin keeps ``LIVE_BUILD``. +""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import os +from pathlib import Path +from typing import Any, Callable + +LIVE_BUILD = "LIVE_BUILD" +PINNED_EXPORT = "PINNED_EXPORT" + +PATH_ENV = "HLX_TRAIN_EXPORT_PATH" +SHA_ENV = "HLX_TRAIN_EXPORT_SHA256" +ROWS_ENV = "HLX_TRAIN_EXPORT_ROWS" +EXPERIMENT_ID_ENV = "HLX_EXPERIMENT_ID" + + +class TrainInputAdmissionError(SystemExit): + """Fail closed before training. ``SystemExit`` so the CLI does not swallow it.""" + + +def _env(name: str) -> str: + return os.environ.get(name, "").strip() + + +def pinned_request() -> tuple[str, str, int | None]: + """Return ``(path, expected_sha256, expected_rows)``. Rows are optional.""" + path = _env(PATH_ENV) + digest = _env(SHA_ENV) + raw_rows = _env(ROWS_ENV) + expected_rows: int | None = None + if raw_rows: + if not raw_rows.isdigit(): + raise TrainInputAdmissionError( + f"ADMISSION FAIL: {ROWS_ENV} must be a non-negative integer" + ) + expected_rows = int(raw_rows) + return path, digest, expected_rows + + +def peek_train_input_mode() -> str: + """Env-only admission. Does not read an export or call ``export_dataset``. + + Incomplete pins fail closed. A controlled experiment with no pin fails + closed. Neither case falls through to ``LIVE_BUILD``. + """ + experiment_id = _env(EXPERIMENT_ID_ENV) + path, digest, _expected_rows = pinned_request() + if experiment_id and (not path or not digest): + raise TrainInputAdmissionError( + "ADMISSION FAIL: experiment id present but no pinned export" + ) + if path or digest: + if not path or not digest: + raise TrainInputAdmissionError( + "ADMISSION FAIL: pinned export requires " + f"{PATH_ENV} and {SHA_ENV}" + ) + return PINNED_EXPORT + return LIVE_BUILD + + +def _digest_matches(actual: str, expected: str) -> bool: + if len(actual) != len(expected): + return False + return hmac.compare_digest(actual, expected) + + +def _load_pinned(path: Path, expected: str, expected_rows: int | None) -> dict[str, Any]: + if not path.is_file(): + raise TrainInputAdmissionError(f"ADMISSION FAIL: pinned export missing: {path}") + raw = path.read_bytes() + actual = hashlib.sha256(raw).hexdigest() + if not _digest_matches(actual, expected): + raise TrainInputAdmissionError( + "ADMISSION FAIL: pinned export digest mismatch: " + f"expected {expected} actual {actual}" + ) + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as exc: + raise TrainInputAdmissionError( + "ADMISSION FAIL: pinned export is not UTF-8 JSONL" + ) from exc + rows: list[Any] = [] + for line in text.splitlines(): + if not line.strip(): + continue + try: + rows.append(json.loads(line)) + except json.JSONDecodeError as exc: + raise TrainInputAdmissionError( + "ADMISSION FAIL: pinned export is not JSONL" + ) from exc + if expected_rows is not None and len(rows) != expected_rows: + raise TrainInputAdmissionError( + "ADMISSION FAIL: pinned export row count mismatch: " + f"expected {expected_rows} actual {len(rows)}" + ) + return { + "rows": rows, + "sha256": actual, + "counts": {"n": len(rows), "live_included": 0}, + "payload": text, + "train_input": { + "mode": PINNED_EXPORT, + "path": str(path.resolve()), + "sha256_expected": expected, + "sha256_actual": actual, + "rows": len(rows), + "live_export_generation_enabled": False, + }, + } + + +def _tag_live(bundle: dict[str, Any]) -> dict[str, Any]: + rows = bundle.get("rows") or [] + digest = bundle.get("sha256") + bundle["train_input"] = { + "mode": LIVE_BUILD, + "path": None, + "sha256_expected": None, + "sha256_actual": digest, + "rows": len(rows), + "live_export_generation_enabled": True, + } + return bundle + + +def load_training_bundle( + root: Path, + *, + include_live: bool, + live_store: Path | None, + export_dataset: Callable[..., dict[str, Any]], +) -> dict[str, Any]: + """Return the bundle ``run_loop`` trains on. + + ``PINNED_EXPORT`` reads the sealed file and does not call ``export_dataset``. + ``LIVE_BUILD`` is the existing generator, including ``include_live``. + """ + mode = peek_train_input_mode() + if mode == PINNED_EXPORT: + path, digest, expected_rows = pinned_request() + return _load_pinned(Path(path), digest, expected_rows) + bundle = export_dataset(root, include_live=include_live, live_store=live_store) + return _tag_live(bundle) + + +def train_input_receipt(bundle: dict[str, Any]) -> dict[str, Any]: + """Fields copied onto the train receipt. Digest is the artifact actually loaded.""" + info = bundle.get("train_input") or {} + mode = info.get("mode") or LIVE_BUILD + actual = info.get("sha256_actual") + if actual is None: + actual = bundle.get("sha256") + rows = info.get("rows") + if rows is None: + rows = len(bundle.get("rows") or []) + generated = info.get("live_export_generation_enabled") + if generated is None: + generated = mode == LIVE_BUILD + return { + "data_sha256": actual if mode == PINNED_EXPORT else bundle.get("sha256", actual), + "training_input_mode": mode, + "training_export_path": info.get("path"), + "training_export_sha256_expected": info.get("sha256_expected"), + "training_export_sha256_actual": actual, + "training_export_rows": rows, + "live_export_generation_enabled": bool(generated), + } diff --git a/scripts/shadow/hyperlexical/training_routing.py b/scripts/shadow/hyperlexical/training_routing.py new file mode 100644 index 00000000..5ec00354 --- /dev/null +++ b/scripts/shadow/hyperlexical/training_routing.py @@ -0,0 +1,80 @@ +"""Loop task selection, not source or review approval. + +Provenance: Notion Sprint 001 Hub NOT_COMPUTABLE + Loop 805 Slice N/A ++ Hash: b3eee725054c1ed1dae16fad3464af005edad0cc (base). +""" +import hashlib +import json +from collections import Counter + +TASKS = {"classify": ("classify",), "unbind": ("unbind",), + "classify+unbind": ("classify", "unbind")} +HEADS = {"classify": "family", "unbind": "structure"} + + +def combined_surface_targets(row): + """Only activate new combined unbind rows with unambiguous literal targets. + + Semantic tags, missing targets and repeated occurrences need separate review + or occurrence-aware alignment. Never send them to token-1 fallback. + """ + text, fillers, roles = row.get("text"), row.get("fillers"), row.get("roles") + return (isinstance(text, str) and isinstance(fillers, list) and bool(fillers) + and isinstance(roles, list) and len(roles) == len(fillers) + and all(isinstance(r, str) and r.strip() for r in roles) + and all(isinstance(f, str) and f.strip() and text.count(f) == 1 for f in fillers) + and len(set(fillers)) == len(fillers)) + + +def route_rows(rows): + """Preserve input order and rows; missing masks retain declared legacy tasks. + + A present loss_masks object must specify both supported heads explicitly. + Other heads are unsupported by this loop and rejected, not silently dropped. + Counts describe selection before recipes/upsampling, not optimizer consumption. + """ + selected = {task: {p: [] for p in ("train", "val")} for task in HEADS} + reasons = Counter() + task_counts = Counter() + explicit = 0 + suppressed_unbind = 0 + for row in rows: + if not isinstance(row, dict) or row.get("task") not in TASKS: + raise ValueError("unsupported task declaration") + if row.get("split") not in {"train", "val", "test"}: + raise ValueError("unsupported split declaration") + masks = row.get("loss_masks") + if "loss_masks" in row: + if (not isinstance(masks, dict) or set(masks) != {"family", "structure"} + or any(type(v) is not bool for v in masks.values())): + raise ValueError("loss_masks requires explicit family/structure booleans") + explicit += 1 + declared = TASKS[row["task"]] + if masks is not None and any(masks[HEADS[t]] and t not in declared for t in HEADS): + raise ValueError("active mask has no declared task") + active = [t for t in declared if masks is None or masks[HEADS[t]]] + invalid_combined = (row["task"] == "classify+unbind" and "unbind" in active + and not combined_surface_targets(row)) + if invalid_combined: + active.remove("unbind") + suppressed_unbind += 1 + if row["split"] == "test": + reasons["reserved_test"] += 1 + elif not active: + reasons["invalid_combined_unbind_targets" if invalid_combined else "all_declared_heads_masked"] += 1 + else: + reasons["selected"] += 1 + for task in active: + selected[task][row["split"]].append(row) + task_counts[task + ":" + row["split"]] += 1 + accounting = { + "input_rows": len(rows), "row_outcomes": dict(reasons), + "task_assignments": dict(task_counts), "explicit_mask_rows": explicit, + "combined_unbind_suppressed": suppressed_unbind, + "legacy_declared_rows": len(rows) - explicit, + "input_sha256": hashlib.sha256(json.dumps(rows, sort_keys=True, separators=(",", ":"), + ensure_ascii=True, allow_nan=False).encode()).hexdigest(), + "unit": "pre_recipe_row_selection", "review_proof": "NOT_COMPUTABLE", + } + assert sum(reasons.values()) == len(rows) + return selected, accounting diff --git a/scripts/shadow/hyperlexical/training_schedule.py b/scripts/shadow/hyperlexical/training_schedule.py new file mode 100644 index 00000000..c8cdf4d6 --- /dev/null +++ b/scripts/shadow/hyperlexical/training_schedule.py @@ -0,0 +1,304 @@ +"""Default training schedule for comparable Hyperlex runs. + +Comparable runs select ``classify_macro_f1_nonnone``. When they do not +preregister a schedule, they use the SELECT-006 candidate schedule: +12 epochs, patience 4, minimum 4 scored epochs, strict improvement, +ties keep the earlier checkpoint, and the best checkpoint is restored. + +A preregistered schedule replaces that default. Pass it as a mapping or +set its ``HYPERLEX_TRAIN_EPOCHS`` / ``HYPERLEX_EARLY_STOP`` variables. +Those explicit values are not rewritten. + +Runs that do not select the classify metric keep the historical unset +schedule: 2 epochs and early stopping off. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Any, Mapping, MutableMapping + +from .classify_metrics import SELECT_METRIC_CLASSIFY + +SOURCE_DEFAULT = "select-006-candidate" +SOURCE_PREREGISTERED = "preregistered" +SOURCE_HISTORICAL = "historical" + +TRAIN_EPOCHS_ENV = "HYPERLEX_TRAIN_EPOCHS" +EARLY_STOP_ENV = "HYPERLEX_EARLY_STOP" +EARLY_STOP_PATIENCE_ENV = "HYPERLEX_EARLY_STOP_PATIENCE" +EARLY_STOP_MIN_EPOCHS_ENV = "HYPERLEX_EARLY_STOP_MIN_EPOCHS" + +SCHEDULE_ENV_KEYS = ( + TRAIN_EPOCHS_ENV, + EARLY_STOP_ENV, + EARLY_STOP_PATIENCE_ENV, + EARLY_STOP_MIN_EPOCHS_ENV, +) + +_OFF = frozenset({"0", "false", "no", "off", "disabled"}) +_ON = frozenset({"1", "true", "yes", "on", "enabled"}) + +HISTORICAL_MAX_EPOCHS = 2 + + +class ScheduleError(ValueError): + """A schedule override is incomplete or disagrees with the process.""" + + +@dataclass(frozen=True) +class TrainingSchedule: + max_epochs: int + early_stopping: bool + patience: int | None + minimum_epochs: int | None + improvement: str + ties: str + restore_best: bool + source: str + + def as_dict(self) -> dict[str, Any]: + return { + "early_stopping": self.early_stopping, + "improvement": self.improvement, + "max_epochs": self.max_epochs, + "minimum_epochs": self.minimum_epochs, + "patience": self.patience, + "restore_best": self.restore_best, + "source": self.source, + "ties": self.ties, + } + + +def select_006_candidate_schedule() -> TrainingSchedule: + """The sealed SELECT-006 candidate schedule, as the comparable-run default.""" + return TrainingSchedule( + max_epochs=12, + early_stopping=True, + patience=4, + minimum_epochs=4, + improvement="strict", + ties="keep_earlier", + restore_best=True, + source=SOURCE_DEFAULT, + ) + + +def _blank(value: Any) -> bool: + return value is None or (isinstance(value, str) and not value.strip()) + + +def _integer(label: str, value: Any) -> int: + if isinstance(value, bool) or isinstance(value, float): + raise ScheduleError(f"{label} must be an integer, got {value!r}") + if isinstance(value, int): + return value + text = str(value).strip() + sign = "" + digits = text + if text[:1] in "+-": + sign, digits = text[0], text[1:] + if not digits.isdigit(): + raise ScheduleError(f"{label} must be an integer, got {value!r}") + return int(sign + digits) + + +def _enabled(value: Any, *, label: str) -> bool: + if isinstance(value, bool): + return value + token = str(value).strip().lower() + if token in _OFF: + return False + if token in _ON: + return True + raise ScheduleError(f"{label} must be on or off, got {value!r}") + + +def _check_rules(schedule: Mapping[str, Any]) -> None: + improvement = schedule.get("improvement") + if improvement is not None and improvement != "strict": + raise ScheduleError("improvement must be strict") + ties = schedule.get("ties") + if ties is not None and ties != "keep_earlier": + raise ScheduleError("ties keep the earlier checkpoint") + if "restore_best" in schedule and schedule.get("restore_best") is not True: + raise ScheduleError("restore_best stays on") + + +def _finish( + *, + max_epochs: int, + early_stopping: bool, + patience: int | None, + minimum_epochs: int | None, + source: str, +) -> TrainingSchedule: + if max_epochs < 1: + raise ScheduleError(f"{TRAIN_EPOCHS_ENV} must be >= 1, got {max_epochs}") + if early_stopping: + if patience is None or minimum_epochs is None: + raise ScheduleError("early stopping requires patience and minimum epochs") + if patience < 0: + raise ScheduleError(f"{EARLY_STOP_PATIENCE_ENV} must be >= 0, got {patience}") + if minimum_epochs < 1: + raise ScheduleError(f"{EARLY_STOP_MIN_EPOCHS_ENV} must be >= 1, got {minimum_epochs}") + if minimum_epochs > max_epochs: + raise ScheduleError( + f"{EARLY_STOP_MIN_EPOCHS_ENV}={minimum_epochs} exceeds " + f"{TRAIN_EPOCHS_ENV}={max_epochs}" + ) + elif patience is not None or minimum_epochs is not None: + raise ScheduleError("patience requires early stopping") + return TrainingSchedule( + max_epochs=max_epochs, + early_stopping=early_stopping, + patience=patience, + minimum_epochs=minimum_epochs, + improvement="strict", + ties="keep_earlier", + restore_best=True, + source=source, + ) + + +def _schedule_body(preregistered: Mapping[str, Any]) -> Mapping[str, Any]: + if "training_schedule" in preregistered: + body = preregistered["training_schedule"] + if not isinstance(body, Mapping): + raise ScheduleError("training_schedule must be an object") + return body + if "max_epochs" in preregistered or TRAIN_EPOCHS_ENV in preregistered: + return preregistered + if "control_schedule" in preregistered or "candidate_schedule" in preregistered: + arm = preregistered.get("schedule_arm") + if arm == "control": + return preregistered["control_schedule"] + if arm == "candidate": + return preregistered["candidate_schedule"] + raise ScheduleError("preregistration names two schedules; set schedule_arm") + raise ScheduleError("preregistration has no training schedule") + + +def parse_preregistered_schedule(preregistered: Mapping[str, Any]) -> TrainingSchedule: + """One explicit schedule. It replaces the SELECT-006 candidate default.""" + body = _schedule_body(preregistered) + _check_rules(body) + if TRAIN_EPOCHS_ENV in body or EARLY_STOP_ENV in body: + return _from_environment(body, source=SOURCE_PREREGISTERED) + if "max_epochs" not in body: + raise ScheduleError("preregistered schedule needs max_epochs") + max_epochs = _integer("max_epochs", body["max_epochs"]) + if "early_stopping" in body: + enabled = _enabled(body["early_stopping"], label="early_stopping") + else: + enabled = "early_stopping_patience" in body or "minimum_epochs" in body + patience = None + minimum = None + if enabled: + if "early_stopping_patience" not in body or "minimum_epochs" not in body: + raise ScheduleError("early stopping requires patience and minimum epochs") + patience = _integer("early_stopping_patience", body["early_stopping_patience"]) + minimum = _integer("minimum_epochs", body["minimum_epochs"]) + return _finish( + max_epochs=max_epochs, + early_stopping=enabled, + patience=patience, + minimum_epochs=minimum, + source=SOURCE_PREREGISTERED, + ) + + +def _from_environment(env: Mapping[str, Any], *, source: str) -> TrainingSchedule: + epochs_raw = env.get(TRAIN_EPOCHS_ENV) + max_epochs = HISTORICAL_MAX_EPOCHS if _blank(epochs_raw) else _integer(TRAIN_EPOCHS_ENV, epochs_raw) + early_raw = env.get(EARLY_STOP_ENV) + patience_raw = env.get(EARLY_STOP_PATIENCE_ENV) + minimum_raw = env.get(EARLY_STOP_MIN_EPOCHS_ENV) + enabled = False if _blank(early_raw) else _enabled(early_raw, label=EARLY_STOP_ENV) + if not enabled and (not _blank(patience_raw) or not _blank(minimum_raw)): + raise ScheduleError("patience requires early stopping") + patience = None if not enabled else _integer(EARLY_STOP_PATIENCE_ENV, patience_raw) + minimum = None if not enabled else _integer(EARLY_STOP_MIN_EPOCHS_ENV, minimum_raw) + return _finish( + max_epochs=max_epochs, + early_stopping=enabled, + patience=patience, + minimum_epochs=minimum, + source=source, + ) + + +def schedule_is_explicit(env: Mapping[str, Any]) -> bool: + return any(not _blank(env.get(key)) for key in SCHEDULE_ENV_KEYS) + + +def _same_schedule(left: TrainingSchedule, right: TrainingSchedule) -> bool: + return ( + left.max_epochs == right.max_epochs + and left.early_stopping == right.early_stopping + and left.patience == right.patience + and left.minimum_epochs == right.minimum_epochs + ) + + +def resolve_training_schedule( + *, + select_metric: str, + environ: Mapping[str, str] | None = None, + preregistered: Mapping[str, Any] | None = None, +) -> TrainingSchedule: + """Default, historical, or the preregistered override. + + An explicit environment and a preregistered mapping must describe the + same epochs and early-stop bounds. Neither one is filled in from the + SELECT-006 candidate default. + """ + env = os.environ if environ is None else environ + explicit = schedule_is_explicit(env) + if preregistered is not None and explicit: + declared = parse_preregistered_schedule(preregistered) + from_env = _from_environment(env, source=SOURCE_PREREGISTERED) + if not _same_schedule(declared, from_env): + raise ScheduleError("preregistered schedule does not match the process environment") + return declared + if preregistered is not None: + return parse_preregistered_schedule(preregistered) + if explicit: + return _from_environment(env, source=SOURCE_PREREGISTERED) + if select_metric == SELECT_METRIC_CLASSIFY: + return select_006_candidate_schedule() + return _finish( + max_epochs=HISTORICAL_MAX_EPOCHS, + early_stopping=False, + patience=None, + minimum_epochs=None, + source=SOURCE_HISTORICAL, + ) + + +def environment_for_schedule(schedule: TrainingSchedule) -> dict[str, str]: + """Environment overlay a launcher exports for a preregistered schedule.""" + overlay = {TRAIN_EPOCHS_ENV: str(schedule.max_epochs)} + if schedule.early_stopping: + overlay[EARLY_STOP_ENV] = "1" + overlay[EARLY_STOP_PATIENCE_ENV] = str(schedule.patience) + overlay[EARLY_STOP_MIN_EPOCHS_ENV] = str(schedule.minimum_epochs) + return overlay + overlay[EARLY_STOP_ENV] = "0" + return overlay + + +def install_default_schedule(environ: MutableMapping[str, str] | None = None) -> None: + """Publish the candidate default without replacing an explicit schedule.""" + env: MutableMapping[str, str] + if environ is None: + env = os.environ + else: + env = environ + if schedule_is_explicit(env): + return + default = select_006_candidate_schedule() + overlay = environment_for_schedule(default) + for key, value in overlay.items(): + env.setdefault(key, value) diff --git a/scripts/shadow/hyperlexical/unbind_curriculum.py b/scripts/shadow/hyperlexical/unbind_curriculum.py new file mode 100644 index 00000000..b12d5186 --- /dev/null +++ b/scripts/shadow/hyperlexical/unbind_curriculum.py @@ -0,0 +1,241 @@ +"""Scheme-split unbind curriculum. Composes with shape_unbind_train. + +Default OFF. Existing trains stay identity (full mix every epoch). +Classify rows are out of scope — this module never sees them. +ne0l0gist harvest / export SoT unchanged. name_gate stays false. + +Morph1 OBSERVED civilian val dump (operator BEST ``seed-morph1``). +Not SoT gold. Do not invent rows from these counts. + +* unbind_exact 0.321 (115/358) +* scheme totals positional 185 / type_slot 173 +* scheme failures positional 135 / type_slot 108 +* themes: positional_head_filler_miss dominant; type_slot TOKEN miss; + residual morph bleed; status-vocab distractors (bum/bolt/burn/mid) + across lineages; many INFERRED type_slot rows are proper-noun noise + +Spend early epochs on positional, then type_slot, then joint. Keep +morph hard-negs. INFERRED cap stays default-off (operator Spark card). +""" + +from __future__ import annotations + +import os +from typing import Any, Iterable, Mapping + +UNBIND_CURRICULUM_ENV = "HYPERLEX_UNBIND_CURRICULUM" +UNBIND_CURRICULUM_POS_EPOCHS_ENV = "HYPERLEX_UNBIND_CURRICULUM_POS_EPOCHS" +UNBIND_CURRICULUM_TYPE_EPOCHS_ENV = "HYPERLEX_UNBIND_CURRICULUM_TYPE_EPOCHS" +UNBIND_CURRICULUM_DEFAULT = 0 +# Used only when curriculum is on. Remainder of HYPERLEX_TRAIN_EPOCHS is joint. +# 1/1 so a 2-epoch smoke still hits both schemes. Longer Spark cards should +# spend more early epochs on positional (Morph1 fail 135/185 vs 108/173). +UNBIND_CURRICULUM_POS_EPOCHS_DEFAULT = 1 +UNBIND_CURRICULUM_TYPE_EPOCHS_DEFAULT = 1 + +# OBSERVED seed-morph1 val dump. Documentation only — not train gold. +MORPH1_VAL_UNBIND_EXACT = 0.321 +MORPH1_VAL_HIT = 115 +MORPH1_VAL_N = 358 +MORPH1_VAL_POSITIONAL_N = 185 +MORPH1_VAL_TYPE_SLOT_N = 173 +MORPH1_VAL_POSITIONAL_FAIL = 135 +MORPH1_VAL_TYPE_SLOT_FAIL = 108 +# Operator-optional distractor denylist surfaces (empty default). +MORPH1_STATUS_VOCAB_DISTRACTORS = ("bum", "bolt", "burn", "mid") + +PHASE_POSITIONAL = "positional" +PHASE_TYPE_SLOT = "type_slot" +PHASE_JOINT = "joint" +PHASES = (PHASE_POSITIONAL, PHASE_TYPE_SLOT, PHASE_JOINT) + + +def resolve_unbind_curriculum(raw: str | int | None = None) -> bool: + """0|1 gate. Default 0 (off). Fail-closed if invalid.""" + if raw is None: + raw = os.environ.get(UNBIND_CURRICULUM_ENV) + if raw is None or (isinstance(raw, str) and not raw.strip()): + return False + token = str(raw).strip() + if token == "0": + return False + if token == "1": + return True + raise ValueError(f"{UNBIND_CURRICULUM_ENV} must be 0 or 1, got {raw!r}") + + +def _resolve_nonneg_int(raw: str | int | None, env_name: str, default: int) -> int: + if raw is None: + raw = os.environ.get(env_name) + if raw is None or (isinstance(raw, str) and not raw.strip()): + return default + try: + n = int(str(raw).strip(), 10) + except (TypeError, ValueError) as exc: + raise ValueError(f"{env_name} must be an int >= 0, got {raw!r}") from exc + if n < 0: + raise ValueError(f"{env_name} must be an int >= 0, got {n}") + return n + + +def resolve_unbind_curriculum_pos_epochs(raw: str | int | None = None) -> int: + """Positional-first epoch count when curriculum is on. Default 1.""" + return _resolve_nonneg_int( + raw, UNBIND_CURRICULUM_POS_EPOCHS_ENV, UNBIND_CURRICULUM_POS_EPOCHS_DEFAULT + ) + + +def resolve_unbind_curriculum_type_epochs(raw: str | int | None = None) -> int: + """type_slot epoch count when curriculum is on. Default 1.""" + return _resolve_nonneg_int( + raw, UNBIND_CURRICULUM_TYPE_EPOCHS_ENV, UNBIND_CURRICULUM_TYPE_EPOCHS_DEFAULT + ) + + +def resolve_curriculum_schedule( + raw: str | int | None = None, + *, + pos_epochs: str | int | None = None, + type_epochs: str | int | None = None, +) -> dict[str, Any]: + """Resolved gate + phase lengths. Off → pos/type recorded as 0 (unused).""" + enabled = resolve_unbind_curriculum(raw) + if not enabled: + return {"enabled": False, "pos_epochs": 0, "type_epochs": 0} + return { + "enabled": True, + "pos_epochs": resolve_unbind_curriculum_pos_epochs(pos_epochs), + "type_epochs": resolve_unbind_curriculum_type_epochs(type_epochs), + } + + +def is_type_slot_row(row: Mapping[str, Any]) -> bool: + return row.get("role_scheme") == PHASE_TYPE_SLOT + + +def is_positional_phase_row(row: Mapping[str, Any]) -> bool: + """positional / non-type_slot. TOKEN:/SLOT rows stay in the type_slot bucket.""" + return not is_type_slot_row(row) + + +def phase_name_for_epoch(epoch: int, schedule: Mapping[str, Any]) -> str: + if epoch < 0: + raise ValueError(f"epoch must be >= 0, got {epoch}") + if not schedule.get("enabled"): + return PHASE_JOINT + pos_n = int(schedule.get("pos_epochs") or 0) + type_n = int(schedule.get("type_epochs") or 0) + if epoch < pos_n: + return PHASE_POSITIONAL + if epoch < pos_n + type_n: + return PHASE_TYPE_SLOT + return PHASE_JOINT + + +def filter_rows_for_phase(rows: Iterable[Mapping[str, Any]], phase: str) -> list[Any]: + """Exclusive subset. joint = full mix (current behavior).""" + pool = list(rows) + if phase == PHASE_JOINT: + return pool + if phase == PHASE_TYPE_SLOT: + return [r for r in pool if is_type_slot_row(r)] + if phase == PHASE_POSITIONAL: + return [r for r in pool if is_positional_phase_row(r)] + raise ValueError(f"unknown unbind curriculum phase {phase!r}") + + +def select_unbind_for_epoch( + rows: Iterable[Mapping[str, Any]], + epoch: int, + schedule: Mapping[str, Any] | None = None, +) -> tuple[list[Any], dict[str, Any]]: + """Rows for one epoch. Empty exclusive subset → full mix (fail-closed).""" + sched = dict(schedule) if schedule is not None else resolve_curriculum_schedule() + pool = list(rows) + phase = phase_name_for_epoch(epoch, sched) + selected = filter_rows_for_phase(pool, phase) + fallback = False + if not selected and pool and phase != PHASE_JOINT: + selected = list(pool) + fallback = True + meta = { + "phase": phase, + "n_rows": len(selected), + "fallback_full_mix": fallback, + } + return selected, meta + + +def _clamp_phases(epochs: int, pos_epochs: int, type_epochs: int) -> list[tuple[str, int, int]]: + """Inclusive-start / exclusive-end epoch ranges actually used.""" + if epochs < 0: + raise ValueError(f"epochs must be >= 0, got {epochs}") + pos_end = min(max(0, pos_epochs), epochs) + type_end = min(max(pos_end, pos_epochs + type_epochs), epochs) + out: list[tuple[str, int, int]] = [] + if pos_end > 0: + out.append((PHASE_POSITIONAL, 0, pos_end)) + if type_end > pos_end: + out.append((PHASE_TYPE_SLOT, pos_end, type_end)) + if epochs > type_end: + out.append((PHASE_JOINT, type_end, epochs)) + return out + + +def plan_unbind_curriculum( + rows: Iterable[Mapping[str, Any]], + epochs: int, + schedule: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """Receipt-facing phase boundaries + n rows per phase. + + n_rows is the subset that will be trained (after empty-subset fallback). + """ + sched = dict(schedule) if schedule is not None else resolve_curriculum_schedule() + pool = list(rows) + n_pos = sum(1 for r in pool if is_positional_phase_row(r)) + n_type = sum(1 for r in pool if is_type_slot_row(r)) + n_joint = len(pool) + enabled = bool(sched.get("enabled")) + pos_n = int(sched.get("pos_epochs") or 0) if enabled else 0 + type_n = int(sched.get("type_epochs") or 0) if enabled else 0 + if not enabled: + ranges = [(PHASE_JOINT, 0, max(0, epochs))] if epochs > 0 else [] + else: + ranges = _clamp_phases(epochs, pos_n, type_n) + phases: list[dict[str, Any]] = [] + for name, start, end in ranges: + selected = filter_rows_for_phase(pool, name) + fallback = False + n_used = len(selected) + if not selected and pool and name != PHASE_JOINT: + n_used = n_joint + fallback = True + phases.append( + { + "name": name, + "start_epoch": start, + "end_epoch": end, + "n_rows": n_used, + "fallback_full_mix": fallback, + } + ) + return { + "enabled": enabled, + "pos_epochs": pos_n, + "type_epochs": type_n, + "n_unbind_positional": n_pos, + "n_unbind_type_slot": n_type, + "n_unbind_joint": n_joint, + "phases": phases, + } + + +def curriculum_env_counts() -> dict[str, int]: + """Export-facing ints. Does not mutate rows.""" + sched = resolve_curriculum_schedule() + return { + "unbind_curriculum": 1 if sched["enabled"] else 0, + "unbind_curriculum_pos_epochs": int(sched["pos_epochs"]), + "unbind_curriculum_type_epochs": int(sched["type_epochs"]), + } diff --git a/scripts/shadow/hyperlexical/unbind_head_slot.py b/scripts/shadow/hyperlexical/unbind_head_slot.py new file mode 100644 index 00000000..f91529c2 --- /dev/null +++ b/scripts/shadow/hyperlexical/unbind_head_slot.py @@ -0,0 +1,55 @@ +"""Head-slot (position-0) filler CE upweight for unbind train. + +Default identity (1.0). Morph1 OBSERVED dump: positional_head_filler_miss +dominated civilian misses; morph14 still shows F1≫exact. Scaling only the +first slot CE pushes the head without inventing gold or flipping name_gate. + +Env: ``HYPERLEX_UNBIND_HEAD_SLOT_WEIGHT`` — finite float in (0, 4]. Fail closed. +""" + +from __future__ import annotations + +import math +import os +from typing import Sequence, TypeVar + +UNBIND_HEAD_SLOT_WEIGHT_ENV = "HYPERLEX_UNBIND_HEAD_SLOT_WEIGHT" +UNBIND_HEAD_SLOT_WEIGHT_DEFAULT = 1.0 +UNBIND_HEAD_SLOT_WEIGHT_MAX = 4.0 + +_T = TypeVar("_T") + + +def resolve_unbind_head_slot_weight(raw: str | float | int | None = None) -> float: + """Identity 1.0. Fail-closed finite in (0, 4].""" + if raw is None: + raw = os.environ.get(UNBIND_HEAD_SLOT_WEIGHT_ENV) + if raw is None or (isinstance(raw, str) and not raw.strip()): + return UNBIND_HEAD_SLOT_WEIGHT_DEFAULT + try: + weight = float(str(raw).strip()) + except (TypeError, ValueError) as exc: + raise ValueError( + f"{UNBIND_HEAD_SLOT_WEIGHT_ENV} must be a finite number in " + f"(0, {UNBIND_HEAD_SLOT_WEIGHT_MAX}], got {raw!r}" + ) from exc + if not math.isfinite(weight) or weight <= 0 or weight > UNBIND_HEAD_SLOT_WEIGHT_MAX: + raise ValueError( + f"{UNBIND_HEAD_SLOT_WEIGHT_ENV} must be a finite number in " + f"(0, {UNBIND_HEAD_SLOT_WEIGHT_MAX}], got {raw!r}" + ) + return weight + + +def apply_head_slot_weight( + slot_ces: Sequence[_T], + weight: float, +) -> list[_T]: + """Scale index-0 CE by ``weight``. Empty → []. Weight 1.0 is a no-op copy.""" + slots = list(slot_ces) + if not slots: + return [] + if weight == 1.0: + return slots + slots[0] = slots[0] * weight # type: ignore[operator] + return slots diff --git a/scripts/shadow/hyperlexical/unbind_metrics.py b/scripts/shadow/hyperlexical/unbind_metrics.py new file mode 100644 index 00000000..62ceb710 --- /dev/null +++ b/scripts/shadow/hyperlexical/unbind_metrics.py @@ -0,0 +1,165 @@ +"""Token/slot F1 beside unbind_exact. No torch. Does not invent gold. + +Civilian unbind_exact is all-or-nothing on the full filler list. These +secondary scores use the same gold/pred alignment the train val loop +already uses (one predicted filler per gold slot, positional / type_slot +order). name_gate stays false. + +Strict keys sit beside the legacy ones. Legacy gold may already be +``mapped_filler`` (OOV → ````), so predicting ```` on that slot +is a legacy hit. Strict gold is the raw lowercased filler, and a ```` +prediction is always a miss. Ladder selection still reads ``unbind_exact``. +""" + +from __future__ import annotations + +from collections import Counter +from typing import Any, Iterable, Mapping, Sequence + +from .layout import UNK + + +def _tokens(fillers: Sequence[str] | None) -> list[str]: + return [str(item) for item in (fillers or [])] + + +# Not a filler. Keeps an unk prediction from matching gold or another unk. +_UNK_MISS = "\x00" + + +def strict_pair(gold: Sequence[str], pred: Sequence[str]) -> tuple[list[str], list[str]]: + """Lowercased raw gold. A ```` prediction never matches.""" + g = [str(item).lower() for item in gold] + p: list[str] = [] + for item in pred: + tok = str(item).lower() + p.append(_UNK_MISS if tok == UNK else tok) + return g, p + + +def mapped_filler(maps: Mapping[str, Any], fill: str) -> str: + """Gold surface through filler_vocab. Missing → UNK, same as exact.""" + vocab = maps["filler_vocab"] + idx = maps["filler_of"].get(fill, maps["filler_of"][UNK]) + return vocab[idx] + + +def mapped_pred(maps: Mapping[str, Any], pred_id: int) -> str: + """Predicted id through filler_vocab. Out of range → UNK.""" + vocab = maps["filler_vocab"] + idx = int(pred_id) + if 0 <= idx < len(vocab): + return vocab[idx] + return vocab[maps["filler_of"][UNK]] + + +def token_multiset_counts( + gold: Sequence[str], pred: Sequence[str] +) -> tuple[int, int, int]: + """Bag-of-filler overlap. Returns (tp, pred_n, gold_n).""" + g, p = _tokens(gold), _tokens(pred) + return sum((Counter(g) & Counter(p)).values()), len(p), len(g) + + +def slot_counts(gold: Sequence[str], pred: Sequence[str]) -> tuple[int, int, int]: + """Order-sensitive slot match. Index is the role when roles exist. + + Treats (position, filler) as the slot. Same-length alignment as exact: + zip for TP, leftover pred slots as FP, leftover gold slots as FN. + """ + g, p = _tokens(gold), _tokens(pred) + n = min(len(g), len(p)) + tp = sum(1 for i in range(n) if g[i] == p[i]) + return tp, len(p), len(g) + + +def _prf(tp: int, pred_n: int, gold_n: int) -> tuple[float, float, float]: + precision = tp / pred_n if pred_n else 0.0 + recall = tp / gold_n if gold_n else 0.0 + if precision + recall == 0: + return precision, recall, 0.0 + return precision, recall, 2 * precision * recall / (precision + recall) + + +def _score_pairs( + pairs: Iterable[tuple[Sequence[str], Sequence[str]]], + *, + strict: bool, +) -> dict[str, float | int]: + """Row-level exact plus micro token/slot F1. + + Skips rows with an empty gold list (same as train val). Empty series + returns zeros, matching unbind_exact = 0 / max(1, n). + """ + exact_hit = exact_n = 0 + tok_tp = tok_p = tok_g = 0 + slot_tp = slot_p = slot_g = 0 + for gold, pred in pairs: + if strict: + g, p = strict_pair(gold, pred) + else: + g, p = _tokens(gold), _tokens(pred) + if not g: + continue + exact_hit += int(g == p) + exact_n += 1 + tp, pn, gn = token_multiset_counts(g, p) + tok_tp += tp + tok_p += pn + tok_g += gn + stp, spn, sgn = slot_counts(g, p) + slot_tp += stp + slot_p += spn + slot_g += sgn + tok_p_, tok_r, tok_f1 = _prf(tok_tp, tok_p, tok_g) + _, _, slot_f1 = _prf(slot_tp, slot_p, slot_g) + return { + "unbind_exact": exact_hit / max(1, exact_n), + "n_unbind_eval": exact_n, + "unbind_token_f1": tok_f1, + "unbind_token_precision": tok_p_, + "unbind_token_recall": tok_r, + "unbind_slot_f1": slot_f1, + } + + +def summarize_unbind_pairs( + pairs: Iterable[tuple[Sequence[str], Sequence[str]]], + *, + strict_pairs: Iterable[tuple[Sequence[str], Sequence[str]]] | None = None, +) -> dict[str, float | int]: + """Legacy unbind metrics plus strict keys. + + ``strict_pairs`` is the raw-filler series (one row per legacy row). + When omitted, strict rules apply to ``pairs`` themselves. Legacy keys + are scored on ``pairs`` unchanged. + """ + rows = list(pairs) + legacy = _score_pairs(rows, strict=False) + strict_src = rows if strict_pairs is None else strict_pairs + strict = _score_pairs(strict_src, strict=True) + legacy.update( + { + "unbind_exact_strict": strict["unbind_exact"], + "unbind_token_f1_strict": strict["unbind_token_f1"], + "unbind_token_precision_strict": strict["unbind_token_precision"], + "unbind_token_recall_strict": strict["unbind_token_recall"], + "unbind_slot_f1_strict": strict["unbind_slot_f1"], + } + ) + return legacy + + +def null_unbind_secondary() -> dict[str, None]: + """004 probe stub/digest: no civilian filler lists, so F1 is not computable.""" + return { + "unbind_token_f1": None, + "unbind_token_precision": None, + "unbind_token_recall": None, + "unbind_slot_f1": None, + "unbind_exact_strict": None, + "unbind_token_f1_strict": None, + "unbind_token_precision_strict": None, + "unbind_token_recall_strict": None, + "unbind_slot_f1_strict": None, + } diff --git a/scripts/shadow/hyperlexical/unbind_recipe.py b/scripts/shadow/hyperlexical/unbind_recipe.py new file mode 100644 index 00000000..0c743eee --- /dev/null +++ b/scripts/shadow/hyperlexical/unbind_recipe.py @@ -0,0 +1,704 @@ +"""Hyperlexical unbind train recipe: morph hard-negs + OBSERVED upsample. + +Train-loop multiplicity / sample weights only. Does not invent OBSERVED +gold in the SoT. ne0l0gist harvest is unchanged. name_gate stays false. +""" + +from __future__ import annotations + +import json +import math +import os +from pathlib import Path +from typing import Any, Iterable, Mapping, Sequence + +UNBIND_OBSERVED_UPSAMPLE_ENV = "HYPERLEX_UNBIND_OBSERVED_UPSAMPLE" +UNBIND_INFERRED_CAP_ENV = "HYPERLEX_UNBIND_INFERRED_CAP" +UNBIND_INFERRED_WEIGHT_ENV = "HYPERLEX_UNBIND_INFERRED_WEIGHT" +UNBIND_MORPH_MARGIN_ENV = "HYPERLEX_UNBIND_MORPH_MARGIN" +UNBIND_FILLER_DENYLIST_ENV = "HYPERLEX_UNBIND_FILLER_DENYLIST" +UNBIND_FILLER_DENYLIST_PATH_ENV = "HYPERLEX_UNBIND_FILLER_DENYLIST_PATH" +UNBIND_HARD_ATOMS_PATH_ENV = "HYPERLEX_UNBIND_HARD_ATOMS_PATH" +UNBIND_HARD_UPSAMPLE_ENV = "HYPERLEX_UNBIND_HARD_UPSAMPLE" +UNBIND_FORCE_TRAIN_PATH_ENV = "HYPERLEX_UNBIND_FORCE_TRAIN_PATH" +UNBIND_OBSERVED_UPSAMPLE_DEFAULT = 1 +# Extra copies of already-OBSERVED hard phrases after the normal upsample. +# 1 = identity (no extra). Operator JSONL is env-path only — not SoT gold. +UNBIND_HARD_UPSAMPLE_DEFAULT = 1 +# 0 = off. Hard low caps starve morph-negs (do not default a cap). +UNBIND_INFERRED_CAP_DEFAULT = 0 +# Soft scale on unbind loss for class != OBSERVED. 1.0 = identity. +# Prefer this over a hard INFERRED_CAP (Morph4 CAP=1000 lost). +UNBIND_INFERRED_WEIGHT_DEFAULT = 1.0 +UNBIND_INFERRED_WEIGHT_MAX = 2.0 +UNBIND_MORPH_MARGIN_DEFAULT = 0.5 + +# Seeded from civilian unbind failure themes (live5 + Morph1 + Morph3 dump). +# Surfaces only — pairing is gated to fillers already present on unbind +# rows (no new slang atoms). Merge, do not duplicate, existing clusters. +# Morph3 residual bleed: aped↔aping, fanum taxed↔tax/fanumtax, looksmaxxed↔ +# looksmaxxing, rizzed↔rizz/rizzless, quiet quitter↔quitting, mewing↔mew*. +# vibe coded↔cooked is not a morph — do not pair. +MORPH_CLUSTERS: tuple[frozenset[str], ...] = ( + frozenset({"aped", "aping"}), + frozenset({"looksmax", "looksmaxx", "looksmaxxing", "looksmaxxed", "looksmaxxer"}), + frozenset({"rizz", "rizzed", "rizzless", "rizzing"}), + frozenset({"fanum", "fanum tax", "fanumtax", "fanum taxed", "tax", "taxed"}), + frozenset({"quiet quit", "quiet quitter", "quiet quitting"}), + frozenset({"mew", "mewing", "mewed"}), + frozenset({"crash", "crashed", "crashout"}), + frozenset( + { + "aura", + "aura farming", + "aura farm", + "aura farmed", + "aura farmer", + "aura points", + "negative aura", + "minus aura", + } + ), +) + +# ``tax`` / ``taxed`` sit on the fanum cluster so gold-is-fanum* can take +# them as siblings. Gold ``tax`` / ``taxed`` must not pull fanum* — too broad. +_FANUM_BROAD_ONLY: frozenset[str] = frozenset({"tax", "taxed"}) + +# Longest-first inflectional / productive slang suffixes. +_AUTO_SUFFIXES = ( + "maxxing", + "maxxed", + "maxxer", + "maxx", + "xing", + "ing", + "ers", + "er", + "ed", + "es", + "s", +) + + +def _norm_surface(text: str) -> str: + return " ".join((text or "").strip().lower().split()) + + +def inflection_stem(token: str) -> str: + """Conservative stem. Too-short leftovers stay the surface (no invent).""" + t = _norm_surface(token) + if " " in t or len(t) < 4: + return t + for suf in _AUTO_SUFFIXES: + if t.endswith(suf) and len(t) - len(suf) >= 3: + return t[: -len(suf)] + return t + + +def resolve_unbind_observed_upsample(raw: str | int | None = None) -> int: + """Repeat factor for OBSERVED unbind train rows. Default 1. Fail-closed.""" + if raw is None: + raw = os.environ.get(UNBIND_OBSERVED_UPSAMPLE_ENV) + if raw is None or (isinstance(raw, str) and not raw.strip()): + return UNBIND_OBSERVED_UPSAMPLE_DEFAULT + try: + n = int(str(raw).strip(), 10) + except (TypeError, ValueError) as exc: + raise ValueError(f"{UNBIND_OBSERVED_UPSAMPLE_ENV} must be a positive int, got {raw!r}") from exc + if n < 1: + raise ValueError(f"{UNBIND_OBSERVED_UPSAMPLE_ENV} must be a positive int, got {n}") + return n + + +def resolve_unbind_hard_upsample(raw: str | int | None = None) -> int: + """Extra copies of matched OBSERVED hard atoms. Default 1 = none. Fail-closed.""" + if raw is None: + raw = os.environ.get(UNBIND_HARD_UPSAMPLE_ENV) + if raw is None or (isinstance(raw, str) and not raw.strip()): + return UNBIND_HARD_UPSAMPLE_DEFAULT + try: + n = int(str(raw).strip(), 10) + except (TypeError, ValueError) as exc: + raise ValueError(f"{UNBIND_HARD_UPSAMPLE_ENV} must be an int >= 1, got {raw!r}") from exc + if n < 1: + raise ValueError(f"{UNBIND_HARD_UPSAMPLE_ENV} must be an int >= 1, got {n}") + return n + + +def resolve_unbind_hard_atoms_path(raw: str | Path | None = None) -> str: + """Operator JSONL path. Empty default. Does not invent a file.""" + if raw is None: + raw = os.environ.get(UNBIND_HARD_ATOMS_PATH_ENV) + if raw is None or (isinstance(raw, str) and not str(raw).strip()): + return "" + return str(raw).strip() + + +def hard_atoms_receipt_path(path: str | Path | None) -> str: + """Basename when a path is set; empty string otherwise.""" + token = str(path or "").strip() + if not token: + return "" + return Path(token).name + + +def resolve_unbind_force_train_path(raw: str | Path | None = None) -> str: + """Operator JSONL of authorized exacts to move val→train. Empty = off.""" + if raw is None: + raw = os.environ.get(UNBIND_FORCE_TRAIN_PATH_ENV) + if raw is None or (isinstance(raw, str) and not str(raw).strip()): + return "" + return str(raw).strip() + + +def force_train_receipt_path(path: str | Path | None) -> str: + token = str(path or "").strip() + if not token: + return "" + return Path(token).name + + +def load_force_train_keys(path: str | Path | None = None) -> frozenset[tuple[str, str]]: + """Load ``(text, role_scheme)`` keys from operator force-train JSONL. + + Unset / empty path → empty set (identity). Configured path must exist and + be valid JSONL with string ``text`` and ``role_scheme`` in + {positional, type_slot}. Does not invent OBSERVED class labels. + """ + raw = resolve_unbind_force_train_path(path) + if not raw: + return frozenset() + p = Path(raw) + if not p.is_file(): + raise ValueError(f"{UNBIND_FORCE_TRAIN_PATH_ENV} is not a file: {p}") + try: + body = p.read_text(encoding="utf-8") + except OSError as exc: + raise ValueError(f"{UNBIND_FORCE_TRAIN_PATH_ENV} is unreadable: {p}") from exc + keys: set[tuple[str, str]] = set() + for index, line in enumerate(body.splitlines(), start=1): + if not line.strip(): + continue + try: + obj = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError( + f"{UNBIND_FORCE_TRAIN_PATH_ENV} line {index} is not valid JSON" + ) from exc + if not isinstance(obj, dict): + raise ValueError( + f"{UNBIND_FORCE_TRAIN_PATH_ENV} line {index} must be a JSON object" + ) + text = obj.get("text") + scheme = obj.get("role_scheme") + if not isinstance(text, str) or not text.strip(): + raise ValueError( + f"{UNBIND_FORCE_TRAIN_PATH_ENV} line {index} needs non-empty string text" + ) + if scheme not in {"positional", "type_slot"}: + raise ValueError( + f"{UNBIND_FORCE_TRAIN_PATH_ENV} line {index} role_scheme must be " + f"positional|type_slot, got {scheme!r}" + ) + keys.add((text.strip(), str(scheme))) + return frozenset(keys) + + +def apply_unbind_force_train( + train: list[dict[str, Any]], + val: list[dict[str, Any]], + *, + path: str | Path | None = None, +) -> tuple[list[dict[str, Any]], list[dict[str, Any]], dict[str, Any]]: + """Move authorized OBSERVED val exacts into train (accept-style force-train). + + Only rows with ``class==OBSERVED`` and ``(text, role_scheme)`` in the + force-train JSONL leave val. Val changes → fair-eval baseline required. + Empty path → identity. Missing configured path fails closed. + """ + keys = load_force_train_keys(path) + receipt_path = force_train_receipt_path(resolve_unbind_force_train_path(path)) + if not keys: + return train, val, { + "unbind_force_train_path": receipt_path, + "n_unbind_force_train": 0, + "n_unbind_force_train_keys": 0, + "n_unbind_val_after_force_train": len(val), + } + keep_val: list[dict[str, Any]] = [] + moved: list[dict[str, Any]] = [] + for row in val: + key = (str(row.get("text") or "").strip(), str(row.get("role_scheme") or "")) + if key in keys and row.get("class") == "OBSERVED": + copy = dict(row) + copy["split"] = "train" + moved.append(copy) + else: + keep_val.append(row) + return train + moved, keep_val, { + "unbind_force_train_path": receipt_path, + "n_unbind_force_train": len(moved), + "n_unbind_force_train_keys": len(keys), + "n_unbind_val_after_force_train": len(keep_val), + } + + +def unbind_row_hard_atom_keys(row: Mapping[str, Any]) -> set[str]: + """Normalized row text plus joined filler atom text. No invented surfaces.""" + keys: set[str] = set() + text = _norm_surface(str(row.get("text") or "")) + if text: + keys.add(text) + fillers = [str(item).strip() for item in (row.get("fillers") or []) if str(item).strip()] + if fillers: + joined = _norm_surface(" ".join(fillers)) + if joined: + keys.add(joined) + return keys + + +def unbind_row_matches_hard_atoms(row: Mapping[str, Any], atoms: Iterable[str]) -> bool: + atom_set = {_norm_surface(str(item)) for item in atoms if _norm_surface(str(item))} + if not atom_set: + return False + return bool(unbind_row_hard_atom_keys(row) & atom_set) + + +def load_hard_atom_texts(path: str | Path | None = None) -> frozenset[str]: + """Load normalized ``text`` fields from operator JSONL. Fail-closed if set. + + Unset / empty path → empty set (identity). A configured path that is + missing, unreadable, or invalid raises. Does not invent OBSERVED gold. + """ + raw = resolve_unbind_hard_atoms_path(path) + if not raw: + return frozenset() + p = Path(raw) + if not p.is_file(): + raise ValueError(f"{UNBIND_HARD_ATOMS_PATH_ENV} is not a file: {p}") + try: + body = p.read_text(encoding="utf-8") + except OSError as exc: + raise ValueError(f"{UNBIND_HARD_ATOMS_PATH_ENV} is unreadable: {p}") from exc + atoms: set[str] = set() + for index, line in enumerate(body.splitlines(), start=1): + if not line.strip(): + continue + try: + obj = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError(f"{UNBIND_HARD_ATOMS_PATH_ENV} line {index} is not valid JSON") from exc + if not isinstance(obj, dict): + raise ValueError( + f"{UNBIND_HARD_ATOMS_PATH_ENV} line {index} must be a JSON object " + "with a string text field" + ) + if "text" not in obj: + raise ValueError( + f"{UNBIND_HARD_ATOMS_PATH_ENV} line {index} is missing required field 'text'" + ) + raw_text = obj.get("text") + if not isinstance(raw_text, str): + raise ValueError( + f"{UNBIND_HARD_ATOMS_PATH_ENV} line {index} field 'text' must be a string, " + f"got {type(raw_text).__name__}" + ) + norm = _norm_surface(raw_text) + if not norm: + raise ValueError(f"{UNBIND_HARD_ATOMS_PATH_ENV} line {index} has an empty text field") + atoms.add(norm) + return frozenset(atoms) + + +def hard_atom_upsample_stats( + unbind_rows: Iterable[Mapping[str, Any]], + *, + hard_upsample: int, + atoms: Iterable[str], + path: str = "", +) -> dict[str, Any]: + """Receipt counts. Only class==OBSERVED rows already in ``unbind_rows``.""" + atom_set = {_norm_surface(str(item)) for item in atoms if _norm_surface(str(item))} + observed = [r for r in unbind_rows if r.get("class") == "OBSERVED"] + matched_rows = [r for r in observed if unbind_row_matches_hard_atoms(r, atom_set)] + hit_atoms: set[str] = set() + for row in matched_rows: + hit_atoms.update(unbind_row_hard_atom_keys(row) & atom_set) + extra = 0 + if atom_set and hard_upsample > 1: + extra = (hard_upsample - 1) * len(matched_rows) + return { + "unbind_hard_atoms_path": hard_atoms_receipt_path(path), + "unbind_hard_upsample": hard_upsample, + "n_unbind_hard_atoms_matched": len(hit_atoms), + "n_unbind_hard_extra_copies": extra, + } + + +def resolve_unbind_inferred_cap(raw: str | int | None = None) -> int: + """Max INFERRED unbind train rows. 0 = off. Fail-closed. + + Hard low caps can starve morph-negs by dropping INFERRED rows that + carry sibling fillers. Default stays off; do not ship a hard cap. + """ + if raw is None: + raw = os.environ.get(UNBIND_INFERRED_CAP_ENV) + if raw is None or (isinstance(raw, str) and not raw.strip()): + return UNBIND_INFERRED_CAP_DEFAULT + try: + n = int(str(raw).strip(), 10) + except (TypeError, ValueError) as exc: + raise ValueError(f"{UNBIND_INFERRED_CAP_ENV} must be an int >= 0, got {raw!r}") from exc + if n < 0: + raise ValueError(f"{UNBIND_INFERRED_CAP_ENV} must be an int >= 0, got {n}") + return n + + +def _parse_filler_denylist_obj(raw: Any, *, source: str) -> dict[str, frozenset[str]]: + if raw is None: + return {} + if not isinstance(raw, dict): + raise ValueError(f"{source} must be a JSON object of lineage → filler list") + out: dict[str, frozenset[str]] = {} + for key, val in raw.items(): + lineage = str(key).strip() + if not lineage: + raise ValueError(f"{source} has an empty lineage key") + if not isinstance(val, list): + raise ValueError(f"{source} value for {lineage!r} must be a list of surfaces") + surfaces = {_norm_surface(str(item)) for item in val if _norm_surface(str(item))} + if surfaces: + out[lineage] = frozenset(surfaces) + return out + + +def resolve_filler_denylist( + raw: str | Mapping[str, Iterable[str]] | None = None, + *, + path: str | Path | None = None, +) -> dict[str, frozenset[str]]: + """Per-lineage banned fillers for hard-neg / CE distractors only. + + Fail-closed empty default. Does not invent slang atoms. Invalid JSON or + a missing path raises. Gold fillers are never stripped. + """ + if raw is not None and not isinstance(raw, str): + return _parse_filler_denylist_obj(dict(raw), source=UNBIND_FILLER_DENYLIST_ENV) + text = raw + if text is None: + text = os.environ.get(UNBIND_FILLER_DENYLIST_ENV) + path_raw = path + if path_raw is None: + path_raw = os.environ.get(UNBIND_FILLER_DENYLIST_PATH_ENV) + if path_raw is not None and str(path_raw).strip(): + p = Path(str(path_raw).strip()) + if not p.is_file(): + raise ValueError(f"{UNBIND_FILLER_DENYLIST_PATH_ENV} is not a file: {p}") + try: + loaded = json.loads(p.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError(f"{UNBIND_FILLER_DENYLIST_PATH_ENV} is not valid JSON") from exc + return _parse_filler_denylist_obj(loaded, source=str(p)) + if text is None or (isinstance(text, str) and not text.strip()): + return {} + try: + loaded = json.loads(text) + except json.JSONDecodeError as exc: + raise ValueError(f"{UNBIND_FILLER_DENYLIST_ENV} must be a JSON object") from exc + return _parse_filler_denylist_obj(loaded, source=UNBIND_FILLER_DENYLIST_ENV) + + +def banned_fillers_for( + lineage: str | None, + denylist: Mapping[str, Iterable[str]] | None = None, +) -> frozenset[str]: + if not lineage or not denylist: + return frozenset() + raw = denylist.get(lineage) + if not raw: + return frozenset() + return frozenset(_norm_surface(str(x)) for x in raw if _norm_surface(str(x))) + + +def resolve_unbind_morph_margin(raw: str | float | int | None = None) -> float: + """Margin for gold-vs-sibling filler ranking. Default 0.5. Fail-closed.""" + if raw is None: + raw = os.environ.get(UNBIND_MORPH_MARGIN_ENV) + if raw is None or (isinstance(raw, str) and not raw.strip()): + return UNBIND_MORPH_MARGIN_DEFAULT + try: + margin = float(str(raw).strip()) + except (TypeError, ValueError) as exc: + raise ValueError(f"{UNBIND_MORPH_MARGIN_ENV} must be a finite number >= 0, got {raw!r}") from exc + if margin < 0 or margin != margin or margin == float("inf"): + raise ValueError(f"{UNBIND_MORPH_MARGIN_ENV} must be a finite number >= 0, got {raw!r}") + return margin + + +def resolve_unbind_inferred_weight(raw: str | float | int | None = None) -> float: + """Soft unbind loss scale for class != OBSERVED. Default 1.0. Fail-closed. + + Finite and in (0, 2]. Does not drop rows. Try 0.4–0.5 on Spark instead + of a hard ``HYPERLEX_UNBIND_INFERRED_CAP`` (Morph4 CAP=1000 rejected). + """ + if raw is None: + raw = os.environ.get(UNBIND_INFERRED_WEIGHT_ENV) + if raw is None or (isinstance(raw, str) and not raw.strip()): + return UNBIND_INFERRED_WEIGHT_DEFAULT + try: + weight = float(str(raw).strip()) + except (TypeError, ValueError) as exc: + raise ValueError( + f"{UNBIND_INFERRED_WEIGHT_ENV} must be a finite number in (0, 2], got {raw!r}" + ) from exc + if not math.isfinite(weight) or weight <= 0 or weight > UNBIND_INFERRED_WEIGHT_MAX: + raise ValueError( + f"{UNBIND_INFERRED_WEIGHT_ENV} must be a finite number in (0, 2], got {raw!r}" + ) + return weight + + +def unbind_row_sample_weight( + row: Mapping[str, Any], + inferred_weight: float, +) -> float: + """Per-row unbind scale. OBSERVED stays 1.0. Non-OBSERVED gets ``inferred_weight``. + + Scales CE + morph-margin for that row. Does not drop the row. + """ + if row.get("class") == "OBSERVED": + return 1.0 + return inferred_weight + + +def known_fillers(unbind_rows: Iterable[dict[str, Any]]) -> set[str]: + out: set[str] = set() + for row in unbind_rows: + for fill in row.get("fillers") or []: + norm = _norm_surface(str(fill)) + if norm: + out.add(norm) + return out + + +def _is_fanum_lineage_surface(surface: str) -> bool: + return _norm_surface(surface).startswith("fanum") + + +def _explicit_siblings(surface: str) -> set[str]: + hit = _norm_surface(surface) + out: set[str] = set() + for cluster in MORPH_CLUSTERS: + if hit in cluster: + out.update(cluster) + # Pair taxed only when gold is fanum* lineage. Standalone tax/taxed + # must not pull fanum / fanum tax / fanumtax. + if hit in _FANUM_BROAD_ONLY and not _is_fanum_lineage_surface(hit): + out = {s for s in out if not _is_fanum_lineage_surface(s)} + out.discard(hit) + return out + + +def _auto_siblings(surface: str, known: set[str]) -> set[str]: + """Same stem, different surface. Only surfaces already in ``known``.""" + hit = _norm_surface(surface) + if not hit or " " in hit: + return set() + stem = inflection_stem(hit) + if stem == hit and len(hit) < 4: + return set() + out: set[str] = set() + for other in known: + if other == hit or " " in other: + continue + if inflection_stem(other) == stem: + out.add(other) + return out + + +def hard_negatives_for(filler: str, known: Iterable[str]) -> list[str]: + """Near-morph siblings that already exist as fillers. Never invents atoms. + + ``tax`` / ``taxed`` pair with fanum* only when gold is a fanum* surface. + """ + known_set = {_norm_surface(x) for x in known if _norm_surface(x)} + hit = _norm_surface(filler) + if not hit or hit not in known_set: + return [] + sibs = (_explicit_siblings(hit) | _auto_siblings(hit, known_set)) & known_set + sibs.discard(hit) + return sorted(sibs) + + +def distractor_fillers_for( + filler: str, + known: Iterable[str], + *, + lineage: str | None = None, + denylist: Mapping[str, Iterable[str]] | None = None, +) -> list[str]: + """Hard-neg / CE distractors. Denylist filters only; never invents atoms.""" + negs = hard_negatives_for(filler, known) + banned = banned_fillers_for(lineage, denylist) + if not banned: + return negs + return [n for n in negs if n not in banned] + + +def morph_pairs_for_rows(unbind_rows: Iterable[dict[str, Any]]) -> list[dict[str, str]]: + """Contrastive (gold, neg) pairs from fillers already on the rows.""" + rows = list(unbind_rows) + known = known_fillers(rows) + pairs: list[dict[str, str]] = [] + seen: set[tuple[str, str]] = set() + for row in rows: + for fill in row.get("fillers") or []: + gold = _norm_surface(str(fill)) + if not gold: + continue + explicit = _explicit_siblings(gold) & known + auto = _auto_siblings(gold, known) + for neg in hard_negatives_for(gold, known): + key = (gold, neg) + if key in seen: + continue + seen.add(key) + source = "explicit" if neg in explicit else "auto" + if neg in explicit and neg in auto: + source = "explicit" + pairs.append({"gold": gold, "neg": neg, "source": source}) + return pairs + + +def attach_hard_neg_fillers( + unbind_rows: Iterable[dict[str, Any]], + known: Iterable[str] | None = None, + *, + denylist: Mapping[str, Iterable[str]] | None = None, +) -> tuple[list[dict[str, Any]], int]: + """Copy rows and attach ``hard_neg_fillers``. Does not mint gold rows.""" + rows = list(unbind_rows) + known_set = set(known) if known is not None else known_fillers(rows) + known_set = {_norm_surface(x) for x in known_set if _norm_surface(x)} + deny = denylist if denylist is not None else resolve_filler_denylist() + out: list[dict[str, Any]] = [] + n_added = 0 + for row in rows: + copy = dict(row) + negs: list[str] = [] + seen_neg: set[str] = set() + lineage = str(row.get("lineage") or "") or None + for fill in row.get("fillers") or []: + for neg in distractor_fillers_for( + str(fill), known_set, lineage=lineage, denylist=deny + ): + if neg in seen_neg: + continue + seen_neg.add(neg) + negs.append(neg) + copy["hard_neg_fillers"] = negs + n_added += len(negs) + out.append(copy) + return out, n_added + + +def shape_unbind_train( + unbind_rows: Iterable[dict[str, Any]], + *, + upsample: int | str | None = None, + inferred_cap: int | str | None = None, + known: Iterable[str] | None = None, + denylist: Mapping[str, Iterable[str]] | None = None, + hard_upsample: int | str | None = None, + hard_atoms_path: str | Path | None = None, +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + """Train-only recipe. Callers must not pass val/test rows. + + OBSERVED copies are loop multiplicity, not new SoT gold. INFERRED cap + keeps first-seen order (no reshuffle). Hard-negs attach in memory. + Hard-atom extras copy already-OBSERVED train rows only. + """ + factor = resolve_unbind_observed_upsample(upsample) + cap = resolve_unbind_inferred_cap(inferred_cap) + hard_factor = resolve_unbind_hard_upsample(hard_upsample) + path = resolve_unbind_hard_atoms_path(hard_atoms_path) + atoms = load_hard_atom_texts(path) + rows = list(unbind_rows) + observed = [r for r in rows if r.get("class") == "OBSERVED"] + inferred = [r for r in rows if r.get("class") != "OBSERVED"] + if cap > 0: + inferred = inferred[:cap] + shaped: list[dict[str, Any]] = [] + for _ in range(factor): + shaped.extend(observed) + shaped.extend(inferred) + hard_stats = hard_atom_upsample_stats( + observed, hard_upsample=hard_factor, atoms=atoms, path=path + ) + if hard_stats["n_unbind_hard_extra_copies"]: + matched = [r for r in observed if unbind_row_matches_hard_atoms(r, atoms)] + extra_rounds = hard_factor - 1 + for _ in range(extra_rounds): + shaped.extend(dict(row) for row in matched) + pool = known if known is not None else known_fillers(rows) + deny = denylist if denylist is not None else resolve_filler_denylist() + attached, _n_row_negs = attach_hard_neg_fillers(shaped, pool, denylist=deny) + pairs = morph_pairs_for_rows(observed + inferred) + stats = { + "n_unbind_observed": len(observed), + "n_unbind_inferred": len(inferred), + "unbind_observed_upsample": factor, + "unbind_inferred_cap": cap, + "n_unbind_morph_negatives": len(pairs), + "n_train_unbind": len(attached), + "unbind_filler_denylist_lineages": len(deny), + } + stats.update(hard_stats) + return attached, stats + + +def morph_margin_loss( + gold_logit: float, + neg_logits: Sequence[float], + margin: float = UNBIND_MORPH_MARGIN_DEFAULT, +) -> float: + """sum(relu(margin + neg - gold)). Torch loop uses the same formula.""" + total = 0.0 + for neg in neg_logits: + gap = margin + float(neg) - float(gold_logit) + if gap > 0: + total += gap + return total + + +def recipe_env_counts(unbind_rows: Iterable[dict[str, Any]]) -> dict[str, Any]: + """Export-facing counts. Does not mutate rows or invent OBSERVED gold.""" + from .unbind_curriculum import curriculum_env_counts + + rows = list(unbind_rows) + n_obs = sum(1 for r in rows if r.get("class") == "OBSERVED") + n_inf = sum(1 for r in rows if r.get("class") != "OBSERVED") + train = [r for r in rows if r.get("split") == "train"] + path = resolve_unbind_hard_atoms_path() + atoms = load_hard_atom_texts(path) + hard_factor = resolve_unbind_hard_upsample() + counts = { + "n_unbind_observed": n_obs, + "n_unbind_inferred": n_inf, + "unbind_observed_upsample": resolve_unbind_observed_upsample(), + "unbind_inferred_cap": resolve_unbind_inferred_cap(), + "unbind_inferred_weight": resolve_unbind_inferred_weight(), + "unbind_morph_negatives": len(morph_pairs_for_rows(train)), + "unbind_filler_denylist_lineages": len(resolve_filler_denylist()), + } + counts.update( + hard_atom_upsample_stats( + [r for r in train if r.get("class") == "OBSERVED"], + hard_upsample=hard_factor, + atoms=atoms, + path=path, + ) + ) + counts.update(curriculum_env_counts()) + return counts diff --git a/scripts/shadow/hyperlexical/unbind_residual.py b/scripts/shadow/hyperlexical/unbind_residual.py new file mode 100644 index 00000000..48cb0595 --- /dev/null +++ b/scripts/shadow/hyperlexical/unbind_residual.py @@ -0,0 +1,205 @@ +"""Civilian unbind residual dump for Spark operators. + +Default OFF. When ``HYPERLEX_UNBIND_RESIDUAL_DUMP`` is set to a path, the +train loop writes a JSONL of val rows that miss ``unbind_exact``, plus a +theme summary on the receipt. Does not invent gold. Does not flip +``name_gate``. Ladder metric stays ``unbind_exact``. +""" + +from __future__ import annotations + +import json +import os +from collections import Counter +from pathlib import Path +from typing import Any, Iterable, Mapping, Sequence + +from .unbind_metrics import slot_counts, token_multiset_counts +from .unbind_recipe import hard_negatives_for + +UNBIND_RESIDUAL_DUMP_ENV = "HYPERLEX_UNBIND_RESIDUAL_DUMP" + +THEME_EXACT = "exact_hit" +THEME_POS_HEAD = "positional_head_filler_miss" +THEME_TYPE_TOKEN = "type_slot_token_miss" +THEME_MORPH = "morph_bleed" +THEME_ORDER = "token_hit_order_miss" +THEME_LENGTH = "length_mismatch" +THEME_FULL = "full_miss" +THEME_PARTIAL = "partial_slot_miss" + + +def resolve_unbind_residual_dump_path(raw: str | None = None) -> str: + """Empty default. Non-empty path is the dump target (parent may be created).""" + if raw is None: + raw = os.environ.get(UNBIND_RESIDUAL_DUMP_ENV) + if raw is None or (isinstance(raw, str) and not raw.strip()): + return "" + return str(raw).strip() + + +def _scheme(row: Mapping[str, Any] | None) -> str: + if not row: + return "" + return str(row.get("role_scheme") or "").strip() + + +def _roles(row: Mapping[str, Any] | None, n: int) -> list[str]: + if not row: + return [""] * n + roles = list(row.get("roles") or []) + if len(roles) >= n: + return [str(r) for r in roles[:n]] + return [str(r) for r in roles] + [""] * (n - len(roles)) + + +def classify_residual_themes( + gold: Sequence[str], + pred: Sequence[str], + *, + role_scheme: str = "", + roles: Sequence[str] | None = None, +) -> list[str]: + """Theme tags for one gold/pred filler list. Exact hit → [exact_hit].""" + g = [str(x) for x in gold] + p = [str(x) for x in pred] + if not g: + return [] + if g == p: + return [THEME_EXACT] + + themes: list[str] = [] + if len(g) != len(p): + themes.append(THEME_LENGTH) + + tok_tp, tok_p, tok_g = token_multiset_counts(g, p) + slot_tp, _, _ = slot_counts(g, p) + if tok_tp == tok_g == tok_p and tok_g > 0 and slot_tp < len(g): + themes.append(THEME_ORDER) + + scheme = (role_scheme or "").strip() + role_list = list(roles or []) + known = list({*g, *p}) + n = min(len(g), len(p)) + morph_hit = False + for i in range(n): + if g[i] == p[i]: + continue + sibs = set(hard_negatives_for(g[i], known)) + if p[i] in sibs: + morph_hit = True + if i == 0 and scheme == "positional": + themes.append(THEME_POS_HEAD) + role_i = str(role_list[i]) if i < len(role_list) else "" + if scheme == "type_slot" and ( + role_i.upper().startswith("TOKEN") or role_i.upper().startswith("SLOT") + ): + if THEME_TYPE_TOKEN not in themes: + themes.append(THEME_TYPE_TOKEN) + if morph_hit: + themes.append(THEME_MORPH) + + if slot_tp == 0 and THEME_ORDER not in themes: + themes.append(THEME_FULL) + elif THEME_POS_HEAD not in themes and THEME_ORDER not in themes: + themes.append(THEME_PARTIAL) + + # De-dupe preserve order + seen: set[str] = set() + out: list[str] = [] + for t in themes: + if t not in seen: + seen.add(t) + out.append(t) + return out + + +def residual_row_record( + *, + text: str, + gold: Sequence[str], + pred: Sequence[str], + row: Mapping[str, Any] | None = None, +) -> dict[str, Any] | None: + """One JSONL record for a miss. Exact hits return None.""" + g = [str(x) for x in gold] + p = [str(x) for x in pred] + if not g or g == p: + return None + scheme = _scheme(row) + roles = _roles(row, len(g)) + themes = classify_residual_themes(g, p, role_scheme=scheme, roles=roles) + slot_tp, slot_p, slot_g = slot_counts(g, p) + tok_tp, tok_p, tok_g = token_multiset_counts(g, p) + return { + "text": text, + "gold": g, + "pred": p, + "role_scheme": scheme or None, + "roles": roles, + "lineage": (str(row.get("lineage")) if row and row.get("lineage") else None), + "class": (str(row.get("class")) if row and row.get("class") else None), + "themes": themes, + "slot_tp": slot_tp, + "slot_n_gold": slot_g, + "token_tp": tok_tp, + "token_n_gold": tok_g, + "token_n_pred": tok_p, + } + + +def summarize_residual_records( + records: Iterable[Mapping[str, Any]], +) -> dict[str, Any]: + """Theme + scheme counters for the receipt. Not SoT gold.""" + theme_c: Counter[str] = Counter() + scheme_c: Counter[str] = Counter() + class_c: Counter[str] = Counter() + n = 0 + for rec in records: + n += 1 + for theme in rec.get("themes") or []: + theme_c[str(theme)] += 1 + scheme = str(rec.get("role_scheme") or "unknown") + scheme_c[scheme] += 1 + cls = str(rec.get("class") or "unknown") + class_c[cls] += 1 + return { + "n_residual": n, + "themes": dict(theme_c.most_common()), + "by_scheme": dict(scheme_c.most_common()), + "by_class": dict(class_c.most_common()), + } + + +def write_residual_dump( + path: str | Path, + records: Sequence[Mapping[str, Any]], +) -> dict[str, Any]: + """Write JSONL + sibling ``.summary.json``. Returns receipt fields.""" + target = Path(path) + if not str(target): + return { + "unbind_residual_dump": "", + "n_unbind_residual": 0, + "unbind_residual_themes": {}, + } + target.parent.mkdir(parents=True, exist_ok=True) + with target.open("w", encoding="utf-8") as fh: + for rec in records: + fh.write(json.dumps(rec, sort_keys=True) + "\n") + summary = summarize_residual_records(records) + summary_path = target.with_suffix(target.suffix + ".summary.json") + if target.suffix == ".jsonl": + summary_path = target.with_name(target.stem + ".summary.json") + summary_path.write_text( + json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return { + "unbind_residual_dump": target.name, + "n_unbind_residual": summary["n_residual"], + "unbind_residual_themes": summary["themes"], + "unbind_residual_by_scheme": summary["by_scheme"], + "unbind_residual_by_class": summary["by_class"], + "unbind_residual_summary": summary_path.name, + } diff --git a/scripts/shadow/hyperlexical/unbind_second_slot.py b/scripts/shadow/hyperlexical/unbind_second_slot.py new file mode 100644 index 00000000..d8897605 --- /dev/null +++ b/scripts/shadow/hyperlexical/unbind_second_slot.py @@ -0,0 +1,57 @@ +"""Second-slot (position-1) filler CE upweight for unbind train. + +Default identity (1.0). Morph19 residual: slot-index misses peak at i=1 +(119) over head i=0 (82). Head-slot weight=2 already cleared ladder 0.45; +scaling only index-1 pushes the dominant leftover without inventing gold, +flipping name_gate, or uniformly rescaling every slot (which collapses to +unbind_loss_weight). + +Env: ``HYPERLEX_UNBIND_SECOND_SLOT_WEIGHT`` — finite float in (0, 4]. Fail closed. +""" + +from __future__ import annotations + +import math +import os +from typing import Sequence, TypeVar + +UNBIND_SECOND_SLOT_WEIGHT_ENV = "HYPERLEX_UNBIND_SECOND_SLOT_WEIGHT" +UNBIND_SECOND_SLOT_WEIGHT_DEFAULT = 1.0 +UNBIND_SECOND_SLOT_WEIGHT_MAX = 4.0 + +_T = TypeVar("_T") + + +def resolve_unbind_second_slot_weight(raw: str | float | int | None = None) -> float: + """Identity 1.0. Fail-closed finite in (0, 4].""" + if raw is None: + raw = os.environ.get(UNBIND_SECOND_SLOT_WEIGHT_ENV) + if raw is None or (isinstance(raw, str) and not raw.strip()): + return UNBIND_SECOND_SLOT_WEIGHT_DEFAULT + try: + weight = float(str(raw).strip()) + except (TypeError, ValueError) as exc: + raise ValueError( + f"{UNBIND_SECOND_SLOT_WEIGHT_ENV} must be a finite number in " + f"(0, {UNBIND_SECOND_SLOT_WEIGHT_MAX}], got {raw!r}" + ) from exc + if not math.isfinite(weight) or weight <= 0 or weight > UNBIND_SECOND_SLOT_WEIGHT_MAX: + raise ValueError( + f"{UNBIND_SECOND_SLOT_WEIGHT_ENV} must be a finite number in " + f"(0, {UNBIND_SECOND_SLOT_WEIGHT_MAX}], got {raw!r}" + ) + return weight + + +def apply_second_slot_weight( + slot_ces: Sequence[_T], + weight: float, +) -> list[_T]: + """Scale index-1 CE by ``weight`` when present. Empty/len<2 → copy. Weight 1.0 no-op.""" + slots = list(slot_ces) + if len(slots) < 2: + return slots + if weight == 1.0: + return slots + slots[1] = slots[1] * weight # type: ignore[operator] + return slots diff --git a/scripts/shadow/hyperlexical/unbind_settlement.py b/scripts/shadow/hyperlexical/unbind_settlement.py new file mode 100644 index 00000000..20811076 --- /dev/null +++ b/scripts/shadow/hyperlexical/unbind_settlement.py @@ -0,0 +1,133 @@ +"""Unbind target settlement. Separate from classify settlement. + +Decisions are about the unbind target only. They do not set +``semantic_family``, ``attest``, or ``evaluation.enabled``. + +A correction is an appended event. Earlier events stay in the log. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Mapping, Sequence + +SCHEMA = "hyperlex.eval_unbind_settlement.v1" +RECEIPT_SCHEMA = "hyperlex.eval_unbind_settlement_receipt.v1" +DECISIONS = ("ACCEPT", "CORRECT_TARGET", "REJECT", "UNRESOLVED") +ADMIT_DECISIONS = frozenset({"ACCEPT", "CORRECT_TARGET"}) +VENDOR_CALLS = 0 + + +def refuse(message: str) -> None: + raise SystemExit(f"REFUSE: {message}") + + +def _hex64(value: Any, field: str) -> str: + text = str(value or "") + if len(text) != 64 or any(ch not in "0123456789abcdef" for ch in text): + refuse(f"{field} must be 64 lowercase hex characters") + return text + + +def _forbid_text(value: Mapping[str, Any]) -> None: + for key in value: + if key in {"text", "normalized_text", "raw_text", "fillers", "surface"}: + refuse(f"unbind settlement must not carry {key}") + + +def event_from_decision( + *, + normalized_text_sha256: str, + decision: str, + target_sha256: str, + target_provenance: str, + operator: str, + settled_at: str, + decision_basis: str, + previous_target_sha256: str = "", +) -> dict[str, Any]: + """Build one append-only settlement event. No surface text.""" + if decision not in DECISIONS: + refuse(f"unbind decision must be one of {DECISIONS}") + if not str(operator or "").strip() or not str(settled_at or "").strip(): + refuse("operator and settled_at are required") + if not str(target_provenance or "").strip(): + refuse("target provenance is required") + if not str(decision_basis or "").strip(): + refuse("decision basis is required") + digest = _hex64(normalized_text_sha256, "normalized_text_sha256") + target = _hex64(target_sha256, "target_sha256") + previous = str(previous_target_sha256 or "") + if decision == "CORRECT_TARGET": + previous = _hex64(previous, "previous_target_sha256") + if previous == target: + refuse("CORRECT_TARGET requires a different target") + elif previous: + refuse("previous_target_sha256 is only valid on CORRECT_TARGET") + body: dict[str, Any] = { + "schema": SCHEMA, + "normalized_text_sha256": digest, + "decision": decision, + "target_sha256": target, + "target_provenance": str(target_provenance), + "operator": str(operator), + "settled_at": str(settled_at), + "decision_basis": str(decision_basis), + "vendor_calls": VENDOR_CALLS, + } + if decision == "CORRECT_TARGET": + body["previous_target_sha256"] = previous + _forbid_text(body) + return body + + +def append_events(path: str | Path, events: Sequence[Mapping[str, Any]]) -> int: + """Append events. Refuse if the existing prefix changes.""" + dest = Path(path) + dest.parent.mkdir(parents=True, exist_ok=True) + prior = dest.read_bytes() if dest.exists() else b"" + encoded: list[bytes] = [] + for event in events: + if not isinstance(event, Mapping): + refuse("settlement event must be an object") + _forbid_text(event) + if event.get("schema") != SCHEMA: + refuse("settlement event schema mismatch") + encoded.append((json.dumps(dict(event), sort_keys=True) + "\n").encode("utf-8")) + with dest.open("ab") as handle: + for line in encoded: + handle.write(line) + current = dest.read_bytes() + if not current.startswith(prior): + refuse("unbind settlement log was not append-only") + return len(encoded) + + +def load_events(path: str | Path) -> list[dict[str, Any]]: + dest = Path(path) + if not dest.is_file(): + return [] + events: list[dict[str, Any]] = [] + for index, line in enumerate(dest.read_text(encoding="utf-8").splitlines(), start=1): + if not line.strip(): + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + refuse(f"settlement log line {index} is not JSON") + if not isinstance(obj, dict) or obj.get("schema") != SCHEMA: + refuse(f"settlement log line {index} has the wrong schema") + _forbid_text(obj) + events.append(obj) + return events + + +def latest_by_hash(events: Sequence[Mapping[str, Any]]) -> dict[str, dict[str, Any]]: + """Last event for each surface hash wins. Earlier lines stay in the file.""" + latest: dict[str, dict[str, Any]] = {} + for event in events: + digest = str(event.get("normalized_text_sha256") or "") + if digest: + latest[digest] = dict(event) + return latest diff --git a/scripts/shadow/hyperlexical/unbind_slot_ce.py b/scripts/shadow/hyperlexical/unbind_slot_ce.py new file mode 100644 index 00000000..9a669780 --- /dev/null +++ b/scripts/shadow/hyperlexical/unbind_slot_ce.py @@ -0,0 +1,128 @@ +"""Per-slot filler CE as the primary unbind train signal. + +Default OFF. Unset env keeps the historical mixed mean (filler CE, role +CE, and morph-margin share one divisor). When armed, mean per-position +filler CE is the primary term; existing list/margin leftovers are +constant-λ additive aux. λ is a fixed receipt constant, not a search. + +Civilian unbind_exact stays the ladder metric. Token/slot F1 still emit. +ne0l0gist harvest / export SoT unchanged. name_gate stays false. +""" + +from __future__ import annotations + +import os +from typing import Any, Sequence, TypeVar + +UNBIND_PRIMARY_ENV = "HYPERLEX_UNBIND_PRIMARY" +UNBIND_SLOT_CE_ENV = "HYPERLEX_UNBIND_SLOT_CE" +UNBIND_PRIMARY_MIXED = "mixed" +UNBIND_PRIMARY_SLOT_CE = "slot_ce" +# Fixed aux scale when slot_ce is primary. Not env-overridable. +UNBIND_SLOT_CE_AUX_LAMBDA = 0.25 + +_T = TypeVar("_T") + + +def _token_or_none(raw: str | int | None, env_name: str) -> str | None: + if raw is None: + raw = os.environ.get(env_name) + if raw is None or (isinstance(raw, str) and not raw.strip()): + return None + return str(raw).strip() + + +def _resolve_primary_token(raw: str | None) -> str: + if raw is None: + return UNBIND_PRIMARY_MIXED + if raw in {UNBIND_PRIMARY_MIXED, UNBIND_PRIMARY_SLOT_CE}: + return raw + raise ValueError( + f"{UNBIND_PRIMARY_ENV} must be {UNBIND_PRIMARY_MIXED!r} or " + f"{UNBIND_PRIMARY_SLOT_CE!r}, got {raw!r}" + ) + + +def _resolve_slot_ce_flag(raw: str | None) -> bool | None: + if raw is None: + return None + if raw == "0": + return False + if raw == "1": + return True + raise ValueError(f"{UNBIND_SLOT_CE_ENV} must be 0 or 1, got {raw!r}") + + +def resolve_unbind_primary_mode( + primary_raw: str | None = None, + slot_ce_raw: str | int | None = None, +) -> dict[str, Any]: + """Resolve primary mode. Default mixed / armed false. Fail-closed. + + Arm with ``HYPERLEX_UNBIND_PRIMARY=slot_ce`` or + ``HYPERLEX_UNBIND_SLOT_CE=1``. Explicit mixed + SLOT_CE=1, or + slot_ce + SLOT_CE=0, is a conflict. + """ + primary_token = _token_or_none(primary_raw, UNBIND_PRIMARY_ENV) + slot_token = _token_or_none(slot_ce_raw, UNBIND_SLOT_CE_ENV) + primary = _resolve_primary_token(primary_token) + slot_flag = _resolve_slot_ce_flag(slot_token) + if primary == UNBIND_PRIMARY_SLOT_CE and slot_flag is False: + raise ValueError( + f"{UNBIND_PRIMARY_ENV}={UNBIND_PRIMARY_SLOT_CE!r} conflicts with " + f"{UNBIND_SLOT_CE_ENV}=0" + ) + if primary_token == UNBIND_PRIMARY_MIXED and slot_flag is True: + raise ValueError( + f"{UNBIND_PRIMARY_ENV}={UNBIND_PRIMARY_MIXED!r} conflicts with " + f"{UNBIND_SLOT_CE_ENV}=1" + ) + armed = primary == UNBIND_PRIMARY_SLOT_CE or slot_flag is True + mode = UNBIND_PRIMARY_SLOT_CE if armed else UNBIND_PRIMARY_MIXED + return { + "unbind_primary": mode, + "unbind_slot_ce_armed": armed, + "unbind_slot_ce_aux_lambda": UNBIND_SLOT_CE_AUX_LAMBDA if armed else None, + } + + +def _mean(values: Sequence[_T]) -> _T | None: + if not values: + return None + total = values[0] + for item in values[1:]: + total = total + item # type: ignore[operator] + return total / len(values) # type: ignore[operator, return-value] + + +def combine_unbind_train_terms( + slot_ces: Sequence[_T], + aux_terms: Sequence[_T], + *, + primary: str, + aux_lambda: float = UNBIND_SLOT_CE_AUX_LAMBDA, +) -> _T | None: + """Compose per-slot filler CE with list/margin leftovers. + + ``mixed`` (default): mean of slot CE + aux, same divisor the Spark + loop used before this knob. ``slot_ce``: mean(slot CE) + λ * mean(aux). + Empty slot list → None (no unbind step). Empty aux → primary only. + """ + slots = list(slot_ces) + aux = list(aux_terms) + if not slots: + return None + if primary == UNBIND_PRIMARY_SLOT_CE: + slot_mean = _mean(slots) + if slot_mean is None: + return None + aux_mean = _mean(aux) + if aux_mean is None: + return slot_mean + return slot_mean + aux_lambda * aux_mean # type: ignore[operator] + if primary != UNBIND_PRIMARY_MIXED: + raise ValueError( + f"unbind primary must be {UNBIND_PRIMARY_MIXED!r} or " + f"{UNBIND_PRIMARY_SLOT_CE!r}, got {primary!r}" + ) + return _mean(slots + aux) diff --git a/scripts/shadow/hyperlexical/zero_init_loader.py b/scripts/shadow/hyperlexical/zero_init_loader.py new file mode 100644 index 00000000..1aa56906 --- /dev/null +++ b/scripts/shadow/hyperlexical/zero_init_loader.py @@ -0,0 +1,331 @@ +"""Deterministic role/filler row expansion. + +Mapped names keep the warm-start weight and bias. Names that exist only in the +target vocabulary are exact zeros. Names that exist only in the warm start are +omitted. Nothing in this module trains or constructs an optimizer. +""" + +from __future__ import annotations + +import hashlib +import json +import os +from typing import Any + +SELECT006_EXPERIMENT = "HLX-EXP-2026-09-29-SELECT-006" +STOP_BEFORE_OPTIMIZER_ENV = "HLX_STOP_BEFORE_OPTIMIZER" +VERIFIED = "ZERO_INIT_LOADER_VERIFIED" + +SOURCE_MISMATCH = "ZERO_INIT_LOADER_SOURCE_MISMATCH" +TARGET_MISMATCH = "ZERO_INIT_LOADER_TARGET_MISMATCH" +DUPLICATE_NAME = "ZERO_INIT_LOADER_DUPLICATE_NAME" +DIMENSION_MISMATCH = "ZERO_INIT_LOADER_DIMENSION_MISMATCH" +MAPPING_FAILURE = "ZERO_INIT_LOADER_MAPPING_FAILURE" +NONZERO_NEW_ROWS = "ZERO_INIT_LOADER_NONZERO_NEW_ROWS" +COPY_MISMATCH = "ZERO_INIT_LOADER_COPY_MISMATCH" +NONDETERMINISTIC = "ZERO_INIT_LOADER_NONDETERMINISTIC" +ENTRYPOINT_BYPASS = "ZERO_INIT_LOADER_ENTRYPOINT_BYPASS" +VERIFICATION_FAILURE = "ZERO_INIT_LOADER_VERIFICATION_FAILURE" + + +class ZeroInitLoaderError(ValueError): + def __init__(self, code: str, detail: str) -> None: + self.code = code + super().__init__(f"{code}: {detail}") + + +def select006_experiment_active() -> bool: + return os.environ.get("HLX_EXPERIMENT_ID", "").strip() == SELECT006_EXPERIMENT + + +def pre_optimizer_stop_requested() -> bool: + return os.environ.get(STOP_BEFORE_OPTIMIZER_ENV) == "1" + + +def _duplicates(labels: list[str]) -> list[str]: + seen: set[str] = set() + found: list[str] = [] + for label in labels: + if label in seen and label not in found: + found.append(label) + seen.add(label) + return found + + +def _require_string_labels(labels: list[Any], code: str, head: str, side: str) -> list[str]: + if not isinstance(labels, list) or any(not isinstance(label, str) or label == "" for label in labels): + raise ZeroInitLoaderError(code, f"{head} {side} vocabulary is not a list of names") + duplicates = _duplicates(labels) + if duplicates: + raise ZeroInitLoaderError(DUPLICATE_NAME, f"{head} {side} vocabulary repeats {duplicates[0]}") + return labels + + +def _tensor_sha256(tensor) -> str: + raw = tensor.detach().cpu().contiguous() + payload = raw.numpy().tobytes() + return hashlib.sha256(payload).hexdigest() + + +def _labels_sha256(labels: list[str]) -> str: + return hashlib.sha256("\n".join(labels).encode("utf-8")).hexdigest() + + +def expansion_expected(head: str) -> dict[str, Any] | None: + """Sealed SELECT-006 counts. Other experiments keep the general name policy.""" + if not select006_experiment_active(): + return None + from .select_006_efficiency_preservation import vocabulary_expansion_pin + + pin = vocabulary_expansion_pin() + if head == "role_head": + return { + "source_count": pin["warm_start_role_count"], + "target_count": pin["target_role_count"], + "mapped": pin["mapped_existing_roles"], + "new": pin["newly_initialized_roles"], + "removed": pin["warm_start_only_roles"], + "new_names": list(pin["new_role_names_in_sorted_order"]), + } + if head == "filler_head": + filler = pin["filler_expansion"] + return { + "source_count": filler["warm_start_filler_count"], + "target_count": filler["target_filler_count"], + "mapped": filler["mapped_existing_fillers"], + "new": filler["newly_initialized_fillers"], + "removed": filler["skipped_warm_start_only"], + } + raise ZeroInitLoaderError(VERIFICATION_FAILURE, f"unknown head {head}") + + +def select006_guard_vocab(expand_vocab: bool, vocab_match: bool) -> None: + if not select006_experiment_active(): + return + if not expand_vocab: + raise ZeroInitLoaderError( + ENTRYPOINT_BYPASS, + "SELECT-006 warm load requires vocabulary expansion", + ) + if vocab_match: + raise ZeroInitLoaderError( + TARGET_MISMATCH, + "SELECT-006 target vocabulary matches the warm start", + ) + + +def expand_named_linear( + module, + state: dict, + source_labels: list[str], + target_labels: list[str], + head: str, + *, + expected: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Copy shared rows by name and set target-only rows to exact zero.""" + import torch + + source = _require_string_labels(list(source_labels), SOURCE_MISMATCH, head, "source") + target = _require_string_labels(list(target_labels), TARGET_MISMATCH, head, "target") + weight = state.get("weight") + if weight is None: + raise ZeroInitLoaderError(SOURCE_MISMATCH, f"{head} source weight is missing") + if int(getattr(weight, "ndim", 0)) != 2: + raise ZeroInitLoaderError(DIMENSION_MISMATCH, f"{head} source weight is not a matrix") + if int(weight.shape[0]) != len(source): + raise ZeroInitLoaderError( + SOURCE_MISMATCH, + f"{head} source rows {int(weight.shape[0])} != source vocabulary {len(source)}", + ) + if int(module.weight.ndim) != 2 or int(module.weight.shape[0]) != len(target): + raise ZeroInitLoaderError( + TARGET_MISMATCH, + f"{head} target rows {tuple(module.weight.shape)} != target vocabulary {len(target)}", + ) + if int(weight.shape[1]) != int(module.weight.shape[1]): + raise ZeroInitLoaderError( + DIMENSION_MISMATCH, + f"{head} hidden {int(weight.shape[1])} != target hidden {int(module.weight.shape[1])}", + ) + source_bias = state.get("bias") + if module.bias is None or source_bias is None: + raise ZeroInitLoaderError(DIMENSION_MISMATCH, f"{head} weight/bias pair is incomplete") + if int(getattr(source_bias, "ndim", 0)) != 1 or int(source_bias.shape[0]) != len(source): + raise ZeroInitLoaderError(SOURCE_MISMATCH, f"{head} source bias length != source vocabulary") + if int(module.bias.shape[0]) != len(target): + raise ZeroInitLoaderError(TARGET_MISMATCH, f"{head} target bias length != target vocabulary") + + if expected is not None: + if int(expected["source_count"]) != len(source): + raise ZeroInitLoaderError(SOURCE_MISMATCH, f"{head} source count != sealed contract") + if int(expected["target_count"]) != len(target): + raise ZeroInitLoaderError(TARGET_MISMATCH, f"{head} target count != sealed contract") + + source_of = {label: index for index, label in enumerate(source)} + target_set = set(target) + src_weight = weight.detach().to(device=module.weight.device, dtype=module.weight.dtype) + src_bias = source_bias.detach().to(device=module.bias.device, dtype=module.bias.dtype) + mapped: list[str] = [] + new_names: list[str] = [] + with torch.no_grad(): + for index, label in enumerate(target): + source_index = source_of.get(label) + if source_index is None: + module.weight.data[index].zero_() + module.bias.data[index].zero_() + new_names.append(label) + continue + module.weight.data[index].copy_(src_weight[source_index]) + module.bias.data[index].copy_(src_bias[source_index]) + mapped.append(label) + for index, label in enumerate(target): + source_index = source_of.get(label) + if source_index is None: + if not torch.equal(module.weight.data[index], torch.zeros_like(module.weight.data[index])): + raise ZeroInitLoaderError(NONZERO_NEW_ROWS, f"{head} new weight row {label}") + if not torch.equal(module.bias.data[index], torch.zeros_like(module.bias.data[index])): + raise ZeroInitLoaderError(NONZERO_NEW_ROWS, f"{head} new bias row {label}") + continue + if not torch.equal(module.weight.data[index], src_weight[source_index]): + raise ZeroInitLoaderError(COPY_MISMATCH, f"{head} weight row {label}") + if not torch.equal(module.bias.data[index], src_bias[source_index]): + raise ZeroInitLoaderError(COPY_MISMATCH, f"{head} bias row {label}") + + removed = [label for label in source if label not in target_set] + if not mapped: + raise ZeroInitLoaderError(MAPPING_FAILURE, f"{head} matched no source rows") + if expected is not None: + if int(expected["mapped"]) != len(mapped): + raise ZeroInitLoaderError(MAPPING_FAILURE, f"{head} mapped count != sealed contract") + if int(expected["new"]) != len(new_names): + raise ZeroInitLoaderError(TARGET_MISMATCH, f"{head} new-row count != sealed contract") + if int(expected["removed"]) != len(removed): + raise ZeroInitLoaderError(SOURCE_MISMATCH, f"{head} removed count != sealed contract") + sealed_new = expected.get("new_names") + if sealed_new is not None and list(sealed_new) != new_names: + raise ZeroInitLoaderError(TARGET_MISMATCH, f"{head} new names != sealed order") + + return { + "mapped": len(mapped), + "skipped_init_only": len(removed), + "new_current_rows": len(new_names), + "init_n": len(source), + "current_n": len(target), + "new_names": new_names, + "new_row_policy": "exact_zero", + "copy_comparison": "exact", + "zero_comparison": "exact", + } + + +def _require_zero_rows(module, labels: list[str], names: list[str]) -> None: + import torch + + index_of = {label: index for index, label in enumerate(labels)} + for name in names: + index = index_of.get(name) + if index is None: + raise ZeroInitLoaderError(TARGET_MISMATCH, f"new row {name} is not in the target vocabulary") + if not torch.equal(module.weight.data[index], torch.zeros_like(module.weight.data[index])): + raise ZeroInitLoaderError(NONZERO_NEW_ROWS, f"weight row {name}") + if module.bias is None or not torch.equal( + module.bias.data[index], torch.zeros_like(module.bias.data[index]) + ): + raise ZeroInitLoaderError(NONZERO_NEW_ROWS, f"bias row {name}") + + +def select006_training_init_guard(maps: dict, role_head, filler_head, receipt: dict) -> None: + """Fail closed unless SELECT-006 initialization used the zero-row loader.""" + if not select006_experiment_active(): + return + if not receipt.get("expand_vocab"): + raise ZeroInitLoaderError( + ENTRYPOINT_BYPASS, + "SELECT-006 training initialization did not expand through the loader", + ) + from .select_006_efficiency_preservation import vocabulary_expansion_pin + + pin = vocabulary_expansion_pin() + filler_pin = pin["filler_expansion"] + role = receipt.get("role") or {} + filler = receipt.get("filler") or {} + if role.get("new_row_policy") != "exact_zero" or filler.get("new_row_policy") != "exact_zero": + raise ZeroInitLoaderError(ENTRYPOINT_BYPASS, "training initialization left new rows uninitialized") + if int(role.get("mapped", -1)) != int(pin["mapped_existing_roles"]): + raise ZeroInitLoaderError(MAPPING_FAILURE, "role mapped count") + if int(role.get("new_current_rows", -1)) != int(pin["newly_initialized_roles"]): + raise ZeroInitLoaderError(TARGET_MISMATCH, "role new-row count") + if list(role.get("new_names") or []) != list(pin["new_role_names_in_sorted_order"]): + raise ZeroInitLoaderError(TARGET_MISMATCH, "role new names") + if int(filler.get("mapped", -1)) != int(filler_pin["mapped_existing_fillers"]): + raise ZeroInitLoaderError(MAPPING_FAILURE, "filler mapped count") + if int(filler.get("new_current_rows", -1)) != int(filler_pin["newly_initialized_fillers"]): + raise ZeroInitLoaderError(TARGET_MISMATCH, "filler new-row count") + if int(filler.get("skipped_init_only", -1)) != int(filler_pin["skipped_warm_start_only"]): + raise ZeroInitLoaderError(SOURCE_MISMATCH, "filler removed count") + role_vocab = list(maps["role_vocab"]) + filler_vocab = list(maps["filler_vocab"]) + if len(role_vocab) != int(pin["target_role_count"]) or role_vocab[0] != pin["unk_token"]: + raise ZeroInitLoaderError(TARGET_MISMATCH, "role target vocabulary") + if role_vocab[1:] != sorted(role_vocab[1:]): + raise ZeroInitLoaderError(TARGET_MISMATCH, "role row order") + if len(filler_vocab) != int(filler_pin["target_filler_count"]) or filler_vocab[0] != pin["unk_token"]: + raise ZeroInitLoaderError(TARGET_MISMATCH, "filler target vocabulary") + if filler_vocab[1:] != sorted(filler_vocab[1:]): + raise ZeroInitLoaderError(TARGET_MISMATCH, "filler row order") + _require_zero_rows(role_head, role_vocab, list(role["new_names"])) + _require_zero_rows(filler_head, filler_vocab, list(filler["new_names"])) + + +def expanded_loader_hashes(role_head, filler_head, maps: dict) -> dict[str, str]: + role_weight = _tensor_sha256(role_head.weight.data) + role_bias = _tensor_sha256(role_head.bias.data) + filler_weight = _tensor_sha256(filler_head.weight.data) + filler_bias = _tensor_sha256(filler_head.bias.data) + body = json.dumps( + { + "filler_bias_sha256": filler_bias, + "filler_vocab_sha256": _labels_sha256(list(maps["filler_vocab"])), + "filler_weight_sha256": filler_weight, + "role_bias_sha256": role_bias, + "role_vocab_sha256": _labels_sha256(list(maps["role_vocab"])), + "role_weight_sha256": role_weight, + }, + sort_keys=True, + separators=(",", ":"), + ) + return { + "expanded_role_weight_sha256": role_weight, + "expanded_role_bias_sha256": role_bias, + "expanded_filler_weight_sha256": filler_weight, + "expanded_filler_bias_sha256": filler_bias, + "expanded_loader_witness_sha256": hashlib.sha256(body.encode("utf-8")).hexdigest(), + } + + +def pre_optimizer_receipt(role_head, filler_head, maps: dict, init_receipt: dict) -> dict[str, Any]: + role = init_receipt.get("role") or {} + filler = init_receipt.get("filler") or {} + return { + "schema": "hyperlex.select_006_pre_optimizer.v1", + "stopped_before_optimizer": True, + "optimizer_constructed": False, + "training_started": False, + "training_launch_authorized": False, + "epochs": 0, + "gradient_steps": 0, + "weights_mutated_by_training": False, + "execution_loader_status": VERIFIED, + "init_from": init_receipt.get("init_from"), + "expand_vocab": bool(init_receipt.get("expand_vocab")), + "role_mapped": role.get("mapped"), + "role_new": role.get("new_current_rows"), + "role_new_names": list(role.get("new_names") or []), + "filler_mapped": filler.get("mapped"), + "filler_new": filler.get("new_current_rows"), + "filler_removed": filler.get("skipped_init_only"), + "role_count": len(maps["role_vocab"]), + "filler_count": len(maps["filler_vocab"]), + **expanded_loader_hashes(role_head, filler_head, maps), + } diff --git a/tests/heldout/test_build_exposure_snapshot.py b/tests/heldout/test_build_exposure_snapshot.py new file mode 100644 index 00000000..57fdb8b1 --- /dev/null +++ b/tests/heldout/test_build_exposure_snapshot.py @@ -0,0 +1,185 @@ +"""The exposure successor is a superset of the historical pair and does not harvest.""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "scripts" / "shadow")) +sys.path.insert(0, str(ROOT / "scripts" / "heldout")) + +from build_exposure_snapshot import ( + ABSENT_BY_DEFAULT, + MONOTONICITY_FAILURE, + PLANNED, + RECONSTRUCTED, + assess_monotonicity, + build_receipt, + main, + plan_reconstruction, + publish_pair, +) + + +def _box(*digests: str) -> str: + return "".join(item + "\n" for item in digests) + + +def _vern(*values: str) -> str: + return "".join(json.dumps(value) + "\n" for value in values) + + +def _plan(box: str, vern: str, **kwargs): + return plan_reconstruction( + baseline_box_text=box, + baseline_vern_text=vern, + baseline_snapshot_id="20260925T211351Z", + baseline_box_sha256="ab" * 32, + baseline_vern_sha256="cd" * 32, + absent_families=ABSENT_BY_DEFAULT, + **kwargs, + ) + + +def test_successor_keeps_every_historical_hash_and_adds_only_new_ones() -> None: + old = "a" * 64 + extra = "b" * 64 + plan = _plan( + _box(old), + _vern("kept"), + box_increments=( + { + "artifact": "agent-context-exposed-normtext-sha256.txt", + "family": "agent_context_exposed_hashes", + "source_sha256": "ef" * 32, + "text": _box(old, extra), + }, + ), + ) + assert plan["monotonicity"]["ok"] is True + assert plan["box_added"] == 1 + assert plan["box_count"] == 2 + assert extra in plan["box_text"] + assert old in plan["box_text"] + assert plan["vern_bytes_carried_forward"] is True + assert plan["vern_added"] == 0 + receipt = build_receipt(plan, snapshot_id=None, published=False, fence=None) + assert receipt["state"] == PLANNED + assert receipt["harvest_run"] is False + assert receipt["training_authorized"] is False + assert "notion_not_a_box_hash_source" in receipt["absent_families"] + + +def test_a_dropped_historical_hash_is_not_publishable() -> None: + previous = {"a" * 64, "b" * 64} + successor = {"b" * 64} + assessment = assess_monotonicity(previous, successor, {"kept"}, {"kept"}) + assert assessment["ok"] is False + assert assessment["box_missing"] == 1 + plan = _plan(_box("a" * 64), _vern("kept")) + plan["monotonicity"] = assessment + receipt = build_receipt(plan, snapshot_id=None, published=False, fence=None) + assert receipt["state"] == MONOTONICITY_FAILURE + with pytest.raises(SystemExit, match="cannot be published"): + build_receipt(plan, snapshot_id="20260929T000000Z", published=True, fence=None) + + +def test_notion_titles_are_refused_as_snapshot_sources() -> None: + with pytest.raises(SystemExit, match="notion"): + _plan( + _box("a" * 64), + _vern("kept"), + box_increments=( + { + "artifact": "notion", + "family": "notion", + "source_sha256": "11" * 32, + "text": _box("c" * 64), + }, + ), + ) + + +def test_publish_writes_both_names_and_refuses_a_second_copy(tmp_path: Path) -> None: + plan = _plan(_box("d" * 64), _vern("kept\nline")) + snapshot_id = "20260929T010203Z" + vern_path, box_path = publish_pair(tmp_path, snapshot_id, plan["box_text"], plan["vern_text"]) + assert vern_path.name == f"vern-texts-{snapshot_id}.jsonl" + assert box_path.name == f"box-jev-hashes-{snapshot_id}.txt" + assert oct(vern_path.stat().st_mode & 0o777) == "0o600" + assert oct(box_path.stat().st_mode & 0o777) == "0o600" + assert "20260925T211351Z" not in vern_path.name + with pytest.raises(SystemExit, match="already published"): + publish_pair(tmp_path, snapshot_id, plan["box_text"], plan["vern_text"]) + + +def test_new_vern_strings_are_appended_without_rewriting_history() -> None: + historical = json.dumps("kept", ensure_ascii=False) + "\n" + plan = _plan( + _box("e" * 64), + historical, + vern_increments=( + { + "artifact": "lane", + "family": "observed_vernacular", + "source_sha256": "22" * 32, + "text": _vern("added"), + }, + ), + ) + assert plan["vern_text"].splitlines() == [json.dumps("kept"), json.dumps("added")] + assert plan["vern_added"] == 1 + assert plan["vern_bytes_carried_forward"] is False + + +def test_module_does_not_fetch_or_harvest() -> None: + source = Path(__file__).resolve().parents[2] / "scripts/heldout/build_exposure_snapshot.py" + module = source.read_text(encoding="utf-8") + for banned in ("subprocess", "urllib", "urlopen", "import hs_run", "Popen", "httpx"): + assert banned not in module + assert "does not harvest" in module + + +def test_cli_publish_is_fresh_and_leaves_the_baseline(tmp_path: Path) -> None: + baseline_box = tmp_path / "box-jev-hashes-20260925T211351Z.txt" + baseline_vern = tmp_path / "vern-texts-20260925T211351Z.jsonl" + extra = tmp_path / "extra.txt" + baseline_box.write_text(_box("a" * 64), encoding="utf-8") + baseline_vern.write_text(_vern("kept"), encoding="utf-8") + extra.write_text(_box("f" * 64), encoding="utf-8") + before_box = baseline_box.read_bytes() + before_vern = baseline_vern.read_bytes() + receipt_path = tmp_path / "receipt.json" + code = main( + [ + "--baseline-box", + str(baseline_box), + "--baseline-vern", + str(baseline_vern), + "--box-increment", + f"agent_context_exposed_hashes:{extra}", + "--out-dir", + str(tmp_path), + "--snapshot-id", + "20260929T020304Z", + "--receipt", + str(receipt_path), + "--publish", + ] + ) + assert code == 0 + assert baseline_box.read_bytes() == before_box + assert baseline_vern.read_bytes() == before_vern + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt["state"] == RECONSTRUCTED + assert receipt["published"] is True + assert receipt["harvest_run"] is False + assert receipt["box_added"] == 1 + assert receipt["freshness"]["state"] == "EXPOSURE_SNAPSHOT_FRESH" + assert receipt["previous_pair_overwritten"] is False + assert os.stat(receipt_path).st_mode & 0o777 == 0o600 diff --git a/tests/shadow/test_hyperlexical_holdout_disjoint.py b/tests/shadow/test_hyperlexical_holdout_disjoint.py new file mode 100644 index 00000000..03ef720b --- /dev/null +++ b/tests/shadow/test_hyperlexical_holdout_disjoint.py @@ -0,0 +1,273 @@ +"""Pinned training rows and a controlled holdout must be disjoint.""" + +from __future__ import annotations + +import hashlib +import json +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "scripts" / "shadow")) + +from hyperlexical.heldout_census import normalize_group_text # noqa: E402 +from hyperlexical.holdout_eligibility import census, exclusion_reason # noqa: E402 +from hyperlexical.holdout_guard import ( # noqa: E402 + assert_pinned_holdout_disjoint, + filter_holdout_rows, + load_holdout_spec, + normalized_text_sha256, + require_holdout_for_training, +) +from hyperlexical.preflight import main as preflight_main # noqa: E402 +from hyperlexical.selection_surface import row_id # noqa: E402 +from hyperlexical.train_input import load_training_bundle # noqa: E402 + + +def _row(text, *, split="train", task="classify", cls="OBSERVED", lineage="brainrot"): + return { + "text": text, + "fillers": ["atom"], + "roles": ["pos_0"], + "role_scheme": "positional", + "class": cls, + "split": split, + "task": task, + "lineage": lineage, + } + + +def _sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _write_pin(path: Path, rows: list[dict]) -> str: + payload = "".join(json.dumps(row, sort_keys=True) + "\n" for row in rows) + path.write_text(payload, encoding="utf-8") + return _sha(path) + + +def _manifest(path: Path, rows: list[dict], *, experiment_id: str = "HLX-EXP-TEST") -> Path: + path.write_text( + json.dumps( + { + "schema": "hyperlex.holdout_manifest.v1", + "status": "UNSCORED_SEALED", + "experiment_id": experiment_id, + "row_ids": [row_id(row) for row in rows], + "normalized_text_sha256": [normalized_text_sha256(row["text"]) for row in rows], + } + ), + encoding="utf-8", + ) + return path + + +def _arm(monkeypatch, tmp_path: Path, rows: list[dict], manifest: Path) -> None: + pinned = tmp_path / "pinned.jsonl" + digest = _write_pin(pinned, rows) + monkeypatch.setenv("HLX_EXPERIMENT_ID", "HLX-EXP-TEST") + monkeypatch.setenv("HLX_TRAIN_EXPORT_PATH", str(pinned)) + monkeypatch.setenv("HLX_TRAIN_EXPORT_SHA256", digest) + monkeypatch.setenv("HLX_TRAIN_EXPORT_ROWS", str(len(rows))) + monkeypatch.setenv("HLX_HOLDOUT_MANIFESTS", str(manifest)) + monkeypatch.setenv("HYPERLEX_ALLOW_TRAIN", "1") + + +def test_same_row_id_is_not_eligible(): + row = _row("cobalt lantern") + reason = exclusion_reason( + row, + train_ids={row_id(row)}, + train_hashes=set(), + other_ids=set(), + other_hashes=set(), + ) + assert reason == "train_row_id" + + +def test_same_normalized_text_different_row_id_is_not_eligible(): + train = _row("Blue Quartz Lantern!", split="train") + hold = _row("blue quartz lantern", split="val", cls="INFERRED", lineage="none") + assert row_id(train) != row_id(hold) + assert normalized_text_sha256(train["text"]) == normalized_text_sha256(hold["text"]) + reason = exclusion_reason( + hold, + train_ids=set(), + train_hashes={normalized_text_sha256(train["text"])}, + other_ids=set(), + other_hashes=set(), + ) + assert reason == "train_text_hash" + + +def test_different_text_is_eligible(): + row = _row("north cobble path", split="val") + reason = exclusion_reason( + row, + train_ids=set(), + train_hashes={normalized_text_sha256("other phrase")}, + other_ids=set(), + other_hashes=set(), + ) + assert reason == "eligible" + + +def test_draw_and_filter_share_the_canonical_normalizer(): + surface = "Hello, World! https://example.com/a" + assert normalize_group_text(surface) == "hello world" + digest = hashlib.sha256(normalize_group_text(surface).encode("utf-8")).hexdigest() + assert digest == normalized_text_sha256(surface) + assert normalized_text_sha256("Hello World") == digest + + +def test_text_collision_fails_controlled_preflight(monkeypatch, tmp_path, capsys): + train = [_row(f"train row {i}") for i in range(3)] + collided = _row("train row 1", split="val", task="unbind", lineage="none") + manifest = _manifest(tmp_path / "holdout.json", [collided]) + _arm(monkeypatch, tmp_path, train, manifest) + trunk = tmp_path / "trunk" + trunk.mkdir() + (trunk / "config.json").write_text("{}\n", encoding="utf-8") + monkeypatch.setenv("HYPERLEX_TRUNK_DIR", str(trunk)) + monkeypatch.setattr("hyperlexical.preflight.export_dataset", lambda *_a, **_k: (_ for _ in ()).throw(AssertionError("export"))) + assert preflight_main() == 2 + report = json.loads(capsys.readouterr().out) + assert report["status"] == "ADMISSION_FAIL" + assert "CONTROLLED_RESERVE" in report["error"] + + +def test_row_id_collision_fails_controlled_preflight(monkeypatch, tmp_path, capsys): + train = [_row("unique cobalt")] + manifest = _manifest(tmp_path / "holdout.json", train) + _arm(monkeypatch, tmp_path, train, manifest) + trunk = tmp_path / "trunk" + trunk.mkdir() + (trunk / "config.json").write_text("{}\n", encoding="utf-8") + monkeypatch.setenv("HYPERLEX_TRUNK_DIR", str(trunk)) + monkeypatch.setattr( + "hyperlexical.preflight.export_dataset", + lambda *_a, **_k: (_ for _ in ()).throw(AssertionError("export")), + ) + assert preflight_main() == 2 + report = json.loads(capsys.readouterr().out) + assert report["status"] == "ADMISSION_FAIL" + assert "CONTROLLED_RESERVE" in report["error"] + + +def test_disjoint_holdout_is_admissible(monkeypatch, tmp_path, capsys): + train = [_row(f"train row {i}") for i in range(4)] + hold = _row("fresh holdout phrase", split="val") + manifest = _manifest(tmp_path / "holdout.json", [hold]) + _arm(monkeypatch, tmp_path, train, manifest) + trunk = tmp_path / "trunk" + trunk.mkdir() + (trunk / "config.json").write_text("{}\n", encoding="utf-8") + monkeypatch.setenv("HYPERLEX_TRUNK_DIR", str(trunk)) + monkeypatch.setattr( + "hyperlexical.preflight.export_dataset", + lambda *_a, **_k: (_ for _ in ()).throw(AssertionError("export")), + ) + assert preflight_main() == 2 + report = json.loads(capsys.readouterr().out) + assert report["status"] == "ADMISSION_FAIL" + assert report["status"] != "TRAINING_READY" + assert "CONTROLLED_RESERVE" in report["error"] + + +def test_pinned_disjoint_holdout_keeps_all_9150_rows(monkeypatch, tmp_path): + rows = [_row(f"pinned sentence {i}") for i in range(9150)] + hold = _row("outside the pin", split="val", task="unbind", lineage="none") + manifest = _manifest(tmp_path / "holdout.json", [hold]) + _arm(monkeypatch, tmp_path, rows, manifest) + called = {"n": 0} + + def _export(*_args, **_kwargs): + called["n"] += 1 + raise AssertionError("export_dataset") + + bundle = load_training_bundle( + tmp_path, + include_live=True, + live_store=None, + export_dataset=_export, + ) + assert called["n"] == 0 + assert len(bundle["rows"]) == 9150 + spec = load_holdout_spec() + report = assert_pinned_holdout_disjoint(bundle["rows"], spec) + kept, removed = filter_holdout_rows(list(bundle["rows"]), spec) + assert report["holdout_filter_training_rows_removed"] == 0 + assert removed == 0 + assert len(kept) == 9150 + + +def test_colliding_holdout_never_reaches_the_trainer(monkeypatch, tmp_path): + train = [_row("shared surface")] + hold = _row("Shared Surface", split="val") + manifest = _manifest(tmp_path / "holdout.json", [hold]) + _arm(monkeypatch, tmp_path, train, manifest) + bundle = load_training_bundle( + tmp_path, + include_live=True, + live_store=None, + export_dataset=lambda *_a, **_k: (_ for _ in ()).throw(AssertionError("export")), + ) + spec = load_holdout_spec() + reached = {"trainer": False} + with pytest.raises(SystemExit, match="ADMISSION FAIL"): + assert_pinned_holdout_disjoint(bundle["rows"], spec) + reached["trainer"] = True + assert reached["trainer"] is False + kept, removed = filter_holdout_rows(list(bundle["rows"]), spec) + assert removed == 1 + assert len(kept) == 0 + + +def test_abandoned_lifecycle_is_not_admissible(monkeypatch, tmp_path): + manifest = _manifest(tmp_path / "holdout.json", [_row("historical phrase", split="val")]) + digest = _sha(manifest) + before = manifest.read_bytes() + (tmp_path / "holdout-lifecycle.json").write_text( + json.dumps( + { + "schema": "hyperlex.holdout_lifecycle.v1", + "status": "UNSCORED_ABANDONED", + "reason": "EXPERIMENT_CLOSED_BEFORE_VALID_EXECUTION", + "manifest_sha256": digest, + "reusable": False, + "scored": False, + } + ), + encoding="utf-8", + ) + monkeypatch.setenv("HYPERLEX_ALLOW_TRAIN", "1") + monkeypatch.setenv("HLX_EXPERIMENT_ID", "HLX-EXP-TEST") + monkeypatch.setenv("HLX_HOLDOUT_MANIFESTS", str(manifest)) + with pytest.raises(SystemExit, match="UNSCORED_ABANDONED"): + require_holdout_for_training() + assert manifest.read_bytes() == before + + +def test_census_excludes_text_collisions_and_abandoned_ids(): + train = [_row("shared surface"), _row("train only")] + abandoned = _row("abandoned phrase", split="val") + fresh = _row("fresh phrase", split="val", task="unbind", lineage="none") + source = [ + _row("test row", split="test"), + train[0], + _row("Shared Surface", split="val", cls="INFERRED"), + abandoned, + fresh, + ] + report = census(source, train, extra_rows=[abandoned]) + assert report["source_rows"] == 5 + assert report["excluded_by_split_test"] == 1 + assert report["excluded_by_training_row_id"] == 1 + assert report["excluded_by_training_normalized_text_hash"] == 1 + assert report["excluded_by_spent_or_abandoned"] == 1 + assert report["eligible_rows"] == 1 + assert report["unbind_eligible"] == 1 + assert report["waterfall_sums_to_source"] is True diff --git a/tests/shadow/test_hyperlexical_holdout_guard.py b/tests/shadow/test_hyperlexical_holdout_guard.py new file mode 100644 index 00000000..af1e24a5 --- /dev/null +++ b/tests/shadow/test_hyperlexical_holdout_guard.py @@ -0,0 +1,431 @@ +"""Holdout manifests stay out of unbind train, val, vocab, and injection.""" + +from __future__ import annotations + +import hashlib +import json +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "scripts" / "shadow")) + +from hyperlexical import train as train_mod # noqa: E402 +from hyperlexical.heldout_census import normalize_group_text # noqa: E402 +from hyperlexical.holdout_guard import ( # noqa: E402 + assert_no_holdout, + holdout_receipt, + load_holdout_spec, + normalized_text_sha256, + require_holdout_for_training, +) +from hyperlexical.layout import label_maps # noqa: E402 +from hyperlexical.loop import prepare_unbind_splits, run_loop # noqa: E402 +from hyperlexical.selection_surface import row_id # noqa: E402 +from hyperlexical.unbind_recipe import ( # noqa: E402 + apply_unbind_force_train, + load_hard_atom_texts, + unbind_row_matches_hard_atoms, +) + + +@pytest.fixture(autouse=True) +def _clear_holdout_env(monkeypatch): + for key in ( + "HLX_HOLDOUT_MANIFESTS", + "HLX_ALLOW_NO_HOLDOUT", + "HLX_EXPERIMENT_ID", + "HLX_TRAIN_EXPORT_PATH", + "HLX_TRAIN_EXPORT_SHA256", + "HLX_TRAIN_EXPORT_ROWS", + "HYPERLEX_ALLOW_TRAIN", + "HYPERLEX_RELEASE_SET", + "HYPERLEX_TASK_ROUTING", + "HYPERLEX_FILLER_FILTER", + "HYPERLEX_UNBIND_FORCE_TRAIN_PATH", + "HYPERLEX_UNBIND_HARD_ATOMS_PATH", + "HYPERLEX_UNBIND_HARD_UPSAMPLE", + "HYPERLEX_UNBIND_OBSERVED_UPSAMPLE", + "HYPERLEX_UNBIND_INFERRED_CAP", + "HYPERLEX_EXPORT_DIR", + ): + monkeypatch.delenv(key, raising=False) + + +def _row(text, fillers, *, split="val", cls="OBSERVED", task="unbind"): + return { + "text": text, + "fillers": list(fillers), + "roles": [f"pos_{i}" for i in range(len(fillers))], + "role_scheme": "positional", + "class": cls, + "split": split, + "task": task, + "lineage": "none", + } + + +def _write_manifest( + path: Path, + *, + row_ids=(), + text_hashes=(), + status=None, + experiment_id=None, +) -> Path: + body = { + "schema": "hyperlex.holdout_manifest.v2", + "row_ids": list(row_ids), + "normalized_text_sha256": list(text_hashes), + } + if status is not None: + body["status"] = status + if experiment_id is not None: + body["experiment_id"] = experiment_id + path.write_text(json.dumps(body), encoding="utf-8") + return path + + +def _arm(monkeypatch, path: Path) -> None: + monkeypatch.setenv("HLX_HOLDOUT_MANIFESTS", str(path)) + + +def test_id_match_removed_from_val(monkeypatch, tmp_path): + secret = _row("blue quartz lantern", ["quartz"], split="val") + keep = _row("north cobble path", ["cobble"], split="val") + manifest = _write_manifest( + tmp_path / "ids.json", + row_ids=[row_id(secret)], + text_hashes=[normalized_text_sha256("unrelated copper kettle")], + ) + _arm(monkeypatch, manifest) + _train, val, stats = prepare_unbind_splits([secret, keep]) + assert [r["text"] for r in val] == ["north cobble path"] + assert stats["n_holdout_removed_val"] == 1 + assert stats["n_holdout_removed_train"] == 0 + assert stats["holdout_manifests"][0]["sha256"] == hashlib.sha256(manifest.read_bytes()).hexdigest() + assert stats["holdout_manifests"][0]["n_row_ids"] == 1 + + +def test_text_hash_removed_from_train_val_and_vocab(monkeypatch, tmp_path): + surface = "Blue Quartz Lantern!" + assert normalize_group_text(surface) == "blue quartz lantern" + digest = hashlib.sha256(normalize_group_text(surface).encode("utf-8")).hexdigest() + assert digest == normalized_text_sha256(surface) + val_row = _row(surface, ["quartz"], split="val") + train_row = _row("blue quartz lantern", ["quartz"], split="train") + keep = _row("north cobble path", ["cobble"], split="train") + assert row_id(val_row) != row_id(train_row) + assert row_id(val_row) != "0123456789abcdef" + assert row_id(train_row) != "0123456789abcdef" + manifest = _write_manifest( + tmp_path / "hashes.json", + row_ids=["0123456789abcdef"], + text_hashes=[digest], + ) + _arm(monkeypatch, manifest) + train, val, stats = prepare_unbind_splits([train_row, val_row, keep]) + assert [r["text"] for r in val] == [] + assert [r["text"] for r in train] == ["north cobble path"] + assert stats["n_holdout_removed_val"] == 1 + assert stats["n_holdout_removed_train"] == 1 + vocab = label_maps(train + val)["filler_vocab"] + assert "quartz" not in vocab + assert "cobble" in vocab + + +def test_force_train_cannot_reinject(monkeypatch, tmp_path): + holdout = _row("blue quartz lantern", ["quartz"], split="val") + keep = _row("north cobble path", ["cobble"], split="train") + force = tmp_path / "force.jsonl" + force.write_text( + json.dumps({"text": "blue quartz lantern", "role_scheme": "positional"}) + "\n", + encoding="utf-8", + ) + monkeypatch.setenv("HYPERLEX_UNBIND_FORCE_TRAIN_PATH", str(force)) + moved, _val, moved_stats = apply_unbind_force_train([keep], [holdout]) + assert moved_stats["n_unbind_force_train"] == 1 + assert any(r["text"] == "blue quartz lantern" for r in moved) + manifest = _write_manifest( + tmp_path / "force-holdout.json", + text_hashes=[normalized_text_sha256(holdout["text"])], + ) + _arm(monkeypatch, manifest) + train, val, stats = prepare_unbind_splits([keep, holdout]) + assert all(r["text"] != "blue quartz lantern" for r in train + val) + assert stats["n_unbind_force_train"] == 0 + assert stats["n_holdout_removed_val"] == 1 + + +def test_hard_atoms_cannot_copy(monkeypatch, tmp_path): + holdout = _row("blue quartz lantern", ["quartz"], split="train") + keep = _row("north cobble path", ["cobble"], split="train") + atoms = tmp_path / "atoms.jsonl" + atoms.write_text(json.dumps({"text": "blue quartz lantern"}) + "\n", encoding="utf-8") + monkeypatch.setenv("HYPERLEX_UNBIND_HARD_ATOMS_PATH", str(atoms)) + monkeypatch.setenv("HYPERLEX_UNBIND_HARD_UPSAMPLE", "4") + assert unbind_row_matches_hard_atoms(holdout, load_hard_atom_texts(atoms)) + manifest = _write_manifest(tmp_path / "atoms-holdout.json", row_ids=[row_id(holdout)]) + _arm(monkeypatch, manifest) + train, val, stats = prepare_unbind_splits([holdout, keep]) + assert [r["text"] for r in train] == ["north cobble path"] + assert val == [] + assert stats["n_unbind_hard_extra_copies"] == 0 + assert stats["n_holdout_removed_train"] == 1 + assert "quartz" not in label_maps(train + val)["filler_vocab"] + + +def test_two_manifests_union(monkeypatch, tmp_path): + by_id = _row("north cobble path", ["cobble"], split="val") + by_text = _row("blue quartz lantern", ["quartz"], split="train") + first = _write_manifest(tmp_path / "a.json", row_ids=[row_id(by_id)]) + second = _write_manifest( + tmp_path / "b.json", + text_hashes=[normalized_text_sha256(by_text["text"])], + ) + monkeypatch.setenv("HLX_HOLDOUT_MANIFESTS", f"{first}, {second}") + train, val, stats = prepare_unbind_splits([by_id, by_text]) + assert train == [] and val == [] + assert stats["n_holdout_removed_val"] == 1 + assert stats["n_holdout_removed_train"] == 1 + receipt = holdout_receipt( + load_holdout_spec(), + {"classify_train": 0, "classify_val": 0, "unbind_train": 1, "unbind_val": 1}, + ) + assert receipt["manifest_sha256"] == [ + hashlib.sha256(first.read_bytes()).hexdigest(), + hashlib.sha256(second.read_bytes()).hexdigest(), + ] + assert receipt["removed"]["unbind_val"] == 1 + assert receipt["allow_no_holdout"] is False + + +def test_overlap_assertion_names_the_split_and_hides_text(tmp_path): + row = _row("blue quartz lantern", ["quartz"], split="val") + manifest = _write_manifest(tmp_path / "overlap.json", row_ids=[row_id(row)]) + spec = load_holdout_spec(str(manifest)) + assert_no_holdout([], spec, "unbind val") + with pytest.raises(SystemExit, match="unbind val") as raised: + assert_no_holdout([row], spec, "unbind val") + message = str(raised.value) + assert "row_id" in message + assert "blue quartz lantern" not in message + + +def test_prepare_asserts_when_filter_is_bypassed(monkeypatch, tmp_path): + row = _row("blue quartz lantern", ["quartz"], split="val") + manifest = _write_manifest(tmp_path / "bypass.json", row_ids=[row_id(row)]) + _arm(monkeypatch, manifest) + monkeypatch.setattr( + "hyperlexical.loop.filter_holdout_rows", + lambda rows, spec: (list(rows), 0), + ) + with pytest.raises(SystemExit, match="holdout overlap remains in unbind val"): + prepare_unbind_splits([row]) + + +def test_manifest_refuses_embedded_rows_bad_hash_and_missing(tmp_path): + embedded = tmp_path / "embedded.json" + embedded.write_text( + json.dumps({"text": "plain synthetic sentence", "fillers": ["plain"], "task": "unbind"}), + encoding="utf-8", + ) + with pytest.raises(SystemExit, match="row content"): + load_holdout_spec(str(embedded)) + bad = _write_manifest(tmp_path / "bad.json", text_hashes=["abcd"]) + with pytest.raises(SystemExit, match="64-char hex"): + load_holdout_spec(str(bad)) + with pytest.raises(SystemExit, match="not a file"): + load_holdout_spec(str(tmp_path / "missing.json")) + + +def test_run_loop_fails_closed_without_manifest(monkeypatch, tmp_path): + monkeypatch.setenv("HYPERLEX_ALLOW_TRAIN", "1") + + def _export(*_args, **_kwargs): + raise AssertionError("export ran") + + monkeypatch.setattr("hyperlexical.loop.export_dataset", _export) + with pytest.raises(SystemExit, match="HLX_HOLDOUT_MANIFESTS"): + run_loop(tmp_path, tmp_path / "out") + + +def test_gate_without_run_does_not_require_manifest(monkeypatch, tmp_path): + monkeypatch.setenv("HYPERLEX_ALLOW_TRAIN", "1") + monkeypatch.setenv("HYPERLEX_TRUNK_DIR", str(tmp_path)) + (tmp_path / "config.json").write_text("{}\n", encoding="utf-8") + assert train_mod.main(["--offline"]) == 0 + + +def test_train_run_fails_closed_without_manifest(monkeypatch, tmp_path): + monkeypatch.setenv("HYPERLEX_ALLOW_TRAIN", "1") + monkeypatch.setenv("HYPERLEX_TRUNK_DIR", str(tmp_path)) + (tmp_path / "config.json").write_text("{}\n", encoding="utf-8") + with pytest.raises(SystemExit, match="no holdout manifest"): + train_mod.main(["--offline", "--run"]) + + +def test_allow_no_holdout_reaches_training(monkeypatch, tmp_path, capsys): + monkeypatch.setenv("HYPERLEX_ALLOW_TRAIN", "1") + monkeypatch.setenv("HLX_ALLOW_NO_HOLDOUT", "1") + monkeypatch.setenv("HYPERLEX_TRUNK_DIR", str(tmp_path)) + (tmp_path / "config.json").write_text("{}\n", encoding="utf-8") + + def _export(*_args, **_kwargs): + raise RuntimeError("past-holdout") + + monkeypatch.setattr("hyperlexical.loop.export_dataset", _export) + assert train_mod.main(["--offline", "--run"]) == 4 + assert "past-holdout" in capsys.readouterr().err + + +def test_cli_manifest_and_run_loop_log(monkeypatch, tmp_path, capsys): + surface = "Blue Quartz Lantern!" + classify = _row(surface, ["quartz"], split="val", task="classify") + unbind = _row("north cobble path", ["cobble"], split="val") + manifest = _write_manifest( + tmp_path / "cli.json", + row_ids=[row_id(unbind)], + text_hashes=[normalized_text_sha256(surface)], + status="UNSCORED_SEALED", + experiment_id="HLX-EXP-TEST", + ) + monkeypatch.setenv("HLX_EXPERIMENT_ID", "HLX-EXP-TEST") + monkeypatch.setenv("HYPERLEX_ALLOW_TRAIN", "1") + monkeypatch.setenv("HYPERLEX_TRUNK_DIR", str(tmp_path)) + monkeypatch.setenv("HYPERLEX_EXPORT_DIR", str(tmp_path / "export")) + (tmp_path / "config.json").write_text("{}\n", encoding="utf-8") + pinned = tmp_path / "pinned.jsonl" + payload = "".join(json.dumps(row, sort_keys=True) + "\n" for row in (classify, unbind)) + pinned.write_text(payload, encoding="utf-8") + monkeypatch.setenv("HLX_TRAIN_EXPORT_PATH", str(pinned)) + monkeypatch.setenv("HLX_TRAIN_EXPORT_SHA256", hashlib.sha256(pinned.read_bytes()).hexdigest()) + + def _export(*_args, **_kwargs): + raise AssertionError("export_dataset must not run for a pinned experiment") + + written = {"n": 0} + + def _write(*_args, **_kwargs): + written["n"] += 1 + + monkeypatch.setattr("hyperlexical.loop.export_dataset", _export) + monkeypatch.setattr("hyperlexical.loop.write_export", _write) + with pytest.raises(SystemExit, match="ADMISSION FAIL"): + train_mod.main(["--offline", "--run", "--holdout-manifest", str(manifest)]) + assert written["n"] == 0 + captured = capsys.readouterr() + assert "blue quartz" not in captured.out + assert surface not in captured.out + assert surface not in captured.err + + +def _admit(monkeypatch, path: Path, experiment_id: str = "HLX-EXP-TEST") -> None: + monkeypatch.setenv("HYPERLEX_ALLOW_TRAIN", "1") + monkeypatch.setenv("HLX_EXPERIMENT_ID", experiment_id) + monkeypatch.setenv("HLX_HOLDOUT_MANIFESTS", str(path)) + + +def test_unscored_sealed_with_matching_binding_is_admissible(monkeypatch, tmp_path): + manifest = _write_manifest( + tmp_path / "fresh.json", + row_ids=["abc123"], + status="UNSCORED_SEALED", + experiment_id="HLX-EXP-TEST", + ) + _admit(monkeypatch, manifest) + spec = require_holdout_for_training() + assert spec.row_ids == frozenset({"abc123"}) + assert spec.manifests[0]["status"] == "UNSCORED_SEALED" + assert spec.manifests[0]["experiment_id"] == "HLX-EXP-TEST" + + +@pytest.mark.parametrize("status", ["SCORED_SPENT", "SCORED", "FROZEN_NOT_SCORED"]) +def test_spent_scored_and_unknown_status_are_rejected(monkeypatch, tmp_path, status): + manifest = _write_manifest( + tmp_path / "closed.json", + row_ids=["abc123"], + status=status, + experiment_id="HLX-EXP-TEST", + ) + _admit(monkeypatch, manifest) + with pytest.raises(SystemExit, match="not admissible"): + require_holdout_for_training() + + +def test_missing_status_is_rejected(monkeypatch, tmp_path): + manifest = _write_manifest(tmp_path / "bare.json", row_ids=["abc123"]) + _admit(monkeypatch, manifest) + with pytest.raises(SystemExit, match="status is missing"): + require_holdout_for_training() + + +def test_wrong_experiment_binding_is_rejected(monkeypatch, tmp_path): + manifest = _write_manifest( + tmp_path / "wrong.json", + row_ids=["abc123"], + status="UNSCORED_SEALED", + experiment_id="HLX-EXP-OTHER", + ) + _admit(monkeypatch, manifest, "HLX-EXP-TEST") + with pytest.raises(SystemExit, match="experiment binding"): + require_holdout_for_training() + + +def test_unbound_experiment_is_rejected(monkeypatch, tmp_path): + manifest = _write_manifest( + tmp_path / "unbound.json", + row_ids=["abc123"], + status="UNSCORED_SEALED", + ) + monkeypatch.setenv("HYPERLEX_ALLOW_TRAIN", "1") + monkeypatch.setenv("HLX_HOLDOUT_MANIFESTS", str(manifest)) + with pytest.raises(SystemExit, match="experiment binding"): + require_holdout_for_training() + + +def test_no_holdout_is_rejected_when_override_is_unset(monkeypatch): + monkeypatch.setenv("HYPERLEX_ALLOW_TRAIN", "1") + with pytest.raises(SystemExit, match="no holdout manifest"): + require_holdout_for_training() + + +def test_spent_companion_does_not_admit(monkeypatch, tmp_path): + fresh = _write_manifest( + tmp_path / "fresh.json", + row_ids=["abc123"], + status="UNSCORED_SEALED", + experiment_id="HLX-EXP-TEST", + ) + spent = _write_manifest( + tmp_path / "spent.json", + row_ids=["def456"], + status="SCORED_SPENT", + experiment_id="HLX-EXP-TEST", + ) + monkeypatch.setenv("HYPERLEX_ALLOW_TRAIN", "1") + monkeypatch.setenv("HLX_EXPERIMENT_ID", "HLX-EXP-TEST") + monkeypatch.setenv("HLX_HOLDOUT_MANIFESTS", f"{fresh},{spent}") + with pytest.raises(SystemExit, match="SCORED_SPENT"): + require_holdout_for_training() + + +def test_spent_manifest_still_excludes_rows(monkeypatch, tmp_path): + secret = _row("blue quartz lantern", ["quartz"], split="val") + keep = _row("north cobble path", ["cobble"], split="val") + manifest = _write_manifest( + tmp_path / "spent.json", + row_ids=[row_id(secret)], + status="SCORED_SPENT", + experiment_id="HLX-EXP-TEST", + ) + _arm(monkeypatch, manifest) + _train, val, stats = prepare_unbind_splits([secret, keep]) + assert [r["text"] for r in val] == ["north cobble path"] + assert stats["n_holdout_removed_val"] == 1 + monkeypatch.setenv("HYPERLEX_ALLOW_TRAIN", "1") + monkeypatch.setenv("HLX_EXPERIMENT_ID", "HLX-EXP-TEST") + with pytest.raises(SystemExit, match="SCORED_SPENT"): + require_holdout_for_training() diff --git a/tests/shadow/test_hyperlexical_holdout_wiring.py b/tests/shadow/test_hyperlexical_holdout_wiring.py new file mode 100644 index 00000000..d0c7d513 --- /dev/null +++ b/tests/shadow/test_hyperlexical_holdout_wiring.py @@ -0,0 +1,29 @@ +"""Fail-closed wiring: the training entrypoints must keep calling the holdout guard. + +Static check so a refactor cannot silently drop the guard (P0, 2026-10-12).""" + +from pathlib import Path + +PKG = Path(__file__).resolve().parents[2] / "scripts" / "shadow" / "hyperlexical" + + +def test_loop_admits_and_filters_holdout(): + src = (PKG / "loop.py").read_text(encoding="utf-8") + assert "admit_training_run(" in src + assert "filter_holdout_rows(train, spec)" in src + assert "assert_no_holdout(" in src + + +def test_admission_requires_holdout(): + assert "require_holdout_for_training()" in (PKG / "admission.py").read_text(encoding="utf-8") + + +def test_train_entrypoint_runs_through_loop(): + src = (PKG / "train.py").read_text(encoding="utf-8") + assert "merge_manifest_env" in src and "run_loop(" in src + + +def test_no_new_trainer_bypasses_the_guard(): + allowed = {"train.py", "train_input.py", "training_routing.py", "training_schedule.py"} + for path in PKG.glob("train*.py"): + assert path.name in allowed, f"new training entrypoint {path.name}: route it through loop.run_loop/admission" From 098c7f707671fc47c4e1fbf7feccb831f941fbf8 Mon Sep 17 00:00:00 2001 From: scrimshawlife-ctrl Date: Mon, 12 Oct 2026 03:54:03 -0700 Subject: [PATCH 2/4] feat(eval): run_report.v1 for eval_classify/eval_unbind (report-only, HLX_RUN_REPORT) + parse validity separate from task score --- .github/workflows/holdout-guard.yml | 2 +- scripts/shadow/hyperlexical/eval_classify.py | 543 ++++++++++++++++++ scripts/shadow/hyperlexical/eval_unbind.py | 7 +- scripts/shadow/hyperlexical/run_report.py | 97 ++++ .../test_hyperlexical_eval_run_report.py | 50 ++ tests/shadow/test_hyperlexical_run_report.py | 99 ++++ 6 files changed, 796 insertions(+), 2 deletions(-) create mode 100644 scripts/shadow/hyperlexical/eval_classify.py create mode 100644 scripts/shadow/hyperlexical/run_report.py create mode 100644 tests/shadow/test_hyperlexical_eval_run_report.py create mode 100644 tests/shadow/test_hyperlexical_run_report.py diff --git a/.github/workflows/holdout-guard.yml b/.github/workflows/holdout-guard.yml index 223f85f7..fb3d40cf 100644 --- a/.github/workflows/holdout-guard.yml +++ b/.github/workflows/holdout-guard.yml @@ -18,4 +18,4 @@ jobs: with: python-version: ${{ matrix.python-version }} - run: python -m pip install --upgrade pip pytest - - run: python -m pytest -q tests/shadow/test_hyperlexical_holdout_disjoint.py tests/shadow/test_hyperlexical_holdout_guard.py tests/shadow/test_hyperlexical_holdout_wiring.py tests/heldout + - run: python -m pytest -q tests/shadow tests/heldout diff --git a/scripts/shadow/hyperlexical/eval_classify.py b/scripts/shadow/hyperlexical/eval_classify.py new file mode 100644 index 00000000..56deacdc --- /dev/null +++ b/scripts/shadow/hyperlexical/eval_classify.py @@ -0,0 +1,543 @@ +"""Classify harness. Predictions plus an eval split, against three baselines. + +Does not train Hyperlexical weights and does not read a held-out manifest. +Slices are reporting-only: they are not a checkpoint rule. + +Baselines, all scored on the eval rows and refit only on the train split +passed in (never on eval text or eval labels): + (a) bag-of-terms logistic regression, two fits of the same guarded split + - ``bow_lr_nonnone``: non-none train rows only, output space the 8 families + - ``bow_lr_with_none``: every train row, including ``none`` + ``--bow-nonnone`` defaults to auto: on when eval gold has no ``none``. + On, ``bow_lr_nonnone`` is the gated baseline and ``bow_lr_with_none`` + is reference. Off, those roles swap. Both still get an exact McNemar. + (b) registry rule ``match_lineage`` (lexical, vector re-rank off) + (c) Spec 004 linear probe, refit on this split when the rows carry span + encodings; otherwise NOT_COMPUTABLE + +sklearn is not a project dependency. The logistic regression is pure Python +(zero init, full-batch softmax cross-entropy) so the public test job can run +it without numpy. +""" + +from __future__ import annotations + +import argparse +import importlib +import json +import math +import sys +from pathlib import Path +from typing import Any, Mapping, Sequence + +from .classify_metrics import ( + NONE_LABEL, + accuracy, + macro_f1_nonnone, + mcnemar_paired, + none_rates, + per_class_table, +) +from .layout import FAMILIES + +NONNONE_FAMILIES = tuple(name for name in FAMILIES if name != NONE_LABEL) + +SLICES_WARNING = "slices are reporting-only" +_PRED_FIELDS = ("pred", "lineage_pred", "prediction") +_GOLD_FIELDS = ("lineage", "label") +_LOGREG_STEPS = 400 +_LOGREG_LR = 1.0 +_LOGREG_L2 = 1e-2 +_SPEC004_REASON = ( + "Spec 004 linear probe cannot be refit on this split: " + "recoverable_structure.fit.fit_scheme needs span item_ids and encoder " + "encodings, which these rows do not carry" +) + + +def _read_jsonl(path: Path) -> list[dict[str, Any]]: + if not path.is_file(): + raise SystemExit(f"REFUSE: not a file: {path}") + rows: list[dict[str, Any]] = [] + for index, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + if not line.strip(): + continue + try: + obj = json.loads(line) + except json.JSONDecodeError as exc: + raise SystemExit(f"REFUSE: {path} line {index} is not JSON") from exc + if not isinstance(obj, dict): + raise SystemExit(f"REFUSE: {path} line {index} must be an object") + rows.append(obj) + return rows + + +def _join_key(row: Mapping[str, Any]) -> tuple[str, str]: + for field in ("id", "row_id"): + token = row.get(field) + if isinstance(token, str) and token.strip(): + return ("id", token.strip()) + text = row.get("text") + if isinstance(text, str) and text.strip(): + return ("text", text) + raise SystemExit("REFUSE: row needs id, row_id, or text") + + +def _field(row: Mapping[str, Any], names: Sequence[str], what: str) -> str: + for name in names: + token = row.get(name) + if isinstance(token, str) and token.strip(): + return token.strip() + raise SystemExit(f"REFUSE: {what} needs one of {', '.join(names)}") + + +def _tags(row: Mapping[str, Any]) -> set[str]: + raw = row.get("tags") + if raw is None: + return set() + if isinstance(raw, str): + token = raw.strip() + return {token} if token else set() + if not isinstance(raw, list): + return set() + return {str(item).strip() for item in raw if str(item).strip()} + + +def _terms(text: str) -> list[str]: + return [tok.casefold() for tok in str(text or "").split() if tok] + + +def resolve_bow_nonnone(gold: Sequence[str], raw: str | None = None) -> tuple[bool, str]: + """Whether ``bow_lr_nonnone`` is the gated baseline. + + ``auto`` (the default) is on exactly when eval gold contains no ``none``. + The decision reads eval labels only to choose which already-fit baseline + is gated. It does not choose features, classes, or steps. + """ + mode = "auto" if raw is None else str(raw).strip().lower() + if mode in {"", "auto"}: + has_none = any(label == NONE_LABEL for label in gold) + if has_none: + return False, "eval gold includes none" + return True, "eval gold has no none" + if mode in {"1", "on", "true"}: + return True, "forced on" + if mode in {"0", "off", "false"}: + return False, "forced off" + raise SystemExit( + "REFUSE: --bow-nonnone must be auto, on, or off, " + f"got {raw!r}" + ) + + +def fit_bag_of_terms_logreg( + train_texts: Sequence[str], + train_labels: Sequence[str], + eval_texts: Sequence[str], + *, + classes: Sequence[str] | None = None, + steps: int = _LOGREG_STEPS, + lr: float = _LOGREG_LR, + l2: float = _LOGREG_L2, +) -> tuple[list[str] | None, dict[str, Any]]: + """Binary bag-of-terms logistic regression. + + Vocab and rows come from the train arguments only. ``eval_texts`` is scored + after the fit. A fixed ``classes`` list (the 8 families) drops train rows + whose label is outside that list and does not add labels from eval. + Steps, learning rate, and L2 are constants. + """ + if len(train_texts) != len(train_labels): + raise ValueError("train texts and labels differ in length") + n_train = len(train_labels) + if classes is None: + paired = list(zip(train_texts, train_labels, strict=True)) + class_list = sorted({label for _text, label in paired}) + empty_reason = "train split is empty" + else: + allowed = set(classes) + paired = [(text, label) for text, label in zip(train_texts, train_labels, strict=True) if label in allowed] + class_list = list(classes) + empty_reason = "no train rows in the requested class list" + base_meta = { + "n_train_rows": n_train, + "n_train_rows_used": len(paired), + "classes": class_list, + "eval_used_for_fit": False, + "steps": steps, + } + if not paired or not class_list: + return None, {"status": "NOT_COMPUTABLE", "reason": empty_reason, **base_meta} + used_texts = [text for text, _label in paired] + used_labels = [label for _text, label in paired] + vocab = sorted({tok for text in used_texts for tok in _terms(text)}) + index = {tok: i for i, tok in enumerate(vocab)} + width = len(vocab) + 1 + n_class = len(class_list) + class_of = {label: i for i, label in enumerate(class_list)} + weights = [[0.0] * width for _ in range(n_class)] + + def featurize(texts: Sequence[str]) -> list[list[float]]: + rows = [] + for text in texts: + vec = [0.0] * width + for tok in _terms(text): + slot = index.get(tok) + if slot is not None: + vec[slot] = 1.0 + vec[-1] = 1.0 + rows.append(vec) + return rows + + features = featurize(used_texts) + targets = [class_of[label] for label in used_labels] + n = float(len(features)) + for _step in range(steps): + grad = [[0.0] * width for _ in range(n_class)] + for vec, target in zip(features, targets): + logits = [sum(w * x for w, x in zip(weights[c], vec)) for c in range(n_class)] + pivot = max(logits) + exps = [math.exp(item - pivot) for item in logits] + total = sum(exps) or 1.0 + for c in range(n_class): + err = exps[c] / total - (1.0 if c == target else 0.0) + if err == 0.0: + continue + for j, value in enumerate(vec): + if value: + grad[c][j] += err * value + for c in range(n_class): + for j in range(width): + penalty = 0.0 if j == width - 1 else l2 * weights[c][j] + weights[c][j] -= lr * (grad[c][j] / n + penalty) + preds: list[str] = [] + for vec in featurize(eval_texts): + logits = [sum(w * x for w, x in zip(weights[c], vec)) for c in range(n_class)] + preds.append(class_list[max(range(n_class), key=lambda c: (logits[c], -c))]) + return preds, { + "status": "OK", + "n_features": len(vocab), + **base_meta, + } + + +def _registry_preds(texts: Sequence[str]) -> tuple[list[str] | None, dict[str, Any]]: + """Lexical registry rule. Imported lazily so this package stays import-free.""" + try: + match_lineage = importlib.import_module("hyperlex.analysis").match_lineage + except Exception as exc: + return None, { + "status": "NOT_COMPUTABLE", + "reason": f"registry matcher could not be imported: {type(exc).__name__}: {exc}", + } + preds: list[str] = [] + for text in texts: + try: + hit = match_lineage(text, use_vector=False) + except Exception as exc: + return None, { + "status": "NOT_COMPUTABLE", + "reason": f"match_lineage failed: {type(exc).__name__}: {exc}", + } + if not hit: + preds.append(NONE_LABEL) + continue + family = hit.get("family_id") + preds.append(str(family) if family else NONE_LABEL) + return preds, {"status": "OK", "rule": "match_lineage", "use_vector": False} + + +def _span_ready(rows: Sequence[Mapping[str, Any]]) -> bool: + width: int | None = None + if not rows: + return False + for row in rows: + items = row.get("item_ids") + encoding = row.get("encoding") + if not isinstance(items, list) or not items or not all(isinstance(item, str) for item in items): + return False + if ( + not isinstance(encoding, list) + or not encoding + or not all(isinstance(value, (int, float)) and not isinstance(value, bool) for value in encoding) + ): + return False + if width is None: + width = len(encoding) + elif len(encoding) != width: + return False + return True + + +def _import_probe(): + try: + from recoverable_structure.fit import fit_scheme, swap_accuracy + except ImportError: + shadow = Path(__file__).resolve().parents[1] + if str(shadow) not in sys.path: + sys.path.insert(0, str(shadow)) + from recoverable_structure.fit import fit_scheme, swap_accuracy + return fit_scheme, swap_accuracy + + +def _spec004_preds( + train_rows: Sequence[Mapping[str, Any]], + eval_rows: Sequence[Mapping[str, Any]], +) -> tuple[list[str] | None, dict[str, Any]]: + if not _span_ready(train_rows) or not _span_ready(eval_rows): + return None, {"status": "NOT_COMPUTABLE", "reason": _SPEC004_REASON} + try: + fit_scheme, swap_accuracy = _import_probe() + spans = [] + encodings = [] + for row in list(train_rows) + list(eval_rows): + spans.append( + { + "item_ids": list(row["item_ids"]), + "type_tags": list(row.get("type_tags") or []), + "label": _field(row, _GOLD_FIELDS, "span row"), + } + ) + encodings.append([float(value) for value in row["encoding"]]) + snap = { + "spans": spans, + "encodings": encodings, + "max_len": max(len(span["item_ids"]) for span in spans), + "encoder_id": "eval_classify_refit", + } + n_train = len(train_rows) + train_idx = list(range(n_train)) + test_idx = list(range(n_train, n_train + len(eval_rows))) + fitted = fit_scheme("positional", snap, train_idx, test_idx) + _acc, predict = swap_accuracy(fitted, snap, train_idx, test_idx) + preds = [str(item) for item in predict(fitted["pred_test"])] + except Exception as exc: + return None, { + "status": "NOT_COMPUTABLE", + "reason": f"Spec 004 linear probe could not be refit: {type(exc).__name__}: {exc}", + } + if len(preds) != len(eval_rows): + return None, { + "status": "NOT_COMPUTABLE", + "reason": "Spec 004 probe returned a different number of predictions than eval rows", + } + return preds, {"status": "OK", "scheme": "positional", "refit": "fit_scheme+swap_accuracy"} + + +def _baseline_block( + gold: Sequence[str], + model: Sequence[str], + preds: list[str] | None, + meta: Mapping[str, Any], +) -> dict[str, Any]: + if preds is None: + return { + "status": str(meta.get("status") or "NOT_COMPUTABLE"), + "reason": str(meta.get("reason") or ""), + "mcnemar": None, + } + return { + "status": "OK", + **{key: value for key, value in meta.items() if key != "status"}, + "accuracy": accuracy(gold, preds), + "macro_f1_nonnone": macro_f1_nonnone(gold, preds), + "mcnemar": mcnemar_paired(gold, model, preds), + } + + +def _score_block(gold: Sequence[str], pred: Sequence[str]) -> dict[str, Any]: + rates = none_rates(gold, pred) + return { + "n": len(gold), + "accuracy": accuracy(gold, pred), + "macro_f1_nonnone": macro_f1_nonnone(gold, pred), + "none_fpr": rates["none_fpr"], + "none_fnr": rates["none_fnr"], + "per_class": per_class_table(gold, pred), + } + + +def _resolve_preds_path(preds: str, checkpoint: str) -> Path: + if preds: + return Path(preds) + if checkpoint: + path = Path(checkpoint) + if path.is_file(): + return path + candidate = path / "classify-preds.jsonl" + if candidate.is_file(): + return candidate + raise SystemExit( + "NOT_COMPUTABLE: checkpoint has no classify-preds.jsonl; pass --preds" + ) + raise SystemExit("REFUSE: pass --preds or --checkpoint") + + +def _bow_pair( + train_texts: Sequence[str], + train_labels: Sequence[str], + eval_texts: Sequence[str], + gold: Sequence[str], + model: Sequence[str], + *, + nonnone_gated: bool, +) -> dict[str, dict[str, Any]]: + """Both BoW fits on one train split. Eval text is scored, never fit.""" + with_preds, with_meta = fit_bag_of_terms_logreg(train_texts, train_labels, eval_texts) + non_preds, non_meta = fit_bag_of_terms_logreg( + train_texts, + train_labels, + eval_texts, + classes=NONNONE_FAMILIES, + ) + with_block = _baseline_block(gold, model, with_preds, with_meta) + non_block = _baseline_block(gold, model, non_preds, non_meta) + with_block["role"] = "reference" if nonnone_gated else "gated" + non_block["role"] = "gated" if nonnone_gated else "reference" + return {"bow_lr_with_none": with_block, "bow_lr_nonnone": non_block} + + +def run_eval_classify( + *, + preds_path: str | Path, + eval_path: str | Path, + train_path: str | Path, + slices: Sequence[str] = (), + bow_nonnone: str | None = None, +) -> dict[str, Any]: + """Score model predictions. Train rows fit the bag-of-terms baseline only.""" + pred_rows = _read_jsonl(Path(preds_path)) + eval_rows = _read_jsonl(Path(eval_path)) + train_rows = _read_jsonl(Path(train_path)) + if not eval_rows: + raise SystemExit("REFUSE: eval split is empty") + pred_of: dict[tuple[str, str], str] = {} + for row in pred_rows: + key = _join_key(row) + if key in pred_of: + raise SystemExit(f"REFUSE: duplicate prediction key {key[1]!r}") + pred_of[key] = _field(row, _PRED_FIELDS, "prediction row") + gold: list[str] = [] + model: list[str] = [] + texts: list[str] = [] + for row in eval_rows: + key = _join_key(row) + if key not in pred_of: + raise SystemExit(f"REFUSE: missing prediction for {key[1]!r}") + gold.append(_field(row, _GOLD_FIELDS, "eval row")) + model.append(pred_of[key]) + texts.append(str(row.get("text") or "")) + train_texts = [str(row.get("text") or "") for row in train_rows] + train_labels = [_field(row, _GOLD_FIELDS, "train row") for row in train_rows] + nonnone_gated, nonnone_reason = resolve_bow_nonnone(gold, bow_nonnone) + bow = _bow_pair( + train_texts, + train_labels, + texts, + gold, + model, + nonnone_gated=nonnone_gated, + ) + rule_preds, rule_meta = _registry_preds(texts) + probe_preds, probe_meta = _spec004_preds(train_rows, eval_rows) + report: dict[str, Any] = { + "schema": "hyperlex.classify_harness.v0.1", + **_score_block(gold, model), + "macro_f1_nonnone_definition": ( + "unweighted mean of per-class F1 over gold labels other than none with n>0" + ), + "none_fpr_definition": "predicted none among gold that is not none", + "none_fnr_definition": "predicted not-none among gold none", + "bow_nonnone": nonnone_gated, + "bow_nonnone_reason": nonnone_reason, + "gated_baseline": "bow_lr_nonnone" if nonnone_gated else "bow_lr_with_none", + "baselines": { + **bow, + "match_lineage": _baseline_block(gold, model, rule_preds, rule_meta), + "spec004_linear_probe": _baseline_block(gold, model, probe_preds, probe_meta), + }, + "brier": None, + } + slice_names = [item.strip() for item in slices if item and item.strip()] + if slice_names: + print(SLICES_WARNING, flush=True) + packed = [] + for row, g, p in zip(eval_rows, gold, model): + packed.append((g, p, _tags(row))) + report["slices_warning"] = SLICES_WARNING + report["slices"] = {} + for name in slice_names: + kept = [(g, p) for g, p, tags in packed if name in tags] + if not kept: + report["slices"][name] = { + "n": 0, + "status": "NOT_COMPUTABLE", + "reason": f"no eval rows tagged {name}", + "reporting_only": True, + } + continue + block = _score_block([g for g, _p in kept], [p for _g, p in kept]) + block["reporting_only"] = True + report["slices"][name] = block + return report + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="hyperlexical-eval-classify") + parser.add_argument("--preds", default="", help="jsonl of model predictions") + parser.add_argument( + "--checkpoint", + default="", + help="checkpoint dir containing classify-preds.jsonl, or a preds jsonl path", + ) + parser.add_argument("--eval", required=True, help="eval split jsonl (gold lineage)") + parser.add_argument("--train", required=True, help="train split jsonl for the bag-of-terms baseline") + parser.add_argument( + "--slices", + default="", + help="comma-separated tags to report. Reporting-only; not a selection rule.", + ) + parser.add_argument( + "--bow-nonnone", + nargs="?", + const="on", + default="auto", + metavar="auto|on|off", + help=( + "Gated BoW baseline. auto (default) is on when eval gold has no none: " + "refit on non-none train rows over the 8 families. " + "bow_lr_with_none stays in the report as reference. " + "Pass off to gate on the with-none fit instead." + ), + ) + parser.add_argument("--out", default="") + args = parser.parse_args(argv) + preds_path = _resolve_preds_path(args.preds, args.checkpoint) + from .run_report import RunReport + + # Report-only (HLX_RUN_REPORT): never changes the eval numbers, stdout, or exit code. + with RunReport("hyperlexical/eval_classify", {"preds": str(preds_path), "eval": args.eval, "train": args.train}) as rr: + report = run_eval_classify( + preds_path=preds_path, + eval_path=args.eval, + train_path=args.train, + slices=[part for part in str(args.slices).split(",")], + bow_nonnone=args.bow_nonnone, + ) + if rr.path: + labels = [] + for row in _read_jsonl(Path(preds_path)): + labels.append(next((str(row[f]) for f in _PRED_FIELDS if row.get(f) not in (None, "")), None)) + bad = [x for x in labels if x not in FAMILIES] + rr.set_validity(len(labels) - len(bad), len(bad), {"label_not_in_families": len(bad)} if bad else None) + rr.set_scores(**{k: report[k] for k in ("macro_f1", "macro_f1_nonnone", "accuracy") if k in report}) + text = json.dumps(report, indent=2, sort_keys=True) + if args.out: + Path(args.out).write_text(text + "\n", encoding="utf-8") + else: + print(text) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/shadow/hyperlexical/eval_unbind.py b/scripts/shadow/hyperlexical/eval_unbind.py index b573aa10..1384e2f8 100644 --- a/scripts/shadow/hyperlexical/eval_unbind.py +++ b/scripts/shadow/hyperlexical/eval_unbind.py @@ -380,8 +380,13 @@ def main(argv=None) -> int: help="load local ModernBERT + trained heads and score real unbind_exact (Spark)", ) args = p.parse_args(argv) + from .run_report import RunReport + + rr = RunReport("hyperlexical/eval_unbind", {"model_dir": args.model_dir, "trunk_forward": args.trunk_forward}) try: - report = run_eval(model_dir=args.model_dir or None, trunk_forward=args.trunk_forward) + with rr: + report = run_eval(model_dir=args.model_dir or None, trunk_forward=args.trunk_forward) + rr.set_scores(**{k: report[k] for k in ("e2_pass", "brier") if k in report}) except TrunkForwardError as exc: print( json.dumps( diff --git a/scripts/shadow/hyperlexical/run_report.py b/scripts/shadow/hyperlexical/run_report.py new file mode 100644 index 00000000..e1b47c1e --- /dev/null +++ b/scripts/shadow/hyperlexical/run_report.py @@ -0,0 +1,97 @@ +import json +import os +import sys +from datetime import datetime + +ENV = "HLX_RUN_REPORT" + + +def safe_inputs(d): + """Return a copy of *d* with sensitive or large list values sanitized.""" + out = {} + for k, v in d.items(): + if k in ("text", "rows"): + continue + if isinstance(v, list) and len(v) > 50: + out[k] = f"" + else: + out[k] = v + return out + + +class RunReport: + def __init__(self, tool, inputs, path=None): + self.tool = tool + self.inputs = safe_inputs(inputs) + self.path = path if path is not None else os.environ.get(ENV) + self.counts = {} + self.scores = {} + self.validity = None + self.warnings = [] + self.status = None + self.started = None + + def __enter__(self): + self.started = datetime.now().astimezone() + return self + + def set_counts(self, **kw): + self.counts.update(kw) + + def set_scores(self, **kw): + self.scores.update(kw) + + def set_validity(self, valid, invalid, reasons=None): + total = valid + invalid + valid_rate = round(valid / total, 4) if total > 0 else None + self.validity = { + "valid": valid, + "invalid": invalid, + "valid_rate": valid_rate, + } + if reasons is not None: + self.validity["reasons"] = reasons + + def warn(self, msg): + self.warnings.append(msg) + + def set_status(self, s): + self.status = s + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.path is None: + return False + + finished = datetime.now().astimezone() + duration_ms = int(round((finished - self.started).total_seconds() * 1000)) + status = self.status if self.status is not None else ("error" if exc_type is not None else "ok") + errors = [] + if exc_type is not None: + errors.append(f"{exc_type.__name__}: {exc_val}") + + data = { + "schema": "run_report.v1", + "tool": self.tool, + "started_at": self.started.isoformat(timespec="seconds"), + "finished_at": finished.isoformat(timespec="seconds"), + "duration_ms": duration_ms, + "status": status, + "inputs": self.inputs, + "counts": self.counts, + "scores": self.scores, + "validity": self.validity, + "warnings": self.warnings, + "errors": errors, + } + + try: + path = os.path.abspath(self.path) + parent = os.path.dirname(path) + if parent: + os.makedirs(parent, exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2) + except Exception as e: + sys.stderr.write(f"Failed to write run report: {e}\n") + + return False diff --git a/tests/shadow/test_hyperlexical_eval_run_report.py b/tests/shadow/test_hyperlexical_eval_run_report.py new file mode 100644 index 00000000..c03b0a05 --- /dev/null +++ b/tests/shadow/test_hyperlexical_eval_run_report.py @@ -0,0 +1,50 @@ +"""run_report.v1 wiring for eval_classify (report-only; P3/P4, 2026-10-12).""" + +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "scripts" / "shadow")) + +from hyperlexical import eval_classify # noqa: E402 + + +def _w(path, rows): + path.write_text("".join(json.dumps(r) + "\n" for r in rows), encoding="utf-8") + + +def _fixture(tmp_path, bad_pred=False): + fams = ["ai-native", "gaming-meta", "none"] + ev = [{"id": f"e{i}", "text": f"eval {i} {fams[i % 3]}", "lineage": fams[i % 3]} for i in range(9)] + tr = [{"id": f"t{i}", "text": f"train {i} {fams[i % 3]}", "lineage": fams[i % 3]} for i in range(30)] + pr = [{"id": r["id"], "pred": r["lineage"]} for r in ev] + if bad_pred: + pr[0]["pred"] = "not-a-family" + _w(tmp_path / "eval.jsonl", ev) + _w(tmp_path / "train.jsonl", tr) + _w(tmp_path / "preds.jsonl", pr) + return ["--preds", str(tmp_path / "preds.jsonl"), "--eval", str(tmp_path / "eval.jsonl"), "--train", str(tmp_path / "train.jsonl")] + + +def test_no_env_no_report_same_stdout(tmp_path, monkeypatch, capsys): + argv = _fixture(tmp_path) + monkeypatch.delenv("HLX_RUN_REPORT", raising=False) + assert eval_classify.main(argv) == 0 + first = capsys.readouterr().out + monkeypatch.setenv("HLX_RUN_REPORT", str(tmp_path / "rr.json")) + assert eval_classify.main(argv) == 0 + assert capsys.readouterr().out == first + rep = json.loads((tmp_path / "rr.json").read_text()) + assert rep["schema"] == "run_report.v1" and rep["status"] == "ok" + assert rep["validity"]["valid_rate"] == 1.0 + + +def test_invalid_label_counted_separately(tmp_path, monkeypatch, capsys): + argv = _fixture(tmp_path, bad_pred=True) + monkeypatch.setenv("HLX_RUN_REPORT", str(tmp_path / "rr.json")) + eval_classify.main(argv) + capsys.readouterr() + rep = json.loads((tmp_path / "rr.json").read_text()) + assert rep["validity"]["invalid"] == 1 and rep["validity"]["valid"] == 8 + assert "text" not in json.dumps(rep["inputs"]) diff --git a/tests/shadow/test_hyperlexical_run_report.py b/tests/shadow/test_hyperlexical_run_report.py new file mode 100644 index 00000000..086028dd --- /dev/null +++ b/tests/shadow/test_hyperlexical_run_report.py @@ -0,0 +1,99 @@ +import json +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "scripts" / "shadow")) + +from hyperlexical import run_report as rr + + +def test_noop_when_env_unset(tmp_path, monkeypatch): + monkeypatch.delenv(rr.ENV, raising=False) + with rr.RunReport("tool", {}, path=None): + pass + assert list(tmp_path.iterdir()) == [] + + +def test_writes_with_env(tmp_path, monkeypatch): + path = tmp_path / "sub" / "report.json" + monkeypatch.setenv(rr.ENV, str(path)) + with rr.RunReport("mytool", {"a": 1}, path=None) as r: + r.set_counts(rows=10) + r.set_scores(score=0.9) + r.set_validity(8, 2) + r.warn("hello") + assert path.exists() + data = json.loads(path.read_text()) + assert data["schema"] == "run_report.v1" + assert data["tool"] == "mytool" + assert data["inputs"] == {"a": 1} + assert data["counts"] == {"rows": 10} + assert data["scores"] == {"score": 0.9} + assert data["validity"] == {"valid": 8, "invalid": 2, "valid_rate": 0.8} + assert data["warnings"] == ["hello"] + assert data["status"] == "ok" + assert "started_at" in data and "finished_at" in data and "duration_ms" in data + assert data["errors"] == [] + + +def test_error_status_and_exception_propagates(tmp_path, monkeypatch): + path = tmp_path / "report.json" + monkeypatch.setenv(rr.ENV, str(path)) + with pytest.raises(ValueError): + with rr.RunReport("tool", {}, path=None): + raise ValueError("boom") + data = json.loads(path.read_text()) + assert data["status"] == "error" + assert data["errors"] == ["ValueError: boom"] + + +def test_explicit_status(tmp_path, monkeypatch): + path = tmp_path / "report.json" + monkeypatch.setenv(rr.ENV, str(path)) + with rr.RunReport("tool", {}, path=None) as r: + r.set_status("skipped") + data = json.loads(path.read_text()) + assert data["status"] == "skipped" + + +def test_validity_rate_math_and_none_when_zero(tmp_path, monkeypatch): + path = tmp_path / "report.json" + monkeypatch.setenv(rr.ENV, str(path)) + with rr.RunReport("tool", {}, path=None) as r: + r.set_validity(3, 1) + data = json.loads(path.read_text()) + assert data["validity"]["valid_rate"] == 0.75 + + with rr.RunReport("tool2", {}, path=None) as r2: + r2.set_validity(0, 0) + data2 = json.loads(path.read_text()) + assert data2["validity"]["valid_rate"] is None + + +def test_write_failure_does_not_raise(tmp_path, monkeypatch, capsys): + path = tmp_path / "dir" + path.mkdir() + monkeypatch.setenv(rr.ENV, str(path)) + with rr.RunReport("tool", {}, path=None): + pass + captured = capsys.readouterr() + assert "Failed to write" in captured.err + + +def test_safe_inputs(): + d = { + "text": "secret", + "rows": [1, 2, 3], + "list50": list(range(50)), + "list51": list(range(51)), + "other": "ok", + } + safe = rr.safe_inputs(d) + assert "text" not in safe + assert "rows" not in safe + assert safe["list50"] == list(range(50)) + assert safe["list51"] == "" + assert safe["other"] == "ok" From a89a3d1d15dae2e134a2bf7ab153e678c48e8501 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 17 Oct 2026 18:22:58 +0000 Subject: [PATCH 3/4] fix(guard): close clean-checkout and fail-open holdout gaps Port recoverable_structure from archived ad4075b so the training export imports in a clean checkout. Resolve the bundled hyperlexical module before an exposure snapshot is published. Refuse holdout identity fields that are not lists of strings. Co-authored-by: Daniel Meyer --- .github/workflows/holdout-guard.yml | 2 +- scripts/heldout/build_exposure_snapshot.py | 19 +++ scripts/shadow/hyperlexical/PORTED.md | 2 +- scripts/shadow/hyperlexical/holdout_guard.py | 11 +- .../shadow/recoverable_structure/__init__.py | 9 ++ scripts/shadow/recoverable_structure/cli.py | 44 ++++++ scripts/shadow/recoverable_structure/fit.py | 148 ++++++++++++++++++ .../shadow/recoverable_structure/fixtures.py | 79 ++++++++++ .../shadow/recoverable_structure/intervene.py | 31 ++++ .../shadow/recoverable_structure/linalg.py | 82 ++++++++++ .../shadow/recoverable_structure/receipt.py | 100 ++++++++++++ .../shadow/recoverable_structure/schemes.py | 41 +++++ scripts/shadow/recoverable_structure/tpr.py | 20 +++ tests/heldout/test_build_exposure_snapshot.py | 63 ++++++++ .../shadow/test_hyperlexical_holdout_guard.py | 24 +++ .../test_recoverable_structure_dependency.py | 40 +++++ 16 files changed, 709 insertions(+), 6 deletions(-) create mode 100644 scripts/shadow/recoverable_structure/__init__.py create mode 100644 scripts/shadow/recoverable_structure/cli.py create mode 100644 scripts/shadow/recoverable_structure/fit.py create mode 100644 scripts/shadow/recoverable_structure/fixtures.py create mode 100644 scripts/shadow/recoverable_structure/intervene.py create mode 100644 scripts/shadow/recoverable_structure/linalg.py create mode 100644 scripts/shadow/recoverable_structure/receipt.py create mode 100644 scripts/shadow/recoverable_structure/schemes.py create mode 100644 scripts/shadow/recoverable_structure/tpr.py create mode 100644 tests/shadow/test_recoverable_structure_dependency.py diff --git a/.github/workflows/holdout-guard.yml b/.github/workflows/holdout-guard.yml index 223f85f7..ae686c04 100644 --- a/.github/workflows/holdout-guard.yml +++ b/.github/workflows/holdout-guard.yml @@ -18,4 +18,4 @@ jobs: with: python-version: ${{ matrix.python-version }} - run: python -m pip install --upgrade pip pytest - - run: python -m pytest -q tests/shadow/test_hyperlexical_holdout_disjoint.py tests/shadow/test_hyperlexical_holdout_guard.py tests/shadow/test_hyperlexical_holdout_wiring.py tests/heldout + - run: python -m pytest -q tests/shadow/test_hyperlexical_holdout_disjoint.py tests/shadow/test_hyperlexical_holdout_guard.py tests/shadow/test_hyperlexical_holdout_wiring.py tests/shadow/test_recoverable_structure_dependency.py tests/heldout diff --git a/scripts/heldout/build_exposure_snapshot.py b/scripts/heldout/build_exposure_snapshot.py index e5f085dd..1279bee5 100644 --- a/scripts/heldout/build_exposure_snapshot.py +++ b/scripts/heldout/build_exposure_snapshot.py @@ -14,6 +14,7 @@ import os import re import stat +import sys import time from datetime import datetime, timezone from pathlib import Path @@ -266,6 +267,23 @@ def publish_pair(out_dir: Path, snapshot_id: str, box_text: str, vern_text: str) return vern_path, box_path +def resolve_bundled_hyperlexical() -> None: + """Import the publish dependency before any snapshot file is written. + + ``python scripts/heldout/build_exposure_snapshot.py`` does not put + ``scripts/shadow`` on ``sys.path``. A missing ``hyperlexical`` module + must fail here, before ``publish_pair`` installs successor files. + """ + shadow = Path(__file__).resolve().parents[1] / "shadow" + entry = str(shadow) + if entry not in sys.path: + sys.path.insert(0, entry) + from hyperlexical.select_005_harvest import exposure_fence + + if not callable(exposure_fence): + raise SystemExit("REFUSE: bundled exposure_fence is missing") + + def freshness_record( *, vern_path: Path, @@ -412,6 +430,7 @@ def main(argv: Sequence[str] | None = None) -> int: snapshot_id = snapshot_id or utc_snapshot_id() if snapshot_id == args.baseline_id: raise SystemExit("REFUSE: successor snapshot id collides with the baseline") + resolve_bundled_hyperlexical() vern_path, box_path = publish_pair(args.out_dir, snapshot_id, plan["box_text"], plan["vern_text"]) published = True fence = freshness_record( diff --git a/scripts/shadow/hyperlexical/PORTED.md b/scripts/shadow/hyperlexical/PORTED.md index 3dce0023..d82e95f3 100644 --- a/scripts/shadow/hyperlexical/PORTED.md +++ b/scripts/shadow/hyperlexical/PORTED.md @@ -1,4 +1,4 @@ # Ported from archived scrimshawlife-ctrl/Hyperlex @ ad4075b (2026-10-12) -Verbatim copy of the holdout guard (PR #120) and the 57 modules its tests and the training entrypoints import (`train.py` -> `loop.run_loop` -> `admission.admit_training_run` -> `holdout_guard`). No logic changes. Torch is imported lazily; the guard tests run with pytest only. +Verbatim copy of the holdout guard (PR #120) and the modules its tests and the training entrypoints import (`train.py` -> `loop.run_loop` -> `admission.admit_training_run` -> `holdout_guard`). `scripts/shadow/recoverable_structure/` is the same archive commit: `export.harvest_unbind_spans` and `eval_unbind` import it, and a clean checkout was missing the package. Torch is imported lazily; the guard tests run with pytest only. Tests: `tests/shadow/test_hyperlexical_holdout_{disjoint,guard,wiring}.py`, `tests/heldout/` (CI: `.github/workflows/holdout-guard.yml`). diff --git a/scripts/shadow/hyperlexical/holdout_guard.py b/scripts/shadow/hyperlexical/holdout_guard.py index d5cbf19d..dbd4fa11 100644 --- a/scripts/shadow/hyperlexical/holdout_guard.py +++ b/scripts/shadow/hyperlexical/holdout_guard.py @@ -174,10 +174,13 @@ def _take_strings(value: list[Any], found: set[str], key: str) -> None: def _collect(obj: Any, ids: set[str], hashes: set[str]) -> None: if isinstance(obj, dict): for key, value in obj.items(): - if key in _ID_KEYS and isinstance(value, list): - _take_strings(value, ids, key) - elif key in _HASH_KEYS and isinstance(value, list): - _take_strings(value, hashes, key) + if key in _ID_KEYS or key in _HASH_KEYS: + if not isinstance(value, list) or not all(isinstance(item, str) for item in value): + raise SystemExit( + f"REFUSE: holdout manifest {key} must be a list of strings" + ) + target = ids if key in _ID_KEYS else hashes + _take_strings(value, target, key) elif isinstance(value, (dict, list)): _collect(value, ids, hashes) elif isinstance(obj, list): diff --git a/scripts/shadow/recoverable_structure/__init__.py b/scripts/shadow/recoverable_structure/__init__.py new file mode 100644 index 00000000..319619f5 --- /dev/null +++ b/scripts/shadow/recoverable_structure/__init__.py @@ -0,0 +1,9 @@ +"""SHADOW recoverable-structure probe. + +Not part of hyperlex API_V1. No Abraxas import. No Brier. +""" + +__all__ = ["ALLOWED_SCHEMES", "Abort", "run_probe"] + +from .schemes import ALLOWED_SCHEMES +from .fit import Abort, run_probe diff --git a/scripts/shadow/recoverable_structure/cli.py b/scripts/shadow/recoverable_structure/cli.py new file mode 100644 index 00000000..5a77178c --- /dev/null +++ b/scripts/shadow/recoverable_structure/cli.py @@ -0,0 +1,44 @@ +"""Optional CLI. No network flags.""" + +from __future__ import annotations + +import argparse +import sys + +from .fit import Abort, run_probe +from .fixtures import snapshot +from .receipt import dumps +from .schemes import ALLOWED_SCHEMES + + +def main(argv=None): + p = argparse.ArgumentParser(prog="recoverable-structure-probe") + p.add_argument("--fixture", choices=("tpr", "atomic_pair"), default="tpr") + p.add_argument("--schemes", default="positional,type_slot") + p.add_argument("--selection-proxy", default="unspecified", choices=("weakness", "mdl", "unspecified")) + p.add_argument("--out", default="") + p.add_argument("--human", action="store_true") + args = p.parse_args(argv) + schemes = [s.strip() for s in args.schemes.split(",") if s.strip()] + bad = [s for s in schemes if s not in ALLOWED_SCHEMES] + if bad: + print(f"abort: forbidden scheme {bad}", file=sys.stderr) + return 2 + try: + snap = snapshot(args.fixture) + result = run_probe(snap, schemes=schemes, selection_proxy=args.selection_proxy) + except Abort as exc: + print(f"abort: {exc}", file=sys.stderr) + return 2 + text = result["card"] if args.human else dumps(result["receipt"]) + if args.out: + with open(args.out, "w", encoding="utf-8") as fh: + fh.write(text) + fh.write("\n") + else: + print(text) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/shadow/recoverable_structure/fit.py b/scripts/shadow/recoverable_structure/fit.py new file mode 100644 index 00000000..19f55eec --- /dev/null +++ b/scripts/shadow/recoverable_structure/fit.py @@ -0,0 +1,148 @@ +import hashlib +import time + +from . import linalg as L +from .receipt import build_receipt, render_card +from .schemes import UnknownScheme, validate_schemes +from .tpr import phi_span, reconstruct + + +class Abort(RuntimeError): + pass + + +def _split(n, seed=7, train_frac=0.75): + idx = list(range(n)) + x = seed % (2**31 - 1) or 1 + for i in range(n - 1, 0, -1): + x = (1103515245 * x + 12345) % (2**31) + j = x % (i + 1) + idx[i], idx[j] = idx[j], idx[i] + cut = int(n * train_frac) + train, test = sorted(idx[:cut]), sorted(idx[cut:]) + raw = ",".join(map(str, train)) + "|" + ",".join(map(str, test)) + return train, test, hashlib.sha256(raw.encode()).hexdigest()[:16] + + +def _init_fillers(spans, df=4): + items = sorted({it for sp in spans for it in sp["item_ids"]}) + fillers = {} + for i, it in enumerate(items): + v = L.zeros(df) + v[i % df] = 1.0 + fillers[it] = v + return fillers + + +def _stack_phi(spans, scheme, fillers, max_len): + return [phi_span(scheme, sp["item_ids"], sp["type_tags"], fillers, max_len) for sp in spans] + + +def _fit_affine(phis, targets, ridge=1e-4): + n = len(phis) + d = len(targets[0]) + X = [phi + [1.0] for phi in phis] + W, b = [], [] + for dim in range(d): + y = [targets[i][dim] for i in range(n)] + w = L.lstsq(X, y, ridge=ridge) + W.append(w[:-1]) + b.append(w[-1]) + return W, b + + +def fit_scheme(scheme, snap, train_idx, test_idx): + t0 = time.monotonic() + spans = snap["spans"] + enc = snap["encodings"] + max_len = snap["max_len"] + fillers = _init_fillers(spans) + train_spans = [spans[i] for i in train_idx] + train_tgt = [enc[i] for i in train_idx] + phis_tr = _stack_phi(train_spans, scheme, fillers, max_len) + W, b = _fit_affine(phis_tr, train_tgt) + pred_tr = [reconstruct(W, b, p) for p in phis_tr] + train_mse = L.mse(pred_tr, train_tgt) + test_spans = [spans[i] for i in test_idx] + test_tgt = [enc[i] for i in test_idx] + phis_te = _stack_phi(test_spans, scheme, fillers, max_len) + pred_te = [reconstruct(W, b, p) for p in phis_te] + test_mse = L.mse(pred_te, test_tgt) + elapsed = time.monotonic() - t0 + if elapsed >= 30: + raise Abort(f"fit exceeded 30s: {elapsed:.2f}") + return { + "scheme": scheme, + "W": W, + "b": b, + "fillers": fillers, + "train_mse": train_mse, + "test_mse": test_mse, + "pred_train": pred_tr, + "pred_test": pred_te, + "elapsed": elapsed, + } + + +def swap_accuracy(fit_res, snap, train_idx, test_idx): + labels_tr = [snap["spans"][i]["label"] for i in train_idx] + labels_te = [snap["spans"][i]["label"] for i in test_idx] + classes = sorted(set(labels_tr + labels_te)) + Xtr = [p + [1.0] for p in fit_res["pred_train"]] + weights = {} + for c in classes: + y = [1.0 if lab == c else 0.0 for lab in labels_tr] + weights[c] = L.lstsq(Xtr, y, ridge=1e-4) + + def predict(vecs): + out = [] + for v in vecs: + x = v + [1.0] + scores = {c: L.dot(weights[c], x) for c in classes} + out.append(max(scores, key=scores.get)) + return out + + pred = predict(fit_res["pred_test"]) + hit = sum(int(a == b) for a, b in zip(pred, labels_te)) + return hit / max(1, len(labels_te)), predict + + +def run_probe(snap, schemes=("positional", "type_slot"), selection_proxy="unspecified", seed=7): + try: + schemes = validate_schemes(schemes) + except UnknownScheme as exc: + raise Abort(str(exc)) from exc + train_idx, test_idx, split_hash = _split(len(snap["spans"]), seed=seed) + batch_raw = repr([(s["item_ids"], s["label"]) for s in snap["spans"]]) + batch_hash = hashlib.sha256(batch_raw.encode()).hexdigest()[:16] + scheme_blocks = [] + raw_fits = [] + for sch in schemes: + fr = fit_scheme(sch, snap, train_idx, test_idx) + acc, predictor = swap_accuracy(fr, snap, train_idx, test_idx) + from .intervene import filler_edit + inter = filler_edit(sch, snap, fr, predictor) + scheme_blocks.append({ + "scheme": sch, + "train_mse": fr["train_mse"], + "test_mse": fr["test_mse"], + "swap_accuracy": acc, + "intervention": inter, + }) + raw_fits.append(fr) + receipt = build_receipt( + encoder_id=snap["encoder_id"], + batch_hash=batch_hash, + split_hash=split_hash, + role_schemes=schemes, + selection_proxy=selection_proxy, + scheme_blocks=scheme_blocks, + source_class="AAL-metric", + ) + return { + "receipt": receipt, + "card": render_card(receipt), + "fits": raw_fits, + "train_idx": train_idx, + "test_idx": test_idx, + } diff --git a/scripts/shadow/recoverable_structure/fixtures.py b/scripts/shadow/recoverable_structure/fixtures.py new file mode 100644 index 00000000..85970b8f --- /dev/null +++ b/scripts/shadow/recoverable_structure/fixtures.py @@ -0,0 +1,79 @@ +import hashlib +from .schemes import n_roles + + +ITEMS = ["a", "b", "c", "d"] +TYPES = ["TOKEN", "SLOT", "MARKER"] + + +def _hash_int(s): + return int(hashlib.sha256(s.encode()).hexdigest()[:8], 16) + + +def _vec(seed, dim): + out = [] + x = seed % (2**31 - 1) or 1 + for _ in range(dim): + x = (1103515245 * x + 12345) % (2**31) + out.append((x / 2**31) * 2 - 1) + return out + + +def make_spans(n=64, length=4, seed=7): + spans = [] + x = seed % (2**31 - 1) or 1 + for i in range(n): + items = [] + tags = [] + for k in range(length): + x = (1103515245 * x + 12345) % (2**31) + items.append(ITEMS[(x >> 16) % len(ITEMS)]) + tags.append(TYPES[(i + k) % len(TYPES)]) + label = ITEMS.index(items[0]) + spans.append({"item_ids": items, "type_tags": tags, "label": label}) + return spans + + +def _tpr_encode(spans, dim, max_len, seed): + df = 4 + fillers = {it: _vec(_hash_int(f"tpr-f-{it}-{seed}"), df) for it in ITEMS} + W = [] + for r in range(dim): + W.append(_vec(_hash_int(f"tpr-W-{r}-{seed}"), n_roles("positional", max_len) * df)) + b = _vec(_hash_int(f"tpr-b-{seed}"), dim) + encodings = [] + from .tpr import phi_span, reconstruct + for sp in spans: + phi = phi_span("positional", sp["item_ids"], sp["type_tags"], fillers, max_len) + encodings.append(reconstruct(W, b, phi)) + return encodings + + +def _atomic_encode(spans, dim, max_len, seed): + encodings = [] + for sp in spans: + key = "seq-" + "-".join(sp["item_ids"]) + f"-{seed}" + encodings.append(_vec(_hash_int(key), dim)) + return encodings + + +def snapshot(kind, n=64, length=4, dim=12, seed=7): + if kind not in {"tpr", "atomic_pair"}: + raise ValueError(kind) + spans = make_spans(n=n, length=length, seed=seed) + max_len = length + if kind == "tpr": + encodings = _tpr_encode(spans, dim, max_len, seed) + fid = "fixture:tpr:v0.1" + else: + encodings = _atomic_encode(spans, dim, max_len, seed) + fid = "fixture:atomic_pair:v0.1" + return { + "encoder_id": fid, + "weight_hash": hashlib.sha256(f"{kind}-{seed}-{dim}".encode()).hexdigest()[:16], + "encodings": encodings, + "spans": spans, + "max_len": max_len, + "dim": dim, + "kind": kind, + } diff --git a/scripts/shadow/recoverable_structure/intervene.py b/scripts/shadow/recoverable_structure/intervene.py new file mode 100644 index 00000000..6937c08d --- /dev/null +++ b/scripts/shadow/recoverable_structure/intervene.py @@ -0,0 +1,31 @@ +from .tpr import phi_span, reconstruct + + +def filler_edit(scheme, snap, fit_res, predictor): + fillers = {k: v[:] for k, v in fit_res["fillers"].items()} + items = list(fillers) + if len(items) < 2: + return {"edit": "none", "expected": 0, "observed": 0, "hit": False} + src, dst = items[0], items[1] + span = snap["spans"][0] + edited_items = [dst if tok == src else tok for tok in span["item_ids"]] + phi_before = phi_span(scheme, span["item_ids"], span["type_tags"], fillers, snap["max_len"]) + phi_after = phi_span(scheme, edited_items, span["type_tags"], fillers, snap["max_len"]) + vec_b = reconstruct(fit_res["W"], fit_res["b"], phi_before) + vec_a = reconstruct(fit_res["W"], fit_res["b"], phi_after) + obs_a = predictor([vec_a])[0] + if span["item_ids"][0] == src: + expected = _item_label(dst) + else: + expected = _item_label(span["item_ids"][0]) + return { + "edit": f"{src}->{dst}", + "expected": expected, + "observed": obs_a, + "hit": bool(obs_a == expected), + } + + +def _item_label(item): + order = ["a", "b", "c", "d"] + return order.index(item) if item in order else -1 diff --git a/scripts/shadow/recoverable_structure/linalg.py b/scripts/shadow/recoverable_structure/linalg.py new file mode 100644 index 00000000..d1ff558d --- /dev/null +++ b/scripts/shadow/recoverable_structure/linalg.py @@ -0,0 +1,82 @@ +"""Tiny stdlib linear algebra. No numpy.""" + + +def zeros(n, m=None): + if m is None: + return [0.0] * n + return [[0.0] * m for _ in range(n)] + + +def add(a, b): + return [x + y for x, y in zip(a, b)] + + +def scale(a, s): + return [x * s for x in a] + + +def dot(a, b): + return sum(x * y for x, y in zip(a, b)) + + +def matvec(A, x): + return [dot(row, x) for row in A] + + +def transpose(A): + return [list(col) for col in zip(*A)] + + +def matmul(A, B): + Bt = transpose(B) + return [[dot(row, col) for col in Bt] for row in A] + + +def identity(n): + I = zeros(n, n) + for i in range(n): + I[i][i] = 1.0 + return I + + +def solve(A, b): + n = len(A) + M = [A[i][:] + [b[i]] for i in range(n)] + for i in range(n): + piv = max(range(i, n), key=lambda r: abs(M[r][i])) + M[i], M[piv] = M[piv], M[i] + if abs(M[i][i]) < 1e-12: + raise ZeroDivisionError("singular") + div = M[i][i] + for j in range(i, n + 1): + M[i][j] /= div + for r in range(n): + if r == i: + continue + f = M[r][i] + for j in range(i, n + 1): + M[r][j] -= f * M[i][j] + return [M[i][n] for i in range(n)] + + +def lstsq(X, y, ridge=1e-6): + Xt = transpose(X) + XtX = matmul(Xt, X) + p = len(XtX) + for i in range(p): + XtX[i][i] += ridge + Xty = matvec(Xt, y) + return solve(XtX, Xty) + + +def mse(pred, target): + n = len(pred) + if n == 0: + return 0.0 + s = 0.0 + for a, b in zip(pred, target): + if isinstance(a, list): + s += sum((x - y) ** 2 for x, y in zip(a, b)) / len(a) + else: + s += (a - b) ** 2 + return s / n diff --git a/scripts/shadow/recoverable_structure/receipt.py b/scripts/shadow/recoverable_structure/receipt.py new file mode 100644 index 00000000..41dce412 --- /dev/null +++ b/scripts/shadow/recoverable_structure/receipt.py @@ -0,0 +1,100 @@ +import json +import uuid +from datetime import datetime, timezone + +SCHEMA = "abraxas.recoverable_structure.probe.v0.1" +FORBIDDEN = {"symbolic", "has_symbols"} +PROXIES = {"weakness", "mdl", "unspecified"} +SCHEMES = {"positional", "type_slot"} + + +class ReceiptError(ValueError): + pass + + +def build_receipt(encoder_id, batch_hash, split_hash, role_schemes, selection_proxy, scheme_blocks, source_class="AAL-metric", run_id=None, logged_at=None): + if selection_proxy not in PROXIES: + raise ReceiptError("missing or invalid selection_proxy") + rec = { + "schema": SCHEMA, + "run_id": run_id or str(uuid.uuid4()), + "logged_at": logged_at or datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "encoder_id": encoder_id, + "batch_hash": batch_hash, + "split_hash": split_hash, + "role_schemes": list(role_schemes), + "selection_proxy": selection_proxy, + "schemes": scheme_blocks, + "brier": None, + "forecast_eligible": False, + "auto_fire": False, + "lane": "SHADOW", + "classification_of_result": "OBSERVED", + "source_class": source_class, + } + validate_receipt(rec) + return rec + + +def validate_receipt(rec): + if rec.get("schema") != SCHEMA: + raise ReceiptError("bad schema") + if rec.get("brier") is not None: + raise ReceiptError("brier must be null") + if rec.get("forecast_eligible") is not False: + raise ReceiptError("forecast_eligible must be false") + if rec.get("auto_fire") is not False: + raise ReceiptError("auto_fire must be false") + if rec.get("lane") != "SHADOW": + raise ReceiptError("lane") + if rec.get("classification_of_result") != "OBSERVED": + raise ReceiptError("classification_of_result") + if rec.get("source_class") not in {"lab-paper", "AAL-metric"}: + raise ReceiptError("source_class") + if rec.get("selection_proxy") not in PROXIES: + raise ReceiptError("selection_proxy") + overlap = FORBIDDEN.intersection(rec) + if overlap: + raise ReceiptError(f"forbidden keys: {overlap}") + schemes = rec.get("role_schemes") or [] + if not 1 <= len(schemes) <= 2 or any(s not in SCHEMES for s in schemes): + raise ReceiptError("role_schemes") + blocks = rec.get("schemes") or [] + if not blocks: + raise ReceiptError("schemes empty") + for blk in blocks: + if blk.get("scheme") not in SCHEMES: + raise ReceiptError("block scheme") + inter = blk.get("intervention") or {} + for k in ("edit", "expected", "observed", "hit"): + if k not in inter: + raise ReceiptError("intervention") + required = ("schema", "run_id", "logged_at", "encoder_id", "batch_hash", "split_hash", "role_schemes", "selection_proxy", "schemes", "brier", "forecast_eligible", "auto_fire", "lane", "classification_of_result", "source_class") + for k in required: + if k not in rec: + raise ReceiptError(f"missing {k}") + return rec + + +def render_card(rec): + validate_receipt(rec) + lines = [ + "Recoverable-structure probe card", + "Lane: SHADOW / advisory", + "Instrumentation only. Not P(structure). Not a fire threshold.", + "Brier: null. Forecast eligible: no.", + "Two-scheme cap: positional, type_slot.", + f"Encoder: {rec['encoder_id']}", + f"Schemes: {', '.join(rec['role_schemes'])}", + f"selection_proxy: {rec['selection_proxy']}", + ] + for blk in rec["schemes"]: + lines.append( + f"- {blk['scheme']}: test_mse={blk['test_mse']:.4f} swap={blk['swap_accuracy']:.3f} edit={blk['intervention']['edit']} hit={blk['intervention']['hit']}" + ) + return "\n".join(lines) + + +def dumps(rec): + validate_receipt(rec) + return json.dumps(rec, indent=2, sort_keys=True) diff --git a/scripts/shadow/recoverable_structure/schemes.py b/scripts/shadow/recoverable_structure/schemes.py new file mode 100644 index 00000000..08d375ae --- /dev/null +++ b/scripts/shadow/recoverable_structure/schemes.py @@ -0,0 +1,41 @@ +ALLOWED_SCHEMES = frozenset({"positional", "type_slot"}) + + +class UnknownScheme(ValueError): + pass + + +def validate_schemes(names): + names = list(names) + if not names or len(names) > 2: + raise UnknownScheme("role_schemes must have 1 or 2 entries") + extra = [n for n in names if n not in ALLOWED_SCHEMES] + if extra: + raise UnknownScheme(f"forbidden role scheme(s): {extra}") + if len(set(names)) != len(names): + raise UnknownScheme("duplicate role scheme") + return names + + +def role_ids(scheme, length, type_tags): + if scheme == "positional": + return list(range(length)) + if scheme == "type_slot": + if type_tags is None or len(type_tags) != length: + raise UnknownScheme("type_slot requires type_tags of span length") + table = {"TOKEN": 0, "SLOT": 1, "MARKER": 2} + out = [] + for tag in type_tags: + if tag not in table: + raise UnknownScheme(f"unknown type tag: {tag}") + out.append(table[tag]) + return out + raise UnknownScheme(scheme) + + +def n_roles(scheme, max_len): + if scheme == "positional": + return max_len + if scheme == "type_slot": + return 3 + raise UnknownScheme(scheme) diff --git a/scripts/shadow/recoverable_structure/tpr.py b/scripts/shadow/recoverable_structure/tpr.py new file mode 100644 index 00000000..a018221f --- /dev/null +++ b/scripts/shadow/recoverable_structure/tpr.py @@ -0,0 +1,20 @@ +from . import linalg as L +from .schemes import n_roles, role_ids + + +def phi_span(scheme, item_ids, type_tags, fillers, max_len): + """Flattened TPR: concat over roles of summed fillers on that role.""" + nr = n_roles(scheme, max_len) + df = len(next(iter(fillers.values()))) + acc = L.zeros(nr * df) + roles = role_ids(scheme, len(item_ids), type_tags) + for item, role in zip(item_ids, roles): + f = fillers[item] + base = role * df + for i, val in enumerate(f): + acc[base + i] += val + return acc + + +def reconstruct(W, b, phi): + return L.add(L.matvec(W, phi), b) diff --git a/tests/heldout/test_build_exposure_snapshot.py b/tests/heldout/test_build_exposure_snapshot.py index 57fdb8b1..5bc6adbb 100644 --- a/tests/heldout/test_build_exposure_snapshot.py +++ b/tests/heldout/test_build_exposure_snapshot.py @@ -4,6 +4,7 @@ import json import os +import subprocess import sys from pathlib import Path @@ -183,3 +184,65 @@ def test_cli_publish_is_fresh_and_leaves_the_baseline(tmp_path: Path) -> None: assert receipt["freshness"]["state"] == "EXPOSURE_SNAPSHOT_FRESH" assert receipt["previous_pair_overwritten"] is False assert os.stat(receipt_path).st_mode & 0o777 == 0o600 + + +def _publish_argv(tmp_path: Path, out: Path, receipt_path: Path) -> list[str]: + baseline_box = tmp_path / "box-jev-hashes-20260925T211351Z.txt" + baseline_vern = tmp_path / "vern-texts-20260925T211351Z.jsonl" + extra = tmp_path / "extra.txt" + baseline_box.write_text(_box("a" * 64), encoding="utf-8") + baseline_vern.write_text(_vern("kept"), encoding="utf-8") + extra.write_text(_box("f" * 64), encoding="utf-8") + return [ + "--baseline-box", + str(baseline_box), + "--baseline-vern", + str(baseline_vern), + "--box-increment", + f"agent_context_exposed_hashes:{extra}", + "--out-dir", + str(out), + "--snapshot-id", + "20260929T030405Z", + "--receipt", + str(receipt_path), + "--publish", + ] + + +def test_missing_bundled_module_refuses_before_publish(tmp_path: Path, monkeypatch) -> None: + import build_exposure_snapshot as mod + + def boom() -> None: + raise SystemExit("REFUSE: bundled hyperlexical is not importable") + + monkeypatch.setattr(mod, "resolve_bundled_hyperlexical", boom) + out = tmp_path / "out" + out.mkdir() + with pytest.raises(SystemExit, match="not importable"): + mod.main(_publish_argv(tmp_path, out, tmp_path / "receipt.json")) + assert list(out.iterdir()) == [] + assert not (tmp_path / "receipt.json").exists() + + +def test_direct_script_publish_resolves_bundled_module(tmp_path: Path) -> None: + script = ROOT / "scripts" / "heldout" / "build_exposure_snapshot.py" + out = tmp_path / "out" + out.mkdir() + receipt_path = tmp_path / "receipt.json" + env = os.environ.copy() + env.pop("PYTHONPATH", None) + proc = subprocess.run( + [sys.executable, str(script), *_publish_argv(tmp_path, out, receipt_path)], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + check=False, + ) + assert proc.returncode == 0, proc.stderr + assert (out / "vern-texts-20260929T030405Z.jsonl").is_file() + assert (out / "box-jev-hashes-20260929T030405Z.txt").is_file() + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt["published"] is True + assert receipt["freshness"]["state"] == "EXPOSURE_SNAPSHOT_FRESH" diff --git a/tests/shadow/test_hyperlexical_holdout_guard.py b/tests/shadow/test_hyperlexical_holdout_guard.py index af1e24a5..20789c30 100644 --- a/tests/shadow/test_hyperlexical_holdout_guard.py +++ b/tests/shadow/test_hyperlexical_holdout_guard.py @@ -226,6 +226,30 @@ def test_prepare_asserts_when_filter_is_bypassed(monkeypatch, tmp_path): prepare_unbind_splits([row]) +@pytest.mark.parametrize( + "body", + [ + {"row_ids": "abc123", "normalized_text_sha256": ["ab" * 32]}, + {"row_ids": ["abc123"], "normalized_text_sha256": "ab" * 32}, + {"row_ids": [1, 2], "normalized_text_sha256": ["ab" * 32]}, + {"meta": {"ids": "abc123"}, "normalized_text_sha256": ["ab" * 32]}, + ], +) +def test_malformed_holdout_identity_fields_are_refused(tmp_path, body): + path = tmp_path / "malformed.json" + path.write_text(json.dumps(body), encoding="utf-8") + with pytest.raises(SystemExit, match="must be a list of strings"): + load_holdout_spec(str(path)) + + +def test_holdout_identity_lists_of_strings_are_kept(tmp_path): + digest = "ab" * 32 + path = _write_manifest(tmp_path / "ok.json", row_ids=["abc123"], text_hashes=[digest]) + spec = load_holdout_spec(str(path)) + assert spec.row_ids == frozenset({"abc123"}) + assert spec.text_hashes == frozenset({digest}) + + def test_manifest_refuses_embedded_rows_bad_hash_and_missing(tmp_path): embedded = tmp_path / "embedded.json" embedded.write_text( diff --git a/tests/shadow/test_recoverable_structure_dependency.py b/tests/shadow/test_recoverable_structure_dependency.py new file mode 100644 index 00000000..98ea8907 --- /dev/null +++ b/tests/shadow/test_recoverable_structure_dependency.py @@ -0,0 +1,40 @@ +"""The training export must import recoverable_structure in a clean checkout.""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] + + +def test_training_export_imports_bundled_recoverable_structure(tmp_path: Path) -> None: + shadow = ROOT / "scripts" / "shadow" + env = os.environ.copy() + env["PYTHONPATH"] = str(shadow) + env.pop("HLX_E2_DISJOINT", None) + code = """ +from hyperlexical.export import harvest_unbind_spans +from recoverable_structure.fit import _split, fit_scheme, run_probe, swap_accuracy +from recoverable_structure.fixtures import make_spans, snapshot + +rows = harvest_unbind_spans(4) +assert len(rows) == 4 and rows[0][1]["item_ids"] +spans = make_spans(n=8, length=4, seed=7) +train, test, digest = _split(len(spans), seed=7) +assert train and test and len(digest) == 16 +snap = snapshot("atomic_pair", n=4, length=4, dim=4, seed=7) +assert len(snap["encodings"]) == 4 +assert callable(fit_scheme) and callable(run_probe) and callable(swap_accuracy) +""" + proc = subprocess.run( + [sys.executable, "-c", code], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + check=False, + ) + assert proc.returncode == 0, proc.stderr From 87066f05c67e4581b497eb2a18d0f9cc99d2153a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 17 Oct 2026 18:24:54 +0000 Subject: [PATCH 4/4] fix(eval): score validity from parsed eval labels and record the port intent Validity now uses the labels run_eval_classify actually scored, so padding and extra prediction rows do not change the count. intent/2026-10-12-run-report-v1.md records the accepted harness-learning port. Co-authored-by: Daniel Meyer --- intent/2026-10-12-run-report-v1.md | 54 +++++++++++++++++++ scripts/shadow/hyperlexical/PORTED.md | 2 + scripts/shadow/hyperlexical/eval_classify.py | 23 ++++---- .../test_hyperlexical_eval_run_report.py | 15 ++++++ 4 files changed, 85 insertions(+), 9 deletions(-) create mode 100644 intent/2026-10-12-run-report-v1.md diff --git a/intent/2026-10-12-run-report-v1.md b/intent/2026-10-12-run-report-v1.md new file mode 100644 index 00000000..0aec3c77 --- /dev/null +++ b/intent/2026-10-12-run-report-v1.md @@ -0,0 +1,54 @@ +--- +id: intent-2026-10-12-run-report-v1 +status: accepted +depends_on: ["007"] +--- + +# Intent + +## Problem + +The archived holdout guard and the eval scripts that sit on it were not in this checkout. A training or eval run here could not fail closed on a sealed holdout, and eval numbers had no separate count of predictions that parse to a known family. + +## Proposed outcome + +What this branch already does: + +- Port the archived holdout guard, its training entrypoints, and `scripts/shadow/recoverable_structure/` from scrimshawlife-ctrl/Hyperlex @ ad4075b, so a clean checkout can import the training export and `eval_unbind`. +- Add `run_report.v1` (`scripts/shadow/hyperlexical/run_report.py`). It writes only when `HLX_RUN_REPORT` is set. It records status, inputs without row text, counts, scores, validity, warnings, and errors. It does not change eval numbers, stdout, or exit codes. A failed write does not raise. +- Wire that report into `eval_unbind.main` and `eval_classify.main`. +- Count validity from the predictions actually scored, separate from the task score. + +Out of scope: prereg files, holdout manifests, in-flight runs, operator settlement, live ingest, a `SKILL.md` command change, and official Hermes hub publish. + +## Affected users / systems + +- shadow training entrypoints and holdout admission +- `eval_classify` and `eval_unbind` when `HLX_RUN_REPORT` is set +- holdout-guard CI + +## Constraints / non-goals + +- `SKILL.md` stays the Hermes command contract. This work does not add a skill command. +- Brier stays null until an operator settles an outcome. +- Phase 5 stays SPECULATIVE. +- No Abraxas import. The ported probe keeps its existing receipt schema string; it does not import Abraxas. +- Fail closed on a missing or malformed holdout manifest. +- Do not rewrite historical receipts. + +## Open questions + +None for this port. Danny accepted the harness-learning batch on 2026-10-12 (P0, 2:48 AM PT), including landing this implementation after the review bugs were fixed. + +## Verified / assumed claims + +| Claim | Label | Basis | +|---|---|---| +| Archived source of the port is scrimshawlife-ctrl/Hyperlex @ ad4075b | `OBSERVED` | `scripts/shadow/hyperlexical/PORTED.md` | +| `run_report.v1` is off unless `HLX_RUN_REPORT` is set | `OBSERVED` | `scripts/shadow/hyperlexical/run_report.py` | +| Danny accepted this harness-learning batch on 2026-10-12 | `OBSERVED` | PR #20 body ("Danny yes 2026-10-12 2:48 AM PT"); owner instruction 2026-10-17 to fix these PRs and then merge | + +## Author / date + +- Author: recorded for the existing port after Danny's 2026-10-12 yes +- Date: 2026-10-12 (recorded 2026-10-17) diff --git a/scripts/shadow/hyperlexical/PORTED.md b/scripts/shadow/hyperlexical/PORTED.md index d82e95f3..5135ade7 100644 --- a/scripts/shadow/hyperlexical/PORTED.md +++ b/scripts/shadow/hyperlexical/PORTED.md @@ -2,3 +2,5 @@ Verbatim copy of the holdout guard (PR #120) and the modules its tests and the training entrypoints import (`train.py` -> `loop.run_loop` -> `admission.admit_training_run` -> `holdout_guard`). `scripts/shadow/recoverable_structure/` is the same archive commit: `export.harvest_unbind_spans` and `eval_unbind` import it, and a clean checkout was missing the package. Torch is imported lazily; the guard tests run with pytest only. Tests: `tests/shadow/test_hyperlexical_holdout_{disjoint,guard,wiring}.py`, `tests/heldout/` (CI: `.github/workflows/holdout-guard.yml`). + +Intent for this port and for run_report.v1: `intent/2026-10-12-run-report-v1.md`. diff --git a/scripts/shadow/hyperlexical/eval_classify.py b/scripts/shadow/hyperlexical/eval_classify.py index 56deacdc..18711fc5 100644 --- a/scripts/shadow/hyperlexical/eval_classify.py +++ b/scripts/shadow/hyperlexical/eval_classify.py @@ -404,8 +404,12 @@ def run_eval_classify( train_path: str | Path, slices: Sequence[str] = (), bow_nonnone: str | None = None, -) -> dict[str, Any]: - """Score model predictions. Train rows fit the bag-of-terms baseline only.""" +) -> tuple[dict[str, Any], list[str]]: + """Score model predictions. Train rows fit the bag-of-terms baseline only. + + The second value is the parsed label for each eval row, in eval order. + Those are the predictions that entered the score. + """ pred_rows = _read_jsonl(Path(preds_path)) eval_rows = _read_jsonl(Path(eval_path)) train_rows = _read_jsonl(Path(train_path)) @@ -479,7 +483,7 @@ def run_eval_classify( block = _score_block([g for g, _p in kept], [p for _g, p in kept]) block["reporting_only"] = True report["slices"][name] = block - return report + return report, model def main(argv: list[str] | None = None) -> int: @@ -517,7 +521,7 @@ def main(argv: list[str] | None = None) -> int: # Report-only (HLX_RUN_REPORT): never changes the eval numbers, stdout, or exit code. with RunReport("hyperlexical/eval_classify", {"preds": str(preds_path), "eval": args.eval, "train": args.train}) as rr: - report = run_eval_classify( + report, scored = run_eval_classify( preds_path=preds_path, eval_path=args.eval, train_path=args.train, @@ -525,11 +529,12 @@ def main(argv: list[str] | None = None) -> int: bow_nonnone=args.bow_nonnone, ) if rr.path: - labels = [] - for row in _read_jsonl(Path(preds_path)): - labels.append(next((str(row[f]) for f in _PRED_FIELDS if row.get(f) not in (None, "")), None)) - bad = [x for x in labels if x not in FAMILIES] - rr.set_validity(len(labels) - len(bad), len(bad), {"label_not_in_families": len(bad)} if bad else None) + bad = [label for label in scored if label not in FAMILIES] + rr.set_validity( + len(scored) - len(bad), + len(bad), + {"label_not_in_families": len(bad)} if bad else None, + ) rr.set_scores(**{k: report[k] for k in ("macro_f1", "macro_f1_nonnone", "accuracy") if k in report}) text = json.dumps(report, indent=2, sort_keys=True) if args.out: diff --git a/tests/shadow/test_hyperlexical_eval_run_report.py b/tests/shadow/test_hyperlexical_eval_run_report.py index c03b0a05..b7d7b01d 100644 --- a/tests/shadow/test_hyperlexical_eval_run_report.py +++ b/tests/shadow/test_hyperlexical_eval_run_report.py @@ -40,6 +40,21 @@ def test_no_env_no_report_same_stdout(tmp_path, monkeypatch, capsys): assert rep["validity"]["valid_rate"] == 1.0 +def test_validity_uses_labels_that_were_scored(tmp_path, monkeypatch, capsys): + argv = _fixture(tmp_path) + preds = tmp_path / "preds.jsonl" + rows = [json.loads(line) for line in preds.read_text(encoding="utf-8").splitlines() if line.strip()] + rows[0]["pred"] = " ai-native " + rows.append({"id": "extra-not-scored", "pred": "not-a-family"}) + _w(preds, rows) + monkeypatch.setenv("HLX_RUN_REPORT", str(tmp_path / "rr.json")) + assert eval_classify.main(argv) == 0 + capsys.readouterr() + rep = json.loads((tmp_path / "rr.json").read_text(encoding="utf-8")) + assert rep["validity"]["valid"] == 9 + assert rep["validity"]["invalid"] == 0 + + def test_invalid_label_counted_separately(tmp_path, monkeypatch, capsys): argv = _fixture(tmp_path, bad_pred=True) monkeypatch.setenv("HLX_RUN_REPORT", str(tmp_path / "rr.json"))