Validate quality measurements and exercise attention oracles on CPU

docs/QUALITY_SEED_UNCERTAINTY.md

@@ -7,8 +7,12 @@ Each input must be an existing evaluation report from `kev_laya.evaluation.evalu
 with the same split, record/question IDs, groups, labels, question types and
 declared slice metadata. It requires the same frozen split hash and distinct
 checkpoint hashes from the CLI evaluation reports. It refuses missing or duplicate rows, mismatched
-targets, inconsistent splits, nonfinite metrics and replacement of an existing
-output. Input file hashes are included in the output.
+targets, inconsistent splits and replacement of an existing output. Before
+aggregation, each row must have finite numeric measurements: correctness and
+answer confidence within [0, 1], and nonnegative NLL, Brier and ordinal error
+(or no ordinal error for unordered tasks). Invalid measurements are rejected
+before an output is created. Fractional soft-target correctness remains valid.
+Input file hashes are included in the output.
 
 For each metric, the output gives individual seed values, their mean, sample
 standard deviation and observed range, including per-slice results. It is

src/kev_laya/evaluation.py

@@ -10,9 +10,27 @@
 from .model import DecisionEngine
 
 
+def _validate_metric_rows(rows: list[dict]) -> None:
+    """Reject invalid measurements before binning or bootstrap resampling."""
+    for row in rows:
+        if not isinstance(row, dict):
+            raise ValueError("evaluation_row_metrics_invalid")
+        for name in ("correct", "answer_probability", "nll", "brier", "ordinal_mae"):
+            value = row.get(name)
+            if name == "ordinal_mae" and value is None and name in row:
+                continue
+            try:
+                valid = type(value) in (int, float) and math.isfinite(value) and value >= 0
+            except OverflowError:
+                valid = False
+            if not valid or (name in ("correct", "answer_probability") and value > 1):
+                raise ValueError(f"evaluation_row_metric_invalid:{name}")
+
+
 def summarize(rows: list[dict], bins: int = 10) -> dict:
     if type(bins) is not int or bins < 1:
         raise ValueError("bins must be a positive integer")
+    _validate_metric_rows(rows)
     if not rows:
         return {"count": 0, "nll": None, "brier": None, "ece": None, "accuracy": None,
                 "ordinal_mae": None, "reliability": [], "risk_coverage": []}
@@ -183,6 +201,7 @@ def grouped_bootstrap(rows: list[dict], *, samples: int = 1000, seed: int = 42,
     if len(grouped) < 2:
         result["reason"] = "fewer than two source groups"
         return result
+    _validate_metric_rows(rows)
     groups = [grouped[key] for key in sorted(grouped)]
     rng = random.Random(seed)  # Do not mutate training/evaluation global RNG.
     values = {key: [] for key in fields}

tests/test_evaluation_uncertainty.py

@@ -54,6 +54,35 @@ def test_invalid_bin_count_rejected(bins):
         summarize(rows(), bins=bins)
 
 
+@pytest.mark.parametrize("metric,value", [
+    ("correct", -0.1), ("correct", 1.1), ("correct", True),
+    ("answer_probability", -0.1), ("answer_probability", 1.1),
+    ("answer_probability", float("nan")), ("answer_probability", float("inf")),
+    ("nll", -0.1), ("nll", "0.3"), ("nll", 10 ** 400),
+    ("brier", -0.1), ("brier", float("inf")),
+    ("ordinal_mae", -0.1), ("ordinal_mae", float("nan")),
+])
+def test_invalid_measurements_rejected_before_quality_aggregation(metric, value):
+    data = rows()
+    data[0][metric] = value
+    with pytest.raises(ValueError, match=f"evaluation_row_metric_invalid:{metric}"):
+        summarize(data)
+    with pytest.raises(ValueError, match=f"evaluation_row_metric_invalid:{metric}"):
+        grouped_bootstrap(data, samples=2, seed=1)
+
+
+def test_soft_target_correctness_and_confidence_endpoints_remain_valid():
+    data = rows()[:3]
+    for row, correct, confidence, ordinal in zip(
+            data, (0.0, 0.25, 1.0), (0.0, 0.5, 1.0), (None, 0.0, 1.25), strict=True):
+        row.update(correct=correct, answer_probability=confidence, ordinal_mae=ordinal)
+    result = summarize(data)
+    assert result["accuracy"] == pytest.approx(1.25 / 3)
+    assert result["ece"] == pytest.approx(0.25 / 3)
+    assert result["ordinal_mae"] == 0.625
+    assert sum(bin_["count"] for bin_ in result["reliability"]) == len(data)
+
+
 def test_evaluation_retains_group_identity(tiny, suite):
     data, _ = suite
     report = evaluate(tiny, data["development"][:2], ByteTokenizer(), Limits(512, 8192), split="development")
@@ -138,6 +167,25 @@ def test_seed_report_cli_reads_three_actual_evaluation_reports(tmp_path, tiny, s
         seed_report_cli([*args, "--out", str(output)])
 
 
+@pytest.mark.parametrize("metric,value", [("correct", 2.0), ("answer_probability", -0.1)])
+def test_seed_report_cli_refuses_impossible_quality_without_output(
+        tmp_path, tiny, suite, metric, value):
+    data, _ = suite
+    args = []
+    for seed in (11, 12, 13):
+        path = tmp_path / f"seed-{seed}.json"
+        report = evaluate(tiny, data["development"][:1], ByteTokenizer(), Limits(512, 8192),
+                          split="development")
+        report.update(checkpoint_sha256=f"{seed:064x}", split_sha256="a" * 64)
+        report["rows"][0][metric] = value
+        path.write_text(json.dumps(report), encoding="utf-8")
+        args.extend(("--run", f"{seed}={path}"))
+    output = tmp_path / "summary.json"
+    with pytest.raises(ValueError, match=f"evaluation_row_metric_invalid:{metric}"):
+        seed_report_cli([*args, "--out", str(output)])
+    assert not output.exists()
+
+
 def test_calibration_without_type_does_not_retain_unreported_old_temperature(tiny, suite):
     data, manifest = suite
     datum = copy.deepcopy(data["calibration"][0])

tests/test_native_gradients.py

@@ -7,6 +7,16 @@
 from kev_laya.native_gradients import CanonicalFP32ShortAttention, causal_vjp
 
 
+@pytest.fixture(params=[
+    "cpu",
+    pytest.param("cuda", marks=pytest.mark.skipif(
+        not torch.cuda.is_available(), reason="CUDA device unavailable")),
+])
+def attention_device(request):
+    """Exercise portable helper math without substituting CPU for CUDA proof."""
+    return request.param
+
+
 @pytest.mark.parametrize("prefix", [0, 8])
 def test_causal_vjp_matches_independent_double_autograd(prefix):
     generator = torch.Generator().manual_seed(2618 + prefix)
@@ -28,15 +38,14 @@ def test_causal_vjp_matches_independent_double_autograd(prefix):
         torch.testing.assert_close(observed, oracle, atol=1e-12, rtol=1e-12)
 
 
-@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA device unavailable")
 @pytest.mark.parametrize("prefix", [0, 11])
-def test_fp32_short_grouped_double_cache_matches_explicit_eager_gqa(prefix):
-    generator = torch.Generator(device="cuda").manual_seed(4521 + prefix)
-    q0 = torch.randn(1, 4, 7, 16, generator=generator, device="cuda", dtype=torch.float32)
+def test_fp32_short_grouped_double_cache_matches_explicit_eager_gqa(prefix, attention_device):
+    generator = torch.Generator(device=attention_device).manual_seed(4521 + prefix)
+    q0 = torch.randn(1, 4, 7, 16, generator=generator, device=attention_device, dtype=torch.float32)
     k0 = torch.randn(1, 2, prefix + 7, 16, generator=generator,
-                     device="cuda", dtype=torch.float32).double()
-    v0 = torch.randn(k0.shape, generator=generator, device="cuda", dtype=torch.float32).double()
-    upstream = torch.randn(q0.shape, generator=generator, device="cuda", dtype=torch.float32)
+                     device=attention_device, dtype=torch.float32).double()
+    v0 = torch.randn(k0.shape, generator=generator, device=attention_device, dtype=torch.float32).double()
+    upstream = torch.randn(q0.shape, generator=generator, device=attention_device, dtype=torch.float32)
     q, k, v = [tensor.detach().clone().requires_grad_(True) for tensor in (q0, k0, v0)]
     actual = CanonicalFP32ShortAttention.apply(q, k, v, 2)
     actual_vjp = torch.autograd.grad(actual, (q, k, v), upstream)
@@ -49,7 +58,7 @@ def test_fp32_short_grouped_double_cache_matches_explicit_eager_gqa(prefix):
     padded_k = torch.nn.functional.pad(repeated_k, (0, 0, 0, 256 - total))
     padded_v = torch.nn.functional.pad(repeated_v, (0, 0, 0, 256 - total))
     scores = torch.matmul(padded_q, padded_k.transpose(-2, -1)) * (16 ** -0.5)
-    allowed = torch.ones((256, 256), device="cuda", dtype=torch.bool).tril()
+    allowed = torch.ones((256, 256), device=attention_device, dtype=torch.bool).tril()
     scores = scores.masked_fill(~allowed, torch.finfo(scores.dtype).min)
     probabilities = torch.nn.functional.softmax(scores, dim=-1, dtype=torch.float32).to(rq.dtype)
     expected = torch.matmul(probabilities, padded_v)[..., prefix:prefix + 7, :]
@@ -76,14 +85,13 @@ def test_fp32_short_grouped_double_cache_matches_explicit_eager_gqa(prefix):
                                    atol=2e-5, rtol=2e-5)
 
 
-@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA device unavailable")
 @pytest.mark.parametrize("prefix_length", [1, 11, 62, 127, 200, 254, 255])
-def test_fp32_short_fixed256_matches_any_cached_prefix(prefix_length):
-    generator = torch.Generator(device="cuda").manual_seed(6256 + prefix_length)
-    query = torch.randn(1, 4, 256, 16, generator=generator, device="cuda")
-    key = torch.randn(1, 2, 256, 16, generator=generator, device="cuda").double()
-    value = torch.randn(1, 2, 256, 16, generator=generator, device="cuda").double()
-    upstream = torch.randn(query.shape, generator=generator, device="cuda")
+def test_fp32_short_fixed256_matches_any_cached_prefix(prefix_length, attention_device):
+    generator = torch.Generator(device=attention_device).manual_seed(6256 + prefix_length)
+    query = torch.randn(1, 4, 256, 16, generator=generator, device=attention_device)
+    key = torch.randn(1, 2, 256, 16, generator=generator, device=attention_device).double()
+    value = torch.randn(1, 2, 256, 16, generator=generator, device=attention_device).double()
+    upstream = torch.randn(query.shape, generator=generator, device=attention_device)
     full_inputs = [tensor.detach().clone().requires_grad_(True)
                    for tensor in (query, key, value)]
     split_inputs = [tensor.detach().clone().requires_grad_(True)