From 8519ecd6567af1dc0492a0efe4f4e9e3585b3f55 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 6 Oct 2026 15:33:17 +0000 Subject: [PATCH 1/2] evaluate: fix coverage_delta duplicating rows when subgroups and ragged coverage coexist pivot_table(dropna=False) builds the cartesian product of the index levels, so with a declared subgroup every overall row was repeated per subgroup level and _primary_rows crashed on a Series from cov_auroc.get(). Found running the HELM BoolQ roster (10 models with unequal coverage plus a perturbation subgroup). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01LeQ5eAyfbApPDkXxbExQS9 --- src/predval/evaluate.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/predval/evaluate.py b/src/predval/evaluate.py index 2268207..09b0af6 100644 --- a/src/predval/evaluate.py +++ b/src/predval/evaluate.py @@ -95,9 +95,14 @@ def coverage_delta(metrics: pd.DataFrame) -> pd.DataFrame: # As-published only: coverage restriction is a property of rung0, and mixing recalibrated # rungs into the pivot would compare corrections rather than the effect of the common subset. primary = metrics[(metrics["ci_method"] != "naive_row_bootstrap") & (metrics["rung"] == RUNG0)] - wide = primary.pivot_table( - index=keys, columns="subset", values="value", aggfunc="first", dropna=False - ).reset_index() + # groupby, not pivot_table: pivot_table(dropna=False) takes the cartesian product of the index + # levels, inventing (overall, perturbation=dialect, ...) rows that duplicate each model_id. + wide = ( + primary.groupby([*keys, "subset"], dropna=False, observed=True)["value"] + .first() + .unstack("subset") + .reset_index() + ) if "full" not in wide or "common" not in wide: return pd.DataFrame(columns=[*keys, "cov_delta"]) wide["cov_delta"] = wide["full"] - wide["common"] From 92359fc82ba8620d4bf08a974ad60674e20075c9 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 16:52:07 +0000 Subject: [PATCH 2/2] tests: regression test for coverage_delta with subgroups and ragged coverage Fails on the pre-fix pivot_table (duplicated overall rows per model) and passes with the groupby version. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01LeQ5eAyfbApPDkXxbExQS9 --- tests/test_evaluate.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test_evaluate.py b/tests/test_evaluate.py index 1a82155..c89b769 100644 --- a/tests/test_evaluate.py +++ b/tests/test_evaluate.py @@ -242,6 +242,22 @@ def test_coverage_delta_is_zero_when_coverage_is_identical(result) -> None: assert np.allclose(finite, 0.0), "full == common here, so every delta must be zero" +def test_coverage_delta_has_one_overall_row_per_model_with_ragged_coverage(tmp_path: Path) -> None: + """Subgroups plus unequal coverage must not multiply the overall rows (report.py indexes them). + + pivot_table(dropna=False) crossed the index levels, inventing overall rows carrying subgroup + levels; _primary_rows then got a Series back from .get() and crashed rendering the report. + """ + cohort, preds = build(tmp_path, drop_for_weak=40) + delta = coverage_delta(evaluate(cohort, preds).metrics) + overall = delta[(delta["stratum_kind"] == "overall") & (delta["metric"] == "auroc")] + assert sorted(overall["model_id"]) == sorted(MODELS) + assert overall["subgroup_name"].isna().all() + by_model = overall.set_index("model_id")["cov_delta"] + assert by_model["weak"] == 0.0, "weak's own rows define the common subset" + assert np.isfinite(by_model["good"]) and by_model["good"] != 0.0, "good loses rows to common" + + # -------------------------------------------------------------------------------- the manifest