diff --git a/src/predval/evaluate.py b/src/predval/evaluate.py index 2268207..09b0af6 100644 --- a/src/predval/evaluate.py +++ b/src/predval/evaluate.py @@ -95,9 +95,14 @@ def coverage_delta(metrics: pd.DataFrame) -> pd.DataFrame: # As-published only: coverage restriction is a property of rung0, and mixing recalibrated # rungs into the pivot would compare corrections rather than the effect of the common subset. primary = metrics[(metrics["ci_method"] != "naive_row_bootstrap") & (metrics["rung"] == RUNG0)] - wide = primary.pivot_table( - index=keys, columns="subset", values="value", aggfunc="first", dropna=False - ).reset_index() + # groupby, not pivot_table: pivot_table(dropna=False) takes the cartesian product of the index + # levels, inventing (overall, perturbation=dialect, ...) rows that duplicate each model_id. + wide = ( + primary.groupby([*keys, "subset"], dropna=False, observed=True)["value"] + .first() + .unstack("subset") + .reset_index() + ) if "full" not in wide or "common" not in wide: return pd.DataFrame(columns=[*keys, "cov_delta"]) wide["cov_delta"] = wide["full"] - wide["common"] diff --git a/tests/test_evaluate.py b/tests/test_evaluate.py index 1a82155..c89b769 100644 --- a/tests/test_evaluate.py +++ b/tests/test_evaluate.py @@ -242,6 +242,22 @@ def test_coverage_delta_is_zero_when_coverage_is_identical(result) -> None: assert np.allclose(finite, 0.0), "full == common here, so every delta must be zero" +def test_coverage_delta_has_one_overall_row_per_model_with_ragged_coverage(tmp_path: Path) -> None: + """Subgroups plus unequal coverage must not multiply the overall rows (report.py indexes them). + + pivot_table(dropna=False) crossed the index levels, inventing overall rows carrying subgroup + levels; _primary_rows then got a Series back from .get() and crashed rendering the report. + """ + cohort, preds = build(tmp_path, drop_for_weak=40) + delta = coverage_delta(evaluate(cohort, preds).metrics) + overall = delta[(delta["stratum_kind"] == "overall") & (delta["metric"] == "auroc")] + assert sorted(overall["model_id"]) == sorted(MODELS) + assert overall["subgroup_name"].isna().all() + by_model = overall.set_index("model_id")["cov_delta"] + assert by_model["weak"] == 0.0, "weak's own rows define the common subset" + assert np.isfinite(by_model["good"]) and by_model["good"] != 0.0, "good loses rows to common" + + # -------------------------------------------------------------------------------- the manifest