From b5fe8a39466c469576b5955ae4b6851cf607a5f5 Mon Sep 17 00:00:00 2001 From: aRealGem <203440826+aRealGem@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:05:01 -0400 Subject: [PATCH 01/12] Guard IMPROVE patient-level CV partition integrity Validate grouping metadata across all six random-forest tables before writing fixtures. Add synthetic regression coverage and clarify that metadata checks do not independently audit upstream training or NNAlign. Validation: 269 passed, 20 fixture-dependent skips; 18 focused tests pass. Ruff lint and formatting pass. No numerical estimator or fixture changes. Publication preserves the reviewed file tree; connector-assigned commit metadata differs. Reviewed-local-commit: 99ca3d72f1d4f8340df9a59de3616277e9ebdb7a Co-developed-with: OpenAI Codex --- examples/improve/README.md | 10 +-- examples/improve/build_fixture.py | 21 ++++++ tests/test_improve_builder.py | 112 ++++++++++++++++++++++++++++++ 3 files changed, 139 insertions(+), 4 deletions(-) create mode 100644 tests/test_improve_builder.py diff --git a/examples/improve/README.md b/examples/improve/README.md index b21f5b8..a77aa5f 100644 --- a/examples/improve/README.md +++ b/examples/improve/README.md @@ -27,10 +27,12 @@ through broad-scale validation of T cell recognition*, Frontiers in Immunology. patients; melanoma 5,921 / 26; a mixed "Basket" group 5,362 / 20). - 467 peptides are experimentally immunogenic — **2.666% prevalence**. - Predictions are the paper's own **out-of-fold** scores from its 5-fold cross-validation. That - CV was partitioned **by patient**: every patient sits in exactly one partition, verified in - `build_fixture.py`. No score here comes from a model that had seen another peptide from the - same patient, which is what makes the file admissible as a predictions-only validation input - rather than an apparent-fit artefact. + CV was partitioned **by patient**. `build_fixture.py` checks every released random-forest + table for non-missing patient IDs and integer partitions, and rejects any patient appearing + in more than one partition before writing either fixture. This verifies the tables' grouping + metadata; it does not independently audit upstream training or the NNAlign comparator's + split assignments. Interpreting the released scores as out-of-fold predictions still relies + on the source authors' training procedure. - Seven models: the IMPROVE random forest in its three released feature configurations (`Simple`; `TME_excluded`, adding cellular prevalence and a prioritisation score; `TME_included`, further adding cytolytic activity, HLA expression and MCP-counter estimates), diff --git a/examples/improve/build_fixture.py b/examples/improve/build_fixture.py index e3f2f1b..4effebb 100644 --- a/examples/improve/build_fixture.py +++ b/examples/improve/build_fixture.py @@ -39,11 +39,32 @@ def _sid(frame: pd.DataFrame) -> pd.Series: ) +def validate_patient_partitions(frame: pd.DataFrame, model_id: str) -> None: + """Fail closed on missing grouping metadata or patients split across CV folds. + + This checks the released RF tables, not the upstream training implementation or + the NNAlign comparator (whose Split field is not read by this builder). + """ + required = {"Patient", "Partition"} + if not required.issubset(frame.columns): + raise SystemExit(f"{model_id}: missing Patient or Partition column") + if frame["Patient"].isna().any() or frame["Patient"].astype(str).str.strip().eq("").any(): + raise SystemExit(f"{model_id}: missing patient identifiers") + partition = pd.to_numeric(frame["Partition"], errors="coerce") + if partition.isna().any() or (partition % 1 != 0).any(): + raise SystemExit(f"{model_id}: CV partitions must be non-missing integers") + # Use the same patient representation as the output cohort, without changing the data. + counts = partition.groupby(frame["Patient"].astype(str)).nunique() + if (counts > 1).any(): + raise SystemExit(f"{model_id}: a patient appears in multiple CV partitions") + + def load_rf(src: Path) -> dict[str, pd.DataFrame]: out = {} for model_id, (sub, name) in RF_MODELS.items(): path = src / "5_fold_CV" / sub / name frame = pd.read_csv(path, sep=r"\s+") + validate_patient_partitions(frame, model_id) frame["subject_id"] = _sid(frame) out[model_id] = frame return out diff --git a/tests/test_improve_builder.py b/tests/test_improve_builder.py new file mode 100644 index 0000000..ab6b037 --- /dev/null +++ b/tests/test_improve_builder.py @@ -0,0 +1,112 @@ +"""Synthetic regression tests for the IMPROVE fixture's patient/CV integrity guard. + +No source archive, generated fixture, or opt-in golden run is needed. +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pandas as pd +import pytest + +PATH = Path(__file__).resolve().parents[1] / "examples" / "improve" / "build_fixture.py" +SPEC = importlib.util.spec_from_file_location("improve_build_fixture", PATH) +assert SPEC is not None and SPEC.loader is not None +builder = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(builder) + + +@pytest.fixture +def rf_frame() -> pd.DataFrame: + return pd.DataFrame( + { + "Patient": ["patient_a", "patient_a", "patient_b"], + "Partition": [0, 0, 1], + "Mut_peptide": ["AAAAAAAAA", "CCCCCCCC", "DDDDDDDDD"], + "HLA_allele": ["HLA-A01:01"] * 3, + "cohort": ["synthetic"] * 3, + "response": [0, 1, 0], + "prediction_rf": [0.1, 0.8, 0.2], + } + ) + + +def write_rf_tables(src: Path, frame: pd.DataFrame) -> None: + for sub, name in builder.RF_MODELS.values(): + path = src / "5_fold_CV" / sub / name + path.parent.mkdir(parents=True, exist_ok=True) + frame.to_csv(path, sep="\t", index=False) + + +@pytest.mark.parametrize("partitions", [[0, 0, 1], ["01", "1.0", "2"]]) +def test_patient_partition_guard_preserves_valid_input(rf_frame, partitions) -> None: + rf_frame["Partition"] = partitions + before = rf_frame.copy(deep=True) + builder.validate_patient_partitions(rf_frame, "test_model") + pd.testing.assert_frame_equal(rf_frame, before) + + +@pytest.mark.parametrize("column", ["Patient", "Partition"]) +def test_patient_partition_guard_requires_columns(rf_frame, column) -> None: + with pytest.raises(SystemExit, match="test_model: missing Patient or Partition column"): + builder.validate_patient_partitions(rf_frame.drop(columns=column), "test_model") + + +@pytest.mark.parametrize("missing", [None, "", " "]) +def test_patient_partition_guard_rejects_missing_patient(rf_frame, missing) -> None: + rf_frame.loc[0, "Patient"] = missing + with pytest.raises(SystemExit, match="test_model: missing patient identifiers"): + builder.validate_patient_partitions(rf_frame, "test_model") + + +@pytest.mark.parametrize("invalid", [None, "unknown", 0.5, float("inf")]) +def test_patient_partition_guard_rejects_invalid_partition(rf_frame, invalid) -> None: + rf_frame["Partition"] = rf_frame["Partition"].astype(object) + rf_frame.loc[0, "Partition"] = invalid + with pytest.raises(SystemExit, match="test_model: CV partitions must be non-missing integers"): + builder.validate_patient_partitions(rf_frame, "test_model") + + +@pytest.mark.parametrize("model_id", list(builder.RF_MODELS)) +def test_builder_rejects_split_patient_in_every_model_before_writing( + tmp_path, rf_frame, model_id +) -> None: + src = tmp_path / "source" + out = tmp_path / "output" + write_rf_tables(src, rf_frame) + rf_frame.loc[1, "Partition"] = 1 + sub, name = builder.RF_MODELS[model_id] + rf_frame.to_csv(src / "5_fold_CV" / sub / name, sep="\t", index=False) + with pytest.raises( + SystemExit, match=f"{model_id}: a patient appears in multiple CV partitions" + ): + builder.main(["--src", str(src), "--out", str(out)]) + assert not out.exists(), "invalid CV metadata must fail before either fixture is written" + + +def test_builder_preserves_valid_fixture_values(tmp_path, rf_frame, monkeypatch) -> None: + src = tmp_path / "source" + out = tmp_path / "output" + write_rf_tables(src, rf_frame) + nn = rf_frame.copy() + nn["subject_id"] = builder._sid(nn) + nn["target"] = nn["response"] + nn["predicted"] = nn["prediction_rf"] + monkeypatch.setattr(builder, "load_nnalign", lambda _: nn) + + assert builder.main(["--src", str(src), "--out", str(out)]) == 0 + cohort = pd.read_parquet(out / "cohort.parquet") + predictions = pd.read_parquet(out / "predictions.parquet") + assert cohort["patient"].tolist() == rf_frame["Patient"].tolist() + assert cohort["cv_partition"].tolist() == rf_frame["Partition"].tolist() + assert cohort["response"].tolist() == rf_frame["response"].tolist() + assert cohort["subject_id"].tolist() == [ + "patient_a|AAAAAAAAA|HLA-A01:01", + "patient_a|CCCCCCCC|HLA-A01:01", + "patient_b|DDDDDDDDD|HLA-A01:01", + ] + assert len(predictions) == len(rf_frame) * (len(builder.RF_MODELS) + 1) + for _, group in predictions.groupby("model_id"): + assert group["predicted"].tolist() == rf_frame["prediction_rf"].tolist() From af4c88c913dd45889332a24947061ba51f64fd56 Mon Sep 17 00:00:00 2001 From: aRealGem <203440826+aRealGem@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:06:01 -0400 Subject: [PATCH 02/12] Prespecify bounded alternatives benchmark Publication preserves the reviewed file tree; connector-assigned commit metadata differs. Reviewed-local-commit: dae6e1ad1c9c20bcd179e5db7cf9f71e3d888beb Co-developed-with: OpenAI Codex --- benchmarks/alternatives/PROTOCOL.md | 64 +++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 benchmarks/alternatives/PROTOCOL.md diff --git a/benchmarks/alternatives/PROTOCOL.md b/benchmarks/alternatives/PROTOCOL.md new file mode 100644 index 0000000..e39df8e --- /dev/null +++ b/benchmarks/alternatives/PROTOCOL.md @@ -0,0 +1,64 @@ +# Alternatives benchmark protocol (registered before benchmark execution) + +Date: 2026-10-10. PREDVAL baseline: `99ca3d72f1d4f8340df9a59de3616277e9ebdb7a`. +This is an engineering/method-comparison benchmark, not clinical validation or a claim +that a fitted model is suitable for patient care. No runtime, accuracy, or novelty winner +will be inferred from unavailable software. + +## Fixed scope and inputs + +1. Independent binary rows: Wisconsin Diagnostic Breast Cancer, bundled with scikit-learn, + UCI dataset 17 (CC BY 4.0). Use its original measurements to create strictly held-out + predictions from an explicitly specified logistic regression pipeline. This is a real, + redistributable methods fixture, not an external clinical validation study. +2. Synthetic clustered patients: seed 20261010; 120 patients, unequal 4–20 repeated rows, + patient random effect and patient-correlated binary labels. Published predictions are + deliberately distorted. This controlled fixture exercises clustering, correction, and + missing-coverage failure paths, and is explicitly synthetic. +3. Missing-model-coverage variants: remove predictions from patients in the upper 30% of + mean predicted risk, and separately remove an entire declared model. The cohort and + model roster stay fixed. Compare full available rows and the common intersection. + +## Matched estimands and configuration + +- Positive outcome is coded 1 throughout. Brier is mean squared error over rows, not a + patient-weighted mean. AUROC uses tied-score midranks. Point-metric tolerance: 1e-10. +- Calibration slope is the slope from unpenalized logistic `y ~ 1 + logit(p)`. + Calibration-in-the-large is a separate intercept with logit(p) as an offset (slope 1). + Do not equate an intercept from a free-slope fit with calibration-in-the-large. +- Matched correction: unpenalized logistic intercept+slope on clipped logit probability + (epsilon 1e-6), five disjoint patient folds. Use PREDVAL's fixed sorted-patient round-robin + assignment as an exported common split, and give competitors those exact folds when + their public API supports it. Compare pooled held-out Brier and AUROC, not unweighted + fold-mean metrics. Label non-equivalent fits separately rather than declaring disagreement. +- Intervals: 1,000 paired whole-patient percentile bootstrap replicates; seed 20261011; + 95% nominal intervals. Conditional on the fixed held-out predictions; no claim of full + correction-training uncertainty. Use the exact same resample indices for custom adapters. + Compare row-bootstrap intervals separately as different assumptions, not matched inference. +- Independent-row fixture: stratified 50% held-out split, seed 20261010; StandardScaler plus + LogisticRegression(C=1, max_iter=2000, solver='lbfgs'); no hyperparameter tuning. The positive + class is malignant. A fixed `sigmoid(-0.4 + 1.4*logit(p))` distortion supplies a correction case. + +## Competitors and fair interpretation + +- PyCalEva: installed registry release, public CalibrationEvaluator for available metrics. + It is primarily a calibration evaluator, so missing end-to-end cohort machinery is not + a numerical defect. Document required adapter work rather than handicap it. +- probably + rsample: public `cal_validate_logistic` accepts an rset; grouped rsets and group + bootstraps are supported. Export matched folds and provide a runnable R adapter. If R is + absent or cannot be installed safely, mark execution unavailable, never failed metrics. +- isitfair: use official registry release if available, otherwise inspect official source + without running unrecognized code. Respect explicit folds on its cross-fitted ladder and + its one-row-per-patient convention. Do not call a row bootstrap a grouped-patient bootstrap. +- PREDVAL is not privileged as the numerical oracle: independently calculate metrics with + scikit-learn/statsmodels and a small shared resampling reference. + +## Deliverables and stopping rule + +Commit protocol before results; record exact software versions, input hashes, outputs, +commands, convergence/failure information, supported/configurable/unavailable capabilities, +and adapter source line counts (not invented developer-time savings). Run each executable +path twice and compare stable output hashes. Deliver CSV/JSON summaries, reproducible scripts, +and a bounded findings report. Stop after both fixtures, coverage variants, and clean repeat +have completed, or document specific dependency/access blockers. No paid compute, private +clinical data, data-use agreements, outreach, push, PR, merge, or deployment. From eb22fde21e5d18e6f57f9f354f717ad2a31ee0e5 Mon Sep 17 00:00:00 2001 From: aRealGem <203440826+aRealGem@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:06:20 -0400 Subject: [PATCH 03/12] Clarify correction estimands after competitor source review Publication preserves the reviewed file tree; connector-assigned commit metadata differs. Reviewed-local-commit: 756e5a1956578a340eddb90806813ddee62e30fc Co-developed-with: OpenAI Codex --- benchmarks/alternatives/PROTOCOL.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/benchmarks/alternatives/PROTOCOL.md b/benchmarks/alternatives/PROTOCOL.md index e39df8e..054b05a 100644 --- a/benchmarks/alternatives/PROTOCOL.md +++ b/benchmarks/alternatives/PROTOCOL.md @@ -62,3 +62,17 @@ path twice and compare stable output hashes. Deliver CSV/JSON summaries, reprodu and a bounded findings report. Stop after both fixtures, coverage variants, and clean repeat have completed, or document specific dependency/access blockers. No paid compute, private clinical data, data-use agreements, outreach, push, PR, merge, or deployment. + +## Source-review amendment (before R or isitfair execution) + +Official source inspection found that probably's `smooth=FALSE` fits `y ~ p`, while +PREDVAL rung2 fits `y ~ logit(p)`. These are different correction families. Report probably's +native result separately and cross-check it against a raw-probability statsmodels GLM; +retain a separate R `glm(y ~ logit(p))` adapter as the exactly matched control. Do not feed +logits into a probability API merely to force equality. Isitfair's native Platt fit uses +L2-regularized sklearn logistic regression and epsilon 1e-10, so it too is a separate native +configuration, not an exact unpenalized rung2 equivalent. No executed isitfair result will +be claimed without an authorized executable package. + +An isolated conda-forge R setup may be attempted for at most 15 minutes and less than 1 GB +of planned package downloads. No system-level R installation or security changes. From 9533c0b75ddea5c6a39abc59746d6ae020a7139f Mon Sep 17 00:00:00 2001 From: aRealGem <203440826+aRealGem@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:06:28 -0400 Subject: [PATCH 04/12] Add reproducible alternatives benchmark and explicit evidence limits Publication preserves the reviewed file tree; connector-assigned commit metadata differs. Reviewed-local-commit: 868276ef960f0bc7f2bc75cc79926ff7b3c2e1f1 Co-developed-with: OpenAI Codex --- .gitignore | 5 + benchmarks/alternatives/.gitignore | 10 + benchmarks/alternatives/FIXTURE_LICENSE.md | 22 ++ benchmarks/alternatives/README.md | 243 +++++++++++++ benchmarks/alternatives/conda-linux-64.lock | 229 ++++++++++++ benchmarks/alternatives/isitfair_adapter.py | 60 ++++ benchmarks/alternatives/probably_adapter.R | 90 +++++ benchmarks/alternatives/requirements-lock.txt | 53 +++ .../alternatives/results/glue_inventory.json | 12 + .../results/python/correction_metrics.csv | 13 + .../alternatives/results/python/coverage.csv | 3 + .../python/coverage_selection_metrics.csv | 5 + .../alternatives/results/python/intervals.csv | 11 + .../alternatives/results/python/manifest.json | 31 ++ .../results/python/point_metrics.csv | 21 ++ .../results/python/resource_usage.json | 6 + .../alternatives/results/python/summary.json | 59 +++ .../alternatives/results/r/r_agreement.csv | 3 + .../results/r/r_bootstrap_summary.csv | 3 + .../alternatives/results/r/r_metrics.csv | 9 + .../alternatives/results/r/resource_usage.txt | 2 + .../alternatives/results/r/sessionInfo.txt | 45 +++ .../alternatives/results/r/verification.json | 17 + .../alternatives/results/reproducibility.json | 18 + benchmarks/alternatives/run_benchmark.py | 339 ++++++++++++++++++ .../alternatives/tests/test_benchmark.py | 70 ++++ .../verification/benchmark-contract-tests.log | 10 + .../verification/predval-pytest.log | 5 + .../alternatives/verification/r-parse.log | 1 + .../alternatives/verification/r-repeat.log | 3 + .../alternatives/verification/r-run.log | 3 + .../verification/repeat-check.log | 17 + benchmarks/alternatives/verification/ruff.log | 1 + .../alternatives/verification/status.json | 21 ++ benchmarks/alternatives/verify_results.py | 63 ++++ 35 files changed, 1503 insertions(+) create mode 100644 benchmarks/alternatives/.gitignore create mode 100644 benchmarks/alternatives/FIXTURE_LICENSE.md create mode 100644 benchmarks/alternatives/README.md create mode 100644 benchmarks/alternatives/conda-linux-64.lock create mode 100644 benchmarks/alternatives/isitfair_adapter.py create mode 100644 benchmarks/alternatives/probably_adapter.R create mode 100644 benchmarks/alternatives/requirements-lock.txt create mode 100644 benchmarks/alternatives/results/glue_inventory.json create mode 100644 benchmarks/alternatives/results/python/correction_metrics.csv create mode 100644 benchmarks/alternatives/results/python/coverage.csv create mode 100644 benchmarks/alternatives/results/python/coverage_selection_metrics.csv create mode 100644 benchmarks/alternatives/results/python/intervals.csv create mode 100644 benchmarks/alternatives/results/python/manifest.json create mode 100644 benchmarks/alternatives/results/python/point_metrics.csv create mode 100644 benchmarks/alternatives/results/python/resource_usage.json create mode 100644 benchmarks/alternatives/results/python/summary.json create mode 100644 benchmarks/alternatives/results/r/r_agreement.csv create mode 100644 benchmarks/alternatives/results/r/r_bootstrap_summary.csv create mode 100644 benchmarks/alternatives/results/r/r_metrics.csv create mode 100644 benchmarks/alternatives/results/r/resource_usage.txt create mode 100644 benchmarks/alternatives/results/r/sessionInfo.txt create mode 100644 benchmarks/alternatives/results/r/verification.json create mode 100644 benchmarks/alternatives/results/reproducibility.json create mode 100644 benchmarks/alternatives/run_benchmark.py create mode 100644 benchmarks/alternatives/tests/test_benchmark.py create mode 100644 benchmarks/alternatives/verification/benchmark-contract-tests.log create mode 100644 benchmarks/alternatives/verification/predval-pytest.log create mode 100644 benchmarks/alternatives/verification/r-parse.log create mode 100644 benchmarks/alternatives/verification/r-repeat.log create mode 100644 benchmarks/alternatives/verification/r-run.log create mode 100644 benchmarks/alternatives/verification/repeat-check.log create mode 100644 benchmarks/alternatives/verification/ruff.log create mode 100644 benchmarks/alternatives/verification/status.json create mode 100644 benchmarks/alternatives/verify_results.py diff --git a/.gitignore b/.gitignore index fb80953..c3b7307 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,8 @@ validation/predrupdate/gusto_estimates.csv examples/improve/cohort.parquet examples/improve/predictions.parquet examples/improve/out/ + +# Isolated benchmark environments and scratch reruns +.venv-clean/ +.venv-benchmark/ +benchmarks/alternatives/results/run*/ diff --git a/benchmarks/alternatives/.gitignore b/benchmarks/alternatives/.gitignore new file mode 100644 index 0000000..33a3cf8 --- /dev/null +++ b/benchmarks/alternatives/.gitignore @@ -0,0 +1,10 @@ +# Generated row-level fixtures and predictions regenerate from the locked script. +# Commit compact numerical summaries and hashes, not bulk row-level inputs. +results/python/coverage_inputs/ +results/python/wdbc_heldout.csv +results/python/wdbc_heldout_corrected.csv +results/python/synthetic_clustered.csv +results/python/synthetic_clustered_corrected.csv +results/r/*_r_corrected.csv +results/r/*_rsample_gains.csv +results/r-repeat*/ diff --git a/benchmarks/alternatives/FIXTURE_LICENSE.md b/benchmarks/alternatives/FIXTURE_LICENSE.md new file mode 100644 index 0000000..705422d --- /dev/null +++ b/benchmarks/alternatives/FIXTURE_LICENSE.md @@ -0,0 +1,22 @@ +# Fixture attribution + +Wisconsin Diagnostic Breast Cancer is provided by the UCI Machine Learning Repository +under Creative Commons Attribution 4.0 International: +https://creativecommons.org/licenses/by/4.0/ + +Wolberg, W., Mangasarian, O., Street, N., & Street, W. (1993). +Breast Cancer Wisconsin (Diagnostic). UCI Machine Learning Repository. +https://doi.org/10.24432/C5DW2B + +Source/license: https://archive.ics.uci.edu/dataset/17/breast+cancer+wisconsin+diagnostic + +The benchmark uses scikit-learn's packaged copy. It derives a fixed held-out subset, +probabilities from a model fitted on separate training rows, deliberately distorted +probabilities, and row-derived identifiers. These are alterations; no creator endorsement +is implied. Generated row-level data are not committed or included in the compact bundle. + +The repeated-patient fixture is wholly synthetic and generated by the repository's own +benchmark code. It is not clinical validation and contains no real patient records. + +The benchmark's original code is covered by the repository's Apache-2.0 LICENSE. +No source-only competitor package is redistributed in the compact review bundle. diff --git a/benchmarks/alternatives/README.md b/benchmarks/alternatives/README.md new file mode 100644 index 0000000..98490b3 --- /dev/null +++ b/benchmarks/alternatives/README.md @@ -0,0 +1,243 @@ +# Bounded comparison: PREDVAL, PyCalEva, probably + rsample, and isitfair + +This is a reproducible **methods/engineering comparison**, not clinical validation. +PREDVAL's added value here is explicit cohort/cluster contracts and connected diagnostics. +Its point metrics are not novel, and correct alternatives can reproduce its results. + +The protocol was committed before execution (`dae6e1a`), with a source-review amendment +(`756e5a1`) distinguishing correction families before any R or isitfair execution. +Both commits descend from patient-partition guard `99ca3d7`; no core PREDVAL code changed. + +## What is and is not executed + +- **PREDVAL 0.1.0, PyCalEva 0.8.2:** executable numerical checks on both fixtures. +- **Independent sklearn/statsmodels control:** point metrics, unpenalized grouped correction, + and a separate custom patient bootstrap. PREDVAL is not its own numerical oracle. +- **probably 1.2.0 + rsample 1.3.2 (R 4.5.3):** native adapter executed on both fixtures, + including grouped cross-validation and 1,000 grouped bootstrap replicates. `cal_validate_logistic` accepts an rset, including + grouped rsample splits. It is not fair to describe this stack as lacking grouped validation. +- **isitfair:** official source reviewed at `70ffb8d21c3e24eddfee412220b8a7a701768937`. + No registry package was found; no source code was executed. An unexecuted public-API adapter + is included. Its existing integrated reports and cross-fitted recalibration ladder make it + a close competitor; this benchmark does not establish superiority over it. + +## Fixtures and estimands + +- `wdbc_heldout`: 285 held-out observations (106 malignant), from the 569-row Wisconsin + Diagnostic Breast Cancer dataset bundled with scikit-learn. Train/test separation is + fixed, stratified, and 50:50. Scaling and model fitting use only the training half. + Logistic predictions are deliberately distorted to exercise calibration. This is a + real-data test fixture, not an externally validated clinical model or deployment study. +- `synthetic_clustered`: 1,332 repeated rows in 120 synthetic patients, 560 events, unequal + cluster sizes and correlated outcomes. Its generator and all seeds are in the protocol. + It contains no real patient records and supplies no evidence of clinical effectiveness. +- Row-weighted Brier and AUROC are matched. Calibration intercept fixes slope at one; + calibration slope is from a separate joint intercept+slope logistic fit. +- Correction uses the same exported, disjoint, five-fold patient assignments. Pooled + held-out metrics are used, not averages of fold AUROCs. PREDVAL rung2 and the independent + GLM use unpenalized logit-scale regression, epsilon 1e-6. +- Gain intervals use 1,000 paired patient bootstrap replicates, conditional on fixed + held-out predictions. They do not include correction-training uncertainty. One fixture's + interval is not an empirical confidence-interval coverage study. + +## Python findings + +PREDVAL and PyCalEva agree exactly on the four checked AUROC/Brier point estimates. +Against independent statsmodels, PREDVAL's calibration intercept/slope differ by less than +2e-15. Grouped rung2 probabilities agree within 4e-14 on these fixtures. + +| Fixture | Published Brier | Grouped corrected Brier | Paired gain | 95% patient-bootstrap gain CI | +|---|---:|---:|---:|---:| +| WDBC held out | 0.031263 | 0.029256 | 0.002007 | [-0.005336, 0.009427] | +| Synthetic clustered | 0.181410 | 0.170541 | 0.010869 | [-0.001163, 0.022167] | + +Both gains are uncertain in these fixtures. The synthetic row-bootstrap interval is +[0.005338, 0.016573], which excludes zero; respecting patients gives an interval 2.08 times +as wide that crosses zero. For WDBC, every cluster is a singleton; small row/patient interval differences reflect +ordering and Monte Carlo draws, not clustering. For the synthetic fixture this is a concrete +consequence of changing the independence +assumption, not evidence that PyCalEva is numerically wrong. A PyCalEva-plus-custom-patient- +bootstrap adapter reproduces PREDVAL's paired interval to numerical precision. + +PREDVAL also withholds the WDBC spline rung after a non-convergence signal. This is reported, +not hidden to make the tool look better; this benchmark does not independently diagnose that +spline fit or establish that every PREDVAL unavailable-reason label is correct. + +### Coverage faults + +The synthetic partial model has **identical scores** to the full model where present; only +availability differs. Removing the top 30% of patients by mean predicted risk leaves +919/1,332 rows (68.99%). Thus a separate-model comparison on available rows changes the +cohort while appearing to compare models: + +| Available cohort | n | AUROC | Brier | +|---|---:|---:|---:| +| Full model's available rows | 1,332 | 0.821898 | 0.181410 | +| Shared/partial rows | 919 | 0.783196 | 0.177929 | + +PREDVAL rejects this at a declared 90% coverage floor, warns that the intersection excludes +31.0% of rows, and detects a completely absent declared model. The same checks are easy to +implement around array-based tools, but the caller must supply the full cohort and roster; +aligned y/p arrays alone cannot reveal missing rows or a missing model. These checks exercise +public I/O/contract functions, not a complete rendered-report/UI workflow. + +## Fair capability comparison + +| Capability | PREDVAL | PyCalEva | probably + rsample | isitfair | +|---|---|---|---|---| +| Binary point AUROC/Brier | Native, executed | Native, executed | Native yardstick, executed | Native, source reviewed | +| Patient-grouped correction folds | Native | Caller supplies correction | Configurable grouped rset or explicit splits | Configurable explicit fold vector | +| Correct repeated-patient bootstrap | Native | Custom loop | Native group_bootstraps plus metric mapping | Custom outer cluster loop needed; built-in patient resampling assumes one row per patient | +| Coverage denominator and declared roster | Native contract, executed | Caller glue | Caller glue | Caller glue in reviewed API | +| Integrated report | Native | Calibration PDF | Requires composition | Native integrated HTML/PDF | +| Correction-family equality to rung2 | Reference definition | No native correction in checked evaluator | Native smooth=FALSE uses y~p, a different family | Platt uses L2 regularization and epsilon 1e-10, different configuration | + +“Caller glue” is not “impossible.” No developer-time savings were measured. Code counts +are implementation-specific and not evidence of programmer effort or package quality. +PyCalEva's `metrics()` documentation calls Brier scaled, but the checked 0.8.2 implementation +and `.brier` property return ordinary mean squared error; direct numerical agreement confirms +which quantity was compared. + +## Reproduce Python without R + +From the repository root, using Python 3.12.14 (recorded lock): + +```sh +uv venv .venv-benchmark --python 3.12 +uv pip install --python .venv-benchmark/bin/python -r benchmarks/alternatives/requirements-lock.txt +uv pip install --python .venv-benchmark/bin/python --no-deps -e . +MPLCONFIGDIR=/tmp/predval-mpl XDG_CACHE_HOME=/tmp/predval-cache \ + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 \ + .venv-benchmark/bin/python benchmarks/alternatives/run_benchmark.py --out /tmp/predval-run-a +MPLCONFIGDIR=/tmp/predval-mpl XDG_CACHE_HOME=/tmp/predval-cache \ + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 \ + .venv-benchmark/bin/pytest benchmarks/alternatives/tests +``` + +Use a fresh output directory; the script refuses to overwrite one. Repeat in a second fresh +venv from the same lock and compare `manifest.json`: all stable data and result hashes must +match. `resource_usage.json` is intentionally excluded from that manifest because timing/RSS +are machine- and load-dependent. Its time covers the benchmark body after imports and excludes +installation; it is **not a between-tool speed benchmark**. + +Optional R command, once the registry packages listed in the adapter are installed: + +```sh +Rscript benchmarks/alternatives/probably_adapter.R /tmp/predval-run-a /tmp/predval-r-run-a +``` + +This gives native probably's raw-probability logistic correction the exact same folds and +checks it against R's raw-probability GLM. A separate R logit GLM checks the truly matched +PREDVAL rung2. Native grouped bootstrap uses R/rsample RNG draws, not identical NumPy draws, +so its Monte Carlo interval need not match the Python interval bit for bit. + +## Sources, redistribution, and versions + +The WDBC-derived fixture is redistributed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). +Credit: Wolberg, W., Mangasarian, O., Street, N., & Street, W. (1993), +*Breast Cancer Wisconsin (Diagnostic)*, UCI Machine Learning Repository, +[doi:10.24432/C5DW2B](https://doi.org/10.24432/C5DW2B). +Changes: selected a held-out subset, fitted logistic probabilities using separate training +rows, deliberately distorted probabilities, and replaced identifiers with row-derived labels. +No endorsement by the data creators is implied. + +Primary sources checked 2026-10-10: + +- [UCI dataset and license](https://archive.ics.uci.edu/dataset/17/breast+cancer+wisconsin+diagnostic) +- [scikit-learn bundled data](https://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_breast_cancer.html) +- [PyCalEva official repository](https://github.com/MartinWeigl/pycaleva) +- [probably resampling validation](https://probably.tidymodels.org/reference/cal_validate_logistic.html) +- [probably logistic estimation](https://probably.tidymodels.org/reference/cal_estimate_logistic.html) +- [rsample grouped CV](https://rsample.tidymodels.org/reference/group_vfold_cv.html) +- [rsample grouped bootstrap](https://rsample.tidymodels.org/reference/group_bootstraps.html) +- [isitfair pinned official source](https://github.com/HugoGuillen/isitfair/tree/70ffb8d21c3e24eddfee412220b8a7a701768937) + +Python numerical dependencies: NumPy 2.4.6, pandas 3.0.5, SciPy 1.17.1, scikit-learn 1.9.0, +statsmodels 0.14.6. Every transitive Python dependency is pinned in requirements-lock.txt. +The checked probably GitHub source is commit `974f0154cc7d44ae41d8072161996b23a19d3d2a`; +installed R package versions, when executed, take precedence over that source-only snapshot. + +## Known limitation found during independent review + +PREDVAL's IRLS implementation can return its last iterate after hitting the iteration limit +without convergence. On `y=[0,0,1,1]`, `p=[0.2,0.5,0.5,0.8]`, the quasi-separated likelihood +has no finite slope MLE, but the baseline returns a finite slope near 22.288 and does not +mark rung2 unavailable. A strict expected-failure regression test records this defect. +The matched rung2 estimates on the two main fixtures still agree with independently +converged GLMs; that agreement does not validate the general failure guard. No core fix is +included in this comparison branch. Do not market the tool as universally detecting +non-convergence or statistically validated on the strength of these checks. + +## Reproducibility and measured resource use + +Two isolated Python environments were created from the complete pinned package list. +After removing the shared PREDVAL transform from the independent GLM control, all stable +input/result file hashes matched across the two fresh-environment runs. Their manifest +SHA-256 is `a2a541ec30fbffe83103e04d2c85a5e83114a671e9818488e896a2f34eaf510d`. +Measured benchmark-body wall times were 1.96 and 2.38 seconds, with peak process RSS about +285 MiB. These exclude imports/installations, include the combined benchmark, and cannot +support a between-package speed claim. Exact records: `results/reproducibility.json`. + +`results/glue_inventory.json` counts physical lines for the custom correction, cluster +bootstrap, coverage wrapper, and language-specific adapters. Counts include comments and +validation/export scaffolding; they are not a minimum implementation size or measured time +saved. This deliberately avoids an invented developer-effort advantage. + +Row-level generated fixture/prediction files are not committed or bundled. Regenerate them +with the script; their hashes remain in the manifest for verification. Compact metric tables, +flags, seeds, software locks, and verification records are retained. + +## Executed R comparison + +R 4.5.3 with probably 1.2.0, rsample 1.3.2, and yardstick 1.4.0 ran successfully in an +isolated conda-forge environment. Resolver-planned downloads were 439,535,932 bytes across +227 packages, within the prespecified setup budget. No system-wide installation was used. +`conda-linux-64.lock` records exact official package URLs and checksums; use micromamba's +`create --file` with a fresh prefix to reproduce. Full package versions are in +`results/r/sessionInfo.txt`. + +| Fixture | probably native y~p Brier | PREDVAL y~logit(p) Brier | probably AUROC | PREDVAL AUROC | +|---|---:|---:|---:|---:| +| WDBC held out | 0.027718 | 0.029256 | 0.968747 | 0.984031 | +| Synthetic clustered | 0.172383 | 0.170541 | 0.818836 | 0.818551 | + +These are native different correction families on identical held-out folds, not a contest +with one universally better method. The explicit raw-probability statsmodels control checks +probably's family; the separate R logit GLM checks PREDVAL's family. Prediction-difference +checks and native grouped-bootstrap summaries are recorded under `results/r/`. + +R `group_vfold_cv` passed the no-patient-overlap check. `group_bootstraps` supplied 1,000 +native cluster resamples for each fixture. R's resamples differ from NumPy's; their intervals +are independently generated Monte Carlo results on fixed corrected predictions. + +## Verification status + +- Existing PREDVAL suite: 269 passed, 20 skipped because optional example/golden fixtures + were unavailable. This does not claim those skipped integration cases passed. +- New benchmark contracts: 5 passed, 1 strict expected failure for quasi-separation. +- Ruff and `git diff --check`: passed. +- Two fresh Python environments: identical stable input/result manifests. +- isitfair: source-reviewed only; its adapter has not been executed or validated. + +The independent raw-probability statsmodels checks agree with native probably predictions +within 1.04e-8 (WDBC) and 9.71e-11 (synthetic). R's matched logit-GLM checks agree with PREDVAL +within 6.73e-12 and 5.78e-15. Default IRLS stopping tolerances differ; the benchmark assertions +are 1e-6 and 1e-8 respectively, specified before the final R run. + +For same-final-input repeats, the compact verification/summary command is: + +```sh +python benchmarks/alternatives/verify_results.py PY_RUN_A PY_RUN_B R_RUN_A R_RUN_B +``` + +It verifies every Python input/result hash, checks R metrics, predictions, bootstrap draws +and session information byte for byte, and writes a compact native-R bootstrap summary. + +Final same-input R repeats matched all seven stable files byte for byte (metrics, independent +agreement, both corrected vectors, both 1,000-draw bootstrap sequences, session information). +Native rsample 95% gain intervals were WDBC [-0.005282, 0.009331] and synthetic +[-0.001058, 0.022890], both crossing zero, consistent with the Python conclusion of uncertainty. +The R benchmark bodies took 142.26 and 187.48 seconds after imports; these include native +rsample bootstrap-object construction and are not workload-matched to the Python body. +R timing/session text has trailing whitespace stripped for repository hygiene; numerical +outputs are untouched. The pre-normalization repeats were also byte-identical. diff --git a/benchmarks/alternatives/conda-linux-64.lock b/benchmarks/alternatives/conda-linux-64.lock new file mode 100644 index 0000000..9e86869 --- /dev/null +++ b/benchmarks/alternatives/conda-linux-64.lock @@ -0,0 +1,229 @@ +# platform: linux-64 +@EXPLICIT +https://conda.anaconda.org/conda-forge/linux-64/libgomp-16.2.0-he0feb66_7.conda#d904c7ddd95e026254b4b80808ee9e12 +https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.2-h25fd6f3_3.conda#0de0122d9570a8ab637c6b73db268389 +https://conda.anaconda.org/conda-forge/linux-64/pandoc-3.12.1-ha770c72_0.conda#c426f0a6202837347387cc6fdd4c381e +https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-20_gnu.conda#a9f577daf3de00bca7c3c76c0ecbd1de +https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_7.conda#aa459086047c0e5e27023ab19f8cb86a +https://conda.anaconda.org/conda-forge/linux-64/libgcc-16.2.0-ha9f2e26_7.conda#735629b400b70eeab21a2ba8f5b1c2ce +https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.46.1-default_hbd61a6d_103.conda#2362c70e7e79827c32db35547d9c179b +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxau-1.0.12-h7cc23a3_2.conda#da33efee1a93603d92e06d4a2b68def6 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdmcp-1.1.5-h7cc23a3_2.conda#eabfc45786244de252090cf8d55128a8 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libice-1.1.2-h280c20c_0.conda#85c9442aec283b4e464fa9ecc484a2f3 +https://conda.anaconda.org/conda-forge/linux-64/libffi-3.7.0-h81df57d_1.conda#6525a0b06aa4fd390795f0740636a9dd +https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.34.8-hebe6cf0_2.conda#3ad2b403be8fc5612b9159e2ce621f69 +https://conda.anaconda.org/conda-forge/linux-64/libev-4.33-h280c20c_3.conda#7bc31538d6e3fb349e897353969237ec +https://conda.anaconda.org/conda-forge/linux-64/fribidi-1.0.17-h7cc23a3_0.conda#f9d5302bca196ee87e62f393f2626562 +https://conda.anaconda.org/conda-forge/linux-64/keyutils-1.6.3-h7cc23a3_1.conda#ba55d1b89fd7775e67de8291029b4059 +https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-h7cc23a3_1004.conda#bb66b610707b811e3c65181a6431e7a8 +https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.6-hdb14827_1.conda#ee6c0cd80a60961a1f48aa3e0b91f986 +https://conda.anaconda.org/conda-forge/linux-64/libwebp-base-1.6.0-hd42ef1d_1.conda#9332b53d0ea93c5d39e33be03a0c611a +https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hda65f42_10.conda#e675fabcf81499adc7edf58124fb1e01 +https://conda.anaconda.org/conda-forge/linux-64/libpng-1.6.59-h922cc85_0.conda#8c9caf742a485ac688ffa88bbadbf0be +https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.18-h0cb94f2_3.conda#f92233bf33e24a25668bb2119e2c51f9 +https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.25-hd45a770_1.conda#40f9b31aa9cf007789867df0decd0492 +https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.8.5-hd2095e1_0.conda#72cfd2b95b0540247220b42f58a148d7 +https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.42.4-hcfc3c73_0.conda#54a948337e66706380ff24d6b5fde63f +https://conda.anaconda.org/conda-forge/linux-64/libjpeg-turbo-3.2.0-hb03c661_1.conda#898d1c9793eaa52efc4727bd84d2e39a +https://conda.anaconda.org/conda-forge/linux-64/make-4.4.1-hb03c661_3.conda#b4a198dc40024de2a59c1343e4ae4144 +https://conda.anaconda.org/conda-forge/linux-64/sed-4.10-h19d0853_0.conda#e4a1ff2e59a5d9b38be71d15c93cf1bd +https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.3-hb03c661_1.conda#1390b7c5ac0b1d8e447bc5efa6d3c8c2 +https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h1df4ec4_4.conda#676a2f9b1c4fcf57b70aa5c65f6c60d0 +https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-16.2.0-h6b99dfc_7.conda#7719ed7be38ad8c492c4bd5cc2bce0c4 +https://conda.anaconda.org/conda-forge/linux-64/libuv-1.52.1-h280c20c_1.conda#01c4ed87826af55996768af6dbc936b4 +https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-16.2.0-h934c35e_7.conda#9b6707e0caca917fa6ea9ee473f37a5d +https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-16.2.0-h69a702a_7.conda#2005515ac9340f816fb2ac7e83afe553 +https://conda.anaconda.org/conda-forge/linux-64/libxcb-1.17.0-hb83e432_2.conda#61f0ffba9161cbb3a77722869b21e4bb +https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20260512-pl5321h373387f_0.conda#27dbd7e2984b8fe48e069379636ccfdd +https://conda.anaconda.org/conda-forge/linux-64/readline-8.3-hd6e31c0_1.conda#69c01c781e7c8190bbdaf79e3848c5ca +https://conda.anaconda.org/conda-forge/linux-64/pcre2-10.47-h8b3dc9c_1.conda#06f6113cf1ff4a54b65f87ead132a5f1 +https://conda.anaconda.org/conda-forge/linux-64/libfreetype6-2.14.3-h5e6c136_2.conda#b72a266a9317036fd0464cb98b027cd0 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libsm-1.2.6-h0d788c3_1.conda#aa7459ed9ad086ba11dda843d764b33d +https://conda.anaconda.org/conda-forge/linux-64/bwidget-1.10.1-hfc2019e_3.conda#38d3e0382f3f10344ac30772dec58035 +https://conda.anaconda.org/conda-forge/linux-64/libgfortran-16.2.0-h69a702a_7.conda#3a8d285f5f22138ef826ad54cf267e73 +https://conda.anaconda.org/conda-forge/linux-64/graphite2-1.3.15-h54a6638_1.conda#f9fe2984587fa8235a6af6004760cd18 +https://conda.anaconda.org/conda-forge/linux-64/icu-78.3-py310h44b86e0_2.conda#72a381cbad04f24b1c2a43ef707f45b4 +https://conda.anaconda.org/conda-forge/linux-64/pixman-0.46.4-h54a6638_3.conda#0ee5bb30034b081a1386c1e2c98ab0a7 +https://conda.anaconda.org/conda-forge/linux-64/libsanitizer-16.2.0-h3048135_7.conda#9cdd9af432fa4eb62094aaf6211425bf +https://conda.anaconda.org/conda-forge/linux-64/lerc-4.2.0-hdb68285_0.conda#fb9d356b1a57d6d54768be7ebd5fce09 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libx11-1.8.13-he1eb515_1.conda#8c282bbe4808a3cc80a5c98e9aec1cfc +https://conda.anaconda.org/conda-forge/linux-64/libglib-2.90.0-h569388d_0.conda#c85e9539551f1fd21104e518d940c244 +https://conda.anaconda.org/conda-forge/linux-64/libfreetype-2.14.3-ha770c72_2.conda#f8054e759d0ddddaf34a5c8fedc900a1 +https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.34-pthreads_hf13c14d_2.conda#7e19fcd73d7f0997d3cb3326d2586a86 +https://conda.anaconda.org/conda-forge/linux-64/libpsl-0.23.1-hd9e3e90_1.conda#77be60417aa572afcb48cbe0f967401d +https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.7.2-hcc2c06a_1.conda#0907d876f460ebc941ae590338b6102f +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.12-hb03c661_1.conda#e470d224a7a5be1b1d021bded7abb536 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.7-h7cc23a3_1.conda#e5b6b28536b81b3f4cb20db4668a4642 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxt-1.3.1-h7cc23a3_1.conda#2121765de9c261e2a95ab9fc6efabefb +https://conda.anaconda.org/conda-forge/linux-64/tktable-2.10-h0eb2ad0_11.conda#25d73f1e53349312f316ad82b78b5a7f +https://conda.anaconda.org/conda-forge/linux-64/fontconfig-2.18.3-h4db4eae_1.conda#922776b528a470ab5afa81fd42abfa1d +https://conda.anaconda.org/conda-forge/linux-64/libblas-3.11.0-11_h4a7cf45_openblas.conda#b8cce1486f3c62f33291d6318c193d0d +https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.11.0-11_h0358290_openblas.conda#73a248c30811075059b5aeda766c2624 +https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.11.0-11_h47877c9_openblas.conda#5622a13917855ce4b953b9e655cf9262 +https://conda.anaconda.org/conda-forge/linux-64/gsl-2.7-he838d99_0.tar.bz2#fec079ba39c9cca093bf4c00001825de +https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.7.22-hbd8a1cb_0.conda#0f51e2391ade309db462a55611263e9c +https://conda.anaconda.org/conda-forge/noarch/_r-mutex-1.0.1-anacondar_1.tar.bz2#19f9db5f4f1b7f5ef5f6d67207f25f38 +https://conda.anaconda.org/conda-forge/noarch/tzdata-2026c-h151e31d_0.conda#fcb489df604d100968b737f2cb6076c6 +https://conda.anaconda.org/conda-forge/noarch/libstdcxx-devel_linux-64-16.2.0-h86e191b_107.conda#a713d3c09518a9fc8f603ac0e157263c +https://conda.anaconda.org/conda-forge/noarch/libgcc-devel_linux-64-16.2.0-he3ce08f_107.conda#f689d67c3e828fe80ff290d15f361bc5 +https://conda.anaconda.org/conda-forge/noarch/kernel-headers_linux-64-6.12.0-he073ed8_6.conda#ad5a27ffc9350af2624927dd79edf2be +https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2#0c96522c6bdaed4b1566d11387caaf45 +https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2#34893075a5c9e55cdafac56607368fc6 +https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2#4d59c254e01d9cde7957100457e2d5fb +https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-h77eed37_3.conda#49023d73832ef61042f6a237cb2687e7 +https://conda.anaconda.org/conda-forge/noarch/sysroot_linux-64-2.39-hc4b9eeb_6.conda#bc073c887526f93b09b8ca490192790b +https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-hc364b38_1.conda#a7970cd949a077b7cb9696379d338681 +https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2#fee5683a3f04bd15cbd8318b096a27ab +https://conda.anaconda.org/conda-forge/linux-64/openssl-3.6.5-h781a0a9_0.conda#5a51a2ea653f120240b1a9bf76e0aae9 +https://conda.anaconda.org/conda-forge/linux-64/binutils_impl_linux-64-2.46.1-default_hfdba357_103.conda#e86f9dc615d7533c5be8b15b3a1dee8d +https://conda.anaconda.org/conda-forge/linux-64/cairo-1.18.6-h3c89d7e_0.conda#7920701f9abcb488411495742cf365a7 +https://conda.anaconda.org/conda-forge/linux-64/libnghttp2-1.68.1-h74cf4be_1.conda#75d211f04ac87506f1f43420712a69fe +https://conda.anaconda.org/conda-forge/linux-64/libssh2-1.11.1-h6154650_1.conda#5c526561e1d2a2e071941174eae84557 +https://conda.anaconda.org/conda-forge/linux-64/krb5-1.22.2-hbc21106_2.conda#53318d715316929a574f83591308b1f8 +https://conda.anaconda.org/conda-forge/linux-64/binutils_linux-64-2.46.1-default_h4852527_103.conda#ca26f0a4f8d04fd83147af557fbd932c +https://conda.anaconda.org/conda-forge/linux-64/gcc_impl_linux-64-16.2.0-h9891192_7.conda#f548b09900fed2701931f8cf925d8b9b +https://conda.anaconda.org/conda-forge/linux-64/libharfbuzz-14.6.0-h23af247_1.conda#bd7ae146a629d6c17e4be9365d4ec811 +https://conda.anaconda.org/conda-forge/linux-64/libgit2-1.9.7-hf27fc33_1.conda#1cf3ef849e34bc3283641cfda2675281 +https://conda.anaconda.org/conda-forge/linux-64/libcurl-8.22.0-ha042cf0_0.conda#e617deee7af8eb0c04f6296d12b54157 +https://conda.anaconda.org/conda-forge/linux-64/gxx_impl_linux-64-16.2.0-h0d273dc_7.conda#ea011fbfe6ce5b3a74a8e34fc9c8481a +https://conda.anaconda.org/conda-forge/linux-64/gcc_linux-64-16.2.0-h0ab548f_2.conda#3928593d74a0b17f84fba5977cb7023d +https://conda.anaconda.org/conda-forge/linux-64/gfortran_impl_linux-64-16.2.0-hf930d89_7.conda#b8fa9e9457b9e75f2bf85ad8017fa1d7 +https://conda.anaconda.org/conda-forge/linux-64/pango-1.58.2-h7bb47b9_1.conda#0b889b1c6b472d215d6332c6b195589c +https://conda.anaconda.org/conda-forge/linux-64/curl-8.22.0-ha042cf0_0.conda#399a7c0b7ca511de34c5ee5762cee149 +https://conda.anaconda.org/conda-forge/linux-64/gxx_linux-64-16.2.0-h23aca61_2.conda#38b21695fc42aa9bf6d4883e45a80fd9 +https://conda.anaconda.org/conda-forge/linux-64/r-base-4.5.3-h502d0c9_3.conda#8e584ef995c58d546ca1b2642fa4deae +https://conda.anaconda.org/conda-forge/linux-64/r-bit-4.6.0-r45h54b55ab_1.conda#6b666bedcfbe9dee03efb5fb95ac18e1 +https://conda.anaconda.org/conda-forge/linux-64/r-base64enc-0.1_6-r45h54b55ab_0.conda#e573dc135976197e7f819eec202c93f1 +https://conda.anaconda.org/conda-forge/linux-64/r-mime-0.13-r45h54b55ab_1.conda#26aa2fa52d4caed58336f2b4887916d6 +https://conda.anaconda.org/conda-forge/linux-64/r-xfun-0.61-r45hddc5280_1.conda#fdc3b42e200db84ffe0636a6f0b96f6e +https://conda.anaconda.org/conda-forge/linux-64/r-digest-0.6.39-r45h3697838_0.conda#a0e856537aa7d62a6835e3a528d29517 +https://conda.anaconda.org/conda-forge/linux-64/r-parallelly-1.48.0-r45h54b55ab_0.conda#4db06e4d5b749678d35b346e442d17a5 +https://conda.anaconda.org/conda-forge/linux-64/r-stringi-1.8.9-r45h3d52c89_0.conda#8b4ab64c1ea52b50aded3186643a9f9f +https://conda.anaconda.org/conda-forge/linux-64/r-colorspace-2.1_3-r45h54b55ab_0.conda#472eadbb95c9103963bae1631ec23b0f +https://conda.anaconda.org/conda-forge/linux-64/r-fastmap-1.2.0-r45h3697838_2.conda#245526991ad3b8a1dc97f2dcb5031065 +https://conda.anaconda.org/conda-forge/linux-64/r-data.table-1.18.6.1-r45hbd9402d_0.conda#34e991670e902d15eaeb84790048974b +https://conda.anaconda.org/conda-forge/linux-64/r-kernsmooth-2.23_27-r45hf1ecf5c_0.conda#9c164aed0ddfd01f45273ba00f6f23b5 +https://conda.anaconda.org/conda-forge/linux-64/r-lattice-0.23_1-r45hddc5280_0.conda#902c7c607b9ab6500debac3acfc6b068 +https://conda.anaconda.org/conda-forge/linux-64/r-rpart-4.1.27-r45h54b55ab_0.conda#072cbea77e0705c0126f04a883ab912b +https://conda.anaconda.org/conda-forge/linux-64/r-mass-7.3_66-r45h54b55ab_0.conda#700d7bffd1c317597b574b218b567017 +https://conda.anaconda.org/conda-forge/linux-64/r-warp-0.2.3-r45h54b55ab_0.conda#8f6a66f0eb973fc76ce601063d147048 +https://conda.anaconda.org/conda-forge/linux-64/r-rcpp-1.1.2-r45h3697838_0.conda#1d6e2e4764de896f03fb73e28614022e +https://conda.anaconda.org/conda-forge/linux-64/r-s7-0.2.2-r45h54b55ab_0.conda#598ca5290f9be9b22804436295471201 +https://conda.anaconda.org/conda-forge/linux-64/r-rappdirs-0.3.4-r45h54b55ab_0.conda#9fa763ece102dca3e50892bad36896ff +https://conda.anaconda.org/conda-forge/linux-64/r-fs-2.1.0-r45h3697838_0.conda#67c48aad5ce921ad990b951a1f194f42 +https://conda.anaconda.org/conda-forge/linux-64/r-yaml-2.3.12-r45h54b55ab_0.conda#d0d07c6f14b640a98cf6a5e3e4e2e723 +https://conda.anaconda.org/conda-forge/linux-64/r-sys-3.4.3-r45h54b55ab_1.conda#f23bbac61ab0536f59f4caa4c2ebdf88 +https://conda.anaconda.org/conda-forge/linux-64/r-curl-8.0.0-r45h444973c_0.conda#8f92ea8dfec9d170d50d31797979695d +https://conda.anaconda.org/conda-forge/linux-64/r-dicedesign-1.10-r45h54b55ab_2.conda#8d008a251f7fa40644a71f3fa4a6da35 +https://conda.anaconda.org/conda-forge/linux-64/r-fansi-1.0.7-r45h54b55ab_0.conda#309740f1b6a2d4cb16d395be786c85c5 +https://conda.anaconda.org/conda-forge/linux-64/r-gower-1.0.2-r45h54b55ab_0.conda#48168b9d60326506af819549391b5b6c +https://conda.anaconda.org/conda-forge/linux-64/r-magrittr-2.0.5-r45h54b55ab_0.conda#2e26e018edc1f198aa72a8f2127fab00 +https://conda.anaconda.org/conda-forge/linux-64/r-jsonlite-2.0.0-r45h54b55ab_1.conda#026c72026f431daf8a5719e09e704faa +https://conda.anaconda.org/conda-forge/linux-64/r-utf8-1.2.6-r45h54b55ab_1.conda#75cb2540ac930ea879e92d99e00e97c3 +https://conda.anaconda.org/conda-forge/linux-64/r-farver-2.1.2-r45h3697838_2.conda#7896efcfd50f8c1f207acce5d4ab1cc0 +https://conda.anaconda.org/conda-forge/linux-64/r-cli-3.6.6-r45h3697838_0.conda#ed8dcbbe9d66e9093ba58891e3b6065a +https://conda.anaconda.org/conda-forge/linux-64/r-rlang-1.3.0-r45h3697838_0.conda#afa49304f86d922c157d08e9e9d9ceb8 +https://conda.anaconda.org/conda-forge/linux-64/r-glue-1.8.1-r45h54b55ab_0.conda#65911c2e9cac35ea0235d3d6d4aa9625 +https://conda.anaconda.org/conda-forge/linux-64/r-bit64-4.8.6-r45hddc5280_0.conda#5cea8ea9e1bf30dc0b863e2b2fcb972b +https://conda.anaconda.org/conda-forge/linux-64/r-matrix-1.7_6-r45h0e4624f_0.conda#4b74bbbc18d62c7c93a84c558372b983 +https://conda.anaconda.org/conda-forge/linux-64/r-nnet-7.3_21-r45hddc5280_0.conda#371b62191b9d87d9caead88954d1e772 +https://conda.anaconda.org/conda-forge/linux-64/r-class-7.3_24-r45h54b55ab_0.conda#7ea996d2b2617350c14416537e6a3d18 +https://conda.anaconda.org/conda-forge/linux-64/r-rcpparmadillo-15.6.0_1-r45h1fcd065_0.conda#909cfe430769dd82e7d12cc2b46b3aef +https://conda.anaconda.org/conda-forge/linux-64/r-lbfgs-1.2.1.2-r45h3697838_5.conda#96b87eea8a76283e2cda7468b34ae2e5 +https://conda.anaconda.org/conda-forge/linux-64/r-askpass-1.2.1-r45h54b55ab_1.conda#1ae3c72e90c6a3871c594448d3e152e4 +https://conda.anaconda.org/conda-forge/linux-64/r-zip-3.0.2-r45h54b55ab_0.conda#9998629d3f28b086ed3a69a564ec257f +https://conda.anaconda.org/conda-forge/linux-64/r-cachem-1.1.0-r45h54b55ab_2.conda#42617e710293f0b76ccc08855e0edbc4 +https://conda.anaconda.org/conda-forge/linux-64/r-ellipsis-0.3.3-r45h54b55ab_0.conda#e43b3e7101a90ac57f58a21da67c99a4 +https://conda.anaconda.org/conda-forge/linux-64/r-survival-3.8_12-r45hddc5280_0.conda#2165bf9ffcefc182e65ada290f3dbe29 +https://conda.anaconda.org/conda-forge/linux-64/r-openssl-2.4.2-r45h68c19f5_0.conda#6568a5797c9d74e89ef94b82724665aa +https://conda.anaconda.org/conda-forge/linux-64/r-htmltools-0.5.9-r45h3697838_0.conda#2a2297687ae137e7fa90d3c996895e61 +https://conda.anaconda.org/conda-forge/noarch/r-listenv-1.1.0-r45hacb3a79_0.conda#adc0aad6fbb0d8b89c760054d37dbb22 +https://conda.anaconda.org/conda-forge/noarch/r-squarem-2026.1-r45hc72bb7e_0.conda#b8a3906de179464adc9e2dcf005cfeaf +https://conda.anaconda.org/conda-forge/noarch/r-shape-1.4.6.1-r45ha770c72_2.conda#9f062131995e818b7bcf8fe65d3f4424 +https://conda.anaconda.org/conda-forge/noarch/r-cpp11-0.5.5-r45h785f33e_0.conda#71c68a3993a696b6b75f7a6550b3750f +https://conda.anaconda.org/conda-forge/noarch/r-evaluate-1.0.5-r45hc72bb7e_1.conda#4e0c71ab78d7292372a89d4daecb49af +https://conda.anaconda.org/conda-forge/noarch/r-gitcreds-0.1.2-r45hc72bb7e_4.conda#8864884cea757c51580b45be5c362068 +https://conda.anaconda.org/conda-forge/noarch/r-ini-0.3.1-r45hc72bb7e_1007.conda#e680ce2dae5ccc37ed94fc0696e6f10d +https://conda.anaconda.org/conda-forge/noarch/r-clipr-0.8.1-r45hc72bb7e_0.conda#6d3690eb86e7fdea974915a5ce963561 +https://conda.anaconda.org/conda-forge/noarch/r-rstudioapi-0.19.0-r45hc72bb7e_0.conda#5f1b6b899fa3204ab77403e75b9df5fd +https://conda.anaconda.org/conda-forge/noarch/r-codetools-0.2_20-r45hc72bb7e_2.conda#8e9e5b14f74a6040c77e0b9c8bfa84ca +https://conda.anaconda.org/conda-forge/noarch/r-viridislite-0.4.3-r45hc72bb7e_0.conda#bcadc0a3726e2191e51449f67fa5e0a9 +https://conda.anaconda.org/conda-forge/noarch/r-timedate-4052.112-r45hc72bb7e_0.conda#b9ce34da7db4d27de7a478d629b1b2b0 +https://conda.anaconda.org/conda-forge/noarch/r-assertthat-0.2.1-r45hc72bb7e_6.conda#b5291c60f52b43108a2973ba6f5885d1 +https://conda.anaconda.org/conda-forge/noarch/r-whisker-0.4.1-r45hc72bb7e_3.conda#5cd861608ecbeab0574d59a7754deba7 +https://conda.anaconda.org/conda-forge/noarch/r-rprojroot-2.1.1-r45hc72bb7e_1.conda#e1b7c51411ad3dd2a95aea6d466b12f7 +https://conda.anaconda.org/conda-forge/noarch/r-numderiv-2016.8_1.1-r45hc72bb7e_7.conda#4458f15e0f1edb2009e76322f666d0ec +https://conda.anaconda.org/conda-forge/noarch/r-withr-3.0.3-r45hc72bb7e_0.conda#388d25e67f93a5fec2aebe56d6a0a1d9 +https://conda.anaconda.org/conda-forge/noarch/r-labeling-0.4.3-r45hc72bb7e_2.conda#a41490fdf607381035aa0304c21407c9 +https://conda.anaconda.org/conda-forge/noarch/r-rcolorbrewer-1.1_3-r45h785f33e_4.conda#d8fa238420cb6de47d463b7345a761eb +https://conda.anaconda.org/conda-forge/noarch/r-pkgconfig-2.0.3-r45hc72bb7e_5.conda#40a5fdd06c7e7880758a021cf2df6c12 +https://conda.anaconda.org/conda-forge/noarch/r-crayon-1.5.3-r45hc72bb7e_2.conda#4f111ce078b9690abaad6248b831a370 +https://conda.anaconda.org/conda-forge/noarch/r-r6-2.6.1-r45hc72bb7e_1.conda#750802806d7d640c286ed8491bb395dc +https://conda.anaconda.org/conda-forge/noarch/r-generics-0.1.4-r45hc72bb7e_1.conda#f19c9493b80f63a41fa017ec3b27bc2e +https://conda.anaconda.org/conda-forge/noarch/r-highr-0.12-r45hc72bb7e_0.conda#0b5902d6af02a23bda1794d46090db42 +https://conda.anaconda.org/conda-forge/noarch/r-tinytex-0.60-r45hc72bb7e_0.conda#5ee226c1449c66d1cd2399d942faa04d +https://conda.anaconda.org/conda-forge/noarch/r-progressr-1.0.0-r45hc72bb7e_0.conda#386049538faa3b7679091494a9a1004f +https://conda.anaconda.org/conda-forge/noarch/r-munsell-0.5.1-r45hc72bb7e_2.conda#c78bd534986cde8fc0cb08cc9a1a2cc6 +https://conda.anaconda.org/conda-forge/noarch/r-lifecycle-1.0.5-r45hc72bb7e_0.conda#5f8369dfbdff08878e58bf15529fca3a +https://conda.anaconda.org/conda-forge/noarch/r-splitfngr-0.1.2-r45hc72bb7e_0.conda#35a5489e4d64d783a1fe578bc05c74dd +https://conda.anaconda.org/conda-forge/noarch/r-memoise-2.0.1-r45hc72bb7e_4.conda#2b04206ff6ea5a92e8e36bdaa5feb3cc +https://conda.anaconda.org/conda-forge/noarch/r-credentials-2.0.3-r45hc72bb7e_1.conda#a94730bc5fa7197875f58d4b092a541a +https://conda.anaconda.org/conda-forge/noarch/r-jquerylib-0.1.4-r45hc72bb7e_4.conda#49a9ed6ed01f4ae6067ead552795bfce +https://conda.anaconda.org/conda-forge/noarch/r-fontawesome-0.5.3-r45hc72bb7e_1.conda#e9fccb3617ec9776569c6496fa254e64 +https://conda.anaconda.org/conda-forge/noarch/r-diagram-1.6.5-r45ha770c72_4.conda#11676b1bcbee45cae2dffbb1ffaf0b7e +https://conda.anaconda.org/conda-forge/noarch/r-globals-0.19.1-r45hc72bb7e_0.conda#5bb935fc63812c4978f0ac443b8b3420 +https://conda.anaconda.org/conda-forge/noarch/r-prettyunits-1.2.0-r45hc72bb7e_2.conda#07465728b1fd99d28b286156dac895a3 +https://conda.anaconda.org/conda-forge/noarch/r-desc-1.4.3-r45hc72bb7e_2.conda#2428c825ae5b2fc162edaeb3fa76b486 +https://conda.anaconda.org/conda-forge/noarch/r-knitr-1.52-r45hacb3a79_0.conda#b127436178c659586a1ea6a9e63598da +https://conda.anaconda.org/conda-forge/noarch/r-gtable-0.3.6-r45hc72bb7e_1.conda#f686123cfba49e6299fae7e029a40266 +https://conda.anaconda.org/conda-forge/noarch/r-scales-1.4.0-r45hc72bb7e_1.conda#8bc81cc6fd130cef963bc9e082726a14 +https://conda.anaconda.org/conda-forge/noarch/r-future-1.76.0-r45hacb3a79_0.conda#7ec394598a99c71084f716f3e077847f +https://conda.anaconda.org/conda-forge/noarch/r-future.apply-1.20.2-r45hc72bb7e_0.conda#ec6b2f2d6089e26eef6452f11ee1a67e +https://conda.anaconda.org/conda-forge/noarch/r-lava-1.9.3-r45hacb3a79_0.conda#7bf82c8c677b3d23029f63ac5765d162 +https://conda.anaconda.org/conda-forge/linux-64/r-isoband-0.3.0-r45h3697838_0.conda#fb538d0b46d6a44aa1ff666b15422ba6 +https://conda.anaconda.org/conda-forge/linux-64/r-timechange-0.4.0-r45h3697838_0.conda#77b6b03b30183b189906b08ea0868b21 +https://conda.anaconda.org/conda-forge/linux-64/r-tzdb-0.5.0-r45h3697838_2.conda#54673c8b5186c85794f0bd40d7c49bb4 +https://conda.anaconda.org/conda-forge/linux-64/r-sass-0.4.10-r45h3697838_1.conda#a3ccf52eefa448628535ee758c5a8a37 +https://conda.anaconda.org/conda-forge/linux-64/r-vctrs-0.7.3-r45h3697838_0.conda#372cefc1b05a567d1fbe19633766ab0c +https://conda.anaconda.org/conda-forge/linux-64/r-gert-2.4.1-r45h7df5fc1_0.conda#acb7d0d15da0942696da919ef9181751 +https://conda.anaconda.org/conda-forge/linux-64/r-lobstr-1.2.2-r45hfa9885e_0.conda#d8bf9af57fc598a5b71b365ab30598f6 +https://conda.anaconda.org/conda-forge/linux-64/r-lubridate-1.9.5-r45h54b55ab_0.conda#cec48e713c16eb74b4984019282ad03a +https://conda.anaconda.org/conda-forge/linux-64/r-clock-0.7.4-r45h3697838_0.conda#b704af8c6d1f3c7241debde4c3c2c994 +https://conda.anaconda.org/conda-forge/linux-64/r-slider-0.3.3-r45h3697838_0.conda#d157647300bcd9dd638da5fdc001f173 +https://conda.anaconda.org/conda-forge/linux-64/r-sparsevctrs-0.3.6-r45h54b55ab_0.conda#81c927f4602a8f849d4117b0766f863a +https://conda.anaconda.org/conda-forge/linux-64/r-purrr-1.2.2-r45h54b55ab_0.conda#dc14c484a0415f43dc1b90b518beb41a +https://conda.anaconda.org/conda-forge/noarch/r-bslib-0.12.0-r45hc72bb7e_0.conda#62a97fa00e2f48b594a592dbcbfbe8a5 +https://conda.anaconda.org/conda-forge/noarch/r-httr2-1.3.0-r45hc72bb7e_0.conda#b240e3c0e61de35f1f43729f4ea2888e +https://conda.anaconda.org/conda-forge/noarch/r-hms-1.1.4-r45hc72bb7e_0.conda#4677c1ad37a9452e27e27d9ed1b8ae90 +https://conda.anaconda.org/conda-forge/noarch/r-stringr-1.6.0-r45h785f33e_0.conda#8db438d8aa370726ee1ce8bf458f2e6d +https://conda.anaconda.org/conda-forge/noarch/r-pillar-1.11.1-r45hc72bb7e_0.conda#807ef77a70fc5156f830d6c683d07a29 +https://conda.anaconda.org/conda-forge/noarch/r-ggplot2-4.0.3-r45h785f33e_0.conda#866fbbf3f356140922d618f26f2b889c +https://conda.anaconda.org/conda-forge/noarch/r-tidyselect-1.2.1-r45hc72bb7e_2.conda#15aa0f323385403fe182e46a1d095e1b +https://conda.anaconda.org/conda-forge/noarch/r-furrr-0.4.0-r45hc72bb7e_0.conda#2813f7fa1b9d4e249ee29ed520df180c +https://conda.anaconda.org/conda-forge/noarch/r-rmarkdown-2.32-r45hacb3a79_0.conda#cf4da65121b4a7a64888455ef67ddd53 +https://conda.anaconda.org/conda-forge/noarch/r-gh-1.6.1-r45hc72bb7e_0.conda#c26e14a9021dfadd58a31980087838a0 +https://conda.anaconda.org/conda-forge/noarch/r-progress-1.2.3-r45hc72bb7e_2.conda#619b691b0965c6894eb99d3851857df7 +https://conda.anaconda.org/conda-forge/noarch/r-usethis-3.2.1-r45hc72bb7e_1.conda#dcbfbae5c448a2e54f37457f16075468 +https://conda.anaconda.org/conda-forge/linux-64/r-tibble-3.3.1-r45h54b55ab_0.conda#ad28f67cbb0b10a5beaa7bf968761cad +https://conda.anaconda.org/conda-forge/linux-64/r-prodlim-2026.03.11-r45h3697838_0.conda#00689a494b747f40c7a760108acdb467 +https://conda.anaconda.org/conda-forge/linux-64/r-vroom-1.7.1-r45h3697838_0.conda#8e45deb7cb8502dda9073be1a5e664df +https://conda.anaconda.org/conda-forge/linux-64/r-dplyr-1.2.1-r45h3697838_0.conda#7c8e643f764769a184f0d0d06ac0e789 +https://conda.anaconda.org/conda-forge/linux-64/r-ipred-0.9_15-r45h54b55ab_2.conda#3026e00ad9eead1acf371d94a317496e +https://conda.anaconda.org/conda-forge/linux-64/r-readr-2.2.0-r45h3697838_0.conda#a5f0d7d99d912486b2d93a576fadb8e0 +https://conda.anaconda.org/conda-forge/linux-64/r-tidyr-1.3.2-r45h3697838_0.conda#e571d4d42233dd2aa3bdb0057ffbdc63 +https://conda.anaconda.org/conda-forge/noarch/r-sfd-0.1.0-r45hc72bb7e_2.conda#722c31e970ef82439c4e88449ad86188 +https://conda.anaconda.org/conda-forge/noarch/r-hardhat-1.4.3-r45hc72bb7e_0.conda#d1ddc05be5ae353ecf22442c76cd0bd4 +https://conda.anaconda.org/conda-forge/noarch/r-modelenv-0.2.0-r45hc72bb7e_1.conda#befcb47a71aeb4b07eeb563a45fb4228 +https://conda.anaconda.org/conda-forge/noarch/r-modeldata-1.6.0-r45hacb3a79_0.conda#52003943948b56084913e8551b7e8f9d +https://conda.anaconda.org/conda-forge/noarch/r-mixopt-0.1.3-r45hc72bb7e_0.conda#ee205159250d6501ade0ff1663923e36 +https://conda.anaconda.org/conda-forge/noarch/r-rsample-1.3.2-r45hc72bb7e_0.conda#a361947d7d35dd7806b98a177e79032a +https://conda.anaconda.org/conda-forge/noarch/r-dials-1.4.4-r45hc72bb7e_0.conda#8ad1b317a2795f05ed70a1a6eb3d92d3 +https://conda.anaconda.org/conda-forge/noarch/r-recipes-1.4.0-r45hacb3a79_0.conda#416c22c7a78347754eecf2bb29d3b8d0 +https://conda.anaconda.org/conda-forge/noarch/r-parsnip-1.6.1-r45hacb3a79_0.conda#ef23280eb12fb0838c4d4eb1c06cb48a +https://conda.anaconda.org/conda-forge/noarch/r-tailor-0.1.0-r45hc72bb7e_1.conda#2aa17f9b3dd7eede9d3cd5d1c29f2a45 +https://conda.anaconda.org/conda-forge/noarch/r-butcher-0.4.0-r45hc72bb7e_0.conda#80312754cab7696b126d3d8ccac81b49 +https://conda.anaconda.org/conda-forge/noarch/r-workflows-1.3.0-r45hc72bb7e_1.conda#0ebbb4ce53157cc0ceb23ee5c26e3846 +https://conda.anaconda.org/conda-forge/linux-64/r-yardstick-1.4.0-r45h54b55ab_0.conda#e7e3ee3b2f419f2da415bc02192567c8 +https://conda.anaconda.org/conda-forge/linux-64/r-gaupro-0.2.17-r45h3704496_0.conda#5b4c70799cb7128a5ffbbe7c51ac23a8 +https://conda.anaconda.org/conda-forge/noarch/r-tune-2.1.0-r45hc72bb7e_0.conda#dc4712e9c84d4e8be0554c1fe859e428 +https://conda.anaconda.org/conda-forge/noarch/r-probably-1.2.0-r45hc72bb7e_0.conda#a571554d3d721d9c259053744159ac95 diff --git a/benchmarks/alternatives/isitfair_adapter.py b/benchmarks/alternatives/isitfair_adapter.py new file mode 100644 index 0000000..556d206 --- /dev/null +++ b/benchmarks/alternatives/isitfair_adapter.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Prepared adapter only. No isitfair execution was authorized in this benchmark. + +After installing an approved pinned isitfair build: +python isitfair_adapter.py path/to/python-results output-dir +Native L2-regularized Platt is a DIFFERENT correction from PREDVAL's unpenalized rung2. +The scores API internally computes native intervals; this adapter discards them. +Those intervals assume one row per patient and are invalid for repeated-row data. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import numpy as np +import pandas as pd + + +def main(source: Path, out: Path): + from isitfair import FairnessAudit + from sklearn.metrics import brier_score_loss, roc_auc_score + + out.mkdir(parents=True, exist_ok=False) + rows = [] + for name in ("wdbc_heldout", "synthetic_clustered"): + df = pd.read_csv(source / f"{name}.csv") + # A single overall group requests the common correction; no claim of a + # clinically meaningful subgroup or fairness comparison is made here. + audit = FairnessAudit( + df.y.to_numpy(), + df.p.to_numpy(), + {"cohort": np.repeat("all", len(df))}, + threshold=0.5, + n_bootstrap=20, + random_state=20261010, + ) + corrected = audit.recalibration_ladder_scores( + by="cohort", method="platt", folds=df.fold.to_numpy() + )["common"] + for method, p in [("published", df.p.to_numpy()), ("native_platt", corrected)]: + rows.append( + { + "fixture": name, + "method": method, + "brier": brier_score_loss(df.y, p), + "auroc": roc_auc_score(df.y, p), + } + ) + df["isitfair_native_platt"] = corrected + df.to_csv(out / f"{name}_isitfair.csv", index=False) + pd.DataFrame(rows).to_csv(out / "isitfair_metrics.csv", index=False) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("source", type=Path) + parser.add_argument("out", type=Path) + args = parser.parse_args() + main(args.source, args.out) diff --git a/benchmarks/alternatives/probably_adapter.R b/benchmarks/alternatives/probably_adapter.R new file mode 100644 index 0000000..8399228 --- /dev/null +++ b/benchmarks/alternatives/probably_adapter.R @@ -0,0 +1,90 @@ +#!/usr/bin/env Rscript +# Native probably calibration and matched R GLM controls, with shared patient folds. +# Rscript benchmarks/alternatives/probably_adapter.R path/to/python-results output-dir +# Requires probably, rsample, yardstick, dplyr, readr, tidyr and purrr. +suppressPackageStartupMessages({ + library(probably); library(rsample); library(yardstick) + library(dplyr); library(readr); library(tidyr); library(purrr) +}) +started <- proc.time() +args <- commandArgs(trailingOnly = TRUE) +stopifnot(length(args) == 2) +dir.create(args[[2]], recursive = TRUE, showWarnings = FALSE) +results <- list() +agreements <- list() +for (fixture in c("wdbc_heldout", "synthetic_clustered")) { + d <- read_csv(file.path(args[[1]], paste0(fixture, ".csv")), show_col_types = FALSE) + d <- d |> mutate(truth = factor(if_else(y == 1, "event", "nonevent"), + levels = c("event", "nonevent")), + .pred_event = p, .pred_nonevent = 1 - p, .row = row_number()) + # Exported assignments are identical to PREDVAL, not a newly randomized split. + splits <- lapply(sort(unique(d$fold)), function(f) { + train <- which(d$fold != f); test <- which(d$fold == f) + stopifnot(length(intersect(d$patient_id[train], d$patient_id[test])) == 0) + make_splits(list(analysis = train, assessment = test), d) + }) + folds <- manual_rset(splits, paste0("Fold", seq_along(splits))) + validated <- cal_validate_logistic(folds, truth, c(.pred_event, .pred_nonevent), + smooth = FALSE, save_pred = TRUE, + metrics = metric_set(brier_class, roc_auc)) + # Pool held-out observations; do not average AUROC across unequal folds. + native <- bind_rows(validated$.predictions_cal) |> arrange(.row) + stopifnot(identical(native$.row, seq_len(nrow(d)))) + d$probably_raw_probability <- native$.pred_event + d$r_glm_logit <- NA_real_ + d$r_glm_raw_probability <- NA_real_ + for (f in sort(unique(d$fold))) { + train <- d$fold != f + d$z <- qlogis(pmin(pmax(d$p, 1e-6), 1 - 1e-6)) + logit_fit <- glm(y ~ z, data = d[train, ], family = binomial(), + control = glm.control(epsilon = 1e-11, maxit = 100)) + raw_fit <- glm(y ~ p, data = d[train, ], family = binomial(), + control = glm.control(epsilon = 1e-11, maxit = 100)) + stopifnot(logit_fit$converged, raw_fit$converged) + d$r_glm_logit[!train] <- predict(logit_fit, d[!train, ], type = "response") + d$r_glm_raw_probability[!train] <- predict(raw_fit, d[!train, ], type = "response") + } + stopifnot(max(abs(d$probably_raw_probability - d$r_glm_raw_probability)) < 1e-6) + py <- read_csv(file.path(args[[1]], paste0(fixture, "_corrected.csv")), + show_col_types = FALSE) + stopifnot(max(abs(py$predval_rung2 - d$r_glm_logit)) < 1e-8) + stopifnot(max(abs(py$statsmodels_raw_probability - d$probably_raw_probability)) < 1e-6) + agreements[[length(agreements) + 1]] <- tibble( + fixture = fixture, + max_abs_predval_vs_r_logit = max(abs(py$predval_rung2 - d$r_glm_logit)), + max_abs_statsmodels_vs_probably_raw = max(abs(py$statsmodels_raw_probability - + d$probably_raw_probability)) + ) + for (column in c("p", "probably_raw_probability", "r_glm_logit", "r_glm_raw_probability")) { + p <- d[[column]] + results[[length(results) + 1]] <- tibble( + fixture = fixture, tool = column, brier = mean((d$y - p)^2), + auroc = roc_auc_vec(d$truth, p, event_level = "first") + ) + } + write_csv(d |> select(subject_id, patient_id, fold, y, p, + probably_raw_probability, r_glm_logit, r_glm_raw_probability), + file.path(args[[2]], paste0(fixture, "_r_corrected.csv"))) + # Demonstrate rsample native grouped splitting and grouped bootstrap support. + # These random folds are a capability check, not the matched comparison partition. + set.seed(20261010) + grouped <- group_vfold_cv(d, patient_id, v = 5) + stopifnot(all(map_lgl(grouped$splits, function(s) { + length(intersect(analysis(s)$patient_id, assessment(s)$patient_id)) == 0 + }))) + set.seed(20261011) + boot <- group_bootstraps(d, patient_id, times = 1000) + # Conditional-on-predictions paired gain; whole patients remain intact. + gains <- map_dbl(boot$splits, function(s) { + b <- analysis(s) + mean((b$y - b$p)^2 - (b$y - b$r_glm_logit)^2) + }) + write_csv(tibble(fixture = fixture, replicate = seq_along(gains), gain = gains), + file.path(args[[2]], paste0(fixture, "_rsample_gains.csv"))) +} +write_csv(bind_rows(results), file.path(args[[2]], "r_metrics.csv")) +writeLines(trimws(capture.output(sessionInfo()), which = "right"), + file.path(args[[2]], "sessionInfo.txt")) +write_csv(bind_rows(agreements), file.path(args[[2]], "r_agreement.csv")) +writeLines(trimws(capture.output(proc.time() - started), which = "right"), + file.path(args[[2]], "resource_usage.txt")) diff --git a/benchmarks/alternatives/requirements-lock.txt b/benchmarks/alternatives/requirements-lock.txt new file mode 100644 index 0000000..baacec3 --- /dev/null +++ b/benchmarks/alternatives/requirements-lock.txt @@ -0,0 +1,53 @@ +annotated-types==0.8.0 +asttokens==3.0.2 +cloudpickle==3.1.2 +contourpy==1.4.0 +cycler==0.12.1 +defusedxml==0.7.1 +executing==2.2.1 +fonttools==4.66.1 +fpdf2==2.8.9 +iniconfig==2.3.1 +ipython==9.17.1 +ipython-pygments-lexers==1.1.1 +jedi==0.20.0 +jinja2==3.1.6 +joblib==1.6.0 +kiwisolver==1.5.1 +markupsafe==3.0.4 +matplotlib==3.11.1 +matplotlib-inline==0.2.2 +narwhals==2.27.1 +numpy==2.4.6 +packaging==26.3 +pandas==3.0.5 +parso==0.8.7 +patsy==1.0.3 +pexpect==4.9.0 +pillow==12.3.0 +pluggy==1.6.0 +prompt-toolkit==3.0.53 +psutil==7.2.2 +ptyprocess==0.7.0 +pure-eval==0.2.4 +pyarrow==25.0.1 +pycaleva==0.8.2 +pydantic==2.13.4 +pydantic-core==2.46.4 +pygments==2.21.0 +pyparsing==3.3.3 +pytest==9.1.1 +python-dateutil==2.9.0.post0 +pyyaml==6.0.3 +ruff==0.17.0 +scikit-learn==1.9.0 +scipy==1.17.1 +six==1.17.0 +stack-data==0.6.3 +statsmodels==0.14.6 +threadpoolctl==3.7.0 +tqdm==4.70.1 +traitlets==5.16.1 +typing-extensions==4.16.0 +typing-inspection==0.4.4 +wcwidth==0.9.2 diff --git a/benchmarks/alternatives/results/glue_inventory.json b/benchmarks/alternatives/results/glue_inventory.json new file mode 100644 index 0000000..b0dc698 --- /dev/null +++ b/benchmarks/alternatives/results/glue_inventory.json @@ -0,0 +1,12 @@ +{ + "physical_lines_including_comments": { + "reference_crossfit": 15, + "custom_bootstrap": 11, + "coverage_cases": 70, + "probably_adapter.R": 90, + "isitfair_adapter.py": 60 + }, + "interpretation": "Descriptive implementation size only. Includes validation/export code; not comparable developer time or a minimum required adapter.", + "native_predval_calls": "ladder; paired_brier_gain_interval; coverage_report/check_coverage/check_roster/check_common_selection", + "pycaleva_custom_scope": "External correction, grouped resampling, cohort/roster handling; point metrics native." +} diff --git a/benchmarks/alternatives/results/python/correction_metrics.csv b/benchmarks/alternatives/results/python/correction_metrics.csv new file mode 100644 index 0000000..c9a2da8 --- /dev/null +++ b/benchmarks/alternatives/results/python/correction_metrics.csv @@ -0,0 +1,13 @@ +fixture,tool,metric,value +wdbc_heldout,predval_rung2,auroc,0.98403077896068303 +wdbc_heldout,predval_rung2,brier,0.029256064983711545 +wdbc_heldout,statsmodels_custom_grouped,auroc,0.98403077896068303 +wdbc_heldout,statsmodels_custom_grouped,brier,0.029256064983711881 +wdbc_heldout,statsmodels_raw_probability,auroc,0.96874670601876245 +wdbc_heldout,statsmodels_raw_probability,brier,0.027717535594874616 +synthetic_clustered,predval_rung2,auroc,0.81855107327905252 +synthetic_clustered,predval_rung2,brier,0.17054132084865364 +synthetic_clustered,statsmodels_custom_grouped,auroc,0.81855107327905252 +synthetic_clustered,statsmodels_custom_grouped,brier,0.17054132084865359 +synthetic_clustered,statsmodels_raw_probability,auroc,0.81883558475203566 +synthetic_clustered,statsmodels_raw_probability,brier,0.17238338966503758 diff --git a/benchmarks/alternatives/results/python/coverage.csv b/benchmarks/alternatives/results/python/coverage.csv new file mode 100644 index 0000000..b542e9a --- /dev/null +++ b/benchmarks/alternatives/results/python/coverage.csv @@ -0,0 +1,3 @@ +model_id,n_scored,n_full,coverage_full,n_common,common_fraction,n_dropped_if_common,meets_minimum +full,1332,1332,1,919,0.68993993993993996,413,True +partial,919,1332,0.68993993993993996,919,0.68993993993993996,0,False diff --git a/benchmarks/alternatives/results/python/coverage_selection_metrics.csv b/benchmarks/alternatives/results/python/coverage_selection_metrics.csv new file mode 100644 index 0000000..ba6e87c --- /dev/null +++ b/benchmarks/alternatives/results/python/coverage_selection_metrics.csv @@ -0,0 +1,5 @@ +subset,metric,value,n +full_available,auroc,0.82189813101406362,1332 +full_available,brier,0.18140992475539261,1332 +common_intersection,auroc,0.78319621993512833,919 +common_intersection,brier,0.17792866586674272,919 diff --git a/benchmarks/alternatives/results/python/intervals.csv b/benchmarks/alternatives/results/python/intervals.csv new file mode 100644 index 0000000..2ff15cf --- /dev/null +++ b/benchmarks/alternatives/results/python/intervals.csv @@ -0,0 +1,11 @@ +fixture,tool,scheme,metric,low,high,width +wdbc_heldout,pycaleva_plus_custom_bootstrap,patient,brier,0.015593303087883301,0.048631062684011898,0.033037759596128598 +wdbc_heldout,pycaleva_plus_custom_bootstrap,patient,paired_brier_gain,-0.0053364764046153401,0.0094270783303540603,0.014763554734969399 +wdbc_heldout,pycaleva_plus_custom_bootstrap,row,brier,0.015723847480198692,0.048319757974591579,0.032595910494392887 +wdbc_heldout,pycaleva_plus_custom_bootstrap,row,paired_brier_gain,-0.0055358590458910581,0.0092397263447514654,0.014775585390642523 +wdbc_heldout,predval,patient,paired_brier_gain,-0.0053364764046153375,0.0094270783303540585,0.014763554734969396 +synthetic_clustered,pycaleva_plus_custom_bootstrap,patient,brier,0.1587361211804677,0.20688337470523394,0.048147253524766231 +synthetic_clustered,pycaleva_plus_custom_bootstrap,patient,paired_brier_gain,-0.0011632227475880863,0.022167365927628083,0.02333058867521617 +synthetic_clustered,pycaleva_plus_custom_bootstrap,row,brier,0.16852215094087003,0.19457984262889494,0.026057691688024909 +synthetic_clustered,pycaleva_plus_custom_bootstrap,row,paired_brier_gain,0.005338072340460855,0.016573381895931719,0.011235309555470864 +synthetic_clustered,predval,patient,paired_brier_gain,-0.0011632227475880895,0.022167365927628101,0.023330588675216191 diff --git a/benchmarks/alternatives/results/python/manifest.json b/benchmarks/alternatives/results/python/manifest.json new file mode 100644 index 0000000..c72cb8a --- /dev/null +++ b/benchmarks/alternatives/results/python/manifest.json @@ -0,0 +1,31 @@ +{ + "bootstrap_replicates": 1000, + "bootstrap_seed": 20261011, + "files": { + "correction_metrics.csv": "810d078fed3b8b96c69751ad2c04333e9730e13f64c2d7a9811faa4becacbd8a", + "coverage.csv": "3a6aa0df1d5da3aea520a81e7b1ff5287d12b1bad3ba5d36cbe90955061f8ca5", + "coverage_inputs/cohort.csv": "50127a122204e8fcbae05f4cf597e77f210791060015bbe120c837edc7f421c2", + "coverage_inputs/cohort.yaml": "fd3789802a5c4884f3d05a87ceb67bc67b5fbc97f9c736efe739f135780cf548", + "coverage_inputs/predictions.csv": "effe905a9fdc1117bf6d50f6bdfd9c20857552597d440259d37eb0accba5bc96", + "coverage_selection_metrics.csv": "2633e304ad3c482de0753cefcfce9c5c6835fc6882f0518310b2dc63d1f38712", + "intervals.csv": "65b5b3fc24cf9c603f7ca421d74b4a3353d8b75d92e2a7817d6428b125e44482", + "point_metrics.csv": "ba5cb17b04f400541e09b3c4b0108e41a211e75709707b55c9d4efee29f62480", + "summary.json": "03300c0306c33c4541e437b87426735a3bba16cabc2808a3144b282f54f41ac9", + "synthetic_clustered.csv": "50127a122204e8fcbae05f4cf597e77f210791060015bbe120c837edc7f421c2", + "synthetic_clustered_corrected.csv": "1bb4899c68ad53387f47804f6da41899c76ff5fc87165e5118cfd60a19e78568", + "wdbc_heldout.csv": "61a22060cc9072ef62ad74f7db619b982e8eeb3130c07d4cc275d602aa1ccd9c", + "wdbc_heldout_corrected.csv": "b47c07f8301ba3d7bac38a4ad9396593b0342036d71dabe1970d3630d5ce80c2" + }, + "protocol_sha256": "8630bb9af1f54eb4a7f8e7bacfa5a652dd4915dc60e1e319ac02105cbf920db1", + "python": "3.12.14", + "seed": 20261010, + "versions": { + "numpy": "2.4.6", + "pandas": "3.0.5", + "predval": "0.1.0", + "pycaleva": "0.8.2", + "scikit-learn": "1.9.0", + "scipy": "1.17.1", + "statsmodels": "0.14.6" + } +} diff --git a/benchmarks/alternatives/results/python/point_metrics.csv b/benchmarks/alternatives/results/python/point_metrics.csv new file mode 100644 index 0000000..285b7e5 --- /dev/null +++ b/benchmarks/alternatives/results/python/point_metrics.csv @@ -0,0 +1,21 @@ +fixture,tool,metric,value,abs_diff_reference +wdbc_heldout,predval,auroc,0.9880889638452619,0 +wdbc_heldout,predval,brier,0.031262751693950298,0 +wdbc_heldout,predval,calibration_intercept,1.8547783584918272,8.8817841970012523e-16 +wdbc_heldout,predval,calibration_slope,0.65895543819077251,1.4432899320127035e-15 +wdbc_heldout,pycaleva,auroc,0.9880889638452619,0 +wdbc_heldout,pycaleva,brier,0.031262751693950298,0 +wdbc_heldout,independent_reference,auroc,0.9880889638452619,0 +wdbc_heldout,independent_reference,brier,0.031262751693950298,0 +wdbc_heldout,independent_reference,calibration_intercept,1.854778358491828,0 +wdbc_heldout,independent_reference,calibration_slope,0.65895543819077107,0 +synthetic_clustered,predval,auroc,0.82189813101406362,0 +synthetic_clustered,predval,brier,0.18140992475539261,0 +synthetic_clustered,predval,calibration_intercept,0.71216526049354745,1.1102230246251565e-16 +synthetic_clustered,predval,calibration_slope,0.79441642455613248,2.2204460492503131e-16 +synthetic_clustered,pycaleva,auroc,0.82189813101406362,0 +synthetic_clustered,pycaleva,brier,0.18140992475539261,0 +synthetic_clustered,independent_reference,auroc,0.82189813101406362,0 +synthetic_clustered,independent_reference,brier,0.18140992475539261,0 +synthetic_clustered,independent_reference,calibration_intercept,0.71216526049354756,0 +synthetic_clustered,independent_reference,calibration_slope,0.79441642455613226,0 diff --git a/benchmarks/alternatives/results/python/resource_usage.json b/benchmarks/alternatives/results/python/resource_usage.json new file mode 100644 index 0000000..33735de --- /dev/null +++ b/benchmarks/alternatives/results/python/resource_usage.json @@ -0,0 +1,6 @@ +{ + "wall_seconds": 1.962911616999918, + "cpu_seconds": 1.9628360740000002, + "max_rss_kib": 291520, + "note": "Benchmark body, not a fair tool runtime comparison; excludes imports and installation." +} diff --git a/benchmarks/alternatives/results/python/summary.json b/benchmarks/alternatives/results/python/summary.json new file mode 100644 index 0000000..3bb2ec8 --- /dev/null +++ b/benchmarks/alternatives/results/python/summary.json @@ -0,0 +1,59 @@ +{ + "coverage": { + "errors": [ + { + "check": "minimum_coverage_90_percent", + "error": "1 model(s) fall below coverage.min_fraction; worst shown\n model: partial\n coverage: 0.689940\n required: 0.900000", + "status": "rejected" + }, + { + "check": "declared_absent_model", + "error": "declared in expected_models but absent from predictions: ['absent']", + "status": "rejected" + } + ], + "flags": [ + { + "code": "declared_but_absent", + "message": "declared in expected_models but absent from predictions: ['absent']", + "severity": "warning" + }, + { + "code": "informative_common_selection", + "message": "the common subset excludes 31.0% of cohort rows (> 20%); selection into the common subset may be informative", + "severity": "warning" + } + ], + "pycaleva_scope": "Accepts aligned y/p arrays; roster and cohort denominators need caller glue." + }, + "scope": "Executed: predval and PyCalEva; statsmodels/sklearn independent reference.", + "synthetic_clustered": { + "brier_gain": 0.010868603906738982, + "brier_gain_ci": [ + -0.0011632227475880895, + 0.0221673659276281 + ], + "events": 560, + "max_corrected_prediction_difference": 5.9119376061289586e-15, + "patients": 120, + "rows": 1332, + "suppressed_ladder_rungs": [] + }, + "wdbc_heldout": { + "brier_gain": 0.002006686710238758, + "brier_gain_ci": [ + -0.0053364764046153375, + 0.009427078330354059 + ], + "events": 106, + "max_corrected_prediction_difference": 3.785860513971784e-14, + "patients": 285, + "rows": 285, + "suppressed_ladder_rungs": [ + [ + "rung3", + "non-convergence" + ] + ] + } +} diff --git a/benchmarks/alternatives/results/r/r_agreement.csv b/benchmarks/alternatives/results/r/r_agreement.csv new file mode 100644 index 0000000..8f9cc40 --- /dev/null +++ b/benchmarks/alternatives/results/r/r_agreement.csv @@ -0,0 +1,3 @@ +fixture,max_abs_predval_vs_r_logit,max_abs_statsmodels_vs_probably_raw +wdbc_heldout,6.723954726339798e-12,1.0349785251051458e-8 +synthetic_clustered,5.773159728050814e-15,9.704870240767605e-11 diff --git a/benchmarks/alternatives/results/r/r_bootstrap_summary.csv b/benchmarks/alternatives/results/r/r_bootstrap_summary.csv new file mode 100644 index 0000000..b0a6040 --- /dev/null +++ b/benchmarks/alternatives/results/r/r_bootstrap_summary.csv @@ -0,0 +1,3 @@ +fixture,replicates,low,high,method +wdbc_heldout,1000,-0.0052822820622240177,0.0093309392234624533,rsample native group_bootstraps; fixed R logit-GLM predictions +synthetic_clustered,1000,-0.0010582283577589672,0.022889944278495382,rsample native group_bootstraps; fixed R logit-GLM predictions diff --git a/benchmarks/alternatives/results/r/r_metrics.csv b/benchmarks/alternatives/results/r/r_metrics.csv new file mode 100644 index 0000000..a9c0f5b --- /dev/null +++ b/benchmarks/alternatives/results/r/r_metrics.csv @@ -0,0 +1,9 @@ +fixture,tool,brier,auroc +wdbc_heldout,p,0.031262751693950305,0.9880889638452619 +wdbc_heldout,probably_raw_probability,0.027717535582638633,0.9687467060187626 +wdbc_heldout,r_glm_logit,0.02925606498367934,0.984030778960683 +wdbc_heldout,r_glm_raw_probability,0.027717535594874523,0.9687467060187626 +synthetic_clustered,p,0.18140992475539264,0.8218981310140636 +synthetic_clustered,probably_raw_probability,0.1723833896630457,0.8188355847520356 +synthetic_clustered,r_glm_logit,0.1705413208486536,0.8185510732790525 +synthetic_clustered,r_glm_raw_probability,0.1723833896650324,0.8188355847520356 diff --git a/benchmarks/alternatives/results/r/resource_usage.txt b/benchmarks/alternatives/results/r/resource_usage.txt new file mode 100644 index 0000000..d863511 --- /dev/null +++ b/benchmarks/alternatives/results/r/resource_usage.txt @@ -0,0 +1,2 @@ + user system elapsed +138.449 3.509 142.258 diff --git a/benchmarks/alternatives/results/r/sessionInfo.txt b/benchmarks/alternatives/results/r/sessionInfo.txt new file mode 100644 index 0000000..d51406a --- /dev/null +++ b/benchmarks/alternatives/results/r/sessionInfo.txt @@ -0,0 +1,45 @@ +R version 4.5.3 (2026-03-11) +Platform: x86_64-conda-linux-gnu +Running under: Debian GNU/Linux 13 (trixie) + +Matrix products: default +BLAS/LAPACK: /workspace/shared/predval-r-toolchain/env/lib/libopenblasp-r0.3.34.so; LAPACK version 3.12.0 + +locale: + [1] LC_CTYPE=C.UTF-8 LC_NUMERIC=C LC_TIME=C.UTF-8 + [4] LC_COLLATE=C.UTF-8 LC_MONETARY=C.UTF-8 LC_MESSAGES=C.UTF-8 + [7] LC_PAPER=C.UTF-8 LC_NAME=C LC_ADDRESS=C +[10] LC_TELEPHONE=C LC_MEASUREMENT=C.UTF-8 LC_IDENTIFICATION=C + +time zone: America/New_York +tzcode source: system (glibc) + +attached base packages: +[1] stats graphics grDevices utils datasets methods base + +other attached packages: +[1] purrr_1.2.2 tidyr_1.3.2 readr_2.2.0 dplyr_1.2.1 +[5] yardstick_1.4.0 rsample_1.3.2 probably_1.2.0 + +loaded via a namespace (and not attached): + [1] gtable_0.3.6 ggplot2_4.0.3 recipes_1.4.0 + [4] lattice_0.23-1 tzdb_0.5.0 vctrs_0.7.3 + [7] tools_4.5.3 generics_0.1.4 parallel_4.5.3 +[10] tibble_3.3.1 pkgconfig_2.0.3 Matrix_1.7-6 +[13] data.table_1.18.6.1 RColorBrewer_1.1-3 S7_0.2.2 +[16] lifecycle_1.0.5 compiler_4.5.3 farver_2.1.2 +[19] codetools_0.2-20 DiceDesign_1.10 class_7.3-24 +[22] tune_2.1.0 prodlim_2026.03.11 pillar_1.11.1 +[25] furrr_0.4.0 crayon_1.5.3 MASS_7.3-66 +[28] gower_1.0.2 rpart_4.1.27 parallelly_1.48.0 +[31] lava_1.9.3 dials_1.4.4 tidyselect_1.2.1 +[34] digest_0.6.39 future_1.76.0 listenv_1.1.0 +[37] splines_4.5.3 parsnip_1.6.1 grid_4.5.3 +[40] cli_3.6.6 magrittr_2.0.5 survival_3.8-12 +[43] future.apply_1.20.2 withr_3.0.3 scales_1.4.0 +[46] bit64_4.8.6 lubridate_1.9.5 timechange_0.4.0 +[49] butcher_0.4.0 globals_0.19.1 bit_4.6.0 +[52] nnet_7.3-21 timeDate_4052.112 workflows_1.3.0 +[55] hms_1.1.4 hardhat_1.4.3 rlang_1.3.0 +[58] Rcpp_1.1.2 glue_1.8.1 ipred_0.9-15 +[61] vroom_1.7.1 R6_2.6.1 diff --git a/benchmarks/alternatives/results/r/verification.json b/benchmarks/alternatives/results/r/verification.json new file mode 100644 index 0000000..9520c7c --- /dev/null +++ b/benchmarks/alternatives/results/r/verification.json @@ -0,0 +1,17 @@ +{ + "python_manifest_sha256": "a2a541ec30fbffe83103e04d2c85a5e83114a671e9818488e896a2f34eaf510d", + "python_stable_files_verified": 13, + "r_files_sha256": { + "r_agreement.csv": "2c1e306addc003d941c80f84ad0f2e1ae476902e42eb5a8498152f147600f4bd", + "r_metrics.csv": "7a8d558a65c8d61d43b04406fbf1d46c5c18126e6fc5a2ebd1818fe9dfde0c59", + "sessionInfo.txt": "54be6cde818461ff3da62ff156022848d00cab4ae131e30af1c4c1ef720b3039", + "synthetic_clustered_r_corrected.csv": "5c13c21f257107c6893d001dc80388d1b19ab92448dadf951982e9358414cba6", + "synthetic_clustered_rsample_gains.csv": "bbedbfec1f6085cb31290b43f84feaf1dff7c6049cb1d276cccd2f26c24a3ccb", + "wdbc_heldout_r_corrected.csv": "8bc8df31c97730d9e2acd8671d2d0ae9a0816f9a1580b8e615129dacccacaf3f", + "wdbc_heldout_rsample_gains.csv": "3f1fc4df7f404beedc6255da517ee6e4bd634ff2588833f9299be0bff96237dd" + }, + "r_repeated_stable_files_identical": 7, + "r_timing_first": "user system elapsed\n138.449 3.509 142.258", + "r_timing_repeat": "user system elapsed\n181.453 4.849 187.484", + "scope": "Same final inputs and fixed dependencies. Runtime excluded from hash equality." +} diff --git a/benchmarks/alternatives/results/reproducibility.json b/benchmarks/alternatives/results/reproducibility.json new file mode 100644 index 0000000..56f53f4 --- /dev/null +++ b/benchmarks/alternatives/results/reproducibility.json @@ -0,0 +1,18 @@ +{ + "fresh_python_environments": 2, + "stable_manifests_identical": true, + "manifest_sha256": "a2a541ec30fbffe83103e04d2c85a5e83114a671e9818488e896a2f34eaf510d", + "primary_usage": { + "wall_seconds": 1.962911616999918, + "cpu_seconds": 1.9628360740000002, + "max_rss_kib": 291520, + "note": "Benchmark body, not a fair tool runtime comparison; excludes imports and installation." + }, + "clean_usage": { + "wall_seconds": 2.3836366670002462, + "cpu_seconds": 2.3833308649999996, + "max_rss_kib": 291860, + "note": "Benchmark body, not a fair tool runtime comparison; excludes imports and installation." + }, + "timing_scope": "Benchmark body after imports, excludes dependency setup. Not a tool speed comparison." +} diff --git a/benchmarks/alternatives/run_benchmark.py b/benchmarks/alternatives/run_benchmark.py new file mode 100644 index 0000000..200de3c --- /dev/null +++ b/benchmarks/alternatives/run_benchmark.py @@ -0,0 +1,339 @@ +#!/usr/bin/env python3 +"""Prespecified, bounded alternatives benchmark. See PROTOCOL.md before running.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.metadata +import json +import platform +import resource +import time +from pathlib import Path + +import numpy as np +import pandas as pd +import statsmodels.api as sm +import yaml +from pycaleva import CalibrationEvaluator +from pycaleva.metrics import brier as pycaleva_brier +from scipy.special import expit, logit +from sklearn.datasets import load_breast_cancer +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import brier_score_loss, roc_auc_score +from sklearn.model_selection import train_test_split +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import StandardScaler + +from predval import metrics as pm +from predval import recalibrate as pr +from predval import uncertainty as pu +from predval.errors import CohortSpecError, CoverageViolation +from predval.io import ( + check_common_selection, + check_coverage, + check_roster, + coverage_report, + load_cohort, + load_predictions, +) + +SEED = 20261010 +BOOT_SEED = 20261011 +N_BOOT = 1000 +HERE = Path(__file__).resolve().parent + + +def digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def frame(y, p, patient): + groups = np.asarray(patient).astype(str) + fold_lookup = {g: i % 5 for i, g in enumerate(np.unique(groups))} + return pd.DataFrame( + { + "subject_id": [f"row_{i:05}" for i in range(len(y))], + "patient_id": groups, + "y": np.asarray(y, dtype=int), + "p": p, + "fold": [fold_lookup[g] for g in groups], + } + ) + + +def fixtures(): + data = load_breast_cancer() + y = 1 - data.target # UCI malignant is the positive event. + train, test = train_test_split(np.arange(len(y)), test_size=0.5, stratify=y, random_state=SEED) + model = make_pipeline(StandardScaler(), LogisticRegression(C=1, max_iter=2000, solver="lbfgs")) + model.fit(data.data[train], y[train]) + p = model.predict_proba(data.data[test])[:, 1] + real = frame( + y[test], + expit(-0.4 + 1.4 * logit(np.clip(p, 1e-6, 1 - 1e-6))), + [f"uci_{i:04}" for i in test], + ) + rng = np.random.default_rng(SEED) + sizes = rng.integers(4, 21, 120) + ids = np.repeat(np.arange(120), sizes) + latent = rng.normal(0, 1.2, 120) + x = rng.normal(size=len(ids)) + risk = expit(-0.8 + 0.9 * x + latent[ids]) + # A shared uniform induces positive within-patient outcome dependence while + # preserving each row's conditional Bernoulli marginal probability. + shared = rng.uniform(size=120) + uniforms = np.where(rng.uniform(size=len(ids)) < 0.65, shared[ids], rng.uniform(size=len(ids))) + sy = (uniforms < risk).astype(int) + sp = expit(-0.4 + 1.4 * (-0.8 + 0.9 * x + latent[ids])) + synthetic = frame(sy, sp, [f"patient_{i:03}" for i in ids]) + return {"wdbc_heldout": real, "synthetic_clustered": synthetic} + + +def metrics(y, p): + return {"auroc": roc_auc_score(y, p), "brier": brier_score_loss(y, p)} + + +def reference_calibration(y, p): + z = logit(np.clip(p, 1e-6, 1 - 1e-6)) + offset = sm.GLM(y, np.ones((len(y), 1)), family=sm.families.Binomial(), offset=z) + joint = sm.GLM(y, sm.add_constant(z), family=sm.families.Binomial()) + a = offset.fit(tol=1e-11, maxiter=100) + b = joint.fit(tol=1e-11, maxiter=100) + assert a.converged and b.converged + return {"calibration_intercept": a.params[0], "calibration_slope": b.params[1]} + + +def reference_crossfit(df, *, logit_scale=True): + y, p = df.y.to_numpy(), df.p.to_numpy() + x = logit(np.clip(p, 1e-6, 1 - 1e-6)) if logit_scale else p + out = np.full(len(df), np.nan) + for f in sorted(df.fold.unique()): + train = df.fold.to_numpy() != f + fit = sm.GLM( + y[train], + sm.add_constant(x[train]), + family=sm.families.Binomial(), + ).fit(tol=1e-11, maxiter=100) + assert fit.converged + out[~train] = fit.predict(sm.add_constant(x[~train])) + assert np.isfinite(out).all() + return out + + +def custom_bootstrap(y, p, corrected, groups): + """Adapter code, NOT a native PyCalEva uncertainty feature.""" + clusters = [np.flatnonzero(groups == g) for g in np.unique(groups)] + rng = np.random.default_rng(BOOT_SEED) + base, gain = [], [] + for _ in range(N_BOOT): + idx = np.concatenate([clusters[k] for k in rng.integers(0, len(clusters), len(clusters))]) + b = pycaleva_brier(y[idx], p[idx]) + base.append(b) + gain.append(b - pycaleva_brier(y[idx], corrected[idx])) + return np.quantile(base, [0.025, 0.975]), np.quantile(gain, [0.025, 0.975]) + + +def coverage_cases(df, out): + case = out / "coverage_inputs" + case.mkdir() + df.to_csv(case / "cohort.csv", index=False, float_format="%.17g") + score = df.groupby("patient_id").p.mean() + excluded = set(score[score >= score.quantile(0.7)].index) + keep = ~df.patient_id.isin(excluded) + a = df[["subject_id", "p"]].rename(columns={"p": "predicted"}).assign(model_id="full") + b = df.loc[keep, ["subject_id", "p"]].rename(columns={"p": "predicted"}) + b = b.assign(model_id="partial") # Identical predictions; only availability differs. + pd.concat([a, b]).to_csv(case / "predictions.csv", index=False, float_format="%.17g") + spec = { + "cohort_id": "controlled_coverage", + "version": 0, + "subject_key": "subject_id", + "data": "cohort.csv", + "outcome": {"type": "binary", "field": "y", "positive_label": 1}, + "clustering": {"field": "patient_id", "name": "patient"}, + "coverage": {"min_fraction": 0.9, "compare_on": "both", "common_warn_frac": 0.2}, + "completeness": {"require_outcome": True, "on_violation": "fail"}, + "expected_models": ["full", "partial", "absent"], + "on_missing_model": "warn", + } + spec_path = case / "cohort.yaml" + spec_path.write_text(yaml.safe_dump(spec)) + cohort, preds = load_cohort(spec_path), load_predictions(case / "predictions.csv") + report = coverage_report(cohort, preds) + report.to_csv(out / "coverage.csv", index=False, float_format="%.17g") + flags = check_roster(cohort, preds) + check_common_selection(cohort, preds) + errors = [] + try: + check_coverage(cohort, preds) + except CoverageViolation as exc: + errors.append( + {"check": "minimum_coverage_90_percent", "status": "rejected", "error": str(exc)} + ) + else: + raise AssertionError("Missing coverage was not rejected") + spec["on_missing_model"] = "fail" + spec_path.write_text(yaml.safe_dump(spec)) + try: + check_roster(load_cohort(spec_path), preds) + except CohortSpecError as exc: + # Avoid run-specific absolute paths in deterministic outputs. + errors.append( + { + "check": "declared_absent_model", + "status": "rejected", + "error": exc.message if hasattr(exc, "message") else str(exc).split("\n")[0], + } + ) + else: + raise AssertionError("Missing declared model was not rejected") + rows = [] + for name, selection in [ + ("full_available", np.ones(len(df), dtype=bool)), + ("common_intersection", keep.to_numpy()), + ]: + for m, value in metrics(df.y.to_numpy()[selection], df.p.to_numpy()[selection]).items(): + rows.append({"subset": name, "metric": m, "value": value, "n": int(selection.sum())}) + pd.DataFrame(rows).to_csv( + out / "coverage_selection_metrics.csv", index=False, float_format="%.17g" + ) + return { + "flags": [f.as_dict() for f in flags], + "errors": errors, + "pycaleva_scope": ( + "Accepts aligned y/p arrays; roster and cohort denominators need caller glue." + ), + } + + +def run(out): + start, cpu = time.monotonic(), time.process_time() + out.mkdir(parents=True, exist_ok=False) + point_rows, correction_rows, interval_rows, summary = [], [], [], {} + for name, df in fixtures().items(): + df.to_csv(out / f"{name}.csv", index=False, float_format="%.17g") + y, p, groups = df.y.to_numpy(), df.p.to_numpy(), df.patient_id.to_numpy() + ce = CalibrationEvaluator(y, p, outsample=True, n_groups=10) + estimates = { + "predval": { + "auroc": pm.auroc(y, p), + "brier": pm.brier(y, p), + "calibration_intercept": pm.calibration_intercept(y, p), + "calibration_slope": pm.calibration_slope(y, p), + }, + "pycaleva": {"auroc": ce.auroc, "brier": ce.brier}, + "independent_reference": metrics(y, p) | reference_calibration(y, p), + } + for tool, values in estimates.items(): + for metric, value in values.items(): + ref = estimates["independent_reference"][metric] + tolerance = 1e-10 if metric in {"auroc", "brier"} else 1e-8 + difference = abs(value - ref) + assert difference < tolerance, (name, tool, metric, difference) + point_rows.append( + { + "fixture": name, + "tool": tool, + "metric": metric, + "value": value, + "abs_diff_reference": difference, + } + ) + _, ladder = pr.ladder(y, p, groups, max_folds=5) + predval_corrected = ladder.crossfit_predictions["rung2"] + reference_corrected = reference_crossfit(df) + max_diff = float(np.max(np.abs(predval_corrected - reference_corrected))) + assert max_diff < 1e-8, max_diff + df["predval_rung2"] = predval_corrected + df["statsmodels_rung2"] = reference_corrected + raw_reference = reference_crossfit(df, logit_scale=False) + df["statsmodels_raw_probability"] = raw_reference + df.to_csv(out / f"{name}_corrected.csv", index=False, float_format="%.17g") + for tool, corrected in [ + ("predval_rung2", predval_corrected), + ("statsmodels_custom_grouped", reference_corrected), + ("statsmodels_raw_probability", raw_reference), + ]: + for metric, value in metrics(y, corrected).items(): + correction_rows.append( + {"fixture": name, "tool": tool, "metric": metric, "value": value} + ) + point, ci = pu.paired_brier_gain_interval( + y, p, predval_corrected, groups, 0.95, n_boot=N_BOOT, seed=BOOT_SEED + ) + for scheme, g in [("patient", groups), ("row", np.arange(len(df)))]: + base, gain = custom_bootstrap(y, p, predval_corrected, g) + for metric, interval in [("brier", base), ("paired_brier_gain", gain)]: + interval_rows.append( + { + "fixture": name, + "tool": "pycaleva_plus_custom_bootstrap", + "scheme": scheme, + "metric": metric, + "low": interval[0], + "high": interval[1], + "width": interval[1] - interval[0], + } + ) + if scheme == "patient": + assert np.allclose(gain, [ci.low, ci.high], atol=1e-12, rtol=0) + interval_rows.append( + { + "fixture": name, + "tool": "predval", + "scheme": "patient", + "metric": "paired_brier_gain", + "low": ci.low, + "high": ci.high, + "width": ci.high - ci.low, + } + ) + summary[name] = { + "rows": len(df), + "patients": len(np.unique(groups)), + "events": int(y.sum()), + "max_corrected_prediction_difference": max_diff, + "brier_gain": point, + "brier_gain_ci": [ci.low, ci.high], + "suppressed_ladder_rungs": ladder.suppressed, + } + for name, rows in [ + ("point_metrics", point_rows), + ("correction_metrics", correction_rows), + ("intervals", interval_rows), + ]: + pd.DataFrame(rows).to_csv(out / f"{name}.csv", index=False, float_format="%.17g") + summary["coverage"] = coverage_cases(fixtures()["synthetic_clustered"], out) + summary["scope"] = "Executed: predval and PyCalEva; statsmodels/sklearn independent reference." + (out / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n") + packages = ["predval", "pycaleva", "numpy", "pandas", "scipy", "scikit-learn", "statsmodels"] + manifest = { + "python": platform.python_version(), + "versions": {p: importlib.metadata.version(p) for p in packages}, + "seed": SEED, + "bootstrap_seed": BOOT_SEED, + "bootstrap_replicates": N_BOOT, + "protocol_sha256": digest(HERE / "PROTOCOL.md"), + "files": { + str(p.relative_to(out)): digest(p) for p in sorted(out.rglob("*")) if p.is_file() + }, + } + (out / "manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + usage = { + "wall_seconds": time.monotonic() - start, + "cpu_seconds": time.process_time() - cpu, + "max_rss_kib": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss, + "note": ( + "Benchmark body, not a fair tool runtime comparison; excludes imports and installation." + ), + } + (out / "resource_usage.json").write_text(json.dumps(usage, indent=2) + "\n") + print(json.dumps(summary, indent=2)) + print(json.dumps(usage)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", type=Path, required=True) + run(parser.parse_args().out) diff --git a/benchmarks/alternatives/tests/test_benchmark.py b/benchmarks/alternatives/tests/test_benchmark.py new file mode 100644 index 0000000..3346ba9 --- /dev/null +++ b/benchmarks/alternatives/tests/test_benchmark.py @@ -0,0 +1,70 @@ +"""Benchmark contract/failure tests; no optional competitor executes on import.""" + +import importlib.util +from pathlib import Path + +import numpy as np +import pytest +from pycaleva import CalibrationEvaluator + +SOURCE = Path(__file__).resolve().parents[1] / "run_benchmark.py" +spec = importlib.util.spec_from_file_location("benchmark", SOURCE) +bm = importlib.util.module_from_spec(spec) +spec.loader.exec_module(bm) + + +def test_fixtures_are_deterministic_and_patient_folds_disjoint(): + first, second = bm.fixtures(), bm.fixtures() + assert len(first["wdbc_heldout"]) == 285 + assert first["synthetic_clustered"].patient_id.nunique() == 120 + for name, df in first.items(): + assert df.equals(second[name]) + assert df.groupby("patient_id").fold.nunique().eq(1).all() + for fold in sorted(df.fold.unique()): + train = df[df.fold != fold] + test = df[df.fold == fold] + assert set(train.patient_id).isdisjoint(test.patient_id) + assert set(train.y) == {0, 1} + + +def test_crossfit_matches_independent_glm(): + for df in bm.fixtures().values(): + _, report = bm.pr.ladder(df.y.to_numpy(), df.p.to_numpy(), df.patient_id.to_numpy()) + assert np.isfinite(bm.reference_crossfit(df, logit_scale=False)).all() + assert np.allclose( + report.crossfit_predictions["rung2"], bm.reference_crossfit(df), atol=1e-8, rtol=0 + ) + + +def test_missing_coverage_and_absent_models_reject(tmp_path): + results = bm.coverage_cases(bm.fixtures()["synthetic_clustered"], tmp_path) + assert {f["code"] for f in results["flags"]} == { + "declared_but_absent", + "informative_common_selection", + } + assert all(e["status"] == "rejected" for e in results["errors"]) + assert len(results["errors"]) == 2 + + +def test_pycaleva_rejects_out_of_range_probabilities(): + with pytest.raises(ValueError): + CalibrationEvaluator(np.array([0, 1, 0, 1]), np.array([0.1, 1.2, 0.3, 0.9]), True) + + +def test_invalid_crossfit_is_unavailable_not_fabricated(): + # Every patient is one class, so holding either of the two patients out + # leaves a single-class training fold. PREDVAL must withhold fitted rungs. + y = np.array([0, 0, 1, 1]) + p = np.array([0.1, 0.2, 0.7, 0.8]) + _, report = bm.pr.ladder(y, p, np.array(["a", "a", "b", "b"]), max_folds=2) + assert report.crossfit_predictions == {} + assert len(report.suppressed) == 3 + + +@pytest.mark.xfail(strict=True, reason="Known PREDVAL IRLS iteration-limit/quasi-separation defect") +def test_quasi_separation_must_not_return_finite_converged_slope(): + y = np.array([0, 0, 1, 1]) + p = np.array([0.2, 0.5, 0.5, 0.8]) + # This quasi-separated likelihood has no finite slope MLE. Current core + # returns its last iterate after max_iter rather than marking unavailable. + assert not np.isfinite(bm.pm.calibration_slope(y, p)) diff --git a/benchmarks/alternatives/verification/benchmark-contract-tests.log b/benchmarks/alternatives/verification/benchmark-contract-tests.log new file mode 100644 index 0000000..e4b1d08 --- /dev/null +++ b/benchmarks/alternatives/verification/benchmark-contract-tests.log @@ -0,0 +1,10 @@ +.....x [100%] +==================================== PASSES ==================================== +=========================== short test summary info ============================ +PASSED benchmarks/alternatives/tests/test_benchmark.py::test_fixtures_are_deterministic_and_patient_folds_disjoint +PASSED benchmarks/alternatives/tests/test_benchmark.py::test_crossfit_matches_independent_glm +PASSED benchmarks/alternatives/tests/test_benchmark.py::test_missing_coverage_and_absent_models_reject +PASSED benchmarks/alternatives/tests/test_benchmark.py::test_pycaleva_rejects_out_of_range_probabilities +PASSED benchmarks/alternatives/tests/test_benchmark.py::test_invalid_crossfit_is_unavailable_not_fabricated +XFAIL benchmarks/alternatives/tests/test_benchmark.py::test_quasi_separation_must_not_return_finite_converged_slope - Known PREDVAL IRLS iteration-limit/quasi-separation defect +5 passed, 1 xfailed in 8.12s diff --git a/benchmarks/alternatives/verification/predval-pytest.log b/benchmarks/alternatives/verification/predval-pytest.log new file mode 100644 index 0000000..9764441 --- /dev/null +++ b/benchmarks/alternatives/verification/predval-pytest.log @@ -0,0 +1,5 @@ +........................................................................ [ 24%] +................sss..................................................... [ 49%] +.................sssssssssssss..............ssss........................ [ 74%] +........................................................................ [ 99%] +. [100%] diff --git a/benchmarks/alternatives/verification/r-parse.log b/benchmarks/alternatives/verification/r-parse.log new file mode 100644 index 0000000..a752f4e --- /dev/null +++ b/benchmarks/alternatives/verification/r-parse.log @@ -0,0 +1 @@ +R adapter syntax and output-whitespace normalization: passed diff --git a/benchmarks/alternatives/verification/r-repeat.log b/benchmarks/alternatives/verification/r-repeat.log new file mode 100644 index 0000000..1e6e0f0 --- /dev/null +++ b/benchmarks/alternatives/verification/r-repeat.log @@ -0,0 +1,3 @@ +Registered S3 method overwritten by 'butcher': + method from + as.character.dev_topic generics diff --git a/benchmarks/alternatives/verification/r-run.log b/benchmarks/alternatives/verification/r-run.log new file mode 100644 index 0000000..1e6e0f0 --- /dev/null +++ b/benchmarks/alternatives/verification/r-run.log @@ -0,0 +1,3 @@ +Registered S3 method overwritten by 'butcher': + method from + as.character.dev_topic generics diff --git a/benchmarks/alternatives/verification/repeat-check.log b/benchmarks/alternatives/verification/repeat-check.log new file mode 100644 index 0000000..44e782a --- /dev/null +++ b/benchmarks/alternatives/verification/repeat-check.log @@ -0,0 +1,17 @@ +{ + "python_stable_files_verified": 13, + "python_manifest_sha256": "a2a541ec30fbffe83103e04d2c85a5e83114a671e9818488e896a2f34eaf510d", + "r_repeated_stable_files_identical": 7, + "r_files_sha256": { + "r_metrics.csv": "7a8d558a65c8d61d43b04406fbf1d46c5c18126e6fc5a2ebd1818fe9dfde0c59", + "r_agreement.csv": "2c1e306addc003d941c80f84ad0f2e1ae476902e42eb5a8498152f147600f4bd", + "sessionInfo.txt": "54be6cde818461ff3da62ff156022848d00cab4ae131e30af1c4c1ef720b3039", + "wdbc_heldout_r_corrected.csv": "8bc8df31c97730d9e2acd8671d2d0ae9a0816f9a1580b8e615129dacccacaf3f", + "wdbc_heldout_rsample_gains.csv": "3f1fc4df7f404beedc6255da517ee6e4bd634ff2588833f9299be0bff96237dd", + "synthetic_clustered_r_corrected.csv": "5c13c21f257107c6893d001dc80388d1b19ab92448dadf951982e9358414cba6", + "synthetic_clustered_rsample_gains.csv": "bbedbfec1f6085cb31290b43f84feaf1dff7c6049cb1d276cccd2f26c24a3ccb" + }, + "r_timing_first": "user system elapsed\n138.449 3.509 142.258", + "r_timing_repeat": "user system elapsed\n181.453 4.849 187.484", + "scope": "Same final inputs and fixed dependencies. Runtime excluded from hash equality." +} diff --git a/benchmarks/alternatives/verification/ruff.log b/benchmarks/alternatives/verification/ruff.log new file mode 100644 index 0000000..1f5f344 --- /dev/null +++ b/benchmarks/alternatives/verification/ruff.log @@ -0,0 +1 @@ +All checks passed! diff --git a/benchmarks/alternatives/verification/status.json b/benchmarks/alternatives/verification/status.json new file mode 100644 index 0000000..e87158f --- /dev/null +++ b/benchmarks/alternatives/verification/status.json @@ -0,0 +1,21 @@ +{ + "predval_base_commit": "99ca3d72f1d4f8340df9a59de3616277e9ebdb7a", + "core_code_changed": false, + "existing_suite": { + "passed": 269, + "skipped": 20, + "failure_count": 0, + "note": "Optional example/golden fixture checks skipped; not claimed as passed." + }, + "benchmark_contracts": { + "passed": 5, + "expected_failures": 1, + "note": "Strict xfail: preexisting IRLS quasi-separation/iteration-limit defect." + }, + "ruff": "passed", + "git_diff_check": "passed", + "python_reproduction": "two freshly created isolated environments; stable hashes identical", + "r_numerics": "native probably/grouped rsample and independent controls executed; see r_agreement.csv", + "isitfair": "source-reviewed; no competitor source executed", + "r_reproduction": "Same final inputs; seven stable outputs byte-identical across two runs" +} diff --git a/benchmarks/alternatives/verify_results.py b/benchmarks/alternatives/verify_results.py new file mode 100644 index 0000000..224da99 --- /dev/null +++ b/benchmarks/alternatives/verify_results.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Verify clean repeats and emit compact, regenerable R summaries.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +import numpy as np +import pandas as pd + + +def sha(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def verify(py: Path, py_repeat: Path, r: Path, r_repeat: Path): + assert (py / "manifest.json").read_bytes() == (py_repeat / "manifest.json").read_bytes() + files = json.loads((py / "manifest.json").read_text())["files"] + for relative, expected in files.items(): + assert sha(py / relative) == sha(py_repeat / relative) == expected + shared_r = ["r_metrics.csv", "r_agreement.csv", "sessionInfo.txt"] + summary = [] + for name in ("wdbc_heldout", "synthetic_clustered"): + shared_r.extend([f"{name}_r_corrected.csv", f"{name}_rsample_gains.csv"]) + gains = pd.read_csv(r / f"{name}_rsample_gains.csv").gain.to_numpy() + assert len(gains) == 1000 and np.isfinite(gains).all() + lo, hi = np.quantile(gains, [0.025, 0.975]) + summary.append( + { + "fixture": name, + "replicates": len(gains), + "low": lo, + "high": hi, + "method": "rsample native group_bootstraps; fixed R logit-GLM predictions", + } + ) + for name in shared_r: + assert (r / name).read_bytes() == (r_repeat / name).read_bytes(), name + pd.DataFrame(summary).to_csv(r / "r_bootstrap_summary.csv", index=False, float_format="%.17g") + result = { + "python_stable_files_verified": len(files), + "python_manifest_sha256": sha(py / "manifest.json"), + "r_repeated_stable_files_identical": len(shared_r), + "r_files_sha256": {name: sha(r / name) for name in shared_r}, + "r_timing_first": (r / "resource_usage.txt").read_text().strip(), + "r_timing_repeat": (r_repeat / "resource_usage.txt").read_text().strip(), + "scope": "Same final inputs and fixed dependencies. Runtime excluded from hash equality.", + } + (r / "verification.json").write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") + print(json.dumps(result, indent=2)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("python", type=Path) + parser.add_argument("python_repeat", type=Path) + parser.add_argument("r", type=Path) + parser.add_argument("r_repeat", type=Path) + args = parser.parse_args() + verify(args.python, args.python_repeat, args.r, args.r_repeat) From 7792ba8402556a972d4aa1533233769636e39a35 Mon Sep 17 00:00:00 2001 From: aRealGem <203440826+aRealGem@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:06:36 -0400 Subject: [PATCH 05/12] Reject nonconverged IRLS fits at the iteration limit Return None when the existing IRLS convergence test has not succeeded by the iteration limit. Preserve the existing estimator, tolerance, and converged fit results. Exercise quasi-separation, finite-MLE budget exhaustion, and downstream recalibration unavailability in the ladder and HTML report. Validation: 276 passed, 20 fixture-dependent skips; 71 focused tests pass. Five new regression cases fail against the original implementation. Ruff lint and formatting pass. Independently reviewed; no golden files changed. Publication preserves the reviewed file tree; connector-assigned commit metadata differs. Reviewed-local-commit: 617ddac480849ac48de41c1c1b1d050ddc427492 Co-developed-with: OpenAI Codex --- src/predval/metrics.py | 3 ++- tests/test_metrics.py | 35 +++++++++++++++++++++++++++++++ tests/test_recalibrate.py | 20 ++++++++++++++++++ tests/test_report.py | 44 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 101 insertions(+), 1 deletion(-) diff --git a/src/predval/metrics.py b/src/predval/metrics.py index 5e027db..0f85e84 100644 --- a/src/predval/metrics.py +++ b/src/predval/metrics.py @@ -128,7 +128,8 @@ def _irls_logistic( if np.max(np.abs(new - beta)) < tol: return new beta = new - return beta + # A finite last iterate is not a converged fit, notably under quasi-separation. + return None def calibration_intercept(y: np.ndarray, p: np.ndarray) -> float: diff --git a/tests/test_metrics.py b/tests/test_metrics.py index f93ac07..95d9a37 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -94,6 +94,41 @@ def test_constant_predictions_have_no_slope() -> None: assert np.isnan(M.calibration_slope(y, np.full(4, 0.3))) +# -------------------------------------------------------------------------- IRLS convergence + + +def test_irls_rejects_quasi_separation_at_default_iteration_limit() -> None: + """Mixed outcomes at the tied score keep weights positive without a finite slope MLE.""" + y = np.array([0, 0, 1, 1]) + p = np.array([0.2, 0.5, 0.5, 0.8]) + assert M._irls_logistic(y, M.logit(p), None) is None + assert np.isnan(M.calibration_slope(y, p)) + # The offset-only fit still has a finite solution despite the unavailable slope fit. + assert M.calibration_intercept(y, p) == pytest.approx(0.0, abs=1e-9) + + +@pytest.mark.parametrize("max_iter", [0, 1]) +def test_irls_rejects_exhausted_budget_even_with_finite_mle(max_iter) -> None: + y = np.array([0, 0, 1, 1, 0, 1]) + z = M.logit(np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6])) + assert M._irls_logistic(y, z, None, max_iter=max_iter) is None + + +def test_irls_preserves_converged_finite_mle() -> None: + y = np.array([0, 0, 1, 1, 0, 1]) + z = M.logit(np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6])) + ref = sm.GLM(y, sm.add_constant(z), family=sm.families.Binomial()).fit() + beta = M._irls_logistic(y, z, None) + assert beta is not None + assert beta == pytest.approx(ref.params, rel=1e-6) + + +def test_irls_accepts_convergence_on_last_allowed_iteration() -> None: + beta = M._irls_logistic(np.array([0, 1]), None, None, max_iter=1) + assert beta is not None + assert beta == pytest.approx([0.0]) + + # --------------------------------------------------------------------------- the eps-clip path diff --git a/tests/test_recalibrate.py b/tests/test_recalibrate.py index 6d3577c..22ebc0f 100644 --- a/tests/test_recalibrate.py +++ b/tests/test_recalibrate.py @@ -159,6 +159,26 @@ def test_single_class_cell_yields_no_ladder_rows() -> None: # ------------------------------------------------------------------------- half-pair guard +def test_quasi_separation_withholds_rung2_with_non_convergence_reason() -> None: + """Every training fold has quasi-separation: no finite apparent or held-out slope exists.""" + y = np.tile([0, 0, 1, 1], 5) + p = np.tile([0.2, 0.5, 0.5, 0.8], 5) + groups = np.repeat(np.arange(5), 4) + + cal, reason = R._fit("rung2", y, p) + assert cal is None and reason == R.R_NON_CONVERGENCE + predictions, reason = R._crossfit_predictions("rung2", y, p, groups, k=5) + assert predictions is None and reason == R.R_NON_CONVERGENCE + + rows, report = R.ladder(y, p, groups) + assert ("rung2", R.R_NON_CONVERGENCE) in report.suppressed + assert report.rung2_slope is None + assert "rung2" not in report.crossfit_predictions + assert not any(row.rung == "rung2" for row in rows) + # The valid offset-only correction still emits both halves of its pair. + assert {row.fit_mode for row in rows if row.rung == "rung1"} == {"apparent", "crossfit"} + + def test_no_crossfit_partner_suppresses_the_whole_rung() -> None: """With one cluster there is nothing to hold out, so every rung is withheld, not half-shown.""" rng = np.random.default_rng(7) diff --git a/tests/test_report.py b/tests/test_report.py index 5e9ff43..773933c 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -24,6 +24,7 @@ render_from_dir, ) from predval.report import FRAMING, PENDING, build_context +from tests.conftest import VALID_SPEC, write_cohort, write_predictions N_CLUSTERS = 8 PER_CLUSTER = 80 @@ -220,6 +221,49 @@ def test_gated_subgroup_ladder_is_flagged_in_the_report(html) -> None: assert "recalibration_suppressed" not in html or "suppressed" in html.lower() +def test_quasi_separated_rung_is_unavailable_in_evaluation_and_report(tmp_path: Path) -> None: + """The default fit must expose failure through the report, with no apparent or gain rows.""" + ids = [f"s{i}" for i in range(20)] + cohort_path = write_cohort( + tmp_path, + spec={ + **VALID_SPEC, + "subgroups": [], + "uncertainty": {"n_boot": 12, "seed": 1337}, + }, + table=pd.DataFrame( + { + "subject_id": ids, + "label": np.tile([0, 0, 1, 1], 5), + "site": np.repeat(np.arange(5), 4), + } + ), + ) + predictions_path = write_predictions( + tmp_path, + pd.DataFrame( + { + "subject_id": ids, + "model_id": "quasi", + "predicted": np.tile([0.2, 0.5, 0.5, 0.8], 5), + } + ), + ) + result = evaluate(load_cohort(cohort_path), load_predictions(predictions_path)) + + assert not (result.metrics["rung"] == "rung2").any() + reason = "rung2 withheld for quasi (overall): non-convergence" + assert any( + flag.code == "recalibration_unavailable" and flag.message == reason for flag in result.flags + ) + ctx = build_context( + result.metrics, result.fragility, result.coverage, result.manifest, result.calibration + ) + assert ctx["calibration_rows"] + assert all(row["rung2_gain"] == "n/a" for row in ctx["calibration_rows"]) + assert reason in render_evaluation(result) + + # ----------------------------------------------------------------------------- provenance From 4829570fd3809c1fda779e8ea24fc9017fcb43a4 Mon Sep 17 00:00:00 2001 From: aRealGem <203440826+aRealGem@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:06:44 -0400 Subject: [PATCH 06/12] Validate integrated IRLS guard against the preserved benchmark Remove the strict expected failure for quasi-separation now that the reviewed convergence guard is present. Keep the historical benchmark protocol, results, and verification unchanged; label current integration evidence separately. Validation: 282 passed, 20 optional fixture-dependent skips; 6 benchmark contracts pass separately with no xfail. Two comparator reruns match all 13 original stable file hashes. Seven retained R evidence hashes verify; R computation was not rerun because adapter and inputs are unchanged. Ruff lint, formatting, and diff checks pass. Publication preserves the reviewed file tree; connector-assigned commit metadata differs. Reviewed-local-commit: 150998f76df1141c73b6c3621233e693c4eb0b49 Co-developed-with: OpenAI Codex --- benchmarks/alternatives/INTEGRATION.md | 105 +++++++++++++++ benchmarks/alternatives/README.md | 34 +++-- .../alternatives/tests/test_benchmark.py | 5 +- .../integrated/benchmark-contract-tests.log | 2 + .../verification/integrated/diff-check.log | 0 .../verification/integrated/format.log | 1 + .../verification/integrated/pytest.log | 24 ++++ .../verification/integrated/python-a.log | 61 +++++++++ .../verification/integrated/python-b.log | 60 +++++++++ .../verification/integrated/ruff.log | 1 + .../verification/integrated/status.json | 125 ++++++++++++++++++ 11 files changed, 404 insertions(+), 14 deletions(-) create mode 100644 benchmarks/alternatives/INTEGRATION.md create mode 100644 benchmarks/alternatives/verification/integrated/benchmark-contract-tests.log create mode 100644 benchmarks/alternatives/verification/integrated/diff-check.log create mode 100644 benchmarks/alternatives/verification/integrated/format.log create mode 100644 benchmarks/alternatives/verification/integrated/pytest.log create mode 100644 benchmarks/alternatives/verification/integrated/python-a.log create mode 100644 benchmarks/alternatives/verification/integrated/python-b.log create mode 100644 benchmarks/alternatives/verification/integrated/ruff.log create mode 100644 benchmarks/alternatives/verification/integrated/status.json diff --git a/benchmarks/alternatives/INTEGRATION.md b/benchmarks/alternatives/INTEGRATION.md new file mode 100644 index 0000000..fd80d0a --- /dev/null +++ b/benchmarks/alternatives/INTEGRATION.md @@ -0,0 +1,105 @@ +# Reviewed-change integration: 2026-10-10 + +## Result + +The reviewed patient-partition guard, alternatives comparator, and IRLS iteration-limit +fix are combined on local branch `integrate/reviewed-predval-20261010`. +The combined suite passes **282 tests**, with **20 optional fixture-dependent skips**, +**zero failures**, and **zero expected failures**. The comparator contracts also pass +separately: **6/6**, including the formerly expected-failure quasi-separation case. +Ruff lint, Ruff formatting (63 files), and `git diff --check` pass. + +Both Python comparator reruns reproduce **all 13 stable input/result hashes exactly** +from the original benchmark. Thus the two main fixtures' point estimates, corrections, +intervals, coverage flags, and suppression summaries have not changed. This does not imply +that every possible input is unchanged: exhausted IRLS fits now become unavailable rather +than being reported as converged finite fits. + +## Source and evidence provenance + +- Patient-partition guard: `99ca3d72f1d4f8340df9a59de3616277e9ebdb7a`. +- Original comparator: `868276ef960f0bc7f2bc75cc79926ff7b3c2e1f1`; its original + preregistration commits `dae6e1a` and `756e5a1` remain in the branch history. +- Reviewed convergence fix: `62c8af368824d75e1dd915e78c131f46e333a340`, cherry-picked + without conflicts as `617ddac480849ac48de41c1c1b1d050ddc427492`. +- Integration-specific source edit: remove the strict xfail marker from + `tests/test_benchmark.py` and update its explanatory comment. The core implementation, + benchmark runner, R adapter, and frozen benchmark protocol are unchanged beyond the + reviewed guard. README wording now distinguishes historical and integrated checks. +- Package metadata still says PREDVAL 0.1.0. The source commits and file hashes in + [`verification/integrated/status.json`](verification/integrated/status.json), rather + than the package version alone, identify the code tested here. + +Original compact results and verification logs remain byte-for-byte unchanged. Their old +269-pass/5-pass/1-xfail counts describe the original comparator, not this integrated source. +New integration logs and the hash-comparison record live separately under +`verification/integrated/`. The independent statistical-stress study is **not integrated**; +its frozen baseline and worktree were not changed by this task. + +## What was rerun + +The full combined Python suite ran against this worktree. The comparator ran twice with +`PYTHONPATH` explicitly selecting this worktree's `src`; source imports were checked in +both interpreters. These used two existing isolated Python 3.12.14 environments originally +created from the benchmark's complete requirements lock, not two new installations during +integration. Each run verified 13 stable file hashes, for 26 new file-hash checks. Both +manifests equal the historical manifest, SHA-256: + +`a2a541ec30fbffe83103e04d2c85a5e83114a671e9818488e896a2f34eaf510d` + +The R adapter consumes the exported fixtures and corrected predictions, all byte-identical +here, and its source is unchanged. All seven retained R stable files were checked against +the original recorded hashes. **R computation was not repeated for integration.** The +original repeated R computation is reused evidence, not a newly executed R result. + +The benchmark-body Python wall times were 3.86 and 5.47 seconds. As in the original report, +these exclude imports/installation, reflect concurrent machine load, and do not compare +individual tools' speed. Exact resource records are in the integration JSON. + +## Reproduce the integration checks + +Use the locked Python setup in the main benchmark README. From the repository root: + +```sh +export PYTHONPATH="$PWD/src" +export MPLCONFIGDIR=/tmp/predval-mpl XDG_CACHE_HOME=/tmp/predval-cache +export OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 +.venv-benchmark/bin/python -m pytest tests benchmarks/alternatives/tests -ra +.venv-benchmark/bin/ruff check . +.venv-benchmark/bin/ruff format --check . +git diff --check +.venv-benchmark/bin/python benchmarks/alternatives/run_benchmark.py --out /tmp/predval-integrated-a +.venv-benchmark/bin/python benchmarks/alternatives/run_benchmark.py --out /tmp/predval-integrated-b +cmp benchmarks/alternatives/results/python/manifest.json /tmp/predval-integrated-a/manifest.json +cmp /tmp/predval-integrated-a/manifest.json /tmp/predval-integrated-b/manifest.json +``` + +The output directories must not already exist. To check the contents as well as the manifest: + +```sh +.venv-benchmark/bin/python - <<'PY' +import hashlib, json +from pathlib import Path +baseline = Path('benchmarks/alternatives/results/python/manifest.json') +expected = json.loads(baseline.read_text())['files'] +for directory in ('/tmp/predval-integrated-a', '/tmp/predval-integrated-b'): + root = Path(directory) + assert (root / 'manifest.json').read_bytes() == baseline.read_bytes() + for relative, digest in expected.items(): + assert hashlib.sha256((root / relative).read_bytes()).hexdigest() == digest +print(f'Passed: {2 * len(expected)} file-hash checks and both baseline-manifest comparisons') +PY +``` + +## Remaining limits + +- Twenty tests did not run because optional PCam, GUSTO, and IMPROVE example/golden fixtures + were unavailable. No full golden-output revalidation is claimed. +- The guard retains the same estimator, tolerance, and iteration limit. A valid finite-MLE + fit can be unavailable if it needs more iterations. This change does not prove every + unavailable-reason label is correct or detect every possible separation condition. +- The two comparator gain intervals still cross zero. They condition on fixed held-out + predictions and do not measure repeated-sampling coverage or correction-training uncertainty. +- The original isitfair adapter remains unexecuted; source review is not an execution result. +- These are bounded engineering/method checks, not clinical validation or a superiority claim. +- All work is local. No push, pull request, merge, or deployment was performed. diff --git a/benchmarks/alternatives/README.md b/benchmarks/alternatives/README.md index 98490b3..44b3e39 100644 --- a/benchmarks/alternatives/README.md +++ b/benchmarks/alternatives/README.md @@ -6,7 +6,11 @@ Its point metrics are not novel, and correct alternatives can reproduce its resu The protocol was committed before execution (`dae6e1a`), with a source-review amendment (`756e5a1`) distinguishing correction families before any R or isitfair execution. -Both commits descend from patient-partition guard `99ca3d7`; no core PREDVAL code changed. +Both commits descend from patient-partition guard `99ca3d7`; the original benchmark +changed no core PREDVAL code. This integrated branch also includes the subsequently +reviewed IRLS iteration-limit guard (`62c8af3`, cherry-picked as `617ddac`). Original +results and verification records below remain unchanged and describe the historical +baseline. Separately labeled current-branch checks are in [INTEGRATION.md](INTEGRATION.md). ## What is and is not executed @@ -157,16 +161,21 @@ statsmodels 0.14.6. Every transitive Python dependency is pinned in requirements The checked probably GitHub source is commit `974f0154cc7d44ae41d8072161996b23a19d3d2a`; installed R package versions, when executed, take precedence over that source-only snapshot. -## Known limitation found during independent review +## Historical IRLS defect and integrated guard -PREDVAL's IRLS implementation can return its last iterate after hitting the iteration limit -without convergence. On `y=[0,0,1,1]`, `p=[0.2,0.5,0.5,0.8]`, the quasi-separated likelihood -has no finite slope MLE, but the baseline returns a finite slope near 22.288 and does not -mark rung2 unavailable. A strict expected-failure regression test records this defect. -The matched rung2 estimates on the two main fixtures still agree with independently -converged GLMs; that agreement does not validate the general failure guard. No core fix is -included in this comparison branch. Do not market the tool as universally detecting -non-convergence or statistically validated on the strength of these checks. +In baseline `99ca3d7`, PREDVAL's IRLS implementation could return its last iterate after +hitting the iteration limit without convergence. On `y=[0,0,1,1]`, +`p=[0.2,0.5,0.5,0.8]`, the quasi-separated likelihood has no finite slope MLE, but the +baseline returned a finite slope near 22.288 and did not mark rung2 unavailable. +The original comparison recorded this as a strict expected failure. + +The integrated branch includes the reviewed iteration-limit guard: exhausted IRLS fits +now return `None`, and this regression is an ordinary passing test. This changes the +failure guard, not the estimator, convergence tolerance, or iteration budget. The two +main fixtures' matched rung2 estimates still agree with independently converged GLMs; +all 13 stable Python output/input hashes are unchanged in the integration reruns. +That bounded check does not establish universal non-convergence detection or statistical +validity. See [INTEGRATION.md](INTEGRATION.md) for current-source provenance and caveats. ## Reproducibility and measured resource use @@ -210,7 +219,10 @@ R `group_vfold_cv` passed the no-patient-overlap check. `group_bootstraps` suppl native cluster resamples for each fixture. R's resamples differ from NumPy's; their intervals are independently generated Monte Carlo results on fixed corrected predictions. -## Verification status +## Original baseline verification status (archived) + +These counts refer to the original comparator at `868276e`, before integration of the +IRLS guard. Current combined-suite results are recorded in [INTEGRATION.md](INTEGRATION.md). - Existing PREDVAL suite: 269 passed, 20 skipped because optional example/golden fixtures were unavailable. This does not claim those skipped integration cases passed. diff --git a/benchmarks/alternatives/tests/test_benchmark.py b/benchmarks/alternatives/tests/test_benchmark.py index 3346ba9..b95110e 100644 --- a/benchmarks/alternatives/tests/test_benchmark.py +++ b/benchmarks/alternatives/tests/test_benchmark.py @@ -61,10 +61,9 @@ def test_invalid_crossfit_is_unavailable_not_fabricated(): assert len(report.suppressed) == 3 -@pytest.mark.xfail(strict=True, reason="Known PREDVAL IRLS iteration-limit/quasi-separation defect") def test_quasi_separation_must_not_return_finite_converged_slope(): y = np.array([0, 0, 1, 1]) p = np.array([0.2, 0.5, 0.5, 0.8]) - # This quasi-separated likelihood has no finite slope MLE. Current core - # returns its last iterate after max_iter rather than marking unavailable. + # This quasi-separated likelihood has no finite slope MLE. Exhausting the + # IRLS iteration budget must mark the slope unavailable. assert not np.isfinite(bm.pm.calibration_slope(y, p)) diff --git a/benchmarks/alternatives/verification/integrated/benchmark-contract-tests.log b/benchmarks/alternatives/verification/integrated/benchmark-contract-tests.log new file mode 100644 index 0000000..fd9f43c --- /dev/null +++ b/benchmarks/alternatives/verification/integrated/benchmark-contract-tests.log @@ -0,0 +1,2 @@ +...... [100%] +6 passed in 7.14s diff --git a/benchmarks/alternatives/verification/integrated/diff-check.log b/benchmarks/alternatives/verification/integrated/diff-check.log new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/alternatives/verification/integrated/format.log b/benchmarks/alternatives/verification/integrated/format.log new file mode 100644 index 0000000..4bf16a1 --- /dev/null +++ b/benchmarks/alternatives/verification/integrated/format.log @@ -0,0 +1 @@ +63 files already formatted diff --git a/benchmarks/alternatives/verification/integrated/pytest.log b/benchmarks/alternatives/verification/integrated/pytest.log new file mode 100644 index 0000000..32c46c4 --- /dev/null +++ b/benchmarks/alternatives/verification/integrated/pytest.log @@ -0,0 +1,24 @@ +........................................................................ [ 23%] +................sss..................................................... [ 47%] +......................sssssssssssss..............ssss................... [ 71%] +........................................................................ [ 95%] +.............. [100%] +=========================== short test summary info ============================ +SKIPPED [1] tests/test_golden_gusto.py:69: GUSTO fixture not built; run examples/gusto/fetch_data.py + make_predictions.py +SKIPPED [1] tests/test_golden_improve.py:67: IMPROVE fixture not built; run examples/improve/build_fixture.py +SKIPPED [1] tests/test_golden_pcam.py:69: PCam fixture not built; run examples/pcam/build_fixture.py +SKIPPED [1] tests/test_pcam_fixture.py:40: PCam fixture not built; run examples/pcam/build_fixture.py +SKIPPED [1] tests/test_pcam_fixture.py:46: PCam fixture not built; run examples/pcam/build_fixture.py +SKIPPED [1] tests/test_pcam_fixture.py:52: PCam fixture not built; run examples/pcam/build_fixture.py +SKIPPED [1] tests/test_pcam_fixture.py:58: PCam fixture not built; run examples/pcam/build_fixture.py +SKIPPED [1] tests/test_pcam_fixture.py:63: PCam fixture not built; run examples/pcam/build_fixture.py +SKIPPED [1] tests/test_pcam_fixture.py:68: PCam fixture not built; run examples/pcam/build_fixture.py +SKIPPED [1] tests/test_pcam_fixture.py:73: PCam fixture not built; run examples/pcam/build_fixture.py +SKIPPED [1] tests/test_pcam_fixture.py:79: PCam fixture not built; run examples/pcam/build_fixture.py +SKIPPED [1] tests/test_pcam_fixture.py:85: PCam fixture not built; run examples/pcam/build_fixture.py +SKIPPED [1] tests/test_pcam_fixture.py:92: PCam fixture not built; run examples/pcam/build_fixture.py +SKIPPED [1] tests/test_pcam_fixture.py:97: PCam fixture not built; run examples/pcam/build_fixture.py +SKIPPED [1] tests/test_pcam_fixture.py:108: PCam fixture not built; run examples/pcam/build_fixture.py +SKIPPED [1] tests/test_pcam_fixture.py:144: PCam fixture not built; run examples/pcam/build_fixture.py +SKIPPED [4] tests/test_predrupdate_agreement.py:99: GUSTO fixture not built; run examples/gusto/fetch_data.py + make_predictions.py +282 passed, 20 skipped in 167.91s (0:02:47) diff --git a/benchmarks/alternatives/verification/integrated/python-a.log b/benchmarks/alternatives/verification/integrated/python-a.log new file mode 100644 index 0000000..c56c1e7 --- /dev/null +++ b/benchmarks/alternatives/verification/integrated/python-a.log @@ -0,0 +1,61 @@ +Matplotlib is building the font cache; this may take a moment. +{ + "wdbc_heldout": { + "rows": 285, + "patients": 285, + "events": 106, + "max_corrected_prediction_difference": 3.785860513971784e-14, + "brier_gain": 0.002006686710238758, + "brier_gain_ci": [ + -0.0053364764046153375, + 0.009427078330354059 + ], + "suppressed_ladder_rungs": [ + [ + "rung3", + "non-convergence" + ] + ] + }, + "synthetic_clustered": { + "rows": 1332, + "patients": 120, + "events": 560, + "max_corrected_prediction_difference": 5.9119376061289586e-15, + "brier_gain": 0.010868603906738982, + "brier_gain_ci": [ + -0.0011632227475880895, + 0.0221673659276281 + ], + "suppressed_ladder_rungs": [] + }, + "coverage": { + "flags": [ + { + "code": "declared_but_absent", + "severity": "warning", + "message": "declared in expected_models but absent from predictions: ['absent']" + }, + { + "code": "informative_common_selection", + "severity": "warning", + "message": "the common subset excludes 31.0% of cohort rows (> 20%); selection into the common subset may be informative" + } + ], + "errors": [ + { + "check": "minimum_coverage_90_percent", + "status": "rejected", + "error": "1 model(s) fall below coverage.min_fraction; worst shown\n model: partial\n coverage: 0.689940\n required: 0.900000" + }, + { + "check": "declared_absent_model", + "status": "rejected", + "error": "declared in expected_models but absent from predictions: ['absent']" + } + ], + "pycaleva_scope": "Accepts aligned y/p arrays; roster and cohort denominators need caller glue." + }, + "scope": "Executed: predval and PyCalEva; statsmodels/sklearn independent reference." +} +{"wall_seconds": 3.85623039300026, "cpu_seconds": 3.6602351980000005, "max_rss_kib": 291780, "note": "Benchmark body, not a fair tool runtime comparison; excludes imports and installation."} diff --git a/benchmarks/alternatives/verification/integrated/python-b.log b/benchmarks/alternatives/verification/integrated/python-b.log new file mode 100644 index 0000000..7e9a490 --- /dev/null +++ b/benchmarks/alternatives/verification/integrated/python-b.log @@ -0,0 +1,60 @@ +{ + "wdbc_heldout": { + "rows": 285, + "patients": 285, + "events": 106, + "max_corrected_prediction_difference": 3.785860513971784e-14, + "brier_gain": 0.002006686710238758, + "brier_gain_ci": [ + -0.0053364764046153375, + 0.009427078330354059 + ], + "suppressed_ladder_rungs": [ + [ + "rung3", + "non-convergence" + ] + ] + }, + "synthetic_clustered": { + "rows": 1332, + "patients": 120, + "events": 560, + "max_corrected_prediction_difference": 5.9119376061289586e-15, + "brier_gain": 0.010868603906738982, + "brier_gain_ci": [ + -0.0011632227475880895, + 0.0221673659276281 + ], + "suppressed_ladder_rungs": [] + }, + "coverage": { + "flags": [ + { + "code": "declared_but_absent", + "severity": "warning", + "message": "declared in expected_models but absent from predictions: ['absent']" + }, + { + "code": "informative_common_selection", + "severity": "warning", + "message": "the common subset excludes 31.0% of cohort rows (> 20%); selection into the common subset may be informative" + } + ], + "errors": [ + { + "check": "minimum_coverage_90_percent", + "status": "rejected", + "error": "1 model(s) fall below coverage.min_fraction; worst shown\n model: partial\n coverage: 0.689940\n required: 0.900000" + }, + { + "check": "declared_absent_model", + "status": "rejected", + "error": "declared in expected_models but absent from predictions: ['absent']" + } + ], + "pycaleva_scope": "Accepts aligned y/p arrays; roster and cohort denominators need caller glue." + }, + "scope": "Executed: predval and PyCalEva; statsmodels/sklearn independent reference." +} +{"wall_seconds": 5.465970102000028, "cpu_seconds": 5.272218276, "max_rss_kib": 293760, "note": "Benchmark body, not a fair tool runtime comparison; excludes imports and installation."} diff --git a/benchmarks/alternatives/verification/integrated/ruff.log b/benchmarks/alternatives/verification/integrated/ruff.log new file mode 100644 index 0000000..1f5f344 --- /dev/null +++ b/benchmarks/alternatives/verification/integrated/ruff.log @@ -0,0 +1 @@ +All checks passed! diff --git a/benchmarks/alternatives/verification/integrated/status.json b/benchmarks/alternatives/verification/integrated/status.json new file mode 100644 index 0000000..9aa2d6a --- /dev/null +++ b/benchmarks/alternatives/verification/integrated/status.json @@ -0,0 +1,125 @@ +{ + "benchmark_tests_separate": { + "passed": 6, + "seconds": 7.14, + "skipped": 0, + "xfailed": 0 + }, + "cherry_picked_guard_commit": "617ddac480849ac48de41c1c1b1d050ddc427492", + "combined_tests": { + "command": "python -m pytest tests benchmarks/alternatives/tests -ra", + "failed": 0, + "passed": 282, + "seconds": 167.91, + "skip_reason": "Unavailable PCam, GUSTO, and IMPROVE optional example/golden fixtures.", + "skipped": 20, + "xfailed": 0, + "xpassed": 0 + }, + "date": "2026-10-10", + "lint": { + "formatted_file_count": 63, + "git_diff_check": "passed", + "ruff_check": "passed", + "ruff_format_check": "passed" + }, + "numerical_outcome": "No change in either main benchmark fixture; all 13 stable input/result files match the original in each of two reruns.", + "original_comparator_commit": "868276ef960f0bc7f2bc75cc79926ff7b3c2e1f1", + "original_convergence_guard_commit": "62c8af368824d75e1dd915e78c131f46e333a340", + "original_protocol_result_verification_files": [ + "benchmarks/alternatives/PROTOCOL.md", + "benchmarks/alternatives/results/glue_inventory.json", + "benchmarks/alternatives/results/python/correction_metrics.csv", + "benchmarks/alternatives/results/python/coverage.csv", + "benchmarks/alternatives/results/python/coverage_selection_metrics.csv", + "benchmarks/alternatives/results/python/intervals.csv", + "benchmarks/alternatives/results/python/manifest.json", + "benchmarks/alternatives/results/python/point_metrics.csv", + "benchmarks/alternatives/results/python/resource_usage.json", + "benchmarks/alternatives/results/python/summary.json", + "benchmarks/alternatives/results/r/r_agreement.csv", + "benchmarks/alternatives/results/r/r_bootstrap_summary.csv", + "benchmarks/alternatives/results/r/r_metrics.csv", + "benchmarks/alternatives/results/r/resource_usage.txt", + "benchmarks/alternatives/results/r/sessionInfo.txt", + "benchmarks/alternatives/results/r/verification.json", + "benchmarks/alternatives/results/reproducibility.json", + "benchmarks/alternatives/verification/benchmark-contract-tests.log", + "benchmarks/alternatives/verification/predval-pytest.log", + "benchmarks/alternatives/verification/r-parse.log", + "benchmarks/alternatives/verification/r-repeat.log", + "benchmarks/alternatives/verification/r-run.log", + "benchmarks/alternatives/verification/repeat-check.log", + "benchmarks/alternatives/verification/ruff.log", + "benchmarks/alternatives/verification/status.json" + ], + "original_protocol_result_verification_files_preserved": 25, + "patient_partition_guard": "99ca3d72f1d4f8340df9a59de3616277e9ebdb7a", + "python_environment_note": "Two existing isolated Python 3.12.14 environments created from requirements-lock.txt for the original benchmark were reused. They were not freshly installed for integration. PYTHONPATH selected this integration worktree; both imports were verified.", + "python_reruns": [ + { + "all_stable_hashes_match_original": true, + "manifest_sha256": "a2a541ec30fbffe83103e04d2c85a5e83114a671e9818488e896a2f34eaf510d", + "resource_usage": { + "cpu_seconds": 3.6602351980000005, + "max_rss_kib": 291780, + "note": "Benchmark body, not a fair tool runtime comparison; excludes imports and installation.", + "wall_seconds": 3.85623039300026 + }, + "run": "python-a", + "stable_file_count": 13 + }, + { + "all_stable_hashes_match_original": true, + "manifest_sha256": "a2a541ec30fbffe83103e04d2c85a5e83114a671e9818488e896a2f34eaf510d", + "resource_usage": { + "cpu_seconds": 5.272218276, + "max_rss_kib": 293760, + "note": "Benchmark body, not a fair tool runtime comparison; excludes imports and installation.", + "wall_seconds": 5.465970102000028 + }, + "run": "python-b", + "stable_file_count": 13 + } + ], + "python_stable_file_sha256": { + "correction_metrics.csv": "810d078fed3b8b96c69751ad2c04333e9730e13f64c2d7a9811faa4becacbd8a", + "coverage.csv": "3a6aa0df1d5da3aea520a81e7b1ff5287d12b1bad3ba5d36cbe90955061f8ca5", + "coverage_inputs/cohort.csv": "50127a122204e8fcbae05f4cf597e77f210791060015bbe120c837edc7f421c2", + "coverage_inputs/cohort.yaml": "fd3789802a5c4884f3d05a87ceb67bc67b5fbc97f9c736efe739f135780cf548", + "coverage_inputs/predictions.csv": "effe905a9fdc1117bf6d50f6bdfd9c20857552597d440259d37eb0accba5bc96", + "coverage_selection_metrics.csv": "2633e304ad3c482de0753cefcfce9c5c6835fc6882f0518310b2dc63d1f38712", + "intervals.csv": "65b5b3fc24cf9c603f7ca421d74b4a3353d8b75d92e2a7817d6428b125e44482", + "point_metrics.csv": "ba5cb17b04f400541e09b3c4b0108e41a211e75709707b55c9d4efee29f62480", + "summary.json": "03300c0306c33c4541e437b87426735a3bba16cabc2808a3144b282f54f41ac9", + "synthetic_clustered.csv": "50127a122204e8fcbae05f4cf597e77f210791060015bbe120c837edc7f421c2", + "synthetic_clustered_corrected.csv": "1bb4899c68ad53387f47804f6da41899c76ff5fc87165e5118cfd60a19e78568", + "wdbc_heldout.csv": "61a22060cc9072ef62ad74f7db619b982e8eeb3130c07d4cc275d602aa1ccd9c", + "wdbc_heldout_corrected.csv": "b47c07f8301ba3d7bac38a4ad9396593b0342036d71dabe1970d3630d5ce80c2" + }, + "r_evidence": { + "execution_repeated_for_integration": false, + "original_stable_file_count_verified": 7, + "original_stable_file_sha256": { + "r_agreement.csv": "2c1e306addc003d941c80f84ad0f2e1ae476902e42eb5a8498152f147600f4bd", + "r_metrics.csv": "7a8d558a65c8d61d43b04406fbf1d46c5c18126e6fc5a2ebd1818fe9dfde0c59", + "sessionInfo.txt": "54be6cde818461ff3da62ff156022848d00cab4ae131e30af1c4c1ef720b3039", + "synthetic_clustered_r_corrected.csv": "5c13c21f257107c6893d001dc80388d1b19ab92448dadf951982e9358414cba6", + "synthetic_clustered_rsample_gains.csv": "bbedbfec1f6085cb31290b43f84feaf1dff7c6049cb1d276cccd2f26c24a3ccb", + "wdbc_heldout_r_corrected.csv": "8bc8df31c97730d9e2acd8671d2d0ae9a0816f9a1580b8e615129dacccacaf3f", + "wdbc_heldout_rsample_gains.csv": "3f1fc4df7f404beedc6255da517ee6e4bd634ff2588833f9299be0bff96237dd" + }, + "reason": "The unchanged R adapter consumes byte-identical Python fixture/corrected-prediction inputs. Historical R evidence is reused and explicitly labeled; no new R execution is claimed." + }, + "remote_actions": [], + "scope": "Local integration revalidation; methods/engineering evidence, not clinical or statistical validation.", + "source_sha256": { + "benchmarks/alternatives/PROTOCOL.md": "8630bb9af1f54eb4a7f8e7bacfa5a652dd4915dc60e1e319ac02105cbf920db1", + "benchmarks/alternatives/probably_adapter.R": "b81a00594a0e7407b4af5f023a01be594353f30bc16a090ac1f16d39aa4723e3", + "benchmarks/alternatives/run_benchmark.py": "a62a5f77c04e74ba79d9a043c98f4f2be37dd651112a49cc76b5d5fcbf360375", + "benchmarks/alternatives/tests/test_benchmark.py": "6b2270052c2df675b94638c7bff18b6d14ed64afb44412d1f4c89393449e2d8f", + "src/predval/metrics.py": "5a0f13aee0e1a4de17611f5aa13ff05421382af478b9fff788fd77132d9b81e6" + }, + "statistical_stress_integrated": false, + "tested_source_note": "The code at the cherry-picked guard commit, with the comparator strict xfail removed; documentation and evidence records do not change source code." +} From d17929d104a2cecac18aebc1e7056a34aa8f4aa7 Mon Sep 17 00:00:00 2001 From: aRealGem <203440826+aRealGem@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:09:48 -0400 Subject: [PATCH 07/12] Record prospective local paired-family extension protocol Publication preserves the reviewed file tree; connector-assigned commit metadata differs. Reviewed-local-commit: 4a289159a937c49b1d06469e93db9079fef5f47f Co-developed-with: OpenAI Codex --- benchmarks/family_extension/PROTOCOL.md | 80 +++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 benchmarks/family_extension/PROTOCOL.md diff --git a/benchmarks/family_extension/PROTOCOL.md b/benchmarks/family_extension/PROTOCOL.md new file mode 100644 index 0000000..94e3c54 --- /dev/null +++ b/benchmarks/family_extension/PROTOCOL.md @@ -0,0 +1,80 @@ +# Prospective local family extension, 2026-10-10 + +This extension follows inspection of the original two fixtures and their results. It is +not a retroactive preregistration of those results. This protocol is committed locally +before executing the new extension. Local Git chronology is an audit trail, not independent +proof of preregistration or a trusted external timestamp. Baseline: integrated `150998f`. +Historical benchmarks/alternatives inputs, protocol and results remain untouched. + +## Fixed design + +Use 20 seeds, the consecutive integers 20261020 through 20261039 inclusive. For each seed: +120 independent patient clusters, uniform integer sizes 4–20 inclusive; patient effect +Normal(0,1.2), independent row covariate Normal(0,1), eta=-0.8+0.9*x+patient_effect. +Published p=expit(-0.4+1.4*eta). Share cluster sizes, covariates, p and uniform outcome +randomness between the two generating families: + +- logit_linear: q=expit(eta), so logit(q)=(logit(p)+0.4)/1.4; +- probability_linear: q=expit(-2.5+5*p), so logit(q) is linear in p. + +Outcome uniforms select a patient-shared uniform with probability 0.65, otherwise an +independent row uniform. y=1[uniform Date: Sat, 10 Oct 2026 12:18:39 -0400 Subject: [PATCH 08/12] Add verified paired-family calibration benchmark extension Publication preserves the reviewed file tree; connector-assigned commit metadata differs. Reviewed-local-commit: 470338a5a994211e41103491e4028e4b9b2ce198 Co-developed-with: OpenAI Codex --- benchmarks/family_extension/README.md | 132 + .../family_extension/results/cases.json | 1360 ++++++++ .../family_extension/results/manifest.json | 43 + .../family_extension/results/metrics.csv | 165 + .../family_extension/results/metrics.json | 3098 +++++++++++++++++ .../family_extension/results/summary.json | 799 +++++ benchmarks/family_extension/run_extension.py | 316 ++ .../family_extension/tests/test_extension.py | 95 + .../verification/diff-check.log | 0 .../family_extension/verification/format.log | 1 + .../family_extension/verification/pytest.log | 24 + .../family_extension/verification/repeat.log | 6 + .../verification/reproducibility.json | 7 + .../family_extension/verification/ruff.log | 1 + .../verification/runtime-a.json | 4 + .../verification/runtime-b.json | 4 + .../family_extension/verification/status.json | 52 + benchmarks/family_extension/verify_results.py | 55 + 18 files changed, 6162 insertions(+) create mode 100644 benchmarks/family_extension/README.md create mode 100644 benchmarks/family_extension/results/cases.json create mode 100644 benchmarks/family_extension/results/manifest.json create mode 100644 benchmarks/family_extension/results/metrics.csv create mode 100644 benchmarks/family_extension/results/metrics.json create mode 100644 benchmarks/family_extension/results/summary.json create mode 100644 benchmarks/family_extension/run_extension.py create mode 100644 benchmarks/family_extension/tests/test_extension.py create mode 100644 benchmarks/family_extension/verification/diff-check.log create mode 100644 benchmarks/family_extension/verification/format.log create mode 100644 benchmarks/family_extension/verification/pytest.log create mode 100644 benchmarks/family_extension/verification/repeat.log create mode 100644 benchmarks/family_extension/verification/reproducibility.json create mode 100644 benchmarks/family_extension/verification/ruff.log create mode 100644 benchmarks/family_extension/verification/runtime-a.json create mode 100644 benchmarks/family_extension/verification/runtime-b.json create mode 100644 benchmarks/family_extension/verification/status.json create mode 100644 benchmarks/family_extension/verify_results.py diff --git a/benchmarks/family_extension/README.md b/benchmarks/family_extension/README.md new file mode 100644 index 0000000..77a28de --- /dev/null +++ b/benchmarks/family_extension/README.md @@ -0,0 +1,132 @@ +# Paired calibration-family extension + +A bounded extension of the historical alternatives benchmark, not clinical validation or a +universal ranking. The local prospective protocol was committed as `4a28915` before this +extension's execution, after inspecting the original benchmark. The local commit sequence +is an audit trail, not independently timestamped preregistration. The protocol phrase +“paired between-family method contrasts only within each family” is a wording error: the +implemented, reported contrasts are between methods within each family. The frozen protocol +is retained unchanged. Historical results under +`benchmarks/alternatives` are unchanged. No production code or public branch was changed. + +## Main finding + +Performance reverses when the known generating calibration family changes. On 20 fixed +seeds, PREDVAL rung2 improves the logit-linear fixture but worsens the probability-linear +fixture relative to raw predictions. The probability-scale GLM has the smaller Brier in +its matched family. This supports testing assumptions, not declaring a universal winner. + +| Generating family | Method | Mean Brier | Mean known-risk MSE | Mean pooled AUROC delta | +|---|---|---:|---:|---:| +| Logit-linear | Raw | 0.168878 | 0.007849 | 0 | +| Logit-linear | PREDVAL rung2 | 0.162805 | 0.001155 | -0.004083 | +| Logit-linear | Probability GLM | 0.165599 | 0.004141 | -0.007555 | +| Probability-linear | Raw | 0.138728 | 0.001522 | 0 | +| Probability-linear | PREDVAL rung2 | 0.141261 | 0.004010 | -0.005336 | +| Probability-linear | Probability GLM | 0.138268 | 0.001115 | -0.008699 | + +The independent logit GLM agrees with PREDVAL on every case where both are available. +The probability GLM represents the family used by probably's smooth=FALSE calibrator; +**probably was not executed in this extension**. This is a Python family comparison, not a +new native R package benchmark. + +Within each family, the paired mean Brier difference (PREDVAL minus probability GLM) is: + +- Logit-linear: -0.002795; approximate 95% Monte Carlo interval [-0.003681, -0.001909]. +- Probability-linear: +0.002993; interval [0.001829, 0.004156]. + +These intervals quantify precision of this simulation mean across 20 independent seeds, +not patient-level uncertainty or broad method superiority. The families share covariates, +published scores, patients, folds and outcome uniform draws, but differ in true risks, +difficulty and raw miscalibration. Do not compare their absolute performance or recovery +percentages as if those were matched difficulty settings. No seed or parameter was selected +after looking at this extension's outputs. + +Mean per-seed recovery of expected reducible error is 85.34% (MCSE 3.00 percentage points) +for rung2 and 47.20% (3.70 points) for probability GLM in the logit family. In the probability +family it is -160.99% (31.59 points) and 27.82% (15.91 points), respectively. Negative recovery +means calibration worsens known-risk squared error; the raw-error denominator is much smaller +in the second family. These are means of per-seed ratios, not ratios of pooled means. + +All 164 method-cases (41 fixtures times four methods) were available; no fitting warnings +were captured. Suppression of other native ladder rungs is retained in cases.json. Zero +failures in 20 seeds is weak evidence about rare failures: the maximum binomial MCSE at +this simulation size is about 0.112. This is not an interval-coverage study. + +## Undistorted real-data control + +| Method | Brier | AUROC | AUROC delta | +|---|---:|---:|---:| +| Raw WDBC model | 0.029478 | 0.988089 | 0 | +| PREDVAL rung2 | 0.029271 | 0.984769 | -0.003320 | +| Probability GLM | 0.027865 | 0.979024 | -0.009065 | + +WDBC model probabilities are fitted estimates, not known true risks. No expected-oracle +metric is calculated for WDBC. Brier improvement does not imply discrimination improvement. +Fold-specific increasing calibrations can change ordering between folds, so pooled AUROC +need not be preserved even when each fold's AUROC is unchanged. + +The public Wisconsin Diagnostic Breast Cancer dataset is credited to Wolberg, Mangasarian, +Street and Street (1993), UCI, https://doi.org/10.24432/C5DW2B, CC BY 4.0. We use the +scikit-learn bundled data, a held-out subset and fitted unmodified logistic probabilities. +No row-level WDBC data are bundled; the runner regenerates them. No endorsement is implied. + +## Expected risk versus realized oracle loss + +For synthetic q, the reported fresh-outcome expected Brier is +mean[q*(1-q)] + mean[(prediction-q)^2], holding fitted predictions and covariates fixed. +It averages over new outcome randomness, including patient shared uniforms. It does not +condition on the originally realized shared uniforms. Cluster dependence changes variance, +not this expectation identity. The realized loss of q on one noisy outcome vector is not an +irreducible floor. The original synthetic fixture's roughly 98% realized raw-to-oracle gain +ratio should therefore not be called recovery of expected reducible risk; that quantity is +roughly 81.31% for that historical fixture. The new simulation uses different fixed seeds. + +## Comparator scope and remaining work + +CalibrationCurves supplies native cluster-aware calibration curves via MIXC, CGC and MAC2: +[official API](https://bavodc.github.io/websiteCalibrationCurves/reference/valProbCluster.html), +[official vignette](https://bavodc.github.io/websiteCalibrationCurves/articles/CalibrationCurves.html), +[current CRAN manual](https://cran.r-project.org/web/packages/CalibrationCurves/CalibrationCurves.pdf). +It is **unexecuted here**. The inspected website uses MIXC as default; CRAN 3.1.0 documents a +combined MAC2/MIXC default. Any adapter should pin version and explicit approach. R 4.5.3 is +available, but CalibrationCurves, lme4, metafor and rms were absent. No dependency installation +was attempted. A future bounded run should start with official bundled clustered data, then +an explicitly configured MIXC small-cluster diagnostic. Singleton WDBC is unsuitable for +within-cluster curve fitting. Curve confidence/prediction bands are not interchangeable with +PREDVAL's paired Brier-gain intervals. + +A nonlinear/spline-shaped fixture would test additional flexibility; this two-family +extension does not establish it. Keep rare-event, separation and low-cluster-count stress +separate, with unavailable fits and interval coverage explicitly measured. The existing +coverage/roster contract demonstrates integration rather than unique statistical methods; +no programmer-time savings or clinical utility were measured. + +## Reproduce and verify + +Use the existing Python lock from benchmarks/alternatives/requirements-lock.txt and install +this worktree's package as in that benchmark. In a compatible existing environment: + +```sh +export PYTHONPATH="$PWD/src" +export MPLCONFIGDIR=/tmp/predval-family-mpl XDG_CACHE_HOME=/tmp/predval-cache +export OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 +python benchmarks/family_extension/run_extension.py --out /tmp/family-a +python benchmarks/family_extension/run_extension.py --out /tmp/family-b +python benchmarks/family_extension/verify_results.py /tmp/family-a /tmp/family-b +python -m pytest tests benchmarks/alternatives/tests benchmarks/family_extension/tests -ra +``` + +Output directories must be new. The runner records per-case scores, warning/suppression +states, input and prediction hashes, summary MCSEs, exact versions and source/protocol +hashes. Run-body timing is separate and excludes imports/install; it is not a package-speed +comparison. The manifest records the absolute import path as provenance, so transferring the +worktree changes that field even if the numerical output hashes match. Verification records +and retained results accompany this report. See verification/status.json for exact checks. + +Verified: 291 tests passed, 20 optional fixture-dependent tests skipped; Ruff lint and +formatting passed. Both complete runs reproduced four stable result files and the manifest. +Independent reconstruction of all 40 synthetic cases and GLM refits reproduced losses within +4.45e-15 and all aggregate MCSE/paired-contrast calculations. Unexpected exceptions outside +the selected fitting exception types would abort rather than become an unavailable entry; +none occurred in this fixed design. diff --git a/benchmarks/family_extension/results/cases.json b/benchmarks/family_extension/results/cases.json new file mode 100644 index 0000000..0782dbb --- /dev/null +++ b/benchmarks/family_extension/results/cases.json @@ -0,0 +1,1360 @@ +[ + { + "family": "logit_linear", + "input_sha256": "76729406c969f748161b85d0857903f925cd35732283521e0393d8739dc0f0c3", + "seed": 20261020, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 7.105427357601002e-15, + "prediction_sha256": "b3766903c61f83db347d5d4d9d1d8465586d0ba98cae15bf66ae16f1649db4b6", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "6534088ebda51f272db94acc600a2f4f641fadcd0ae849fcd0ded3ea0ca2fd53", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "91d893e97ec2b84544182e8df861d3966be13d3506b292ad323106df3a763928", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "6495d8694bf45663428ce1e83ef253d3c0bcbed7f9ab3189f4fabe84166ccea2", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "probability_linear", + "input_sha256": "063f6e5fad1b80fc1610f4929b83042548cf1bf3871ce2b4c14ceb8cf96eb9ce", + "seed": 20261020, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 1.2212453270876722e-15, + "prediction_sha256": "4f2e2f5aa278d8960583be30bd37160122e690cd664fce6ca6c177f0cd2aa723", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "6534088ebda51f272db94acc600a2f4f641fadcd0ae849fcd0ded3ea0ca2fd53", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "fc8196f6fa62e1069fa9567c33fe80eb33252a4df2e93bdd3d8fdba636594ecc", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "95920eb427dfd925d33d8b47dbcace75374dad69e08c97f6810dfd32098ea28e", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "logit_linear", + "input_sha256": "cf310ba75c8f3b72e2e4ba0d0a10ab097ad3a8d48228c33e49afa963226132fa", + "seed": 20261021, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 4.274358644806853e-15, + "prediction_sha256": "78ee130b7a66a6e5c623da0363f7d6becd753c707d83b8c84c055484cf69e626", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "b710abac7443713f10b4baf54d426e82bf0770e6f944bea500e1a2cf179d2f0c", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "c3632f404fd7ca51222f0f20604534a1c45cb7c2a77ae531dd80e7810cfcfb35", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "67284f6f73b47c9480d9ad86ceababa9142ffaa195226b5c3dfd1be8627e8986", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "probability_linear", + "input_sha256": "b7b6f4644c24dce8cd0d7f8c0fe4855277719800453f2399b7e81d1c5b7d51b2", + "seed": 20261021, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 8.881784197001252e-16, + "prediction_sha256": "591e9cf499146e966acd146304c576a6e448faa6d1147eb529ae36cc5891813c", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "b710abac7443713f10b4baf54d426e82bf0770e6f944bea500e1a2cf179d2f0c", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "b8f249072f8afe941534040e3d6c77421a82fa093c553dd34a5d74f35da44844", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "e964f7d7a2a4ea173575f973d0ef277232afe625aea5e87f4cef155f17affd5c", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "logit_linear", + "input_sha256": "cee86f1ef27b5be25840a1ceb0d3c5ce52a6fbe95460654cfb0832051145d31f", + "seed": 20261022, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 1.2212453270876722e-15, + "prediction_sha256": "3803fc3fd31ed08d2e4f42d3e478d2063045da9701f4e755881cd9393065498f", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "0523ed7291290b6976b1317f3bc7a5d8ba06f31884c2e4af3f197ba4ad54c448", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "9c2c8f6ab13fe155d8812304cb68f52c91a2e5cf3fd93ea41dcddeb28b9ed542", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "443e6c04ef89356fcfb372788b4139d07f715b0e74ca6de1ff8bd5c4992d00b4", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "probability_linear", + "input_sha256": "d1f32411cbbb872fbd7ebe836d16cfac5873678510699eccb66e4d664fca3496", + "seed": 20261022, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 9.992007221626409e-16, + "prediction_sha256": "3de538c2b5d6019d950c6c396c52b6883c3bb17062dca328f72e1e72ccc3f774", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "0523ed7291290b6976b1317f3bc7a5d8ba06f31884c2e4af3f197ba4ad54c448", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "4b03e3684896fe810699caab33d6d527e2e6045eea552b5fe7fc21e41c3b17ce", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "cd8b5a32d74dc5aca8a7402d74aadf1129664aa1a26adb6c1669a8898e08c4a0", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "logit_linear", + "input_sha256": "70dc053d44974241bfd9172a31a5597fcb7bc4bcc4c0584e107d005894b33222", + "seed": 20261023, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 8.881784197001252e-16, + "prediction_sha256": "e7245f2d8e7646343d18dc4571b2cf8fd70af0181cce561f4659914b2c899146", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "35007152480a980cd24d2ade029ea4a51a585fe6b379bd765f553ef6110a81ac", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "b5d6a2f0bd69bec3d09a05367ea9366480928966e396845cb2fa967ee7fa43c7", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "65c3d5df20e762ba983e5de0d2fdae573fb7b13de763e3d1e3fc2e585384c641", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "probability_linear", + "input_sha256": "fd54a29fa98297cd7475a36dbafe937a9c0a6ef62dfba90ddaa0bd4faac5d7c0", + "seed": 20261023, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 7.771561172376096e-16, + "prediction_sha256": "9e1b51c094d1a9ae0fff6d78a419730f2b663939cafb825f73e6bc543190ba3c", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "35007152480a980cd24d2ade029ea4a51a585fe6b379bd765f553ef6110a81ac", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "db59e49ac09db22ad7d0da05391b454556561fed6898246b13da415dc55f2f09", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "b75ada4929d53b311ea638027379c5e27d28ec606a2a4642567ee33f69ae3eff", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "logit_linear", + "input_sha256": "1a185b870337ddab9d287c00f616b6f7701ef90eeaf0277c04452b400b2075a0", + "seed": 20261024, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 4.551914400963142e-15, + "prediction_sha256": "72b655cd4bf5d998dd71699b78a28544b49ffe42cfca19be9c826c0912cb692e", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "d26bd0fc2f8762ecaf00b32318feae35ba7d315bb87ca506a71e9ca06f2d9711", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "1b07a42448566457e9d68d68b29f49cfe552f4136e73eaf45404531e78337539", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "0e7d8727ebb7684ce903afeeb29522373894a30dd87aa91353237e962343d51d", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "probability_linear", + "input_sha256": "105b7973e7f3df64f50d92e3bf71fba7bce4b00c2eedbb23da3d16a32c2a7c70", + "seed": 20261024, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 2.886579864025407e-15, + "prediction_sha256": "52522b0ea8d049d5c9846807aa6bb06ebc94c41279fd3d115d51cc6a4ae0cfb3", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "d26bd0fc2f8762ecaf00b32318feae35ba7d315bb87ca506a71e9ca06f2d9711", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "8cd4aff250e2e115b16bbe2b5d88d64fa2620fbb546aae20f7f5608d31d194af", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "eb58957a9aa4d1391de73eb3c152472c92109ed7a6dc364309089a306e8391a3", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "logit_linear", + "input_sha256": "6e57c9d3ade17a7c17bfe8d310014b23e9e8e7fc1c38c26a94b9d85fe8bb3f9c", + "seed": 20261025, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 3.1086244689504383e-15, + "prediction_sha256": "e2991f2781ce9968ba2f5df5ba3f637e286b4ca08bcd942b67d012a36570f749", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "5a9592fe80d6eab4b89eb99a8f0a983cca91e7350d9fc47da4f04ceb6a949198", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "6908b2d3993206c835636b19f9b74cb40cae0e203e4e186994d9532d0644a49a", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "438ec9dab3a2539ee8f1575bd89270806c5bba3ea716c902d788c281976bd86f", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "probability_linear", + "input_sha256": "740e48894d8f278a0e8405c08e42702be59f9e8c9516fad1f0e300e182d027e3", + "seed": 20261025, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 9.992007221626409e-16, + "prediction_sha256": "12a4d2b57a2ee01ce0e76057033c3574598708e6ed5b3810ca1812866a989b91", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "5a9592fe80d6eab4b89eb99a8f0a983cca91e7350d9fc47da4f04ceb6a949198", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "8fe9c241b6d0ac53a7f453f8c858b77f92f6206c0d31eec683b0f7ef07939e97", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "05a3c87e48d0d87d43843752d4757c8f5e01a960ad127d71015d5f07e71f3dca", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "logit_linear", + "input_sha256": "6d6d30775ce01614297e48d4894cca36fc03a0207028b480083a97e12bc40418", + "seed": 20261026, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 1.2212453270876722e-15, + "prediction_sha256": "cc7e43d5ca144b063e0afa8376e329a2c159eda9bcb738e1b02c1ff5c55c1ee6", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "060cc3ad59ef9e78fa9e82eb601b080732b2c18e706f1171021cc46810405fda", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "f7e9485d473026924e2ba1831ea2fe0152636ac4674ff41ea214a56fd2567bbb", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "0b87194cfa432abccb252812efc8e0593fc536283026a77718da9b833af2755a", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "probability_linear", + "input_sha256": "4d2ed833bd9e1883b86f73dddaafad60c1fe57e8ad5b65cf09daf0738ee49148", + "seed": 20261026, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 1.4432899320127035e-15, + "prediction_sha256": "88c8ec1548b3b9a5c1e97c16ecd0ebde96b9b38680321be678af5f699314d52f", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "060cc3ad59ef9e78fa9e82eb601b080732b2c18e706f1171021cc46810405fda", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "c8e3986414aaa3c02de7016c7b3f5fc0daa229ee39b2856ec5ef34f71a91886a", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "4fb8cd94fac847a983b001ffdd66da7fabd8a2bc443ee853ce8ce781cb467049", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "logit_linear", + "input_sha256": "2c6f501ce63058003a8be5598d38a4e89689ee2a0fb6684827c148408fd52f2c", + "seed": 20261027, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 5.440092820663267e-15, + "prediction_sha256": "17a7a1cc033871cbaeb17fa40116f99062898556919c9f0bf633a7076c1a6fb3", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "69c46bbea6e174c532826649e16093280feee050e7f403283a5167cde0f8b385", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "8eebcbb9ee00b8d90fb5ba5c0b1b76ecb4c8f9d2dbc12137f5deb435606ac205", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "95dd0befc44a6cdd56f6c825859d284b8ac2d72e996a23a0cd8b3c248279bd91", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "probability_linear", + "input_sha256": "6ab43d562d4833cf7354c7349dc91c7c66c74bf879c6fcc18f81edeec919cb76", + "seed": 20261027, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 5.551115123125783e-16, + "prediction_sha256": "9f8ed6f1310661540add276608c5971ca7a9c7b97a00cb36d44e1715d2471759", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "69c46bbea6e174c532826649e16093280feee050e7f403283a5167cde0f8b385", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "ab61cc8869a8c638eab2c6cc62775198329a28a4ba216617a5d7232f07ef13e2", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "5fcbd54dba7563df6acb0581afa03b7964625ecc6be2502a7f22aaeb4216dc4b", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "logit_linear", + "input_sha256": "72cefaa9db18fc97db2a44507fca0e1ba123b9746ffba5ab615f76f678154207", + "seed": 20261028, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 7.771561172376096e-16, + "prediction_sha256": "809db226e1f2c7fa874eead15a4460e3532353b9574d6a9c4b87d6c3c64a8a3f", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "fd977651715ecc15fb7e1445ac63eaa66ed69af5f9ade0ad43b1c237fcf31ca9", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "b98d980ec759e507d66efd255159725ab348599b218f8f926401f3bd339c4f93", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "5d5e2d5d794c6128aa58bf4bddcdb3be98753325c2b71e12b6ea31eb20ace072", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "probability_linear", + "input_sha256": "0a1e74091e61ffe85e675458a3a73ba044c364baf5c636336037b0f41f0e2fc4", + "seed": 20261028, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 8.881784197001252e-16, + "prediction_sha256": "c02746f2854c2334cc5c0e8d628ddf209ea255775bbdf43e3891610ab138c463", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "fd977651715ecc15fb7e1445ac63eaa66ed69af5f9ade0ad43b1c237fcf31ca9", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "3fd5472ccca5d9457bbb1d3f718c4d6100062a413270747d3996f15620a9cedb", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "4d80a1b0790f32a38f3553fa6d5b98db8d2266be087a452363d50d26960088c2", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "logit_linear", + "input_sha256": "4159f308a7d8e81a909bb8cd1b8cbdd884184153079c708fedb1e2867c653045", + "seed": 20261029, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 4.3298697960381105e-15, + "prediction_sha256": "e9b9ad29a8cc56c0e38496b269f7b5af631938d2751b0311f1c63e56d04d0d19", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "3378a19feb6a3bc7922eddbcff0258f7aea9b74120c266e2ce8e583c3ff6f3fb", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "02f8f73962ddc6b90f1b2b3f0b8e4fb72f4cc69b41bdb80768bb660148c8fb9c", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "8ac089e72f9c7ddc8fd5ecff04c7369748f1c9602d66f1e9866c0606b49b0258", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "probability_linear", + "input_sha256": "89ae5875f7ff7f77e7beea09542b3c6ef3316a525fef0fd7d5c1b58d35f92da7", + "seed": 20261029, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 6.661338147750939e-16, + "prediction_sha256": "3a92a63d2e79c15222044c5169ef6db62898b753ce56e615361f39c8854c519d", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "3378a19feb6a3bc7922eddbcff0258f7aea9b74120c266e2ce8e583c3ff6f3fb", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "16a90d3d7c42ff43c9a4f661c134bf54f188fb276f85ee6b222b7412deb2cd00", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "0aaf775e61f57279001e2e44b27ba33ec818865c15f458263ec1accd2710e311", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "logit_linear", + "input_sha256": "665a675c60a48454479972c2c3b08d06922b00aeb15412326ce0765d312ae437", + "seed": 20261030, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 1.3322676295501878e-15, + "prediction_sha256": "d28004750c864023229a7a26218e552be4c3a7d9dec68429ee4bcbb21c2b0074", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "fa944a96f03cfd0710cb1f65dd63236f7e6273de440df92cdb9b9ba5e480f302", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "d424faed7657cb3ffeb0a9008b44df09f3ee6d6fbe9c7c0841b69908792e6d73", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "002c4f1de57541bbfd24b1605c56a023b9b908c89c60b5d287ff27660030ff6e", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "probability_linear", + "input_sha256": "109f165dc712f80dcb5b08ccf568b8083f14473e056fc7a1a021f2ef7c8ab3e3", + "seed": 20261030, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 7.771561172376096e-16, + "prediction_sha256": "5b7d7abe94ee7c2cb2b9dde75b1dc810b3e031895f695e93576bca05bd812f11", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "fa944a96f03cfd0710cb1f65dd63236f7e6273de440df92cdb9b9ba5e480f302", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "1e3c73d3200e5a1284a24384891427d77ca267852d0ea0406b30514885111e9f", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "9f1ef5cb2659911b33b48fd6fea2dee46064f4ff24845eecf866d3c14946bb60", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "logit_linear", + "input_sha256": "0f0bf52cb63c39522ecc8e530f1dadcf302db638266198d39f0921c9d7cf456f", + "seed": 20261031, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 6.661338147750939e-16, + "prediction_sha256": "042c2048d458aea773345b75aa3a4d6b1cffeb7667210c11cc263bca75ed4c9d", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "1147be9610babed7bc7742d7a1610d4a154159288219d8a1b72ca561963da157", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "a9de268cead4a03069986c75491066b0e6d60ae3b04e6dfac19d3ff1ad3baf10", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "8427d02f514e573fe9ed716fe30b9a89f146f30f9b98cab3fc9122c0f3a11d80", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "probability_linear", + "input_sha256": "28fd410c055e3f63e1f6877c498feb08de2377dce7ed5e19ab8ecaddbd8b1710", + "seed": 20261031, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 8.881784197001252e-16, + "prediction_sha256": "1481cc163331140be1f9c7d40a5b27411ed6b766c188943a2d7c15cb2abbc8d6", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "1147be9610babed7bc7742d7a1610d4a154159288219d8a1b72ca561963da157", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "464b8c682d1c8b49c80117b477e4f5ae058bbe83645ab509b00bf3f9550f52bf", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "c8ac2480ca1754862a15053dfe47fe92850d518337f6941c120ba46f3e073581", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "logit_linear", + "input_sha256": "2b753f4ddaf9c715600429ca11aeb3280f6a4f3ea93ef1ff42dbc58d6dbb7764", + "seed": 20261032, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 1.5543122344752192e-15, + "prediction_sha256": "a31e3f44061edff95c582c71b7175ad2b35460757a0210331ad58820e6a611ea", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "995b5f3ea226250c999a83108b0f3f58560771199c4afda0b7b5750fb9675954", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "14b59c460e4bfbdefab12db021cae9d97bb91e317f7df40c6f86ac7be55ad660", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "e5e468c35efb696d99dd45597372db6051d51b562ba1d6b301af6dc1f41884b5", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "probability_linear", + "input_sha256": "4d0a04370ce594111262ee6606c2913e986e291158edbe531db6361ed8a9a0d6", + "seed": 20261032, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 1.1102230246251565e-15, + "prediction_sha256": "af04eb818dac14e871a5e25844bbb9289afbc0b57350d3e5119d2e12b5598c52", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "995b5f3ea226250c999a83108b0f3f58560771199c4afda0b7b5750fb9675954", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "b162641e91b0bee3b15464819d7215b25c936e81ebd1c6b9e0726937023f0e7a", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "dbd8b6d502528e5ef280213fa0f5a708b996cda4a1ede3c3ad1f9bfb48feee92", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "logit_linear", + "input_sha256": "963d4cd3f01ad027143d30d280d221b691996f526c6801a1e7bb7752ea63aa53", + "seed": 20261033, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 1.7763568394002505e-15, + "prediction_sha256": "9b91493d5937889672c4ace7a846d6c3caab5143fcb2303dbdf513c64a6b51da", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "70140b8862a60ff9502f51a0c5a02926baa7d803002b6b9ab2130417cbe5d162", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "05f628683fce6d0554a0ad2c5438dc0984f2aa54758bac1fa8eabf375ccdaffd", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "227228a7a487abd8eeba64c2ec3b75322c7a068779c2c141a719b952139c7c95", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "probability_linear", + "input_sha256": "2a19d362d458d1d57aa7e80ead6eaa7c2f509652df6e2cecf6f3e5c2ba2fa771", + "seed": 20261033, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 6.661338147750939e-16, + "prediction_sha256": "940ea6fc33021c68484f1a1ce22febd32bbb533bf8875737bef2a0660a139375", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "70140b8862a60ff9502f51a0c5a02926baa7d803002b6b9ab2130417cbe5d162", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "50ca76b881198a2d9e12cea44957f6202e9df542c376fc6d1cfce22bcb7a3113", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "6728875ee5d17d5fdaef3c2ee30757d8aff9383110843665342e6cecf4e99fe0", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "logit_linear", + "input_sha256": "39e893266275a14184cbe247af2d35ac23b88694e076d75f90b77268e2d169ae", + "seed": 20261034, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 1.6653345369377348e-15, + "prediction_sha256": "497d62ade4f8ab2653fe2165a76f189372dc711c5dce5ce55beed859da4c09f8", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "a2db77fbbe6fe7efc9602689a3d6b3edb2ffb89dda2c81b881388559828a442e", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "80f3a7d01e42ce7b6cbf1a01b7d4ba12ec7d6cfd54f7cd8cd03a702931f12668", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "b33291b6bc39549d2aa7f140c7019c266a70b85969d00903f47e2155ac5f6f5e", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "probability_linear", + "input_sha256": "385cb81d952abd56070b112365e55e364d6f34ce3b8006ddabccb6483b370097", + "seed": 20261034, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 1.1102230246251565e-15, + "prediction_sha256": "d0428cdd3ad8a41dbd2c225fb4ac8b9fb6e86e8231b24ea65a6922f52b71ed4e", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "a2db77fbbe6fe7efc9602689a3d6b3edb2ffb89dda2c81b881388559828a442e", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "6d71e637961a65a0bfe6612321fe3be7621e106518f8f83f9f4b29fa94f08397", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "4c159ef3eb5e2087e922d6883064f632ea83264bc608028a6c27a5ecd4d01231", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "logit_linear", + "input_sha256": "c1b857644ce921be9722353e5fec89c1ede61cf116b9baff79d92c4332e8570e", + "seed": 20261035, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 3.774758283725532e-15, + "prediction_sha256": "dd6ca8cb4b12e4615c0d75dd7d5b5e2a1b62e865ab84aeb6227a235a49ca750e", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "da09c44b2d79bab90d5100995e891ea37eadae0a33f554861ed29bf24caf0d37", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "cd5c07ef270f652c1ffffcf4230c903b92a6f4705a0f2638a3f73efd9e0af3e7", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "d55fad6e6ccb9dbd0f490b9d9522bc712825de95cecadb5ae6018d4bd7029e66", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "probability_linear", + "input_sha256": "c4671174e637ba24c19431ca365fcd670502b61483b3d850601e23cbc186ae8c", + "seed": 20261035, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 1.7763568394002505e-15, + "prediction_sha256": "4877f420929bead257312caa3b9eb7f02d8e12cefb925bb25497993d41f44300", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "da09c44b2d79bab90d5100995e891ea37eadae0a33f554861ed29bf24caf0d37", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "98202ea03fa5e7056ecbf16562a7b8b4fa2e3c43fd1983a3b373dee7beb72d6f", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "ee281e892a53fda2e7fb2d8fd5c5fd88578832f46e7b65ef73f0aa467e9a8893", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "logit_linear", + "input_sha256": "f975a75e5452c749629effcee8e059e061e2b2122daecc9072331035a683bd49", + "seed": 20261036, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 6.661338147750939e-16, + "prediction_sha256": "c8fe4ddf2e39d37c34c7ca60f61737e4e1b3a513ff36281f9a4efe853e9890af", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "af21e10965357381952a0670d29cff784da6f4a971e85075420bab7678b51392", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "5a1fce22171273af9949bd55dda828977709f95a04cf4261d29724b40cf3a794", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "06d5f1e1068b70515de25a160ecdc85b888ae50e15bac182dacbd6a2b85be803", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "probability_linear", + "input_sha256": "79a7659866de7787366b39545ebb1c3a68d9c677dbcdd66656fa4b65d53cf1ba", + "seed": 20261036, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 8.881784197001252e-16, + "prediction_sha256": "ba4261b04eac538ff268a7be1a854f7bd5d40d8037fbaad2f9a86e4b60ad0631", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "af21e10965357381952a0670d29cff784da6f4a971e85075420bab7678b51392", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "550365e5722082786c001db1bffb28173c6e287ddd1faca80d07e8cd36a8a82d", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "b06695d529184b8be4d7a53c14ca1475264aa3deb1ade4d53fb0b40bc2b1bede", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "logit_linear", + "input_sha256": "5274b9abf29645bd4de30c2ee46d705ffbe8d4354b2976fcf8a38d3fd85d1e14", + "seed": 20261037, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 1.2212453270876722e-15, + "prediction_sha256": "07d2f39fd48254e0f5ea872948adc78ca22edfc225d8eae43fa131a4c3af7696", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "3fdef14453752e68d0d2d666b93837d1a1da59b1d0ffde2da9e4a3ead50e3cc0", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "c04d96654f29d6095ebb30572279ee3a75e20573e0bcc0a59f8d42d5195856d4", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "fa02a85d28beb53051e212bb0f0e945041e801beb740970d0c3b2bf7ab44e48a", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "probability_linear", + "input_sha256": "73dbb803c8c09f8502a87a7c2ddc34f620c27caf536e87667f1ffe29b972d1e8", + "seed": 20261037, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 7.771561172376096e-16, + "prediction_sha256": "83298dd6daad18a1526bfb2d5274797f3ff02b7e1cfa1da99c3a6902c59ce07c", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "3fdef14453752e68d0d2d666b93837d1a1da59b1d0ffde2da9e4a3ead50e3cc0", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "c59127cb798ecee9293a195cebca1c33b27b40bd5cf7fb7e376275b8d7ce742c", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "f055889cebffadc830612286dbb49cf80e651954a628c484005319960b2d996e", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "logit_linear", + "input_sha256": "9170e706cd9cfc1947d2b5ec41a8941a3766846ae7d3cc63630792e5e655e3c1", + "seed": 20261038, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 1.2212453270876722e-15, + "prediction_sha256": "3720a81f33ebe8ed8636c1b4f29e8819f30c5c58a2a40b3525a8ecf3b07bb3c2", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "96a90709056367ef9f44c8acd569865223ce822886f5e8da8389c76e95ed0470", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "16e5f1b81e6b36e88a53b0788c65c78615cc0be985b2b91ec37e0180cad57ec3", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "3bf9a20100386ab6a64e61b445067be3695c8f16b8a950e3e0ceb29dd9fe7bde", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "probability_linear", + "input_sha256": "5991c42d852eb335be69961701fb55021906532080ed6175a063238de4fe437a", + "seed": 20261038, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 1.6653345369377348e-15, + "prediction_sha256": "e2a99546b9f22b56e2ab4a106374b936946869b6ae21cd70bcf68e3d7b3ddad5", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "96a90709056367ef9f44c8acd569865223ce822886f5e8da8389c76e95ed0470", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "c7156ed56aa0f4d23c0b4093239ee5229272850ae5ec2b2c6d9079f3a3c84a44", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "071062e1f2305f12e5f36c73957b8415316a186e5a6c6d9fd7502b00592e0f72", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "logit_linear", + "input_sha256": "519bff67e1def944da0ccd642e17c665b6b0cfa4c341369839b17bd1465a57f3", + "seed": 20261039, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 7.771561172376096e-16, + "prediction_sha256": "ebdf78d7e02b34e74eb42df8dcbd72315938735c9f2d5bd72c7e430f03e9b3db", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "a575911268fa514e186402ce25e2a08307d3a235c1b38b4fbac80624ac0ee888", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "feb8edb91b2658d790055f80726ccaca6397f51683986576cd933b33366e1c6e", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "95362cee5b1b24d48bead9ff05a444b824e38c03b10a94ca007fbe25b4443cbe", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "probability_linear", + "input_sha256": "3abecb81511ab594d6ecccd5d125069f551c89a0699a637e94e3e220fe401d8a", + "seed": 20261039, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [], + "max_abs_independent_difference": 7.771561172376096e-16, + "prediction_sha256": "449937893f0e17ac3ec1575371cca662fa86f98fc0bfa549efcf8f8442b9eac0", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "a575911268fa514e186402ce25e2a08307d3a235c1b38b4fbac80624ac0ee888", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "35f9a80f05e8f9815b6d392abab603ab3f15dc7ad418d71054e374ff2f896b10", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "3250af499eea776eb3684846c527c8759fa072cc9c0c32a00418fd44654f79af", + "reason": null, + "warnings": [] + } + } + }, + { + "family": "wdbc_undistorted", + "input_sha256": "26699cf382274c15c7fae2f3ea15e43d348bfc50a23199c9d1c7497fc28b6e8c", + "seed": 20261010, + "statuses": { + "predval_rung2": { + "available": true, + "ladder_suppressed": [ + [ + "rung3", + "non-convergence" + ] + ], + "max_abs_independent_difference": 1.4432899320127035e-15, + "prediction_sha256": "c47825af5e581bd855cf122df9538219e7ab76be8dfd5e798a33721e5978a4b0", + "reason": null, + "warnings": [] + }, + "raw": { + "available": true, + "prediction_sha256": "259d2416f9c5099267cc30ce8c18720c851c49f0aa3d926c82a71269316fb4bc", + "reason": null, + "warnings": [] + }, + "statsmodels_logit": { + "available": true, + "prediction_sha256": "d715e4f86ca289ddbcce01547696ac02ed0e5ffff3abb1fe240f6430e6a07d2f", + "reason": null, + "warnings": [] + }, + "statsmodels_probability": { + "available": true, + "prediction_sha256": "ac9b2bd9c03011693acd4d13849489ccaf352e9f5195a5db0c56a8177d7c24f3", + "reason": null, + "warnings": [] + } + } + } +] diff --git a/benchmarks/family_extension/results/manifest.json b/benchmarks/family_extension/results/manifest.json new file mode 100644 index 0000000..ea3e8e1 --- /dev/null +++ b/benchmarks/family_extension/results/manifest.json @@ -0,0 +1,43 @@ +{ + "files": { + "cases.json": "579f26f50eb7cd51413532ae113ecdf484f4fb5645a7dca7ff11239df21f0b25", + "metrics.csv": "39c811a406ee6a42b791db6c9b39b95f397fdba4ba19db404f59f967bcf5cbc5", + "metrics.json": "0f76da5790c41dd780ef71e5e5ec389907b8bfe44bc36b5070146f13da03cfd6", + "summary.json": "9b975a31e79bb5415a8c054004c430eb174f42f59215aa4713584595b38576cd" + }, + "predval_source": "/workspace/shared/predval-family-extension-20261010/src/predval/recalibrate.py", + "protocol_sha256": "8a34e65b43b189e3a845a6c0fc7d92f11a992d57eafb62e199de62fef8954559", + "python": "3.12.14", + "recalibrate_sha256": "513d69580b00cdd2f4633060300210513c79f5524fc0027d038e25810d072a22", + "runner_sha256": "cbffe569b0bd7ae556bbebad65cdebc28b5045c4573f29fc6e1a300bc2978bf2", + "seeds": [ + 20261020, + 20261021, + 20261022, + 20261023, + 20261024, + 20261025, + 20261026, + 20261027, + 20261028, + 20261029, + 20261030, + 20261031, + 20261032, + 20261033, + 20261034, + 20261035, + 20261036, + 20261037, + 20261038, + 20261039 + ], + "versions": { + "numpy": "2.4.6", + "pandas": "3.0.5", + "predval": "0.1.0", + "scikit-learn": "1.9.0", + "scipy": "1.17.1", + "statsmodels": "0.14.6" + } +} diff --git a/benchmarks/family_extension/results/metrics.csv b/benchmarks/family_extension/results/metrics.csv new file mode 100644 index 0000000..797f81c --- /dev/null +++ b/benchmarks/family_extension/results/metrics.csv @@ -0,0 +1,165 @@ +seed,family,method,rows,clusters,events,available,reason,brier,auroc,auroc_delta,brier_gain,risk_mse,oracle_expected_brier,expected_brier,expected_gain,expected_recovery +20261020,logit_linear,raw,1497,120,540,True,,0.16107731913455833,0.82150431518247602,0,0,0.007808613529657212,0.16752891002530854,0.17533752355496574,0,0 +20261020,logit_linear,predval_rung2,1497,120,540,True,,0.15935636225350433,0.8205425906575331,-0.0009617245249429196,0.0017209568810540032,0.00084518829409010695,0.16752891002530854,0.16837409831939865,0.0069634252355671054,0.89176205341958958 +20261020,logit_linear,statsmodels_logit,1497,120,540,True,,0.15935636225350433,0.8205425906575331,-0.0009617245249429196,0.0017209568810540032,0.00084518829409007876,0.16752891002530854,0.16837409831939862,0.0069634252355671332,0.89176205341959325 +20261020,logit_linear,statsmodels_probability,1497,120,540,True,,0.15908628947466405,0.81949185339989938,-0.0020124617825766355,0.0019910296598942834,0.0041912349449848008,0.16752891002530854,0.17172014497029334,0.0036173785846724113,0.46325491342778868 +20261020,probability_linear,raw,1497,120,467,True,,0.13190586105567659,0.85203841915968481,0,0,0.0014120442303316365,0.14276742283729035,0.144179467067622,0,0 +20261020,probability_linear,predval_rung2,1497,120,467,True,,0.13379041585682258,0.84914866634789299,-0.0028897528117918236,-0.0018845548011459934,0.002132190804176785,0.14276742283729035,0.14489961364146714,-0.00072014657384514855,-0.51000284436983456 +20261020,probability_linear,statsmodels_logit,1497,120,467,True,,0.13379041585682258,0.84914866634789299,-0.0028897528117918236,-0.0018845548011459934,0.0021321908041767798,0.14276742283729035,0.14489961364146714,-0.00072014657384514335,-0.51000284436983101 +20261020,probability_linear,statsmodels_probability,1497,120,467,True,,0.13043779368272918,0.84688052223446497,-0.0051578969252198403,0.0014680673729474125,0.00061324468918715796,0.14276742283729035,0.14338066752647752,0.0007987995411444785,0.56570433417434118 +20261021,logit_linear,raw,1333,120,535,True,,0.17046590699206199,0.837092731829574,0,0,0.0070201131506704931,0.15826094317987213,0.16528105633054263,0,0 +20261021,logit_linear,predval_rung2,1333,120,535,True,,0.16054807456148029,0.83517438455952975,-0.0019183472700442517,0.0099178324305816967,0.00082471708884662362,0.15826094317987213,0.15908566026871876,0.0061953960618238695,0.88252082678071153 +20261021,logit_linear,statsmodels_logit,1333,120,535,True,,0.16054807456148035,0.83517438455952975,-0.0019183472700442517,0.0099178324305816412,0.00082471708884665257,0.15826094317987213,0.15908566026871879,0.0061953960618238409,0.88252082678070742 +20261021,logit_linear,statsmodels_probability,1333,120,535,True,,0.163994792294443,0.831080036539948,-0.0060126952896260022,0.0064711146976189904,0.0036598585118350325,0.15826094317987213,0.16192080169170717,0.0033602546388354607,0.47866103675473115 +20261021,probability_linear,raw,1333,120,455,True,,0.13924385623121441,0.85181105910035282,0,0,0.0016033500138655586,0.1388292585937953,0.14043260860766085,0,0 +20261021,probability_linear,predval_rung2,1333,120,455,True,,0.14128816472603203,0.84986607925104507,-0.0019449798493077486,-0.002044308494817626,0.0033048920928629229,0.1388292585937953,0.14213415068665822,-0.0017015420789973643,-1.0612418151262379 +20261021,probability_linear,statsmodels_logit,1333,120,455,True,,0.14128816472603206,0.84986607925104507,-0.0019449798493077486,-0.0020443084948176538,0.003304892092862929,0.1388292585937953,0.14213415068665822,-0.0017015420789973704,-1.0612418151262419 +20261021,probability_linear,statsmodels_probability,1333,120,455,True,,0.13833251946917985,0.84491977271020557,-0.0068912863901472443,0.00091133676203455916,0.00033417150407126555,0.1388292585937953,0.13916343009786658,0.0012691785097942931,0.79157919282664757 +20261022,logit_linear,raw,1263,120,459,True,,0.16571605127396458,0.84498260332325303,0,0,0.0081024039624581681,0.16318958587396784,0.17129198983642602,0,0 +20261022,logit_linear,predval_rung2,1263,120,459,True,,0.15321664568771615,0.84340281164981201,-0.0015797916734410222,0.01249940558624843,0.0028313074337011608,0.16318958587396784,0.166020893307669,0.0052710965287570072,0.65055958122801649 +20261022,logit_linear,statsmodels_logit,1263,120,459,True,,0.15321664568771615,0.84340281164981201,-0.0015797916734410222,0.01249940558624843,0.002831307433701183,0.16318958587396784,0.16602089330766903,0.0052710965287569855,0.65055958122801383 +20261022,logit_linear,statsmodels_probability,1263,120,459,True,,0.15794355631526627,0.84249504113419826,-0.0024875621890547706,0.0077724949586983061,0.0050456454714794576,0.16318958587396784,0.16823523134544729,0.0030567584909787105,0.37726562451612544 +20261022,probability_linear,raw,1263,120,365,True,,0.11976713111415747,0.87727369801995292,0,0,0.0015317442510852197,0.13762883536214698,0.13916057961323219,0,0 +20261022,probability_linear,predval_rung2,1263,120,365,True,,0.11952004569354643,0.87434786588156332,-0.0029258321383895947,0.00024708542061104011,0.0034107725650683808,0.13762883536214698,0.14103960792721537,-0.0018790283139831611,-1.2267245740611696 +20261022,probability_linear,statsmodels_logit,1263,120,365,True,,0.11952004569354643,0.87434786588156332,-0.0029258321383895947,0.00024708542061104011,0.0034107725650683591,0.13762883536214698,0.14103960792721534,-0.0018790283139831394,-1.2267245740611554 +20261022,probability_linear,statsmodels_probability,1263,120,365,True,,0.11919046017725436,0.87138542270494546,-0.0058882753150074629,0.000576670936903112,0.0027271911278039301,0.13762883536214698,0.14035602648995091,-0.0011954468767187104,-0.78044809103853519 +20261023,logit_linear,raw,1468,120,560,True,,0.16102492586755904,0.84662130270610447,0,0,0.0069782210240426487,0.15525482952310388,0.16223305054714651,0,0 +20261023,logit_linear,predval_rung2,1468,120,560,True,,0.15352182364408309,0.84473725613593442,-0.0018840465701700504,0.0075031022234759437,0.00044936055909354917,0.15525482952310388,0.15570419008219744,0.0065288604649490992,0.93560528427727785 +20261023,logit_linear,statsmodels_logit,1468,120,560,True,,0.15352182364408309,0.84473725613593442,-0.0018840465701700504,0.0075031022234759437,0.00044936055909355281,0.15525482952310388,0.15570419008219744,0.0065288604649490958,0.9356052842772774 +20261023,logit_linear,statsmodels_probability,1468,120,560,True,,0.15734146686440126,0.84380703272498414,-0.002814269981120332,0.0036834590031577796,0.0034270967770347717,0.15525482952310388,0.15868192630013864,0.0035511242470078769,0.50888675419894147 +20261023,probability_linear,raw,1468,120,498,True,,0.14333302168416276,0.82660125036227394,0,0,0.001663594105528006,0.13523789741194575,0.13690149151747374,0,0 +20261023,probability_linear,predval_rung2,1468,120,498,True,,0.14805074765092122,0.82367407775431634,-0.0029271726079576066,-0.004717725966758457,0.0050933734568542593,0.13523789741194575,0.14033127086880001,-0.0034297793513262532,-2.0616683720682456 +20261023,probability_linear,statsmodels_logit,1468,120,498,True,,0.14805074765092122,0.82367407775431634,-0.0029271726079576066,-0.004717725966758457,0.0050933734568542558,0.13523789741194575,0.14033127086880001,-0.0034297793513262498,-2.0616683720682434 +20261023,probability_linear,statsmodels_probability,1468,120,498,True,,0.14116575175555704,0.8219289529251026,-0.0046722974371713466,0.0021672699286057195,0.00048753841905313869,0.13523789741194575,0.13572543583099889,0.0011760556864748675,0.70693667557904727 +20261024,logit_linear,raw,1530,120,495,True,,0.17011447778887551,0.79861416093300153,0,0,0.0075198715289420013,0.15829396339227805,0.16581383492122007,0,0 +20261024,logit_linear,predval_rung2,1530,120,495,True,,0.16658973597194313,0.78951837212706777,-0.0090957888059337577,0.0035247418169323808,0.0027634421510904198,0.15829396339227805,0.16105740554336848,0.0047564293778515811,0.63251471245823021 +20261024,logit_linear,statsmodels_logit,1530,120,495,True,,0.16658973597194307,0.78951837212706777,-0.0090957888059337577,0.0035247418169324363,0.0027634421510904054,0.15829396339227805,0.16105740554336845,0.0047564293778515959,0.63251471245823199 +20261024,logit_linear,statsmodels_probability,1530,120,495,True,,0.16918705010198151,0.78270433806665696,-0.01590982286634457,0.00092742768689399457,0.0058151262096525751,0.15829396339227805,0.16410908960193063,0.0017047453192894262,0.22669872919082612 +20261024,probability_linear,raw,1530,120,433,True,,0.15285937375593533,0.78464466390597065,0,0,0.001618776572271234,0.13656869498098234,0.13818747155325359,0,0 +20261024,probability_linear,predval_rung2,1530,120,433,True,,0.15473972610443756,0.77240258441561171,-0.012242079490358937,-0.0018803523485022289,0.0083747641118564267,0.13656869498098234,0.14494345909283876,-0.0067559875395851923,-4.173514526532939 +20261024,probability_linear,statsmodels_logit,1530,120,433,True,,0.15473972610443759,0.77240258441561171,-0.012242079490358937,-0.0018803523485022566,0.0083747641118564666,0.13656869498098234,0.14494345909283882,-0.0067559875395852322,-4.173514526532963 +20261024,probability_linear,statsmodels_probability,1530,120,433,True,,0.15036100604113092,0.76657312300395153,-0.018071540902019123,0.0024983677148044114,0.0025052977814917603,0.13656869498098234,0.13907399276247409,-0.00088652120922052624,-0.54764890004349853 +20261025,logit_linear,raw,1495,120,569,True,,0.17407210831243555,0.82159789255523874,0,0,0.007826241136583369,0.16371700228003758,0.17154324341662094,0,0 +20261025,logit_linear,predval_rung2,1495,120,569,True,,0.16510883980585878,0.81891993455989254,-0.0026779579953462029,0.008963268506576777,0.00069114344529233119,0.16371700228003758,0.16440814572532991,0.0071350976912910383,0.91168896623161588 +20261025,logit_linear,statsmodels_logit,1495,120,569,True,,0.16510883980585878,0.81891993455989254,-0.0026779579953462029,0.008963268506576777,0.00069114344529233357,0.16371700228003758,0.16440814572532991,0.0071350976912910357,0.91168896623161555 +20261025,logit_linear,statsmodels_probability,1495,120,569,True,,0.16676653804641975,0.81837713088401087,-0.0032207616712278675,0.0073055702660158039,0.0032947160951716176,0.16371700228003758,0.1670117183752092,0.0045315250414117514,0.57901679265022477 +20261025,probability_linear,raw,1495,120,489,True,,0.14113753386990624,0.84759947472628439,0,0,0.0015039687552661199,0.13973682611542312,0.14124079487068925,0,0 +20261025,probability_linear,predval_rung2,1495,120,489,True,,0.14188332681782653,0.84473323657238575,-0.002866238153898637,-0.00074579294792029294,0.0033695711170130399,0.13973682611542312,0.14310639723243618,-0.00186560236174692,-1.2404528719194108 +20261025,probability_linear,statsmodels_logit,1495,120,489,True,,0.14188332681782653,0.84473323657238575,-0.002866238153898637,-0.00074579294792029294,0.0033695711170130468,0.13973682611542312,0.14310639723243618,-0.001865602361746927,-1.2404528719194157 +20261025,probability_linear,statsmodels_probability,1495,120,489,True,,0.14028208748997176,0.84291591961523304,-0.0046835551110513407,0.00085544637993448247,0.0010749189039762647,0.13973682611542312,0.14081174501939939,0.00042904985128985515,0.28527843400173358 +20261026,logit_linear,raw,1439,120,376,True,,0.15460662741395059,0.80307389764015946,0,0,0.0081093723678853525,0.15876063494271395,0.16687000731059931,0,0 +20261026,logit_linear,predval_rung2,1439,120,376,True,,0.1526914661434865,0.79689908128340115,-0.006174816356758317,0.0019151612704640941,0.0042841556764160495,0.15876063494271395,0.16304479061912999,0.003825216691469303,0.4717031747879632 +20261026,logit_linear,statsmodels_logit,1439,120,376,True,,0.15269146614348647,0.79689908128340115,-0.006174816356758317,0.0019151612704641219,0.004284155676416046,0.15876063494271395,0.16304479061912999,0.0038252166914693065,0.47170317478796353 +20261026,logit_linear,statsmodels_probability,1439,120,376,True,,0.15580253596308638,0.79134975280718955,-0.011724144832969907,-0.0011959085491357879,0.008148112201388091,0.15876063494271395,0.16690874714410203,-3.8739833502738491e-05,-0.0047771679169839043 +20261026,probability_linear,raw,1439,120,295,True,,0.12447071576425298,0.81672988028920235,0,0,0.0016421177631608071,0.13439118325346794,0.13603330101662875,0,0 +20261026,probability_linear,predval_rung2,1439,120,295,True,,0.12363905785284017,0.81141400971909439,-0.0053158705701079612,0.00083165791141280587,0.0066420551106619753,0.13439118325346794,0.14103323836412993,-0.0049999373475011679,-3.0448104634573285 +20261026,probability_linear,statsmodels_logit,1439,120,295,True,,0.12363905785284016,0.81141400971909439,-0.0053158705701079612,0.00083165791141281975,0.0066420551106619579,0.13439118325346794,0.1410332383641299,-0.0049999373475011506,-3.0448104634573179 +20261026,probability_linear,statsmodels_probability,1439,120,295,True,,0.12201998576962196,0.8060329501007466,-0.010696930188455744,0.0024507299946310152,0.0044024753512347924,0.13439118325346794,0.13879365860470272,-0.0027603575880739851,-1.6809741968570817 +20261027,logit_linear,raw,1434,120,543,True,,0.17960615968882734,0.81149741739060333,0,0,0.00790570255461563,0.1687889958472952,0.17669469840191082,0,0 +20261027,logit_linear,predval_rung2,1434,120,543,True,,0.1717018632773567,0.80905225779381684,-0.0024451595967864836,0.0079042964114706404,0.00023222800797601979,0.1687889958472952,0.16902122385527121,0.0076734745466396099,0.97062525355947815 +20261027,logit_linear,statsmodels_logit,1434,120,543,True,,0.17170186327735662,0.80905225779381684,-0.0024451595967864836,0.0079042964114707237,0.00023222800797600651,0.1687889958472952,0.16902122385527121,0.0076734745466396238,0.97062525355947982 +20261027,logit_linear,statsmodels_probability,1434,120,543,True,,0.17652392302243888,0.80469933631382351,-0.0067980810767798161,0.0030822366663884604,0.0031649110165331215,0.1687889958472952,0.17195390686382833,0.0047407915380825085,0.59966732941586143 +20261027,probability_linear,raw,1434,120,440,True,,0.13890666060678486,0.8450292665081397,0,0,0.0013957935869700804,0.14391178505006047,0.14530757863703056,0,0 +20261027,probability_linear,predval_rung2,1434,120,440,True,,0.14012962915760457,0.84351563929028706,-0.0015136272178526422,-0.0012229685508197141,0.0021060998095103404,0.14391178505006047,0.14601788485957082,-0.00071030622254026005,-0.50889059039320972 +20261027,probability_linear,statsmodels_logit,1434,120,440,True,,0.14012962915760457,0.84351563929028706,-0.0015136272178526422,-0.0012229685508197141,0.0021060998095103361,0.14391178505006047,0.14601788485957082,-0.00071030622254025571,-0.50889059039320661 +20261027,probability_linear,statsmodels_probability,1434,120,440,True,,0.13905559613725615,0.84040378635449065,-0.0046254801536490531,-0.00014893553047129471,0.00043083643050707197,0.14391178505006047,0.14434262148056753,0.00096495715646300836,0.69133227539588404 +20261028,logit_linear,raw,1362,120,437,True,,0.15115390857840855,0.83694477085781427,0,0,0.0078329847535030458,0.16412564094611809,0.17195862569962114,0,0 +20261028,logit_linear,predval_rung2,1362,120,437,True,,0.1513164039532636,0.83149978353639686,-0.0054449873214174049,-0.00016249537485504839,0.0019872713638107854,0.16412564094611809,0.16611291230992886,0.00584571338969226,0.74629449356172395 +20261028,logit_linear,statsmodels_logit,1362,120,437,True,,0.1513164039532636,0.83149978353639686,-0.0054449873214174049,-0.00016249537485504839,0.0019872713638107889,0.16412564094611809,0.16611291230992889,0.0058457133896922565,0.7462944935617235 +20261028,logit_linear,statsmodels_probability,1362,120,437,True,,0.15415532267150719,0.82804873523408984,-0.0088960356237244298,-0.0030014140930986477,0.0059874847815408767,0.16412564094611809,0.17011312572765896,0.0018454999719621692,0.23560622547322463 +20261028,probability_linear,raw,1362,120,376,True,,0.13455048626417901,0.83184799965474088,0,0,0.001517365326666941,0.14061862960859675,0.14213599493526369,0,0 +20261028,probability_linear,predval_rung2,1362,120,376,True,,0.13793526928538702,0.82442762504855205,-0.0074203746061888287,-0.0033847830212080099,0.0033525541668795889,0.14061862960859675,0.14397118377547635,-0.0018351888402126479,-1.2094574773524323 +20261028,probability_linear,statsmodels_logit,1362,120,376,True,,0.13793526928538696,0.82442762504855205,-0.0074203746061888287,-0.0033847830212079544,0.003352554166879585,0.14061862960859675,0.14397118377547632,-0.001835188840212644,-1.2094574773524296 +20261028,probability_linear,statsmodels_probability,1362,120,376,True,,0.13554525423386143,0.82049760476457656,-0.011350394890164317,-0.0009947679696824252,0.0010628841223208798,0.14061862960859675,0.14168151373091764,0.00045448120434606119,0.2995199615799703 +20261029,logit_linear,raw,1389,120,532,True,,0.17106500461507412,0.81718224967319131,0,0,0.0080598187678669574,0.16803294409659267,0.17609276286445963,0,0 +20261029,logit_linear,predval_rung2,1389,120,532,True,,0.16582742814913554,0.81521700985251933,-0.0019652398206719823,0.0052375764659385871,0.00016834552030134116,0.16803294409659267,0.16820128961689401,0.0078914732475656169,0.97911298936739066 +20261029,logit_linear,statsmodels_logit,1389,120,532,True,,0.16582742814913554,0.81521700985251933,-0.0019652398206719823,0.0052375764659385871,0.00016834552030133921,0.16803294409659267,0.16820128961689401,0.0078914732475656186,0.97911298936739088 +20261029,logit_linear,statsmodels_probability,1389,120,532,True,,0.16663707705971004,0.81530474377308504,-0.0018775059001062688,0.0044279275553640862,0.0030293003468433838,0.16803294409659267,0.17106224444343604,0.0050305184210235731,0.62414783333334278 +20261029,probability_linear,raw,1389,120,463,True,,0.14255645847029708,0.83161744468649856,0,0,0.0013744286983071573,0.14157529884154518,0.14294972753985233,0,0 +20261029,probability_linear,predval_rung2,1389,120,463,True,,0.14642408619754843,0.82852231432716472,-0.003095130359333842,-0.0038676277272513548,0.0033899352316571364,0.14157529884154518,0.14496523407320233,-0.002015506533349979,-1.4664322244088894 +20261029,probability_linear,statsmodels_logit,1389,120,463,True,,0.14642408619754843,0.82852231432716472,-0.003095130359333842,-0.0038676277272513548,0.0033899352316571285,0.14157529884154518,0.1449652340732023,-0.0020155065333499712,-1.4664322244088837 +20261029,probability_linear,statsmodels_probability,1389,120,463,True,,0.14180229261273747,0.8277736053253969,-0.0038438393611016641,0.00075416585755960774,0.00038739940821536751,0.14157529884154518,0.14196269824976054,0.00098702929009178994,0.71813786434136906 +20261030,logit_linear,raw,1427,120,465,True,,0.1762970204996186,0.78512731093376253,0,0,0.0082050204616452849,0.16724040363983009,0.17544542410147537,0,0 +20261030,logit_linear,predval_rung2,1427,120,465,True,,0.17232120501660714,0.77749312588022268,-0.0076341850535398548,0.0039758154830114578,0.00070613549701270377,0.16724040363983009,0.16794653913684279,0.0074988849646325812,0.91393860620902001 +20261030,logit_linear,statsmodels_logit,1427,120,465,True,,0.17232120501660717,0.77749312588022268,-0.0076341850535398548,0.0039758154830114301,0.00070613549701271342,0.16724040363983009,0.16794653913684279,0.0074988849646325717,0.91393860620901879 +20261030,logit_linear,statsmodels_probability,1427,120,465,True,,0.17407984245534294,0.76949232110522436,-0.015634989828538171,0.0022171780442756528,0.0037133278692481888,0.16724040363983009,0.17095373150907828,0.004491692592397096,0.54743222316064943 +20261030,probability_linear,raw,1427,120,402,True,,0.15427472405979165,0.78454799174857415,0,0,0.0014991540859313113,0.14250065214551882,0.14399980623145014,0,0 +20261030,probability_linear,predval_rung2,1427,120,402,True,,0.15818335809906339,0.77364640213566316,-0.010901589612910989,-0.0039086340392717467,0.0047080825610009069,0.14250065214551882,0.14720873470651974,-0.0032089284750695954,-2.1404927653424837 +20261030,probability_linear,statsmodels_logit,1427,120,402,True,,0.15818335809906339,0.77364640213566316,-0.010901589612910989,-0.0039086340392717467,0.0047080825610009069,0.14250065214551882,0.14720873470651974,-0.0032089284750695954,-2.1404927653424837 +20261030,probability_linear,statsmodels_probability,1427,120,402,True,,0.15457490073521149,0.76713262953525063,-0.017415362213323515,-0.00030017667541984427,0.00064888749365303805,0.14250065214551882,0.14314953963917187,0.00085026659227827321,0.56716424299378598 +20261031,logit_linear,raw,1394,120,524,True,,0.16141963329581652,0.85267175572519083,0,0,0.0074894512804113108,0.15830125412026977,0.16579070540068108,0,0 +20261031,logit_linear,predval_rung2,1394,120,524,True,,0.1527128401147837,0.85150916907958241,-0.0011625866456084211,0.0087067931810328181,0.00052734832358431166,0.15830125412026977,0.15882860244385408,0.0069621029568269991,0.92958785579344205 +20261031,logit_linear,statsmodels_logit,1394,120,524,True,,0.15271284011478367,0.85150916907958241,-0.0011625866456084211,0.0087067931810328458,0.00052734832358431697,0.15830125412026977,0.15882860244385408,0.0069621029568269939,0.92958785579344128 +20261031,logit_linear,statsmodels_probability,1394,120,524,True,,0.15880926786788699,0.85030709835921736,-0.002364657365973466,0.0026103654279295285,0.0034047156080196095,0.15830125412026977,0.16170596972828938,0.0040847356723917013,0.5453985237977772 +20261031,probability_linear,raw,1394,120,448,True,,0.13102294943217255,0.86550041528239197,0,0,0.0015944681594983726,0.13625553958671963,0.137850007746218,0,0 +20261031,probability_linear,predval_rung2,1394,120,448,True,,0.13154209823190791,0.86474299682875255,-0.00075741845363941351,-0.00051914879973535832,0.0020894628266548043,0.13625553958671963,0.13834500241337444,-0.00049499466715643178,-0.31044499961175731 +20261031,probability_linear,statsmodels_logit,1394,120,448,True,,0.13154209823190793,0.86474299682875255,-0.00075741845363941351,-0.00051914879973538608,0.0020894628266548009,0.13625553958671963,0.13834500241337444,-0.00049499466715642831,-0.31044499961175531 +20261031,probability_linear,statsmodels_probability,1394,120,448,True,,0.13168989552322111,0.86438906297191165,-0.001111352310480318,-0.00066694609104855918,0.00025064813166457883,0.13625553958671963,0.13650618771838421,0.0013438200278337938,0.84280141928739805 +20261032,logit_linear,raw,1439,120,385,True,,0.1528738643980696,0.80410310751866731,0,0,0.008426679166857869,0.1555046858754299,0.16393136504228778,0,0 +20261032,logit_linear,predval_rung2,1439,120,385,True,,0.14967189067177419,0.80171024421498804,-0.0023928633036792712,0.0032019737262954096,0.0011932864159085088,0.1555046858754299,0.15669797229133842,0.0072333927509493599,0.85839185374450888 +20261032,logit_linear,statsmodels_logit,1439,120,385,True,,0.14967189067177419,0.80171024421498804,-0.0023928633036792712,0.0032019737262954096,0.0011932864159084964,0.1555046858754299,0.15669797229133839,0.0072333927509493721,0.85839185374451032 +20261032,logit_linear,statsmodels_probability,1439,120,385,True,,0.15237104445403205,0.80092905197269526,-0.0031740555459720543,0.00050281994403755803,0.0047630480123219557,0.1555046858754299,0.16026773388775187,0.0036636311545359132,0.43476571042896417 +20261032,probability_linear,raw,1439,120,311,True,,0.11970621001886332,0.81095071948188191,0,0,0.0017264119985204889,0.12972553676727858,0.13145194876579908,0,0 +20261032,probability_linear,predval_rung2,1439,120,311,True,,0.12287242927138385,0.80632710770563953,-0.0046236117762423756,-0.0031662192525205302,0.003143565460977191,0.12972553676727858,0.13286910222825576,-0.001417153462456702,-0.82086631908906038 +20261032,probability_linear,statsmodels_logit,1439,120,311,True,,0.12287242927138381,0.80632710770563953,-0.0046236117762423756,-0.0031662192525204885,0.0031435654609771862,0.12972553676727858,0.13286910222825576,-0.0014171534624566973,-0.82086631908905749 +20261032,probability_linear,statsmodels_probability,1439,120,311,True,,0.11994467511678716,0.80367608492394704,-0.007274634557934867,-0.00023846509792384285,0.00067248395307467797,0.12972553676727858,0.13039802072035325,0.0010539280454458109,0.61047307731237543 +20261033,logit_linear,raw,1448,120,569,True,,0.19035127029195961,0.79680536477983643,0,0,0.0081483872161152073,0.1742876411581982,0.18243602837431341,0,0 +20261033,logit_linear,predval_rung2,1448,120,569,True,,0.17946219157754739,0.79400820952072471,-0.0027971552591117277,0.01088907871441222,0.00035003291051367608,0.1742876411581982,0.17463767406871186,0.0077983543056015315,0.95704267590261172 +20261033,logit_linear,statsmodels_logit,1448,120,569,True,,0.17946219157754739,0.79400820952072471,-0.0027971552591117277,0.01088907871441222,0.00035003291051368036,0.1742876411581982,0.17463767406871189,0.0077983543056015271,0.95704267590261127 +20261033,logit_linear,statsmodels_probability,1448,120,569,True,,0.18329486886876653,0.79220875295660709,-0.0045966118232293462,0.0070564014231930816,0.0025800393302457163,0.1742876411581982,0.17686768048844392,0.005568347885869491,0.68336809950033706 +20261033,probability_linear,raw,1448,120,453,True,,0.13698360411259891,0.85104551454845989,0,0,0.0012733848280287177,0.14714167916576223,0.14841506399379095,0,0 +20261033,probability_linear,predval_rung2,1448,120,453,True,,0.13928854339119578,0.84583402664536822,-0.0052114879030916716,-0.0023049392785968736,0.0016384017263794217,0.14714167916576223,0.14878008089214165,-0.00036501689835070406,-0.28665089320702353 +20261033,probability_linear,statsmodels_logit,1448,120,453,True,,0.13928854339119578,0.84583402664536822,-0.0052114879030916716,-0.0023049392785968736,0.0016384017263794215,0.14714167916576223,0.14878008089214165,-0.00036501689835070385,-0.28665089320702353 +20261033,probability_linear,statsmodels_probability,1448,120,453,True,,0.13818787256301304,0.84299200195236679,-0.0080535125960931042,-0.001204268450414131,0.00056139075112271583,0.14714167916576223,0.14770306991688495,0.00071199407690600183,0.55913504011840232 +20261034,logit_linear,raw,1420,120,538,True,,0.19433385877185633,0.79519763295652834,0,0,0.0077490875632242979,0.16207689706451592,0.16982598462774021,0,0 +20261034,logit_linear,predval_rung2,1420,120,538,True,,0.18007137521802935,0.78888804592468964,-0.0063095870318387037,0.014262483553826977,0.0013963471720264211,0.16207689706451592,0.16347324423654233,0.006352740391197877,0.81980495630824712 +20261034,logit_linear,statsmodels_logit,1420,120,538,True,,0.18007137521802932,0.78888804592468964,-0.0063095870318387037,0.014262483553827005,0.0013963471720264389,0.16207689706451592,0.16347324423654236,0.0063527403911978588,0.81980495630824479 +20261034,logit_linear,statsmodels_probability,1420,120,538,True,,0.18604124035470598,0.78368906422544227,-0.011508568731086077,0.0082926184171503436,0.0042056067153507723,0.16207689706451592,0.1662825037798667,0.0035434808478735256,0.45727717217833674 +20261034,probability_linear,raw,1420,120,428,True,,0.14939958351758506,0.82147365842628883,0,0,0.0015950044348788373,0.13960855814065604,0.14120356257553487,0,0 +20261034,probability_linear,predval_rung2,1420,120,428,True,,0.15088167791573759,0.8171823183599638,-0.0042913400663250334,-0.0014820943981525281,0.0037349462620427727,0.13960855814065604,0.1433435044026988,-0.0021399418271639357,-1.3416525875217982 +20261034,probability_linear,statsmodels_logit,1420,120,428,True,,0.15088167791573759,0.8171823183599638,-0.0042913400663250334,-0.0014820943981525281,0.0037349462620427667,0.13960855814065604,0.1433435044026988,-0.0021399418271639296,-1.3416525875217942 +20261034,probability_linear,statsmodels_probability,1420,120,428,True,,0.1502340938068514,0.81479876394332229,-0.0066748944829665424,-0.00083451028926634363,0.0003393626592441403,0.13960855814065604,0.13994792079990018,0.0012556417756346969,0.78723403407344161 +20261035,logit_linear,raw,1395,120,495,True,,0.16276500622013998,0.83443097643097641,0,0,0.0080589285852221172,0.16661223520551655,0.17467116379073866,0,0 +20261035,logit_linear,predval_rung2,1395,120,495,True,,0.1573149264663205,0.83171268237934903,-0.002718294051627379,0.0054500797538194756,0.00062970017287700087,0.16661223520551655,0.16724193537839355,0.0074292284123451159,0.92186304094669613 +20261035,logit_linear,statsmodels_logit,1395,120,495,True,,0.1573149264663205,0.83171268237934903,-0.002718294051627379,0.0054500797538194756,0.00062970017287696889,0.16661223520551655,0.16724193537839352,0.007429228412345148,0.92186304094670013 +20261035,logit_linear,statsmodels_probability,1395,120,495,True,,0.16122503282622702,0.82994612794612788,-0.0044848484848485359,0.0015399733939129578,0.0036479300931206923,0.16661223520551655,0.17026016529863724,0.0044109984921014249,0.54734304262107447 +20261035,probability_linear,raw,1395,120,410,True,,0.12837023084131388,0.85405967562213692,0,0,0.0014150698545028963,0.14033102356854707,0.14174609342304997,0,0 +20261035,probability_linear,predval_rung2,1395,120,410,True,,0.12995193254944318,0.85159836572985015,-0.0024613098922867671,-0.0015817017081292961,0.0015842528911091912,0.14033102356854707,0.14191527645965626,-0.00016918303660629487,-0.11955808122682932 +20261035,probability_linear,statsmodels_logit,1395,120,410,True,,0.12995193254944312,0.85159836572985015,-0.0024613098922867671,-0.0015817017081292406,0.0015842528911091851,0.14033102356854707,0.14191527645965626,-0.0001691830366062888,-0.1195580812268251 +20261035,probability_linear,statsmodels_probability,1395,120,410,True,,0.12848959579259572,0.84877305930419722,-0.0052866163179396963,-0.00011936495128184177,0.00060641455479657999,0.14033102356854707,0.14093743812334364,0.00080865529970631633,0.57145963298779412 +20261036,logit_linear,raw,1445,120,430,True,,0.15680895201474182,0.80209646007561008,0,0,0.007838661521616835,0.16069195138335651,0.16853061290497334,0,0 +20261036,logit_linear,predval_rung2,1445,120,430,True,,0.15678553492640884,0.79527093596059117,-0.0068255241150189105,2.341708833297762e-05,0.0015782214650137034,0.16069195138335651,0.16227017284837023,0.0062604400566031318,0.79866186839916353 +20261036,logit_linear,statsmodels_logit,1445,120,430,True,,0.15678553492640884,0.79527093596059117,-0.0068255241150189105,2.341708833297762e-05,0.0015782214650136963,0.16069195138335651,0.1622701728483702,0.0062604400566031388,0.79866186839916442 +20261036,logit_linear,statsmodels_probability,1445,120,430,True,,0.15687893600704139,0.78680490319624241,-0.015291556879367674,-6.9983992299571529e-05,0.0049404181190858178,0.16069195138335651,0.16563236950244234,0.0028982434025310172,0.3697370264730111 +20261036,probability_linear,raw,1445,120,354,True,,0.13579580988000245,0.79815853386982349,0,0,0.0016312042789875907,0.13718559603332492,0.1388168003123125,0,0 +20261036,probability_linear,predval_rung2,1445,120,354,True,,0.13854284264072655,0.78900816645693828,-0.0091503674128852186,-0.0027470327607241052,0.0049801002867055988,0.13718559603332492,0.14216569632003051,-0.0033488960077180082,-2.0530206123518173 +20261036,probability_linear,statsmodels_logit,1445,120,354,True,,0.13854284264072655,0.78900816645693828,-0.0091503674128852186,-0.0027470327607241052,0.0049801002867055971,0.13718559603332492,0.14216569632003051,-0.0033488960077180064,-2.0530206123518164 +20261036,probability_linear,statsmodels_probability,1445,120,354,True,,0.13479291777209781,0.78062680275702068,-0.017531731112802817,0.0010028921079046349,0.00159866457864132,0.13718559603332492,0.13878426061196625,3.2539700346270609e-05,0.019948268138719194 +20261037,logit_linear,raw,1431,120,525,True,,0.17225504979552506,0.81395143487858723,0,0,0.0076204075302433493,0.16505299786426769,0.17267340539451104,0,0 +20261037,logit_linear,predval_rung2,1431,120,525,True,,0.16822510182326061,0.80696100073583532,-0.0069904341427519112,0.0040299479722644416,0.00032398871600892716,0.16505299786426769,0.16537698658027661,0.0072964188142344219,0.95748406962185384 +20261037,logit_linear,statsmodels_logit,1431,120,525,True,,0.16822510182326061,0.80696100073583532,-0.0069904341427519112,0.0040299479722644416,0.0003239887160089296,0.16505299786426769,0.16537698658027661,0.0072964188142344193,0.95748406962185351 +20261037,logit_linear,statsmodels_probability,1431,120,525,True,,0.17106590327667873,0.80210238620834651,-0.011849048670240725,0.001189146518846329,0.0034235955691945159,0.16505299786426769,0.16847659343346222,0.0041968119610488333,0.55073327041799469 +20261037,probability_linear,raw,1431,120,453,True,,0.14496039900858099,0.82838563180252522,0,0,0.0014836324570238477,0.14257933898331515,0.144062971440339,0,0 +20261037,probability_linear,predval_rung2,1431,120,453,True,,0.14871826901052171,0.82040430305574752,-0.007981328746777705,-0.0037578700019407218,0.0032171827356474395,0.14257933898331515,0.1457965217189626,-0.0017335502786235918,-1.1684499556588812 +20261037,probability_linear,statsmodels_logit,1431,120,453,True,,0.14871826901052171,0.82040430305574752,-0.007981328746777705,-0.0037578700019407218,0.0032171827356474408,0.14257933898331515,0.1457965217189626,-0.0017335502786235931,-1.1684499556588821 +20261037,probability_linear,statsmodels_probability,1431,120,453,True,,0.14700666197157719,0.81636849542021606,-0.012017136382309168,-0.0020462629629962037,0.00035409853306648452,0.14257933898331515,0.14293343751638163,0.0011295339239573631,0.76133001715478588 +20261038,logit_linear,raw,1466,120,567,True,,0.18263284372067984,0.79400588151051621,0,0,0.0082092760133281716,0.16328721412426131,0.17149649013758947,0,0 +20261038,logit_linear,predval_rung2,1466,120,567,True,,0.1758965649633098,0.78683349910639488,-0.0071723824041213247,0.00673627875737004,0.0010725824081500759,0.16328721412426131,0.1643597965324114,0.0071366936051780955,0.86934506692079982 +20261038,logit_linear,statsmodels_logit,1466,120,567,True,,0.17589656496330977,0.78683349910639488,-0.0071723824041213247,0.0067362787573700678,0.0010725824081500709,0.16328721412426131,0.16435979653241137,0.0071366936051781007,0.86934506692080038 +20261038,logit_linear,statsmodels_probability,1466,120,567,True,,0.17536452942941122,0.77862528029380085,-0.015380601216715362,0.0072683142912686183,0.0032986240737581827,0.16328721412426131,0.1665858381980195,0.0049106519395699889,0.59818331502038657 +20261038,probability_linear,raw,1466,120,510,True,,0.16657929387983328,0.77580605463942909,0,0,0.0014629692458767159,0.13675973442152489,0.13822270366740161,0,0 +20261038,probability_linear,predval_rung2,1466,120,510,True,,0.17408761028883663,0.76544630404463043,-0.010359750594798656,-0.0075083164090033461,0.010172788548894675,0.13675973442152489,0.14693252297041956,-0.0087098193030179585,-5.9535218033913022 +20261038,probability_linear,statsmodels_logit,1466,120,510,True,,0.17408761028883657,0.76544630404463043,-0.010359750594798656,-0.0075083164090032906,0.010172788548894671,0.13675973442152489,0.14693252297041956,-0.0087098193030179551,-5.9535218033912995 +20261038,probability_linear,statsmodels_probability,1466,120,510,True,,0.16377843541801357,0.76315325293297231,-0.012652801706456773,0.0028008584618197108,0.0029041079331997657,0.13675973442152489,0.13966384235472465,-0.0014411386873230498,-0.98507791013707635 +20261039,logit_linear,raw,1388,120,502,True,,0.16891388601364496,0.81681850476199047,0,0,0.0080662829180899696,0.16402665079764175,0.17209293371573173,0,0 +20261039,logit_linear,predval_rung2,1388,120,502,True,,0.16375251534607063,0.81329984801201516,-0.003518656749975313,0.0051613706675743276,0.00024067951376043371,0.16402665079764175,0.1642673303114022,0.0078256034043295356,0.97016227719701353 +20261039,logit_linear,statsmodels_logit,1388,120,502,True,,0.16375251534607063,0.81329984801201516,-0.003518656749975313,0.0051613706675743276,0.00024067951376043035,0.16402665079764175,0.16426733031140217,0.007825603404329539,0.97016227719701387 +20261039,logit_linear,statsmodels_probability,1388,120,502,True,,0.16541806596677425,0.81175073970483747,-0.0050677650571530064,0.0034958200468707123,0.0030862638374258276,0.16402665079764175,0.16711291463506758,0.0049800190806641425,0.61738710769685845 +20261039,probability_linear,raw,1388,120,412,True,,0.13873596655130047,0.81821233885086753,0,0,0.0015021648026766682,0.13923138281847094,0.14073354762114762,0,0 +20261039,probability_linear,predval_rung2,1388,120,412,True,,0.14374572159091231,0.810376213592233,-0.0078361252586345298,-0.0050097550396118462,0.0037563414375808186,0.13923138281847094,0.14298772425605177,-0.0022541766349041502,-1.5006187276439253 +20261039,probability_linear,statsmodels_logit,1388,120,412,True,,0.14374572159091228,0.810376213592233,-0.0078361252586345298,-0.0050097550396118185,0.0037563414375808104,0.13923138281847094,0.14298772425605175,-0.0022541766349041424,-1.50061872764392 +20261039,probability_linear,statsmodels_probability,1388,120,412,True,,0.13846697693990537,0.80813554432595891,-0.010076794524908617,0.00026898961139509847,0.00033031108087790991,0.13923138281847094,0.13956169389934886,0.0011718537217987584,0.78010995844840914 +20261010,wdbc_undistorted,raw,285,285,106,True,,0.029478263131405693,0.9880889638452619,0,0,,,,, +20261010,wdbc_undistorted,predval_rung2,285,285,106,True,,0.029271022790021212,0.98476863075787924,-0.0033203330873826653,0.00020724034138448061,,,,, +20261010,wdbc_undistorted,statsmodels_logit,285,285,106,True,,0.02927102279002123,0.98476863075787924,-0.0033203330873826653,0.00020724034138446326,,,,, +20261010,wdbc_undistorted,statsmodels_probability,285,285,106,True,,0.027864603166123696,0.97902392747970912,-0.009065036365552781,0.0016136599652819966,,,,, diff --git a/benchmarks/family_extension/results/metrics.json b/benchmarks/family_extension/results/metrics.json new file mode 100644 index 0000000..1627008 --- /dev/null +++ b/benchmarks/family_extension/results/metrics.json @@ -0,0 +1,3098 @@ +[ + { + "auroc": 0.821504315182476, + "auroc_delta": 0.0, + "available": true, + "brier": 0.16107731913455833, + "brier_gain": 0.0, + "clusters": 120, + "events": 540, + "expected_brier": 0.17533752355496574, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "logit_linear", + "method": "raw", + "oracle_expected_brier": 0.16752891002530854, + "reason": null, + "risk_mse": 0.007808613529657212, + "rows": 1497, + "seed": 20261020 + }, + { + "auroc": 0.8205425906575331, + "auroc_delta": -0.0009617245249429196, + "available": true, + "brier": 0.15935636225350433, + "brier_gain": 0.0017209568810540032, + "clusters": 120, + "events": 540, + "expected_brier": 0.16837409831939865, + "expected_gain": 0.006963425235567105, + "expected_recovery": 0.8917620534195896, + "family": "logit_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.16752891002530854, + "reason": null, + "risk_mse": 0.000845188294090107, + "rows": 1497, + "seed": 20261020 + }, + { + "auroc": 0.8205425906575331, + "auroc_delta": -0.0009617245249429196, + "available": true, + "brier": 0.15935636225350433, + "brier_gain": 0.0017209568810540032, + "clusters": 120, + "events": 540, + "expected_brier": 0.16837409831939862, + "expected_gain": 0.006963425235567133, + "expected_recovery": 0.8917620534195932, + "family": "logit_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.16752891002530854, + "reason": null, + "risk_mse": 0.0008451882940900788, + "rows": 1497, + "seed": 20261020 + }, + { + "auroc": 0.8194918533998994, + "auroc_delta": -0.0020124617825766355, + "available": true, + "brier": 0.15908628947466405, + "brier_gain": 0.0019910296598942834, + "clusters": 120, + "events": 540, + "expected_brier": 0.17172014497029334, + "expected_gain": 0.0036173785846724113, + "expected_recovery": 0.4632549134277887, + "family": "logit_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.16752891002530854, + "reason": null, + "risk_mse": 0.004191234944984801, + "rows": 1497, + "seed": 20261020 + }, + { + "auroc": 0.8520384191596848, + "auroc_delta": 0.0, + "available": true, + "brier": 0.1319058610556766, + "brier_gain": 0.0, + "clusters": 120, + "events": 467, + "expected_brier": 0.144179467067622, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "probability_linear", + "method": "raw", + "oracle_expected_brier": 0.14276742283729035, + "reason": null, + "risk_mse": 0.0014120442303316365, + "rows": 1497, + "seed": 20261020 + }, + { + "auroc": 0.849148666347893, + "auroc_delta": -0.0028897528117918236, + "available": true, + "brier": 0.13379041585682258, + "brier_gain": -0.0018845548011459934, + "clusters": 120, + "events": 467, + "expected_brier": 0.14489961364146714, + "expected_gain": -0.0007201465738451486, + "expected_recovery": -0.5100028443698346, + "family": "probability_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.14276742283729035, + "reason": null, + "risk_mse": 0.002132190804176785, + "rows": 1497, + "seed": 20261020 + }, + { + "auroc": 0.849148666347893, + "auroc_delta": -0.0028897528117918236, + "available": true, + "brier": 0.13379041585682258, + "brier_gain": -0.0018845548011459934, + "clusters": 120, + "events": 467, + "expected_brier": 0.14489961364146714, + "expected_gain": -0.0007201465738451433, + "expected_recovery": -0.510002844369831, + "family": "probability_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.14276742283729035, + "reason": null, + "risk_mse": 0.00213219080417678, + "rows": 1497, + "seed": 20261020 + }, + { + "auroc": 0.846880522234465, + "auroc_delta": -0.00515789692521984, + "available": true, + "brier": 0.13043779368272918, + "brier_gain": 0.0014680673729474125, + "clusters": 120, + "events": 467, + "expected_brier": 0.14338066752647752, + "expected_gain": 0.0007987995411444785, + "expected_recovery": 0.5657043341743412, + "family": "probability_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.14276742283729035, + "reason": null, + "risk_mse": 0.000613244689187158, + "rows": 1497, + "seed": 20261020 + }, + { + "auroc": 0.837092731829574, + "auroc_delta": 0.0, + "available": true, + "brier": 0.170465906992062, + "brier_gain": 0.0, + "clusters": 120, + "events": 535, + "expected_brier": 0.16528105633054263, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "logit_linear", + "method": "raw", + "oracle_expected_brier": 0.15826094317987213, + "reason": null, + "risk_mse": 0.007020113150670493, + "rows": 1333, + "seed": 20261021 + }, + { + "auroc": 0.8351743845595297, + "auroc_delta": -0.0019183472700442517, + "available": true, + "brier": 0.1605480745614803, + "brier_gain": 0.009917832430581697, + "clusters": 120, + "events": 535, + "expected_brier": 0.15908566026871876, + "expected_gain": 0.0061953960618238695, + "expected_recovery": 0.8825208267807115, + "family": "logit_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.15826094317987213, + "reason": null, + "risk_mse": 0.0008247170888466236, + "rows": 1333, + "seed": 20261021 + }, + { + "auroc": 0.8351743845595297, + "auroc_delta": -0.0019183472700442517, + "available": true, + "brier": 0.16054807456148035, + "brier_gain": 0.009917832430581641, + "clusters": 120, + "events": 535, + "expected_brier": 0.1590856602687188, + "expected_gain": 0.006195396061823841, + "expected_recovery": 0.8825208267807074, + "family": "logit_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.15826094317987213, + "reason": null, + "risk_mse": 0.0008247170888466526, + "rows": 1333, + "seed": 20261021 + }, + { + "auroc": 0.831080036539948, + "auroc_delta": -0.006012695289626002, + "available": true, + "brier": 0.163994792294443, + "brier_gain": 0.00647111469761899, + "clusters": 120, + "events": 535, + "expected_brier": 0.16192080169170717, + "expected_gain": 0.0033602546388354607, + "expected_recovery": 0.47866103675473115, + "family": "logit_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.15826094317987213, + "reason": null, + "risk_mse": 0.0036598585118350325, + "rows": 1333, + "seed": 20261021 + }, + { + "auroc": 0.8518110591003528, + "auroc_delta": 0.0, + "available": true, + "brier": 0.1392438562312144, + "brier_gain": 0.0, + "clusters": 120, + "events": 455, + "expected_brier": 0.14043260860766085, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "probability_linear", + "method": "raw", + "oracle_expected_brier": 0.1388292585937953, + "reason": null, + "risk_mse": 0.0016033500138655586, + "rows": 1333, + "seed": 20261021 + }, + { + "auroc": 0.8498660792510451, + "auroc_delta": -0.0019449798493077486, + "available": true, + "brier": 0.14128816472603203, + "brier_gain": -0.002044308494817626, + "clusters": 120, + "events": 455, + "expected_brier": 0.14213415068665822, + "expected_gain": -0.0017015420789973643, + "expected_recovery": -1.061241815126238, + "family": "probability_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.1388292585937953, + "reason": null, + "risk_mse": 0.003304892092862923, + "rows": 1333, + "seed": 20261021 + }, + { + "auroc": 0.8498660792510451, + "auroc_delta": -0.0019449798493077486, + "available": true, + "brier": 0.14128816472603206, + "brier_gain": -0.0020443084948176538, + "clusters": 120, + "events": 455, + "expected_brier": 0.14213415068665822, + "expected_gain": -0.0017015420789973704, + "expected_recovery": -1.061241815126242, + "family": "probability_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.1388292585937953, + "reason": null, + "risk_mse": 0.003304892092862929, + "rows": 1333, + "seed": 20261021 + }, + { + "auroc": 0.8449197727102056, + "auroc_delta": -0.006891286390147244, + "available": true, + "brier": 0.13833251946917985, + "brier_gain": 0.0009113367620345592, + "clusters": 120, + "events": 455, + "expected_brier": 0.13916343009786658, + "expected_gain": 0.001269178509794293, + "expected_recovery": 0.7915791928266476, + "family": "probability_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.1388292585937953, + "reason": null, + "risk_mse": 0.00033417150407126555, + "rows": 1333, + "seed": 20261021 + }, + { + "auroc": 0.844982603323253, + "auroc_delta": 0.0, + "available": true, + "brier": 0.16571605127396458, + "brier_gain": 0.0, + "clusters": 120, + "events": 459, + "expected_brier": 0.17129198983642602, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "logit_linear", + "method": "raw", + "oracle_expected_brier": 0.16318958587396784, + "reason": null, + "risk_mse": 0.008102403962458168, + "rows": 1263, + "seed": 20261022 + }, + { + "auroc": 0.843402811649812, + "auroc_delta": -0.0015797916734410222, + "available": true, + "brier": 0.15321664568771615, + "brier_gain": 0.01249940558624843, + "clusters": 120, + "events": 459, + "expected_brier": 0.166020893307669, + "expected_gain": 0.005271096528757007, + "expected_recovery": 0.6505595812280165, + "family": "logit_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.16318958587396784, + "reason": null, + "risk_mse": 0.002831307433701161, + "rows": 1263, + "seed": 20261022 + }, + { + "auroc": 0.843402811649812, + "auroc_delta": -0.0015797916734410222, + "available": true, + "brier": 0.15321664568771615, + "brier_gain": 0.01249940558624843, + "clusters": 120, + "events": 459, + "expected_brier": 0.16602089330766903, + "expected_gain": 0.0052710965287569855, + "expected_recovery": 0.6505595812280138, + "family": "logit_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.16318958587396784, + "reason": null, + "risk_mse": 0.002831307433701183, + "rows": 1263, + "seed": 20261022 + }, + { + "auroc": 0.8424950411341983, + "auroc_delta": -0.0024875621890547706, + "available": true, + "brier": 0.15794355631526627, + "brier_gain": 0.007772494958698306, + "clusters": 120, + "events": 459, + "expected_brier": 0.1682352313454473, + "expected_gain": 0.0030567584909787105, + "expected_recovery": 0.37726562451612544, + "family": "logit_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.16318958587396784, + "reason": null, + "risk_mse": 0.0050456454714794576, + "rows": 1263, + "seed": 20261022 + }, + { + "auroc": 0.8772736980199529, + "auroc_delta": 0.0, + "available": true, + "brier": 0.11976713111415747, + "brier_gain": 0.0, + "clusters": 120, + "events": 365, + "expected_brier": 0.1391605796132322, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "probability_linear", + "method": "raw", + "oracle_expected_brier": 0.13762883536214698, + "reason": null, + "risk_mse": 0.0015317442510852197, + "rows": 1263, + "seed": 20261022 + }, + { + "auroc": 0.8743478658815633, + "auroc_delta": -0.0029258321383895947, + "available": true, + "brier": 0.11952004569354643, + "brier_gain": 0.0002470854206110401, + "clusters": 120, + "events": 365, + "expected_brier": 0.14103960792721537, + "expected_gain": -0.0018790283139831611, + "expected_recovery": -1.2267245740611696, + "family": "probability_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.13762883536214698, + "reason": null, + "risk_mse": 0.003410772565068381, + "rows": 1263, + "seed": 20261022 + }, + { + "auroc": 0.8743478658815633, + "auroc_delta": -0.0029258321383895947, + "available": true, + "brier": 0.11952004569354643, + "brier_gain": 0.0002470854206110401, + "clusters": 120, + "events": 365, + "expected_brier": 0.14103960792721534, + "expected_gain": -0.0018790283139831394, + "expected_recovery": -1.2267245740611554, + "family": "probability_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.13762883536214698, + "reason": null, + "risk_mse": 0.003410772565068359, + "rows": 1263, + "seed": 20261022 + }, + { + "auroc": 0.8713854227049455, + "auroc_delta": -0.005888275315007463, + "available": true, + "brier": 0.11919046017725436, + "brier_gain": 0.000576670936903112, + "clusters": 120, + "events": 365, + "expected_brier": 0.1403560264899509, + "expected_gain": -0.0011954468767187104, + "expected_recovery": -0.7804480910385352, + "family": "probability_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.13762883536214698, + "reason": null, + "risk_mse": 0.00272719112780393, + "rows": 1263, + "seed": 20261022 + }, + { + "auroc": 0.8466213027061045, + "auroc_delta": 0.0, + "available": true, + "brier": 0.16102492586755904, + "brier_gain": 0.0, + "clusters": 120, + "events": 560, + "expected_brier": 0.1622330505471465, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "logit_linear", + "method": "raw", + "oracle_expected_brier": 0.15525482952310388, + "reason": null, + "risk_mse": 0.006978221024042649, + "rows": 1468, + "seed": 20261023 + }, + { + "auroc": 0.8447372561359344, + "auroc_delta": -0.0018840465701700504, + "available": true, + "brier": 0.1535218236440831, + "brier_gain": 0.007503102223475944, + "clusters": 120, + "events": 560, + "expected_brier": 0.15570419008219744, + "expected_gain": 0.006528860464949099, + "expected_recovery": 0.9356052842772778, + "family": "logit_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.15525482952310388, + "reason": null, + "risk_mse": 0.0004493605590935492, + "rows": 1468, + "seed": 20261023 + }, + { + "auroc": 0.8447372561359344, + "auroc_delta": -0.0018840465701700504, + "available": true, + "brier": 0.1535218236440831, + "brier_gain": 0.007503102223475944, + "clusters": 120, + "events": 560, + "expected_brier": 0.15570419008219744, + "expected_gain": 0.006528860464949096, + "expected_recovery": 0.9356052842772774, + "family": "logit_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.15525482952310388, + "reason": null, + "risk_mse": 0.0004493605590935528, + "rows": 1468, + "seed": 20261023 + }, + { + "auroc": 0.8438070327249841, + "auroc_delta": -0.002814269981120332, + "available": true, + "brier": 0.15734146686440126, + "brier_gain": 0.0036834590031577796, + "clusters": 120, + "events": 560, + "expected_brier": 0.15868192630013864, + "expected_gain": 0.003551124247007877, + "expected_recovery": 0.5088867541989415, + "family": "logit_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.15525482952310388, + "reason": null, + "risk_mse": 0.0034270967770347717, + "rows": 1468, + "seed": 20261023 + }, + { + "auroc": 0.8266012503622739, + "auroc_delta": 0.0, + "available": true, + "brier": 0.14333302168416276, + "brier_gain": 0.0, + "clusters": 120, + "events": 498, + "expected_brier": 0.13690149151747374, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "probability_linear", + "method": "raw", + "oracle_expected_brier": 0.13523789741194575, + "reason": null, + "risk_mse": 0.001663594105528006, + "rows": 1468, + "seed": 20261023 + }, + { + "auroc": 0.8236740777543163, + "auroc_delta": -0.0029271726079576066, + "available": true, + "brier": 0.14805074765092122, + "brier_gain": -0.004717725966758457, + "clusters": 120, + "events": 498, + "expected_brier": 0.1403312708688, + "expected_gain": -0.0034297793513262532, + "expected_recovery": -2.0616683720682456, + "family": "probability_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.13523789741194575, + "reason": null, + "risk_mse": 0.005093373456854259, + "rows": 1468, + "seed": 20261023 + }, + { + "auroc": 0.8236740777543163, + "auroc_delta": -0.0029271726079576066, + "available": true, + "brier": 0.14805074765092122, + "brier_gain": -0.004717725966758457, + "clusters": 120, + "events": 498, + "expected_brier": 0.1403312708688, + "expected_gain": -0.0034297793513262498, + "expected_recovery": -2.0616683720682434, + "family": "probability_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.13523789741194575, + "reason": null, + "risk_mse": 0.005093373456854256, + "rows": 1468, + "seed": 20261023 + }, + { + "auroc": 0.8219289529251026, + "auroc_delta": -0.004672297437171347, + "available": true, + "brier": 0.14116575175555704, + "brier_gain": 0.0021672699286057195, + "clusters": 120, + "events": 498, + "expected_brier": 0.1357254358309989, + "expected_gain": 0.0011760556864748675, + "expected_recovery": 0.7069366755790473, + "family": "probability_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.13523789741194575, + "reason": null, + "risk_mse": 0.0004875384190531387, + "rows": 1468, + "seed": 20261023 + }, + { + "auroc": 0.7986141609330015, + "auroc_delta": 0.0, + "available": true, + "brier": 0.1701144777888755, + "brier_gain": 0.0, + "clusters": 120, + "events": 495, + "expected_brier": 0.16581383492122007, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "logit_linear", + "method": "raw", + "oracle_expected_brier": 0.15829396339227805, + "reason": null, + "risk_mse": 0.007519871528942001, + "rows": 1530, + "seed": 20261024 + }, + { + "auroc": 0.7895183721270678, + "auroc_delta": -0.009095788805933758, + "available": true, + "brier": 0.16658973597194313, + "brier_gain": 0.0035247418169323808, + "clusters": 120, + "events": 495, + "expected_brier": 0.16105740554336848, + "expected_gain": 0.004756429377851581, + "expected_recovery": 0.6325147124582302, + "family": "logit_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.15829396339227805, + "reason": null, + "risk_mse": 0.0027634421510904198, + "rows": 1530, + "seed": 20261024 + }, + { + "auroc": 0.7895183721270678, + "auroc_delta": -0.009095788805933758, + "available": true, + "brier": 0.16658973597194307, + "brier_gain": 0.0035247418169324363, + "clusters": 120, + "events": 495, + "expected_brier": 0.16105740554336845, + "expected_gain": 0.004756429377851596, + "expected_recovery": 0.632514712458232, + "family": "logit_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.15829396339227805, + "reason": null, + "risk_mse": 0.0027634421510904054, + "rows": 1530, + "seed": 20261024 + }, + { + "auroc": 0.782704338066657, + "auroc_delta": -0.01590982286634457, + "available": true, + "brier": 0.1691870501019815, + "brier_gain": 0.0009274276868939946, + "clusters": 120, + "events": 495, + "expected_brier": 0.16410908960193063, + "expected_gain": 0.0017047453192894262, + "expected_recovery": 0.22669872919082612, + "family": "logit_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.15829396339227805, + "reason": null, + "risk_mse": 0.005815126209652575, + "rows": 1530, + "seed": 20261024 + }, + { + "auroc": 0.7846446639059707, + "auroc_delta": 0.0, + "available": true, + "brier": 0.15285937375593533, + "brier_gain": 0.0, + "clusters": 120, + "events": 433, + "expected_brier": 0.1381874715532536, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "probability_linear", + "method": "raw", + "oracle_expected_brier": 0.13656869498098234, + "reason": null, + "risk_mse": 0.001618776572271234, + "rows": 1530, + "seed": 20261024 + }, + { + "auroc": 0.7724025844156117, + "auroc_delta": -0.012242079490358937, + "available": true, + "brier": 0.15473972610443756, + "brier_gain": -0.0018803523485022289, + "clusters": 120, + "events": 433, + "expected_brier": 0.14494345909283876, + "expected_gain": -0.006755987539585192, + "expected_recovery": -4.173514526532939, + "family": "probability_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.13656869498098234, + "reason": null, + "risk_mse": 0.008374764111856427, + "rows": 1530, + "seed": 20261024 + }, + { + "auroc": 0.7724025844156117, + "auroc_delta": -0.012242079490358937, + "available": true, + "brier": 0.1547397261044376, + "brier_gain": -0.0018803523485022566, + "clusters": 120, + "events": 433, + "expected_brier": 0.14494345909283882, + "expected_gain": -0.006755987539585232, + "expected_recovery": -4.173514526532963, + "family": "probability_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.13656869498098234, + "reason": null, + "risk_mse": 0.008374764111856467, + "rows": 1530, + "seed": 20261024 + }, + { + "auroc": 0.7665731230039515, + "auroc_delta": -0.018071540902019123, + "available": true, + "brier": 0.15036100604113092, + "brier_gain": 0.0024983677148044114, + "clusters": 120, + "events": 433, + "expected_brier": 0.1390739927624741, + "expected_gain": -0.0008865212092205262, + "expected_recovery": -0.5476489000434985, + "family": "probability_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.13656869498098234, + "reason": null, + "risk_mse": 0.0025052977814917603, + "rows": 1530, + "seed": 20261024 + }, + { + "auroc": 0.8215978925552387, + "auroc_delta": 0.0, + "available": true, + "brier": 0.17407210831243555, + "brier_gain": 0.0, + "clusters": 120, + "events": 569, + "expected_brier": 0.17154324341662094, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "logit_linear", + "method": "raw", + "oracle_expected_brier": 0.16371700228003758, + "reason": null, + "risk_mse": 0.007826241136583369, + "rows": 1495, + "seed": 20261025 + }, + { + "auroc": 0.8189199345598925, + "auroc_delta": -0.002677957995346203, + "available": true, + "brier": 0.16510883980585878, + "brier_gain": 0.008963268506576777, + "clusters": 120, + "events": 569, + "expected_brier": 0.1644081457253299, + "expected_gain": 0.007135097691291038, + "expected_recovery": 0.9116889662316159, + "family": "logit_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.16371700228003758, + "reason": null, + "risk_mse": 0.0006911434452923312, + "rows": 1495, + "seed": 20261025 + }, + { + "auroc": 0.8189199345598925, + "auroc_delta": -0.002677957995346203, + "available": true, + "brier": 0.16510883980585878, + "brier_gain": 0.008963268506576777, + "clusters": 120, + "events": 569, + "expected_brier": 0.1644081457253299, + "expected_gain": 0.007135097691291036, + "expected_recovery": 0.9116889662316155, + "family": "logit_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.16371700228003758, + "reason": null, + "risk_mse": 0.0006911434452923336, + "rows": 1495, + "seed": 20261025 + }, + { + "auroc": 0.8183771308840109, + "auroc_delta": -0.0032207616712278675, + "available": true, + "brier": 0.16676653804641975, + "brier_gain": 0.007305570266015804, + "clusters": 120, + "events": 569, + "expected_brier": 0.1670117183752092, + "expected_gain": 0.004531525041411751, + "expected_recovery": 0.5790167926502248, + "family": "logit_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.16371700228003758, + "reason": null, + "risk_mse": 0.0032947160951716176, + "rows": 1495, + "seed": 20261025 + }, + { + "auroc": 0.8475994747262844, + "auroc_delta": 0.0, + "available": true, + "brier": 0.14113753386990624, + "brier_gain": 0.0, + "clusters": 120, + "events": 489, + "expected_brier": 0.14124079487068925, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "probability_linear", + "method": "raw", + "oracle_expected_brier": 0.13973682611542312, + "reason": null, + "risk_mse": 0.0015039687552661199, + "rows": 1495, + "seed": 20261025 + }, + { + "auroc": 0.8447332365723857, + "auroc_delta": -0.002866238153898637, + "available": true, + "brier": 0.14188332681782653, + "brier_gain": -0.0007457929479202929, + "clusters": 120, + "events": 489, + "expected_brier": 0.14310639723243618, + "expected_gain": -0.00186560236174692, + "expected_recovery": -1.2404528719194108, + "family": "probability_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.13973682611542312, + "reason": null, + "risk_mse": 0.00336957111701304, + "rows": 1495, + "seed": 20261025 + }, + { + "auroc": 0.8447332365723857, + "auroc_delta": -0.002866238153898637, + "available": true, + "brier": 0.14188332681782653, + "brier_gain": -0.0007457929479202929, + "clusters": 120, + "events": 489, + "expected_brier": 0.14310639723243618, + "expected_gain": -0.001865602361746927, + "expected_recovery": -1.2404528719194157, + "family": "probability_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.13973682611542312, + "reason": null, + "risk_mse": 0.003369571117013047, + "rows": 1495, + "seed": 20261025 + }, + { + "auroc": 0.842915919615233, + "auroc_delta": -0.004683555111051341, + "available": true, + "brier": 0.14028208748997176, + "brier_gain": 0.0008554463799344825, + "clusters": 120, + "events": 489, + "expected_brier": 0.1408117450193994, + "expected_gain": 0.00042904985128985515, + "expected_recovery": 0.2852784340017336, + "family": "probability_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.13973682611542312, + "reason": null, + "risk_mse": 0.0010749189039762647, + "rows": 1495, + "seed": 20261025 + }, + { + "auroc": 0.8030738976401595, + "auroc_delta": 0.0, + "available": true, + "brier": 0.1546066274139506, + "brier_gain": 0.0, + "clusters": 120, + "events": 376, + "expected_brier": 0.1668700073105993, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "logit_linear", + "method": "raw", + "oracle_expected_brier": 0.15876063494271395, + "reason": null, + "risk_mse": 0.008109372367885352, + "rows": 1439, + "seed": 20261026 + }, + { + "auroc": 0.7968990812834011, + "auroc_delta": -0.006174816356758317, + "available": true, + "brier": 0.1526914661434865, + "brier_gain": 0.0019151612704640941, + "clusters": 120, + "events": 376, + "expected_brier": 0.16304479061913, + "expected_gain": 0.003825216691469303, + "expected_recovery": 0.4717031747879632, + "family": "logit_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.15876063494271395, + "reason": null, + "risk_mse": 0.0042841556764160495, + "rows": 1439, + "seed": 20261026 + }, + { + "auroc": 0.7968990812834011, + "auroc_delta": -0.006174816356758317, + "available": true, + "brier": 0.15269146614348647, + "brier_gain": 0.0019151612704641219, + "clusters": 120, + "events": 376, + "expected_brier": 0.16304479061913, + "expected_gain": 0.0038252166914693065, + "expected_recovery": 0.47170317478796353, + "family": "logit_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.15876063494271395, + "reason": null, + "risk_mse": 0.004284155676416046, + "rows": 1439, + "seed": 20261026 + }, + { + "auroc": 0.7913497528071896, + "auroc_delta": -0.011724144832969907, + "available": true, + "brier": 0.15580253596308638, + "brier_gain": -0.001195908549135788, + "clusters": 120, + "events": 376, + "expected_brier": 0.16690874714410203, + "expected_gain": -3.873983350273849e-05, + "expected_recovery": -0.004777167916983904, + "family": "logit_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.15876063494271395, + "reason": null, + "risk_mse": 0.008148112201388091, + "rows": 1439, + "seed": 20261026 + }, + { + "auroc": 0.8167298802892023, + "auroc_delta": 0.0, + "available": true, + "brier": 0.12447071576425298, + "brier_gain": 0.0, + "clusters": 120, + "events": 295, + "expected_brier": 0.13603330101662875, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "probability_linear", + "method": "raw", + "oracle_expected_brier": 0.13439118325346794, + "reason": null, + "risk_mse": 0.0016421177631608071, + "rows": 1439, + "seed": 20261026 + }, + { + "auroc": 0.8114140097190944, + "auroc_delta": -0.005315870570107961, + "available": true, + "brier": 0.12363905785284017, + "brier_gain": 0.0008316579114128059, + "clusters": 120, + "events": 295, + "expected_brier": 0.14103323836412993, + "expected_gain": -0.004999937347501168, + "expected_recovery": -3.0448104634573285, + "family": "probability_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.13439118325346794, + "reason": null, + "risk_mse": 0.006642055110661975, + "rows": 1439, + "seed": 20261026 + }, + { + "auroc": 0.8114140097190944, + "auroc_delta": -0.005315870570107961, + "available": true, + "brier": 0.12363905785284016, + "brier_gain": 0.0008316579114128198, + "clusters": 120, + "events": 295, + "expected_brier": 0.1410332383641299, + "expected_gain": -0.0049999373475011506, + "expected_recovery": -3.044810463457318, + "family": "probability_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.13439118325346794, + "reason": null, + "risk_mse": 0.006642055110661958, + "rows": 1439, + "seed": 20261026 + }, + { + "auroc": 0.8060329501007466, + "auroc_delta": -0.010696930188455744, + "available": true, + "brier": 0.12201998576962196, + "brier_gain": 0.002450729994631015, + "clusters": 120, + "events": 295, + "expected_brier": 0.13879365860470272, + "expected_gain": -0.002760357588073985, + "expected_recovery": -1.6809741968570817, + "family": "probability_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.13439118325346794, + "reason": null, + "risk_mse": 0.004402475351234792, + "rows": 1439, + "seed": 20261026 + }, + { + "auroc": 0.8114974173906033, + "auroc_delta": 0.0, + "available": true, + "brier": 0.17960615968882734, + "brier_gain": 0.0, + "clusters": 120, + "events": 543, + "expected_brier": 0.17669469840191082, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "logit_linear", + "method": "raw", + "oracle_expected_brier": 0.1687889958472952, + "reason": null, + "risk_mse": 0.00790570255461563, + "rows": 1434, + "seed": 20261027 + }, + { + "auroc": 0.8090522577938168, + "auroc_delta": -0.0024451595967864836, + "available": true, + "brier": 0.1717018632773567, + "brier_gain": 0.00790429641147064, + "clusters": 120, + "events": 543, + "expected_brier": 0.1690212238552712, + "expected_gain": 0.00767347454663961, + "expected_recovery": 0.9706252535594782, + "family": "logit_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.1687889958472952, + "reason": null, + "risk_mse": 0.0002322280079760198, + "rows": 1434, + "seed": 20261027 + }, + { + "auroc": 0.8090522577938168, + "auroc_delta": -0.0024451595967864836, + "available": true, + "brier": 0.17170186327735662, + "brier_gain": 0.007904296411470724, + "clusters": 120, + "events": 543, + "expected_brier": 0.1690212238552712, + "expected_gain": 0.007673474546639624, + "expected_recovery": 0.9706252535594798, + "family": "logit_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.1687889958472952, + "reason": null, + "risk_mse": 0.0002322280079760065, + "rows": 1434, + "seed": 20261027 + }, + { + "auroc": 0.8046993363138235, + "auroc_delta": -0.006798081076779816, + "available": true, + "brier": 0.17652392302243888, + "brier_gain": 0.0030822366663884604, + "clusters": 120, + "events": 543, + "expected_brier": 0.17195390686382833, + "expected_gain": 0.0047407915380825085, + "expected_recovery": 0.5996673294158614, + "family": "logit_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.1687889958472952, + "reason": null, + "risk_mse": 0.0031649110165331215, + "rows": 1434, + "seed": 20261027 + }, + { + "auroc": 0.8450292665081397, + "auroc_delta": 0.0, + "available": true, + "brier": 0.13890666060678486, + "brier_gain": 0.0, + "clusters": 120, + "events": 440, + "expected_brier": 0.14530757863703056, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "probability_linear", + "method": "raw", + "oracle_expected_brier": 0.14391178505006047, + "reason": null, + "risk_mse": 0.0013957935869700804, + "rows": 1434, + "seed": 20261027 + }, + { + "auroc": 0.8435156392902871, + "auroc_delta": -0.0015136272178526422, + "available": true, + "brier": 0.14012962915760457, + "brier_gain": -0.0012229685508197141, + "clusters": 120, + "events": 440, + "expected_brier": 0.14601788485957082, + "expected_gain": -0.00071030622254026, + "expected_recovery": -0.5088905903932097, + "family": "probability_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.14391178505006047, + "reason": null, + "risk_mse": 0.0021060998095103404, + "rows": 1434, + "seed": 20261027 + }, + { + "auroc": 0.8435156392902871, + "auroc_delta": -0.0015136272178526422, + "available": true, + "brier": 0.14012962915760457, + "brier_gain": -0.0012229685508197141, + "clusters": 120, + "events": 440, + "expected_brier": 0.14601788485957082, + "expected_gain": -0.0007103062225402557, + "expected_recovery": -0.5088905903932066, + "family": "probability_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.14391178505006047, + "reason": null, + "risk_mse": 0.002106099809510336, + "rows": 1434, + "seed": 20261027 + }, + { + "auroc": 0.8404037863544906, + "auroc_delta": -0.004625480153649053, + "available": true, + "brier": 0.13905559613725615, + "brier_gain": -0.0001489355304712947, + "clusters": 120, + "events": 440, + "expected_brier": 0.14434262148056753, + "expected_gain": 0.0009649571564630084, + "expected_recovery": 0.691332275395884, + "family": "probability_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.14391178505006047, + "reason": null, + "risk_mse": 0.000430836430507072, + "rows": 1434, + "seed": 20261027 + }, + { + "auroc": 0.8369447708578143, + "auroc_delta": 0.0, + "available": true, + "brier": 0.15115390857840855, + "brier_gain": 0.0, + "clusters": 120, + "events": 437, + "expected_brier": 0.17195862569962114, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "logit_linear", + "method": "raw", + "oracle_expected_brier": 0.1641256409461181, + "reason": null, + "risk_mse": 0.007832984753503046, + "rows": 1362, + "seed": 20261028 + }, + { + "auroc": 0.8314997835363969, + "auroc_delta": -0.005444987321417405, + "available": true, + "brier": 0.1513164039532636, + "brier_gain": -0.0001624953748550484, + "clusters": 120, + "events": 437, + "expected_brier": 0.16611291230992886, + "expected_gain": 0.00584571338969226, + "expected_recovery": 0.746294493561724, + "family": "logit_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.1641256409461181, + "reason": null, + "risk_mse": 0.0019872713638107854, + "rows": 1362, + "seed": 20261028 + }, + { + "auroc": 0.8314997835363969, + "auroc_delta": -0.005444987321417405, + "available": true, + "brier": 0.1513164039532636, + "brier_gain": -0.0001624953748550484, + "clusters": 120, + "events": 437, + "expected_brier": 0.1661129123099289, + "expected_gain": 0.0058457133896922565, + "expected_recovery": 0.7462944935617235, + "family": "logit_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.1641256409461181, + "reason": null, + "risk_mse": 0.001987271363810789, + "rows": 1362, + "seed": 20261028 + }, + { + "auroc": 0.8280487352340898, + "auroc_delta": -0.00889603562372443, + "available": true, + "brier": 0.1541553226715072, + "brier_gain": -0.0030014140930986477, + "clusters": 120, + "events": 437, + "expected_brier": 0.17011312572765896, + "expected_gain": 0.0018454999719621692, + "expected_recovery": 0.23560622547322463, + "family": "logit_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.1641256409461181, + "reason": null, + "risk_mse": 0.005987484781540877, + "rows": 1362, + "seed": 20261028 + }, + { + "auroc": 0.8318479996547409, + "auroc_delta": 0.0, + "available": true, + "brier": 0.134550486264179, + "brier_gain": 0.0, + "clusters": 120, + "events": 376, + "expected_brier": 0.1421359949352637, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "probability_linear", + "method": "raw", + "oracle_expected_brier": 0.14061862960859675, + "reason": null, + "risk_mse": 0.001517365326666941, + "rows": 1362, + "seed": 20261028 + }, + { + "auroc": 0.824427625048552, + "auroc_delta": -0.007420374606188829, + "available": true, + "brier": 0.13793526928538702, + "brier_gain": -0.00338478302120801, + "clusters": 120, + "events": 376, + "expected_brier": 0.14397118377547635, + "expected_gain": -0.001835188840212648, + "expected_recovery": -1.2094574773524323, + "family": "probability_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.14061862960859675, + "reason": null, + "risk_mse": 0.003352554166879589, + "rows": 1362, + "seed": 20261028 + }, + { + "auroc": 0.824427625048552, + "auroc_delta": -0.007420374606188829, + "available": true, + "brier": 0.13793526928538696, + "brier_gain": -0.0033847830212079544, + "clusters": 120, + "events": 376, + "expected_brier": 0.14397118377547632, + "expected_gain": -0.001835188840212644, + "expected_recovery": -1.2094574773524296, + "family": "probability_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.14061862960859675, + "reason": null, + "risk_mse": 0.003352554166879585, + "rows": 1362, + "seed": 20261028 + }, + { + "auroc": 0.8204976047645766, + "auroc_delta": -0.011350394890164317, + "available": true, + "brier": 0.13554525423386143, + "brier_gain": -0.0009947679696824252, + "clusters": 120, + "events": 376, + "expected_brier": 0.14168151373091764, + "expected_gain": 0.0004544812043460612, + "expected_recovery": 0.2995199615799703, + "family": "probability_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.14061862960859675, + "reason": null, + "risk_mse": 0.0010628841223208798, + "rows": 1362, + "seed": 20261028 + }, + { + "auroc": 0.8171822496731913, + "auroc_delta": 0.0, + "available": true, + "brier": 0.17106500461507412, + "brier_gain": 0.0, + "clusters": 120, + "events": 532, + "expected_brier": 0.17609276286445963, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "logit_linear", + "method": "raw", + "oracle_expected_brier": 0.16803294409659267, + "reason": null, + "risk_mse": 0.008059818767866957, + "rows": 1389, + "seed": 20261029 + }, + { + "auroc": 0.8152170098525193, + "auroc_delta": -0.0019652398206719823, + "available": true, + "brier": 0.16582742814913554, + "brier_gain": 0.005237576465938587, + "clusters": 120, + "events": 532, + "expected_brier": 0.168201289616894, + "expected_gain": 0.007891473247565617, + "expected_recovery": 0.9791129893673907, + "family": "logit_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.16803294409659267, + "reason": null, + "risk_mse": 0.00016834552030134116, + "rows": 1389, + "seed": 20261029 + }, + { + "auroc": 0.8152170098525193, + "auroc_delta": -0.0019652398206719823, + "available": true, + "brier": 0.16582742814913554, + "brier_gain": 0.005237576465938587, + "clusters": 120, + "events": 532, + "expected_brier": 0.168201289616894, + "expected_gain": 0.007891473247565619, + "expected_recovery": 0.9791129893673909, + "family": "logit_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.16803294409659267, + "reason": null, + "risk_mse": 0.0001683455203013392, + "rows": 1389, + "seed": 20261029 + }, + { + "auroc": 0.815304743773085, + "auroc_delta": -0.0018775059001062688, + "available": true, + "brier": 0.16663707705971004, + "brier_gain": 0.004427927555364086, + "clusters": 120, + "events": 532, + "expected_brier": 0.17106224444343604, + "expected_gain": 0.005030518421023573, + "expected_recovery": 0.6241478333333428, + "family": "logit_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.16803294409659267, + "reason": null, + "risk_mse": 0.003029300346843384, + "rows": 1389, + "seed": 20261029 + }, + { + "auroc": 0.8316174446864986, + "auroc_delta": 0.0, + "available": true, + "brier": 0.14255645847029708, + "brier_gain": 0.0, + "clusters": 120, + "events": 463, + "expected_brier": 0.14294972753985233, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "probability_linear", + "method": "raw", + "oracle_expected_brier": 0.14157529884154518, + "reason": null, + "risk_mse": 0.0013744286983071573, + "rows": 1389, + "seed": 20261029 + }, + { + "auroc": 0.8285223143271647, + "auroc_delta": -0.003095130359333842, + "available": true, + "brier": 0.14642408619754843, + "brier_gain": -0.003867627727251355, + "clusters": 120, + "events": 463, + "expected_brier": 0.14496523407320233, + "expected_gain": -0.002015506533349979, + "expected_recovery": -1.4664322244088894, + "family": "probability_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.14157529884154518, + "reason": null, + "risk_mse": 0.0033899352316571364, + "rows": 1389, + "seed": 20261029 + }, + { + "auroc": 0.8285223143271647, + "auroc_delta": -0.003095130359333842, + "available": true, + "brier": 0.14642408619754843, + "brier_gain": -0.003867627727251355, + "clusters": 120, + "events": 463, + "expected_brier": 0.1449652340732023, + "expected_gain": -0.002015506533349971, + "expected_recovery": -1.4664322244088837, + "family": "probability_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.14157529884154518, + "reason": null, + "risk_mse": 0.0033899352316571285, + "rows": 1389, + "seed": 20261029 + }, + { + "auroc": 0.8277736053253969, + "auroc_delta": -0.003843839361101664, + "available": true, + "brier": 0.14180229261273747, + "brier_gain": 0.0007541658575596077, + "clusters": 120, + "events": 463, + "expected_brier": 0.14196269824976054, + "expected_gain": 0.00098702929009179, + "expected_recovery": 0.7181378643413691, + "family": "probability_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.14157529884154518, + "reason": null, + "risk_mse": 0.0003873994082153675, + "rows": 1389, + "seed": 20261029 + }, + { + "auroc": 0.7851273109337625, + "auroc_delta": 0.0, + "available": true, + "brier": 0.1762970204996186, + "brier_gain": 0.0, + "clusters": 120, + "events": 465, + "expected_brier": 0.17544542410147537, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "logit_linear", + "method": "raw", + "oracle_expected_brier": 0.1672404036398301, + "reason": null, + "risk_mse": 0.008205020461645285, + "rows": 1427, + "seed": 20261030 + }, + { + "auroc": 0.7774931258802227, + "auroc_delta": -0.007634185053539855, + "available": true, + "brier": 0.17232120501660714, + "brier_gain": 0.003975815483011458, + "clusters": 120, + "events": 465, + "expected_brier": 0.1679465391368428, + "expected_gain": 0.007498884964632581, + "expected_recovery": 0.91393860620902, + "family": "logit_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.1672404036398301, + "reason": null, + "risk_mse": 0.0007061354970127038, + "rows": 1427, + "seed": 20261030 + }, + { + "auroc": 0.7774931258802227, + "auroc_delta": -0.007634185053539855, + "available": true, + "brier": 0.17232120501660717, + "brier_gain": 0.00397581548301143, + "clusters": 120, + "events": 465, + "expected_brier": 0.1679465391368428, + "expected_gain": 0.007498884964632572, + "expected_recovery": 0.9139386062090188, + "family": "logit_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.1672404036398301, + "reason": null, + "risk_mse": 0.0007061354970127134, + "rows": 1427, + "seed": 20261030 + }, + { + "auroc": 0.7694923211052244, + "auroc_delta": -0.01563498982853817, + "available": true, + "brier": 0.17407984245534294, + "brier_gain": 0.002217178044275653, + "clusters": 120, + "events": 465, + "expected_brier": 0.17095373150907828, + "expected_gain": 0.004491692592397096, + "expected_recovery": 0.5474322231606494, + "family": "logit_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.1672404036398301, + "reason": null, + "risk_mse": 0.003713327869248189, + "rows": 1427, + "seed": 20261030 + }, + { + "auroc": 0.7845479917485741, + "auroc_delta": 0.0, + "available": true, + "brier": 0.15427472405979165, + "brier_gain": 0.0, + "clusters": 120, + "events": 402, + "expected_brier": 0.14399980623145014, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "probability_linear", + "method": "raw", + "oracle_expected_brier": 0.14250065214551882, + "reason": null, + "risk_mse": 0.0014991540859313113, + "rows": 1427, + "seed": 20261030 + }, + { + "auroc": 0.7736464021356632, + "auroc_delta": -0.010901589612910989, + "available": true, + "brier": 0.1581833580990634, + "brier_gain": -0.003908634039271747, + "clusters": 120, + "events": 402, + "expected_brier": 0.14720873470651974, + "expected_gain": -0.0032089284750695954, + "expected_recovery": -2.1404927653424837, + "family": "probability_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.14250065214551882, + "reason": null, + "risk_mse": 0.004708082561000907, + "rows": 1427, + "seed": 20261030 + }, + { + "auroc": 0.7736464021356632, + "auroc_delta": -0.010901589612910989, + "available": true, + "brier": 0.1581833580990634, + "brier_gain": -0.003908634039271747, + "clusters": 120, + "events": 402, + "expected_brier": 0.14720873470651974, + "expected_gain": -0.0032089284750695954, + "expected_recovery": -2.1404927653424837, + "family": "probability_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.14250065214551882, + "reason": null, + "risk_mse": 0.004708082561000907, + "rows": 1427, + "seed": 20261030 + }, + { + "auroc": 0.7671326295352506, + "auroc_delta": -0.017415362213323515, + "available": true, + "brier": 0.1545749007352115, + "brier_gain": -0.0003001766754198443, + "clusters": 120, + "events": 402, + "expected_brier": 0.14314953963917187, + "expected_gain": 0.0008502665922782732, + "expected_recovery": 0.567164242993786, + "family": "probability_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.14250065214551882, + "reason": null, + "risk_mse": 0.000648887493653038, + "rows": 1427, + "seed": 20261030 + }, + { + "auroc": 0.8526717557251908, + "auroc_delta": 0.0, + "available": true, + "brier": 0.16141963329581652, + "brier_gain": 0.0, + "clusters": 120, + "events": 524, + "expected_brier": 0.16579070540068108, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "logit_linear", + "method": "raw", + "oracle_expected_brier": 0.15830125412026977, + "reason": null, + "risk_mse": 0.007489451280411311, + "rows": 1394, + "seed": 20261031 + }, + { + "auroc": 0.8515091690795824, + "auroc_delta": -0.001162586645608421, + "available": true, + "brier": 0.1527128401147837, + "brier_gain": 0.008706793181032818, + "clusters": 120, + "events": 524, + "expected_brier": 0.15882860244385408, + "expected_gain": 0.006962102956826999, + "expected_recovery": 0.929587855793442, + "family": "logit_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.15830125412026977, + "reason": null, + "risk_mse": 0.0005273483235843117, + "rows": 1394, + "seed": 20261031 + }, + { + "auroc": 0.8515091690795824, + "auroc_delta": -0.001162586645608421, + "available": true, + "brier": 0.15271284011478367, + "brier_gain": 0.008706793181032846, + "clusters": 120, + "events": 524, + "expected_brier": 0.15882860244385408, + "expected_gain": 0.006962102956826994, + "expected_recovery": 0.9295878557934413, + "family": "logit_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.15830125412026977, + "reason": null, + "risk_mse": 0.000527348323584317, + "rows": 1394, + "seed": 20261031 + }, + { + "auroc": 0.8503070983592174, + "auroc_delta": -0.002364657365973466, + "available": true, + "brier": 0.158809267867887, + "brier_gain": 0.0026103654279295285, + "clusters": 120, + "events": 524, + "expected_brier": 0.16170596972828938, + "expected_gain": 0.004084735672391701, + "expected_recovery": 0.5453985237977772, + "family": "logit_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.15830125412026977, + "reason": null, + "risk_mse": 0.0034047156080196095, + "rows": 1394, + "seed": 20261031 + }, + { + "auroc": 0.865500415282392, + "auroc_delta": 0.0, + "available": true, + "brier": 0.13102294943217255, + "brier_gain": 0.0, + "clusters": 120, + "events": 448, + "expected_brier": 0.137850007746218, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "probability_linear", + "method": "raw", + "oracle_expected_brier": 0.13625553958671963, + "reason": null, + "risk_mse": 0.0015944681594983726, + "rows": 1394, + "seed": 20261031 + }, + { + "auroc": 0.8647429968287526, + "auroc_delta": -0.0007574184536394135, + "available": true, + "brier": 0.1315420982319079, + "brier_gain": -0.0005191487997353583, + "clusters": 120, + "events": 448, + "expected_brier": 0.13834500241337444, + "expected_gain": -0.0004949946671564318, + "expected_recovery": -0.3104449996117573, + "family": "probability_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.13625553958671963, + "reason": null, + "risk_mse": 0.0020894628266548043, + "rows": 1394, + "seed": 20261031 + }, + { + "auroc": 0.8647429968287526, + "auroc_delta": -0.0007574184536394135, + "available": true, + "brier": 0.13154209823190793, + "brier_gain": -0.0005191487997353861, + "clusters": 120, + "events": 448, + "expected_brier": 0.13834500241337444, + "expected_gain": -0.0004949946671564283, + "expected_recovery": -0.3104449996117553, + "family": "probability_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.13625553958671963, + "reason": null, + "risk_mse": 0.002089462826654801, + "rows": 1394, + "seed": 20261031 + }, + { + "auroc": 0.8643890629719116, + "auroc_delta": -0.001111352310480318, + "available": true, + "brier": 0.1316898955232211, + "brier_gain": -0.0006669460910485592, + "clusters": 120, + "events": 448, + "expected_brier": 0.13650618771838421, + "expected_gain": 0.0013438200278337938, + "expected_recovery": 0.842801419287398, + "family": "probability_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.13625553958671963, + "reason": null, + "risk_mse": 0.00025064813166457883, + "rows": 1394, + "seed": 20261031 + }, + { + "auroc": 0.8041031075186673, + "auroc_delta": 0.0, + "available": true, + "brier": 0.1528738643980696, + "brier_gain": 0.0, + "clusters": 120, + "events": 385, + "expected_brier": 0.16393136504228778, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "logit_linear", + "method": "raw", + "oracle_expected_brier": 0.1555046858754299, + "reason": null, + "risk_mse": 0.008426679166857869, + "rows": 1439, + "seed": 20261032 + }, + { + "auroc": 0.801710244214988, + "auroc_delta": -0.0023928633036792712, + "available": true, + "brier": 0.1496718906717742, + "brier_gain": 0.0032019737262954096, + "clusters": 120, + "events": 385, + "expected_brier": 0.15669797229133842, + "expected_gain": 0.00723339275094936, + "expected_recovery": 0.8583918537445089, + "family": "logit_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.1555046858754299, + "reason": null, + "risk_mse": 0.0011932864159085088, + "rows": 1439, + "seed": 20261032 + }, + { + "auroc": 0.801710244214988, + "auroc_delta": -0.0023928633036792712, + "available": true, + "brier": 0.1496718906717742, + "brier_gain": 0.0032019737262954096, + "clusters": 120, + "events": 385, + "expected_brier": 0.1566979722913384, + "expected_gain": 0.007233392750949372, + "expected_recovery": 0.8583918537445103, + "family": "logit_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.1555046858754299, + "reason": null, + "risk_mse": 0.0011932864159084964, + "rows": 1439, + "seed": 20261032 + }, + { + "auroc": 0.8009290519726953, + "auroc_delta": -0.0031740555459720543, + "available": true, + "brier": 0.15237104445403205, + "brier_gain": 0.000502819944037558, + "clusters": 120, + "events": 385, + "expected_brier": 0.16026773388775187, + "expected_gain": 0.0036636311545359132, + "expected_recovery": 0.43476571042896417, + "family": "logit_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.1555046858754299, + "reason": null, + "risk_mse": 0.004763048012321956, + "rows": 1439, + "seed": 20261032 + }, + { + "auroc": 0.8109507194818819, + "auroc_delta": 0.0, + "available": true, + "brier": 0.11970621001886332, + "brier_gain": 0.0, + "clusters": 120, + "events": 311, + "expected_brier": 0.13145194876579908, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "probability_linear", + "method": "raw", + "oracle_expected_brier": 0.12972553676727858, + "reason": null, + "risk_mse": 0.001726411998520489, + "rows": 1439, + "seed": 20261032 + }, + { + "auroc": 0.8063271077056395, + "auroc_delta": -0.004623611776242376, + "available": true, + "brier": 0.12287242927138385, + "brier_gain": -0.00316621925252053, + "clusters": 120, + "events": 311, + "expected_brier": 0.13286910222825576, + "expected_gain": -0.001417153462456702, + "expected_recovery": -0.8208663190890604, + "family": "probability_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.12972553676727858, + "reason": null, + "risk_mse": 0.003143565460977191, + "rows": 1439, + "seed": 20261032 + }, + { + "auroc": 0.8063271077056395, + "auroc_delta": -0.004623611776242376, + "available": true, + "brier": 0.12287242927138381, + "brier_gain": -0.0031662192525204885, + "clusters": 120, + "events": 311, + "expected_brier": 0.13286910222825576, + "expected_gain": -0.0014171534624566973, + "expected_recovery": -0.8208663190890575, + "family": "probability_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.12972553676727858, + "reason": null, + "risk_mse": 0.003143565460977186, + "rows": 1439, + "seed": 20261032 + }, + { + "auroc": 0.803676084923947, + "auroc_delta": -0.007274634557934867, + "available": true, + "brier": 0.11994467511678716, + "brier_gain": -0.00023846509792384285, + "clusters": 120, + "events": 311, + "expected_brier": 0.13039802072035325, + "expected_gain": 0.001053928045445811, + "expected_recovery": 0.6104730773123754, + "family": "probability_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.12972553676727858, + "reason": null, + "risk_mse": 0.000672483953074678, + "rows": 1439, + "seed": 20261032 + }, + { + "auroc": 0.7968053647798364, + "auroc_delta": 0.0, + "available": true, + "brier": 0.1903512702919596, + "brier_gain": 0.0, + "clusters": 120, + "events": 569, + "expected_brier": 0.1824360283743134, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "logit_linear", + "method": "raw", + "oracle_expected_brier": 0.1742876411581982, + "reason": null, + "risk_mse": 0.008148387216115207, + "rows": 1448, + "seed": 20261033 + }, + { + "auroc": 0.7940082095207247, + "auroc_delta": -0.0027971552591117277, + "available": true, + "brier": 0.1794621915775474, + "brier_gain": 0.01088907871441222, + "clusters": 120, + "events": 569, + "expected_brier": 0.17463767406871186, + "expected_gain": 0.0077983543056015315, + "expected_recovery": 0.9570426759026117, + "family": "logit_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.1742876411581982, + "reason": null, + "risk_mse": 0.0003500329105136761, + "rows": 1448, + "seed": 20261033 + }, + { + "auroc": 0.7940082095207247, + "auroc_delta": -0.0027971552591117277, + "available": true, + "brier": 0.1794621915775474, + "brier_gain": 0.01088907871441222, + "clusters": 120, + "events": 569, + "expected_brier": 0.1746376740687119, + "expected_gain": 0.007798354305601527, + "expected_recovery": 0.9570426759026113, + "family": "logit_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.1742876411581982, + "reason": null, + "risk_mse": 0.00035003291051368036, + "rows": 1448, + "seed": 20261033 + }, + { + "auroc": 0.7922087529566071, + "auroc_delta": -0.004596611823229346, + "available": true, + "brier": 0.18329486886876653, + "brier_gain": 0.007056401423193082, + "clusters": 120, + "events": 569, + "expected_brier": 0.17686768048844392, + "expected_gain": 0.005568347885869491, + "expected_recovery": 0.6833680995003371, + "family": "logit_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.1742876411581982, + "reason": null, + "risk_mse": 0.0025800393302457163, + "rows": 1448, + "seed": 20261033 + }, + { + "auroc": 0.8510455145484599, + "auroc_delta": 0.0, + "available": true, + "brier": 0.1369836041125989, + "brier_gain": 0.0, + "clusters": 120, + "events": 453, + "expected_brier": 0.14841506399379095, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "probability_linear", + "method": "raw", + "oracle_expected_brier": 0.14714167916576223, + "reason": null, + "risk_mse": 0.0012733848280287177, + "rows": 1448, + "seed": 20261033 + }, + { + "auroc": 0.8458340266453682, + "auroc_delta": -0.005211487903091672, + "available": true, + "brier": 0.13928854339119578, + "brier_gain": -0.0023049392785968736, + "clusters": 120, + "events": 453, + "expected_brier": 0.14878008089214165, + "expected_gain": -0.00036501689835070406, + "expected_recovery": -0.2866508932070235, + "family": "probability_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.14714167916576223, + "reason": null, + "risk_mse": 0.0016384017263794217, + "rows": 1448, + "seed": 20261033 + }, + { + "auroc": 0.8458340266453682, + "auroc_delta": -0.005211487903091672, + "available": true, + "brier": 0.13928854339119578, + "brier_gain": -0.0023049392785968736, + "clusters": 120, + "events": 453, + "expected_brier": 0.14878008089214165, + "expected_gain": -0.00036501689835070385, + "expected_recovery": -0.2866508932070235, + "family": "probability_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.14714167916576223, + "reason": null, + "risk_mse": 0.0016384017263794215, + "rows": 1448, + "seed": 20261033 + }, + { + "auroc": 0.8429920019523668, + "auroc_delta": -0.008053512596093104, + "available": true, + "brier": 0.13818787256301304, + "brier_gain": -0.001204268450414131, + "clusters": 120, + "events": 453, + "expected_brier": 0.14770306991688495, + "expected_gain": 0.0007119940769060018, + "expected_recovery": 0.5591350401184023, + "family": "probability_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.14714167916576223, + "reason": null, + "risk_mse": 0.0005613907511227158, + "rows": 1448, + "seed": 20261033 + }, + { + "auroc": 0.7951976329565283, + "auroc_delta": 0.0, + "available": true, + "brier": 0.19433385877185633, + "brier_gain": 0.0, + "clusters": 120, + "events": 538, + "expected_brier": 0.1698259846277402, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "logit_linear", + "method": "raw", + "oracle_expected_brier": 0.16207689706451592, + "reason": null, + "risk_mse": 0.007749087563224298, + "rows": 1420, + "seed": 20261034 + }, + { + "auroc": 0.7888880459246896, + "auroc_delta": -0.006309587031838704, + "available": true, + "brier": 0.18007137521802935, + "brier_gain": 0.014262483553826977, + "clusters": 120, + "events": 538, + "expected_brier": 0.16347324423654233, + "expected_gain": 0.006352740391197877, + "expected_recovery": 0.8198049563082471, + "family": "logit_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.16207689706451592, + "reason": null, + "risk_mse": 0.0013963471720264211, + "rows": 1420, + "seed": 20261034 + }, + { + "auroc": 0.7888880459246896, + "auroc_delta": -0.006309587031838704, + "available": true, + "brier": 0.18007137521802932, + "brier_gain": 0.014262483553827005, + "clusters": 120, + "events": 538, + "expected_brier": 0.16347324423654236, + "expected_gain": 0.006352740391197859, + "expected_recovery": 0.8198049563082448, + "family": "logit_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.16207689706451592, + "reason": null, + "risk_mse": 0.001396347172026439, + "rows": 1420, + "seed": 20261034 + }, + { + "auroc": 0.7836890642254423, + "auroc_delta": -0.011508568731086077, + "available": true, + "brier": 0.18604124035470598, + "brier_gain": 0.008292618417150344, + "clusters": 120, + "events": 538, + "expected_brier": 0.1662825037798667, + "expected_gain": 0.0035434808478735256, + "expected_recovery": 0.45727717217833674, + "family": "logit_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.16207689706451592, + "reason": null, + "risk_mse": 0.004205606715350772, + "rows": 1420, + "seed": 20261034 + }, + { + "auroc": 0.8214736584262888, + "auroc_delta": 0.0, + "available": true, + "brier": 0.14939958351758506, + "brier_gain": 0.0, + "clusters": 120, + "events": 428, + "expected_brier": 0.14120356257553487, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "probability_linear", + "method": "raw", + "oracle_expected_brier": 0.13960855814065604, + "reason": null, + "risk_mse": 0.0015950044348788373, + "rows": 1420, + "seed": 20261034 + }, + { + "auroc": 0.8171823183599638, + "auroc_delta": -0.004291340066325033, + "available": true, + "brier": 0.1508816779157376, + "brier_gain": -0.001482094398152528, + "clusters": 120, + "events": 428, + "expected_brier": 0.1433435044026988, + "expected_gain": -0.0021399418271639357, + "expected_recovery": -1.3416525875217982, + "family": "probability_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.13960855814065604, + "reason": null, + "risk_mse": 0.0037349462620427727, + "rows": 1420, + "seed": 20261034 + }, + { + "auroc": 0.8171823183599638, + "auroc_delta": -0.004291340066325033, + "available": true, + "brier": 0.1508816779157376, + "brier_gain": -0.001482094398152528, + "clusters": 120, + "events": 428, + "expected_brier": 0.1433435044026988, + "expected_gain": -0.0021399418271639296, + "expected_recovery": -1.3416525875217942, + "family": "probability_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.13960855814065604, + "reason": null, + "risk_mse": 0.0037349462620427667, + "rows": 1420, + "seed": 20261034 + }, + { + "auroc": 0.8147987639433223, + "auroc_delta": -0.006674894482966542, + "available": true, + "brier": 0.1502340938068514, + "brier_gain": -0.0008345102892663436, + "clusters": 120, + "events": 428, + "expected_brier": 0.13994792079990018, + "expected_gain": 0.001255641775634697, + "expected_recovery": 0.7872340340734416, + "family": "probability_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.13960855814065604, + "reason": null, + "risk_mse": 0.0003393626592441403, + "rows": 1420, + "seed": 20261034 + }, + { + "auroc": 0.8344309764309764, + "auroc_delta": 0.0, + "available": true, + "brier": 0.16276500622013998, + "brier_gain": 0.0, + "clusters": 120, + "events": 495, + "expected_brier": 0.17467116379073866, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "logit_linear", + "method": "raw", + "oracle_expected_brier": 0.16661223520551655, + "reason": null, + "risk_mse": 0.008058928585222117, + "rows": 1395, + "seed": 20261035 + }, + { + "auroc": 0.831712682379349, + "auroc_delta": -0.002718294051627379, + "available": true, + "brier": 0.1573149264663205, + "brier_gain": 0.005450079753819476, + "clusters": 120, + "events": 495, + "expected_brier": 0.16724193537839355, + "expected_gain": 0.007429228412345116, + "expected_recovery": 0.9218630409466961, + "family": "logit_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.16661223520551655, + "reason": null, + "risk_mse": 0.0006297001728770009, + "rows": 1395, + "seed": 20261035 + }, + { + "auroc": 0.831712682379349, + "auroc_delta": -0.002718294051627379, + "available": true, + "brier": 0.1573149264663205, + "brier_gain": 0.005450079753819476, + "clusters": 120, + "events": 495, + "expected_brier": 0.16724193537839352, + "expected_gain": 0.007429228412345148, + "expected_recovery": 0.9218630409467001, + "family": "logit_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.16661223520551655, + "reason": null, + "risk_mse": 0.0006297001728769689, + "rows": 1395, + "seed": 20261035 + }, + { + "auroc": 0.8299461279461279, + "auroc_delta": -0.004484848484848536, + "available": true, + "brier": 0.16122503282622702, + "brier_gain": 0.0015399733939129578, + "clusters": 120, + "events": 495, + "expected_brier": 0.17026016529863724, + "expected_gain": 0.004410998492101425, + "expected_recovery": 0.5473430426210745, + "family": "logit_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.16661223520551655, + "reason": null, + "risk_mse": 0.0036479300931206923, + "rows": 1395, + "seed": 20261035 + }, + { + "auroc": 0.8540596756221369, + "auroc_delta": 0.0, + "available": true, + "brier": 0.12837023084131388, + "brier_gain": 0.0, + "clusters": 120, + "events": 410, + "expected_brier": 0.14174609342304997, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "probability_linear", + "method": "raw", + "oracle_expected_brier": 0.14033102356854707, + "reason": null, + "risk_mse": 0.0014150698545028963, + "rows": 1395, + "seed": 20261035 + }, + { + "auroc": 0.8515983657298501, + "auroc_delta": -0.002461309892286767, + "available": true, + "brier": 0.12995193254944318, + "brier_gain": -0.0015817017081292961, + "clusters": 120, + "events": 410, + "expected_brier": 0.14191527645965626, + "expected_gain": -0.00016918303660629487, + "expected_recovery": -0.11955808122682932, + "family": "probability_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.14033102356854707, + "reason": null, + "risk_mse": 0.0015842528911091912, + "rows": 1395, + "seed": 20261035 + }, + { + "auroc": 0.8515983657298501, + "auroc_delta": -0.002461309892286767, + "available": true, + "brier": 0.12995193254944312, + "brier_gain": -0.0015817017081292406, + "clusters": 120, + "events": 410, + "expected_brier": 0.14191527645965626, + "expected_gain": -0.0001691830366062888, + "expected_recovery": -0.1195580812268251, + "family": "probability_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.14033102356854707, + "reason": null, + "risk_mse": 0.0015842528911091851, + "rows": 1395, + "seed": 20261035 + }, + { + "auroc": 0.8487730593041972, + "auroc_delta": -0.005286616317939696, + "available": true, + "brier": 0.12848959579259572, + "brier_gain": -0.00011936495128184177, + "clusters": 120, + "events": 410, + "expected_brier": 0.14093743812334364, + "expected_gain": 0.0008086552997063163, + "expected_recovery": 0.5714596329877941, + "family": "probability_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.14033102356854707, + "reason": null, + "risk_mse": 0.00060641455479658, + "rows": 1395, + "seed": 20261035 + }, + { + "auroc": 0.8020964600756101, + "auroc_delta": 0.0, + "available": true, + "brier": 0.15680895201474182, + "brier_gain": 0.0, + "clusters": 120, + "events": 430, + "expected_brier": 0.16853061290497334, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "logit_linear", + "method": "raw", + "oracle_expected_brier": 0.1606919513833565, + "reason": null, + "risk_mse": 0.007838661521616835, + "rows": 1445, + "seed": 20261036 + }, + { + "auroc": 0.7952709359605912, + "auroc_delta": -0.0068255241150189105, + "available": true, + "brier": 0.15678553492640884, + "brier_gain": 2.341708833297762e-05, + "clusters": 120, + "events": 430, + "expected_brier": 0.16227017284837023, + "expected_gain": 0.006260440056603132, + "expected_recovery": 0.7986618683991635, + "family": "logit_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.1606919513833565, + "reason": null, + "risk_mse": 0.0015782214650137034, + "rows": 1445, + "seed": 20261036 + }, + { + "auroc": 0.7952709359605912, + "auroc_delta": -0.0068255241150189105, + "available": true, + "brier": 0.15678553492640884, + "brier_gain": 2.341708833297762e-05, + "clusters": 120, + "events": 430, + "expected_brier": 0.1622701728483702, + "expected_gain": 0.006260440056603139, + "expected_recovery": 0.7986618683991644, + "family": "logit_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.1606919513833565, + "reason": null, + "risk_mse": 0.0015782214650136963, + "rows": 1445, + "seed": 20261036 + }, + { + "auroc": 0.7868049031962424, + "auroc_delta": -0.015291556879367674, + "available": true, + "brier": 0.1568789360070414, + "brier_gain": -6.998399229957153e-05, + "clusters": 120, + "events": 430, + "expected_brier": 0.16563236950244234, + "expected_gain": 0.002898243402531017, + "expected_recovery": 0.3697370264730111, + "family": "logit_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.1606919513833565, + "reason": null, + "risk_mse": 0.004940418119085818, + "rows": 1445, + "seed": 20261036 + }, + { + "auroc": 0.7981585338698235, + "auroc_delta": 0.0, + "available": true, + "brier": 0.13579580988000245, + "brier_gain": 0.0, + "clusters": 120, + "events": 354, + "expected_brier": 0.1388168003123125, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "probability_linear", + "method": "raw", + "oracle_expected_brier": 0.13718559603332492, + "reason": null, + "risk_mse": 0.0016312042789875907, + "rows": 1445, + "seed": 20261036 + }, + { + "auroc": 0.7890081664569383, + "auroc_delta": -0.009150367412885219, + "available": true, + "brier": 0.13854284264072655, + "brier_gain": -0.002747032760724105, + "clusters": 120, + "events": 354, + "expected_brier": 0.1421656963200305, + "expected_gain": -0.003348896007718008, + "expected_recovery": -2.0530206123518173, + "family": "probability_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.13718559603332492, + "reason": null, + "risk_mse": 0.004980100286705599, + "rows": 1445, + "seed": 20261036 + }, + { + "auroc": 0.7890081664569383, + "auroc_delta": -0.009150367412885219, + "available": true, + "brier": 0.13854284264072655, + "brier_gain": -0.002747032760724105, + "clusters": 120, + "events": 354, + "expected_brier": 0.1421656963200305, + "expected_gain": -0.0033488960077180064, + "expected_recovery": -2.0530206123518164, + "family": "probability_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.13718559603332492, + "reason": null, + "risk_mse": 0.004980100286705597, + "rows": 1445, + "seed": 20261036 + }, + { + "auroc": 0.7806268027570207, + "auroc_delta": -0.017531731112802817, + "available": true, + "brier": 0.13479291777209781, + "brier_gain": 0.001002892107904635, + "clusters": 120, + "events": 354, + "expected_brier": 0.13878426061196625, + "expected_gain": 3.253970034627061e-05, + "expected_recovery": 0.019948268138719194, + "family": "probability_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.13718559603332492, + "reason": null, + "risk_mse": 0.00159866457864132, + "rows": 1445, + "seed": 20261036 + }, + { + "auroc": 0.8139514348785872, + "auroc_delta": 0.0, + "available": true, + "brier": 0.17225504979552506, + "brier_gain": 0.0, + "clusters": 120, + "events": 525, + "expected_brier": 0.17267340539451104, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "logit_linear", + "method": "raw", + "oracle_expected_brier": 0.1650529978642677, + "reason": null, + "risk_mse": 0.007620407530243349, + "rows": 1431, + "seed": 20261037 + }, + { + "auroc": 0.8069610007358353, + "auroc_delta": -0.006990434142751911, + "available": true, + "brier": 0.16822510182326061, + "brier_gain": 0.004029947972264442, + "clusters": 120, + "events": 525, + "expected_brier": 0.1653769865802766, + "expected_gain": 0.007296418814234422, + "expected_recovery": 0.9574840696218538, + "family": "logit_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.1650529978642677, + "reason": null, + "risk_mse": 0.00032398871600892716, + "rows": 1431, + "seed": 20261037 + }, + { + "auroc": 0.8069610007358353, + "auroc_delta": -0.006990434142751911, + "available": true, + "brier": 0.16822510182326061, + "brier_gain": 0.004029947972264442, + "clusters": 120, + "events": 525, + "expected_brier": 0.1653769865802766, + "expected_gain": 0.007296418814234419, + "expected_recovery": 0.9574840696218535, + "family": "logit_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.1650529978642677, + "reason": null, + "risk_mse": 0.0003239887160089296, + "rows": 1431, + "seed": 20261037 + }, + { + "auroc": 0.8021023862083465, + "auroc_delta": -0.011849048670240725, + "available": true, + "brier": 0.17106590327667873, + "brier_gain": 0.001189146518846329, + "clusters": 120, + "events": 525, + "expected_brier": 0.16847659343346222, + "expected_gain": 0.004196811961048833, + "expected_recovery": 0.5507332704179947, + "family": "logit_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.1650529978642677, + "reason": null, + "risk_mse": 0.003423595569194516, + "rows": 1431, + "seed": 20261037 + }, + { + "auroc": 0.8283856318025252, + "auroc_delta": 0.0, + "available": true, + "brier": 0.144960399008581, + "brier_gain": 0.0, + "clusters": 120, + "events": 453, + "expected_brier": 0.144062971440339, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "probability_linear", + "method": "raw", + "oracle_expected_brier": 0.14257933898331515, + "reason": null, + "risk_mse": 0.0014836324570238477, + "rows": 1431, + "seed": 20261037 + }, + { + "auroc": 0.8204043030557475, + "auroc_delta": -0.007981328746777705, + "available": true, + "brier": 0.1487182690105217, + "brier_gain": -0.003757870001940722, + "clusters": 120, + "events": 453, + "expected_brier": 0.1457965217189626, + "expected_gain": -0.0017335502786235918, + "expected_recovery": -1.1684499556588812, + "family": "probability_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.14257933898331515, + "reason": null, + "risk_mse": 0.0032171827356474395, + "rows": 1431, + "seed": 20261037 + }, + { + "auroc": 0.8204043030557475, + "auroc_delta": -0.007981328746777705, + "available": true, + "brier": 0.1487182690105217, + "brier_gain": -0.003757870001940722, + "clusters": 120, + "events": 453, + "expected_brier": 0.1457965217189626, + "expected_gain": -0.001733550278623593, + "expected_recovery": -1.168449955658882, + "family": "probability_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.14257933898331515, + "reason": null, + "risk_mse": 0.003217182735647441, + "rows": 1431, + "seed": 20261037 + }, + { + "auroc": 0.8163684954202161, + "auroc_delta": -0.012017136382309168, + "available": true, + "brier": 0.1470066619715772, + "brier_gain": -0.0020462629629962037, + "clusters": 120, + "events": 453, + "expected_brier": 0.14293343751638163, + "expected_gain": 0.0011295339239573631, + "expected_recovery": 0.7613300171547859, + "family": "probability_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.14257933898331515, + "reason": null, + "risk_mse": 0.0003540985330664845, + "rows": 1431, + "seed": 20261037 + }, + { + "auroc": 0.7940058815105162, + "auroc_delta": 0.0, + "available": true, + "brier": 0.18263284372067984, + "brier_gain": 0.0, + "clusters": 120, + "events": 567, + "expected_brier": 0.17149649013758947, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "logit_linear", + "method": "raw", + "oracle_expected_brier": 0.1632872141242613, + "reason": null, + "risk_mse": 0.008209276013328172, + "rows": 1466, + "seed": 20261038 + }, + { + "auroc": 0.7868334991063949, + "auroc_delta": -0.007172382404121325, + "available": true, + "brier": 0.1758965649633098, + "brier_gain": 0.00673627875737004, + "clusters": 120, + "events": 567, + "expected_brier": 0.1643597965324114, + "expected_gain": 0.0071366936051780955, + "expected_recovery": 0.8693450669207998, + "family": "logit_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.1632872141242613, + "reason": null, + "risk_mse": 0.001072582408150076, + "rows": 1466, + "seed": 20261038 + }, + { + "auroc": 0.7868334991063949, + "auroc_delta": -0.007172382404121325, + "available": true, + "brier": 0.17589656496330977, + "brier_gain": 0.006736278757370068, + "clusters": 120, + "events": 567, + "expected_brier": 0.16435979653241137, + "expected_gain": 0.007136693605178101, + "expected_recovery": 0.8693450669208004, + "family": "logit_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.1632872141242613, + "reason": null, + "risk_mse": 0.001072582408150071, + "rows": 1466, + "seed": 20261038 + }, + { + "auroc": 0.7786252802938008, + "auroc_delta": -0.015380601216715362, + "available": true, + "brier": 0.17536452942941122, + "brier_gain": 0.007268314291268618, + "clusters": 120, + "events": 567, + "expected_brier": 0.1665858381980195, + "expected_gain": 0.004910651939569989, + "expected_recovery": 0.5981833150203866, + "family": "logit_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.1632872141242613, + "reason": null, + "risk_mse": 0.0032986240737581827, + "rows": 1466, + "seed": 20261038 + }, + { + "auroc": 0.7758060546394291, + "auroc_delta": 0.0, + "available": true, + "brier": 0.16657929387983328, + "brier_gain": 0.0, + "clusters": 120, + "events": 510, + "expected_brier": 0.1382227036674016, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "probability_linear", + "method": "raw", + "oracle_expected_brier": 0.1367597344215249, + "reason": null, + "risk_mse": 0.0014629692458767159, + "rows": 1466, + "seed": 20261038 + }, + { + "auroc": 0.7654463040446304, + "auroc_delta": -0.010359750594798656, + "available": true, + "brier": 0.17408761028883663, + "brier_gain": -0.007508316409003346, + "clusters": 120, + "events": 510, + "expected_brier": 0.14693252297041956, + "expected_gain": -0.008709819303017959, + "expected_recovery": -5.953521803391302, + "family": "probability_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.1367597344215249, + "reason": null, + "risk_mse": 0.010172788548894675, + "rows": 1466, + "seed": 20261038 + }, + { + "auroc": 0.7654463040446304, + "auroc_delta": -0.010359750594798656, + "available": true, + "brier": 0.17408761028883657, + "brier_gain": -0.0075083164090032906, + "clusters": 120, + "events": 510, + "expected_brier": 0.14693252297041956, + "expected_gain": -0.008709819303017955, + "expected_recovery": -5.9535218033912995, + "family": "probability_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.1367597344215249, + "reason": null, + "risk_mse": 0.010172788548894671, + "rows": 1466, + "seed": 20261038 + }, + { + "auroc": 0.7631532529329723, + "auroc_delta": -0.012652801706456773, + "available": true, + "brier": 0.16377843541801357, + "brier_gain": 0.0028008584618197108, + "clusters": 120, + "events": 510, + "expected_brier": 0.13966384235472465, + "expected_gain": -0.0014411386873230498, + "expected_recovery": -0.9850779101370764, + "family": "probability_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.1367597344215249, + "reason": null, + "risk_mse": 0.0029041079331997657, + "rows": 1466, + "seed": 20261038 + }, + { + "auroc": 0.8168185047619905, + "auroc_delta": 0.0, + "available": true, + "brier": 0.16891388601364496, + "brier_gain": 0.0, + "clusters": 120, + "events": 502, + "expected_brier": 0.17209293371573173, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "logit_linear", + "method": "raw", + "oracle_expected_brier": 0.16402665079764175, + "reason": null, + "risk_mse": 0.00806628291808997, + "rows": 1388, + "seed": 20261039 + }, + { + "auroc": 0.8132998480120152, + "auroc_delta": -0.003518656749975313, + "available": true, + "brier": 0.16375251534607063, + "brier_gain": 0.005161370667574328, + "clusters": 120, + "events": 502, + "expected_brier": 0.1642673303114022, + "expected_gain": 0.007825603404329536, + "expected_recovery": 0.9701622771970135, + "family": "logit_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.16402665079764175, + "reason": null, + "risk_mse": 0.0002406795137604337, + "rows": 1388, + "seed": 20261039 + }, + { + "auroc": 0.8132998480120152, + "auroc_delta": -0.003518656749975313, + "available": true, + "brier": 0.16375251534607063, + "brier_gain": 0.005161370667574328, + "clusters": 120, + "events": 502, + "expected_brier": 0.16426733031140217, + "expected_gain": 0.007825603404329539, + "expected_recovery": 0.9701622771970139, + "family": "logit_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.16402665079764175, + "reason": null, + "risk_mse": 0.00024067951376043035, + "rows": 1388, + "seed": 20261039 + }, + { + "auroc": 0.8117507397048375, + "auroc_delta": -0.005067765057153006, + "available": true, + "brier": 0.16541806596677425, + "brier_gain": 0.0034958200468707123, + "clusters": 120, + "events": 502, + "expected_brier": 0.16711291463506758, + "expected_gain": 0.0049800190806641425, + "expected_recovery": 0.6173871076968585, + "family": "logit_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.16402665079764175, + "reason": null, + "risk_mse": 0.0030862638374258276, + "rows": 1388, + "seed": 20261039 + }, + { + "auroc": 0.8182123388508675, + "auroc_delta": 0.0, + "available": true, + "brier": 0.13873596655130047, + "brier_gain": 0.0, + "clusters": 120, + "events": 412, + "expected_brier": 0.14073354762114762, + "expected_gain": 0.0, + "expected_recovery": 0.0, + "family": "probability_linear", + "method": "raw", + "oracle_expected_brier": 0.13923138281847094, + "reason": null, + "risk_mse": 0.0015021648026766682, + "rows": 1388, + "seed": 20261039 + }, + { + "auroc": 0.810376213592233, + "auroc_delta": -0.00783612525863453, + "available": true, + "brier": 0.1437457215909123, + "brier_gain": -0.005009755039611846, + "clusters": 120, + "events": 412, + "expected_brier": 0.14298772425605177, + "expected_gain": -0.00225417663490415, + "expected_recovery": -1.5006187276439253, + "family": "probability_linear", + "method": "predval_rung2", + "oracle_expected_brier": 0.13923138281847094, + "reason": null, + "risk_mse": 0.0037563414375808186, + "rows": 1388, + "seed": 20261039 + }, + { + "auroc": 0.810376213592233, + "auroc_delta": -0.00783612525863453, + "available": true, + "brier": 0.14374572159091228, + "brier_gain": -0.0050097550396118185, + "clusters": 120, + "events": 412, + "expected_brier": 0.14298772425605175, + "expected_gain": -0.0022541766349041424, + "expected_recovery": -1.50061872764392, + "family": "probability_linear", + "method": "statsmodels_logit", + "oracle_expected_brier": 0.13923138281847094, + "reason": null, + "risk_mse": 0.0037563414375808104, + "rows": 1388, + "seed": 20261039 + }, + { + "auroc": 0.8081355443259589, + "auroc_delta": -0.010076794524908617, + "available": true, + "brier": 0.13846697693990537, + "brier_gain": 0.00026898961139509847, + "clusters": 120, + "events": 412, + "expected_brier": 0.13956169389934886, + "expected_gain": 0.0011718537217987584, + "expected_recovery": 0.7801099584484091, + "family": "probability_linear", + "method": "statsmodels_probability", + "oracle_expected_brier": 0.13923138281847094, + "reason": null, + "risk_mse": 0.0003303110808779099, + "rows": 1388, + "seed": 20261039 + }, + { + "auroc": 0.9880889638452619, + "auroc_delta": 0.0, + "available": true, + "brier": 0.029478263131405693, + "brier_gain": 0.0, + "clusters": 285, + "events": 106, + "family": "wdbc_undistorted", + "method": "raw", + "reason": null, + "rows": 285, + "seed": 20261010 + }, + { + "auroc": 0.9847686307578792, + "auroc_delta": -0.0033203330873826653, + "available": true, + "brier": 0.029271022790021212, + "brier_gain": 0.0002072403413844806, + "clusters": 285, + "events": 106, + "family": "wdbc_undistorted", + "method": "predval_rung2", + "reason": null, + "rows": 285, + "seed": 20261010 + }, + { + "auroc": 0.9847686307578792, + "auroc_delta": -0.0033203330873826653, + "available": true, + "brier": 0.02927102279002123, + "brier_gain": 0.00020724034138446326, + "clusters": 285, + "events": 106, + "family": "wdbc_undistorted", + "method": "statsmodels_logit", + "reason": null, + "rows": 285, + "seed": 20261010 + }, + { + "auroc": 0.9790239274797091, + "auroc_delta": -0.009065036365552781, + "available": true, + "brier": 0.027864603166123696, + "brier_gain": 0.0016136599652819966, + "clusters": 285, + "events": 106, + "family": "wdbc_undistorted", + "method": "statsmodels_probability", + "reason": null, + "rows": 285, + "seed": 20261010 + } +] diff --git a/benchmarks/family_extension/results/summary.json b/benchmarks/family_extension/results/summary.json new file mode 100644 index 0000000..edbfb77 --- /dev/null +++ b/benchmarks/family_extension/results/summary.json @@ -0,0 +1,799 @@ +{ + "limits": "20 seeds; maximum binomial MCSE about 0.112. Monte Carlo intervals are not patient CIs.", + "paired_method_contrasts": [ + { + "attempted": 20, + "contrast": "predval_minus_probability_glm", + "family": "logit_linear", + "mc95_high": -0.0019085833194652886, + "mc95_low": -0.0036808660554192288, + "mcse": 0.0004233784920486638, + "mean": -0.0027947246874422587, + "metric": "brier", + "n": 20 + }, + { + "attempted": 20, + "contrast": "predval_minus_probability_glm", + "family": "logit_linear", + "mc95_high": 0.004792878543390647, + "mc95_low": 0.0021507730689963337, + "mcse": 0.000631169400282222, + "mean": 0.0034718258061934904, + "metric": "auroc", + "n": 20 + }, + { + "attempted": 20, + "contrast": "predval_minus_probability_glm", + "family": "logit_linear", + "mc95_high": -0.002760721107878252, + "mc95_low": -0.003212436236997838, + "mcse": 0.00010790968411667028, + "mean": -0.002986578672438045, + "metric": "expected_brier", + "n": 20 + }, + { + "attempted": 20, + "contrast": "predval_minus_probability_glm", + "family": "logit_linear", + "mc95_high": -0.00276072110787825, + "mc95_low": -0.003212436236997836, + "mcse": 0.00010790968411667025, + "mean": -0.002986578672438043, + "metric": "risk_mse", + "n": 20 + }, + { + "attempted": 20, + "contrast": "predval_minus_probability_glm", + "family": "probability_linear", + "mc95_high": 0.004156444920676035, + "mc95_low": 0.0018291730117361111, + "mcse": 0.0005559591883423997, + "mean": 0.002992808966206073, + "metric": "brier", + "n": 20 + }, + { + "attempted": 20, + "contrast": "predval_minus_probability_glm", + "family": "probability_linear", + "mc95_high": 0.00429473957971521, + "mc95_low": 0.002431354955927047, + "mcse": 0.00044514171250529096, + "mean": 0.0033630472678211286, + "metric": "auroc", + "n": 20 + }, + { + "attempted": 20, + "contrast": "predval_minus_probability_glm", + "family": "probability_linear", + "mc95_high": 0.0036603447197199653, + "mc95_low": 0.002130555859913125, + "mcse": 0.0003654494215163968, + "mean": 0.0028954502898165453, + "metric": "expected_brier", + "n": 20 + }, + { + "attempted": 20, + "contrast": "predval_minus_probability_glm", + "family": "probability_linear", + "mc95_high": 0.0036603447197199623, + "mc95_low": 0.0021305558599131204, + "mcse": 0.00036544942151639726, + "mean": 0.0028954502898165414, + "metric": "risk_mse", + "n": 20 + } + ], + "simulation_summary": [ + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": 0.17441612247293337, + "mc95_low": 0.1633392649958434, + "mcse": 0.002646137165447279, + "mean": 0.16887769373438838, + "method": "raw", + "metric": "brier", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": 0.8259933367905318, + "mc95_low": 0.8074386403757764, + "mcse": 0.0044325091189649, + "mean": 0.8167159885831541, + "method": "raw", + "metric": "auroc", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": 0.0, + "mc95_low": 0.0, + "mcse": 0.0, + "mean": 0.0, + "method": "raw", + "metric": "auroc_delta", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": 0.0, + "mc95_low": 0.0, + "mcse": 0.0, + "mean": 0.0, + "method": "raw", + "metric": "brier_gain", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": 0.008025154030127294, + "mc95_low": 0.007672398473170633, + "mcse": 8.426935089773344e-05, + "mean": 0.007848776251648964, + "method": "raw", + "metric": "risk_mse", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": 0.1733647311827367, + "mc95_low": 0.1686363594546188, + "mcse": 0.0011295550374012804, + "mean": 0.17100054531867775, + "method": "raw", + "metric": "expected_brier", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": 0.0, + "mc95_low": 0.0, + "mcse": 0.0, + "mean": 0.0, + "method": "raw", + "metric": "expected_gain", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": 0.0, + "mc95_low": 0.0, + "mcse": 0.0, + "mean": 0.0, + "method": "raw", + "metric": "expected_recovery", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": 0.1672908638593121, + "mc95_low": 0.1583184150978819, + "mcse": 0.0021434174974082373, + "mean": 0.162804639478597, + "method": "predval_rung2", + "metric": "brier", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": 0.822753139174774, + "mc95_low": 0.8025118851222557, + "mcse": 0.004835408845370514, + "mean": 0.8126325121485148, + "method": "predval_rung2", + "metric": "auroc", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": -0.002886232425366471, + "mc95_low": -0.0052807204439120495, + "mcse": 0.0005720163639549027, + "mean": -0.00408347643463926, + "method": "predval_rung2", + "metric": "auroc_delta", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": 0.007946359773364303, + "mc95_low": 0.004199748738218462, + "mcse": 0.0008950234057880898, + "mean": 0.0060730542557913825, + "method": "predval_rung2", + "metric": "brier_gain", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": 0.001657421772136416, + "mc95_low": 0.0006521264414109986, + "mcse": 0.00024015379293135055, + "mean": 0.0011547741067737074, + "method": "predval_rung2", + "metric": "risk_mse", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": 0.16644331020245742, + "mc95_low": 0.16216977614514758, + "mcse": 0.0010208994130547517, + "mean": 0.1643065431738025, + "method": "predval_rung2", + "metric": "expected_brier", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": 0.007204199494556759, + "mc95_low": 0.006183804795193755, + "mcse": 0.00024376086295182744, + "mean": 0.006694002144875257, + "method": "predval_rung2", + "metric": "expected_gain", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": 0.9163057180740772, + "mc95_low": 0.7905612425974582, + "mcse": 0.030038946569146456, + "mean": 0.8534334803357677, + "method": "predval_rung2", + "metric": "expected_recovery", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": 0.1672908638593121, + "mc95_low": 0.1583184150978819, + "mcse": 0.0021434174974082365, + "mean": 0.162804639478597, + "method": "statsmodels_logit", + "metric": "brier", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": 0.822753139174774, + "mc95_low": 0.8025118851222557, + "mcse": 0.004835408845370514, + "mean": 0.8126325121485148, + "method": "statsmodels_logit", + "metric": "auroc", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": -0.002886232425366471, + "mc95_low": -0.0052807204439120495, + "mcse": 0.0005720163639549027, + "mean": -0.00408347643463926, + "method": "statsmodels_logit", + "metric": "auroc_delta", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": 0.007946359773364312, + "mc95_low": 0.00419974873821847, + "mcse": 0.00089502340578809, + "mean": 0.006073054255791391, + "method": "statsmodels_logit", + "metric": "brier_gain", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": 0.001657421772136416, + "mc95_low": 0.0006521264414109971, + "mcse": 0.00024015379293135084, + "mean": 0.0011547741067737065, + "method": "statsmodels_logit", + "metric": "risk_mse", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": 0.16644331020245742, + "mc95_low": 0.16216977614514758, + "mcse": 0.0010208994130547525, + "mean": 0.1643065431738025, + "method": "statsmodels_logit", + "metric": "expected_brier", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": 0.00720419949455676, + "mc95_low": 0.006183804795193754, + "mcse": 0.00024376086295182806, + "mean": 0.006694002144875257, + "method": "statsmodels_logit", + "metric": "expected_gain", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": 0.9163057180740772, + "mc95_low": 0.7905612425974582, + "mcse": 0.03003894656914649, + "mean": 0.8534334803357677, + "method": "statsmodels_logit", + "metric": "expected_recovery", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": 0.17012659040686948, + "mc95_low": 0.16107213792520905, + "mcse": 0.0021630072675443016, + "mean": 0.16559936416603926, + "method": "statsmodels_probability", + "metric": "brier", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": 0.8202185077995777, + "mc95_low": 0.7981028648850651, + "mcse": 0.005283179347110898, + "mean": 0.8091606863423214, + "method": "statsmodels_probability", + "metric": "auroc", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": -0.005123963124248554, + "mc95_low": -0.009986641357416948, + "mcse": 0.0011616393569215473, + "mean": -0.007555302240832751, + "method": "statsmodels_probability", + "metric": "auroc_delta", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": 0.004789270986053497, + "mc95_low": 0.0017673881506447512, + "mcse": 0.0007218939574641014, + "mean": 0.003278329568349124, + "method": "statsmodels_probability", + "metric": "brier_gain", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": 0.004762872151085089, + "mc95_low": 0.0035198334073384104, + "mcse": 0.0002969480310387743, + "mean": 0.00414135277921175, + "method": "statsmodels_probability", + "metric": "risk_mse", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": 0.16938037645603002, + "mc95_low": 0.16520586723645106, + "mcse": 0.000997243488622747, + "mean": 0.16729312184624054, + "method": "statsmodels_probability", + "metric": "expected_brier", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": 0.004336165938796567, + "mc95_low": 0.003078681006077862, + "mcse": 0.0003003990637542368, + "mean": 0.0037074234724372144, + "method": "statsmodels_probability", + "metric": "expected_gain", + "n": 20 + }, + { + "attempted": 20, + "family": "logit_linear", + "mc95_high": 0.5495381194605182, + "mc95_low": 0.39446723677342915, + "mcse": 0.03704469672971033, + "mean": 0.47200267811697366, + "method": "statsmodels_probability", + "metric": "expected_recovery", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": 0.14418031315312022, + "mc95_low": 0.13327567385874076, + "mcse": 0.0026049961708304752, + "mean": 0.13872799350593049, + "method": "raw", + "metric": "brier", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": 0.8417389408659913, + "mc95_low": 0.8155944282025568, + "mcse": 0.0062456311976848014, + "mean": 0.8286666845342741, + "method": "raw", + "metric": "auroc", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": 0.0, + "mc95_low": 0.0, + "mcse": 0.0, + "mean": 0.0, + "method": "raw", + "metric": "auroc_delta", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": 0.0, + "mc95_low": 0.0, + "mcse": 0.0, + "mean": 0.0, + "method": "raw", + "metric": "brier_gain", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": 0.001575380304462251, + "mc95_low": 0.0014692844404755696, + "mcse": 2.5345113393040863e-05, + "mean": 0.0015223323724689103, + "method": "raw", + "metric": "risk_mse", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": 0.14242258700418559, + "mc95_low": 0.13888056510938945, + "mcse": 0.0008461493520191356, + "mean": 0.14065157605678752, + "method": "raw", + "metric": "expected_brier", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": 0.0, + "mc95_low": 0.0, + "mcse": 0.0, + "mean": 0.0, + "method": "raw", + "metric": "expected_gain", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": 0.0, + "mc95_low": 0.0, + "mcse": 0.0, + "mean": 0.0, + "method": "raw", + "metric": "expected_recovery", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": 0.14733093474668488, + "mc95_low": 0.13519056048658465, + "mcse": 0.0029001994111176746, + "mean": 0.14126074761663476, + "method": "predval_rung2", + "metric": "brier", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": 0.8377855535737847, + "mc95_low": 0.8088762767424854, + "mcse": 0.006906102385782623, + "mean": 0.8233309151581351, + "method": "predval_rung2", + "metric": "auroc", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": -0.0037370092301654027, + "mc95_low": -0.006934529522112595, + "mcse": 0.0007638517782948082, + "mean": -0.005335769376138999, + "method": "predval_rung2", + "metric": "auroc_delta", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": -0.0016143201943686234, + "mc95_low": -0.0034511880270399952, + "mcse": 0.0004388071481554588, + "mean": -0.0025327541107043093, + "method": "predval_rung2", + "metric": "brier_gain", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": 0.00504278133787225, + "mc95_low": 0.0029773519824811195, + "mcse": 0.0004934079355277695, + "mean": 0.004010066660176685, + "method": "predval_rung2", + "metric": "risk_mse", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": 0.14479297290261287, + "mc95_low": 0.1414856477863777, + "mcse": 0.0007900829207550885, + "mean": 0.14313931034449529, + "method": "predval_rung2", + "metric": "expected_brier", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": -0.0014722864710261028, + "mc95_low": -0.0035031821043894442, + "mcse": 0.0004851582161910387, + "mean": -0.0024877342877077735, + "method": "predval_rung2", + "metric": "expected_gain", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": -0.948666155442277, + "mc95_low": -2.2711810950311806, + "mcse": 0.31593400391252896, + "mean": -1.6099236252367288, + "method": "predval_rung2", + "metric": "expected_recovery", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": 0.14733093474668488, + "mc95_low": 0.13519056048658465, + "mcse": 0.002900199411117674, + "mean": 0.14126074761663476, + "method": "statsmodels_logit", + "metric": "brier", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": 0.8377855535737847, + "mc95_low": 0.8088762767424854, + "mcse": 0.006906102385782623, + "mean": 0.8233309151581351, + "method": "statsmodels_logit", + "metric": "auroc", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": -0.0037370092301654027, + "mc95_low": -0.006934529522112595, + "mcse": 0.0007638517782948082, + "mean": -0.005335769376138999, + "method": "statsmodels_logit", + "metric": "auroc_delta", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": -0.00161432019436862, + "mc95_low": -0.0034511880270399814, + "mcse": 0.0004388071481554564, + "mean": -0.0025327541107043007, + "method": "statsmodels_logit", + "metric": "brier_gain", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": 0.005042781337872249, + "mc95_low": 0.0029773519824811147, + "mcse": 0.0004934079355277703, + "mean": 0.004010066660176682, + "method": "statsmodels_logit", + "metric": "risk_mse", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": 0.14479297290261287, + "mc95_low": 0.1414856477863777, + "mcse": 0.000790082920755089, + "mean": 0.14313931034449529, + "method": "statsmodels_logit", + "metric": "expected_brier", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": -0.001472286471026098, + "mc95_low": -0.003503182104389443, + "mcse": 0.0004851582161910396, + "mean": -0.0024877342877077705, + "method": "statsmodels_logit", + "metric": "expected_gain", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": -0.9486661554422744, + "mc95_low": -2.2711810950311797, + "mcse": 0.31593400391252935, + "mean": -1.609923625236727, + "method": "statsmodels_logit", + "metric": "expected_recovery", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": 0.14362728711405895, + "mc95_low": 0.13290859018679843, + "mcse": 0.0025605766223003716, + "mean": 0.1382679386504287, + "method": "statsmodels_probability", + "metric": "brier", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": 0.8349191002541355, + "mc95_low": 0.8050166355264922, + "mcse": 0.0071433638482709064, + "mean": 0.8199678678903138, + "method": "statsmodels_probability", + "metric": "auroc", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": -0.006412644937968379, + "mc95_low": -0.010984988349951878, + "mcse": 0.0010922816205464216, + "mean": -0.008698816643960128, + "method": "statsmodels_probability", + "metric": "auroc_delta", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": 0.0010880119057419615, + "mc95_low": -0.0001679021947384338, + "mcse": 0.00030002380952937453, + "mean": 0.0004600548555017639, + "method": "statsmodels_probability", + "metric": "brier_gain", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": 0.0016466449726532093, + "mc95_low": 0.0005825877680670746, + "mcse": 0.0002541913463309289, + "mean": 0.001114616370360142, + "method": "statsmodels_probability", + "metric": "risk_mse", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": 0.14191682968337346, + "mc95_low": 0.13857089042598406, + "mcse": 0.0007993074065112134, + "mean": 0.14024386005467876, + "method": "statsmodels_probability", + "metric": "expected_brier", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": 0.0009286761184149918, + "mc95_low": -0.00011324411419745507, + "mcse": 0.00024890307171051467, + "mean": 0.00040771600210876837, + "method": "statsmodels_probability", + "metric": "expected_gain", + "n": 20 + }, + { + "attempted": 20, + "family": "probability_linear", + "mc95_high": 0.6112281651888186, + "mc95_low": -0.0548286321550272, + "mcse": 0.1591135075445031, + "mean": 0.2781997665168957, + "method": "statsmodels_probability", + "metric": "expected_recovery", + "n": 20 + } + ] +} diff --git a/benchmarks/family_extension/run_extension.py b/benchmarks/family_extension/run_extension.py new file mode 100644 index 0000000..7e56000 --- /dev/null +++ b/benchmarks/family_extension/run_extension.py @@ -0,0 +1,316 @@ +"""Run the prospectively specified, local-only paired-family extension.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.metadata +import json +import platform +import time +import warnings +from pathlib import Path + +import numpy as np +import pandas as pd +import statsmodels.api as sm +from scipy.special import expit, logit +from scipy.stats import t +from sklearn.datasets import load_breast_cancer +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import roc_auc_score +from sklearn.model_selection import train_test_split +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import StandardScaler + +from predval import recalibrate + +SEEDS = tuple(range(20261020, 20261040)) +HERE = Path(__file__).resolve().parent +METHODS = ("raw", "predval_rung2", "statsmodels_logit", "statsmodels_probability") +OUTCOMES = ( + "brier", + "auroc", + "auroc_delta", + "brier_gain", + "risk_mse", + "expected_brier", + "expected_gain", + "expected_recovery", +) + + +def sha(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def frame(y, p, groups, q=None): + groups = np.asarray(groups).astype(str) + lookup = {g: i % 5 for i, g in enumerate(np.unique(groups))} + d = pd.DataFrame({"y": y, "p": p, "patient": groups, "fold": [lookup[g] for g in groups]}) + if q is not None: + d["q"] = q + return d + + +def paired_fixtures(seed): + rng = np.random.default_rng(seed) + ids = np.repeat(np.arange(120), rng.integers(4, 21, 120)) + latent = rng.normal(0, 1.2, 120) + x = rng.normal(size=len(ids)) + eta = -0.8 + 0.9 * x + latent[ids] + p = expit(-0.4 + 1.4 * eta) + shared = rng.uniform(size=120) + u = np.where(rng.uniform(size=len(ids)) < 0.65, shared[ids], rng.uniform(size=len(ids))) + groups = [f"patient_{i:03}" for i in ids] + return { + name: frame((u < q).astype(int), p, groups, q) + for name, q in (("logit_linear", expit(eta)), ("probability_linear", expit(-2.5 + 5 * p))) + } + + +def wdbc_control(): + data = load_breast_cancer() + y = 1 - data.target + train, test = train_test_split( + np.arange(len(y)), test_size=0.5, stratify=y, random_state=20261010 + ) + model = make_pipeline(StandardScaler(), LogisticRegression(C=1, max_iter=2000, solver="lbfgs")) + model.fit(data.data[train], y[train]) + return frame(y[test], model.predict_proba(data.data[test])[:, 1], [f"uci_{i:04}" for i in test]) + + +def grouped_glm(d, logit_scale): + x = logit(np.clip(d.p.to_numpy(), 1e-6, 1 - 1e-6)) if logit_scale else d.p.to_numpy() + out = np.full(len(d), np.nan) + for fold in sorted(d.fold.unique()): + train = d.fold.to_numpy() != fold + assert not set(d.patient[train]) & set(d.patient[~train]) + if np.unique(d.y[train]).size < 2: + raise ValueError(f"single-class training fold {fold}") + fit = sm.GLM( + d.y.to_numpy()[train], + sm.add_constant(x[train], has_constant="add"), + family=sm.families.Binomial(), + ).fit(tol=1e-11, maxiter=100) + if not fit.converged: + raise ValueError(f"non-convergence fold {fold}") + out[~train] = fit.predict(sm.add_constant(x[~train], has_constant="add")) + return out + + +def fit_methods(d): + predictions = {"raw": d.p.to_numpy()} + statuses = {"raw": {"available": True, "reason": None, "warnings": []}} + for method in METHODS[1:]: + extra = {} + caught = [] + try: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + if method == "predval_rung2": + _, ladder = recalibrate.ladder( + d.y.to_numpy(), d.p.to_numpy(), d.patient.to_numpy(), max_folds=5 + ) + extra["ladder_suppressed"] = ladder.suppressed + if "rung2" not in ladder.crossfit_predictions: + raise ValueError(f"rung2 unavailable: {ladder.suppressed}") + prediction = ladder.crossfit_predictions["rung2"] + else: + prediction = grouped_glm(d, logit_scale=method == "statsmodels_logit") + if ( + not np.isfinite(prediction).all() + or not ((prediction >= 0) & (prediction <= 1)).all() + ): + raise ValueError("nonfinite or out-of-range prediction") + predictions[method] = prediction + statuses[method] = {"available": True, "reason": None, **extra} + except (ValueError, RuntimeError, np.linalg.LinAlgError) as exc: + statuses[method] = { + "available": False, + "reason": f"{type(exc).__name__}: {exc}", + **extra, + } + statuses[method]["warnings"] = [f"{type(w.message).__name__}: {w.message}" for w in caught] + if "predval_rung2" in predictions and "statsmodels_logit" in predictions: + delta = float( + np.max(np.abs(predictions["predval_rung2"] - predictions["statsmodels_logit"])) + ) + statuses["predval_rung2"]["max_abs_independent_difference"] = delta + assert delta < 1e-8, delta + for method, prediction in predictions.items(): + statuses[method]["prediction_sha256"] = hashlib.sha256( + np.asarray(prediction, dtype=" 0 else None, + ) + return result + + +def mean_precision(values): + a = np.asarray(values, dtype=float) + n = len(a) + if n == 0: + return {"n": 0, "mean": None, "mcse": None, "mc95_low": None, "mc95_high": None} + mean = float(a.mean()) + se = float(a.std(ddof=1) / np.sqrt(n)) if n > 1 else None + half = float(t.ppf(0.975, n - 1) * se) if se is not None else None + return { + "n": n, + "mean": mean, + "mcse": se, + "mc95_low": mean - half if half is not None else None, + "mc95_high": mean + half if half is not None else None, + } + + +def summarize(rows): + synthetic = [r for r in rows if r["family"] != "wdbc_undistorted"] + summaries, contrasts = [], [] + for family in ("logit_linear", "probability_linear"): + for method in METHODS: + subset = [r for r in synthetic if r["family"] == family and r["method"] == method] + for metric in OUTCOMES: + values = [r[metric] for r in subset if r["available"] and r.get(metric) is not None] + summaries.append( + { + "family": family, + "method": method, + "metric": metric, + "attempted": len(subset), + **mean_precision(values), + } + ) + for metric in ("brier", "auroc", "expected_brier", "risk_mse"): + values = [] + for seed in SEEDS: + pair = { + r["method"]: r for r in synthetic if r["family"] == family and r["seed"] == seed + } + a, b = pair["predval_rung2"], pair["statsmodels_probability"] + if ( + a["available"] + and b["available"] + and a.get(metric) is not None + and b.get(metric) is not None + ): + values.append(a[metric] - b[metric]) + contrasts.append( + { + "family": family, + "contrast": "predval_minus_probability_glm", + "metric": metric, + "attempted": len(SEEDS), + **mean_precision(values), + } + ) + return { + "simulation_summary": summaries, + "paired_method_contrasts": contrasts, + "limits": ( + "20 seeds; maximum binomial MCSE about 0.112. " + "Monte Carlo intervals are not patient CIs." + ), + } + + +def run(out): + started = time.monotonic() + out.mkdir(parents=True, exist_ok=False) + rows, cases = [], [] + fixtures = [(seed, name, d) for seed in SEEDS for name, d in paired_fixtures(seed).items()] + fixtures.append((20261010, "wdbc_undistorted", wdbc_control())) + for seed, family, d in fixtures: + predictions, statuses = fit_methods(d) + input_hash = hashlib.sha256( + d.to_csv(index=False, float_format="%.17g").encode() + ).hexdigest() + cases.append( + {"seed": seed, "family": family, "input_sha256": input_hash, "statuses": statuses} + ) + for method in METHODS: + row = { + "seed": seed, + "family": family, + "method": method, + "rows": len(d), + "clusters": int(d.patient.nunique()), + "events": int(d.y.sum()), + "available": statuses[method]["available"], + "reason": statuses[method]["reason"], + } + if method in predictions: + row.update(score(d, predictions[method])) + rows.append(row) + for name, value in ( + ("metrics.json", rows), + ("cases.json", cases), + ("summary.json", summarize(rows)), + ): + (out / name).write_text(json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n") + pd.DataFrame(rows).to_csv(out / "metrics.csv", index=False, float_format="%.17g") + manifest = { + "python": platform.python_version(), + "seeds": SEEDS, + "versions": { + p: importlib.metadata.version(p) + for p in ("numpy", "scipy", "pandas", "statsmodels", "scikit-learn", "predval") + }, + "protocol_sha256": sha(HERE / "PROTOCOL.md"), + "runner_sha256": sha(Path(__file__)), + "predval_source": str(Path(recalibrate.__file__).resolve()), + "recalibrate_sha256": sha(Path(recalibrate.__file__)), + "files": {p.name: sha(p) for p in sorted(out.iterdir())}, + } + (out / "manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + (out / "runtime.json").write_text( + json.dumps( + { + "seconds": time.monotonic() - started, + "scope": "combined workload, excludes imports/install", + }, + indent=2, + ) + + "\n" + ) + print( + json.dumps( + { + "cases": len(cases), + "method_cases": len(rows), + "unavailable": sum(not r["available"] for r in rows), + "out": str(out), + }, + indent=2, + ) + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", required=True, type=Path) + run(parser.parse_args().out) diff --git a/benchmarks/family_extension/tests/test_extension.py b/benchmarks/family_extension/tests/test_extension.py new file mode 100644 index 0000000..f1145b6 --- /dev/null +++ b/benchmarks/family_extension/tests/test_extension.py @@ -0,0 +1,95 @@ +"""Independent contract checks for the prospective family extension.""" + +import importlib.util +from pathlib import Path + +import numpy as np +import pytest +from scipy.special import expit, logit + +SPEC = importlib.util.spec_from_file_location( + "extension", Path(__file__).parents[1] / "run_extension.py" +) +ex = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(ex) + + +def test_fixed_seed_budget(): + assert ex.SEEDS == tuple(range(20261020, 20261040)) + + +def test_shared_design_and_families(): + pair = ex.paired_fixtures(ex.SEEDS[0]) + a, b = pair.values() + for column in ("p", "patient", "fold"): + np.testing.assert_array_equal(a[column], b[column]) + np.testing.assert_allclose(a.q, expit((logit(a.p) + 0.4) / 1.4), atol=1e-14) + np.testing.assert_allclose(logit(b.q), -2.5 + 5 * b.p, atol=1e-14) + # Same outcome uniforms imply monotonic coupling when a row's q changes. + assert np.all(a.y[a.q <= b.q] <= b.y[a.q <= b.q]) + assert np.all(a.y[a.q >= b.q] >= b.y[a.q >= b.q]) + assert a.patient.nunique() == 120 + assert a.groupby("patient").size().between(4, 20).all() + + +def test_grouped_partitions_match_native(): + d = ex.paired_fixtures(ex.SEEDS[1])["logit_linear"] + native = ex.recalibrate._grouped_folds(d.patient.to_numpy(), 5) + for f, rows in enumerate(native): + np.testing.assert_array_equal(rows, np.flatnonzero(d.fold.to_numpy() == f)) + assert not set(d.patient.iloc[rows]) & set(d.patient[d.fold != f]) + + +def test_expected_loss_identity_and_oracle(): + d = ex.paired_fixtures(ex.SEEDS[2])["probability_linear"] + p = np.full(len(d), 0.4) + actual = ex.score(d, p) + direct = np.mean(d.q * (1 - p) ** 2 + (1 - d.q) * p**2) + assert actual["expected_brier"] == pytest.approx(direct, abs=1e-14) + oracle = ex.score(d, d.q.to_numpy()) + assert oracle["risk_mse"] == 0 + assert oracle["expected_recovery"] == 1 + assert ex.score(d, d.p.to_numpy())["expected_recovery"] == 0 + + +@pytest.mark.parametrize("family", ["logit_linear", "probability_linear"]) +def test_independent_agreement_and_probabilities(family): + d = ex.paired_fixtures(ex.SEEDS[3])[family] + preds, statuses = ex.fit_methods(d) + assert all(status["available"] for status in statuses.values()) + assert len(preds) == 4 + np.testing.assert_allclose(preds["predval_rung2"], preds["statsmodels_logit"], atol=1e-8) + assert all(np.isfinite(p).all() and ((p >= 0) & (p <= 1)).all() for p in preds.values()) + + +def test_unavailability_is_explicit(): + d = ex.frame(np.zeros(20, dtype=int), np.linspace(0.1, 0.9, 20), np.arange(20)) + preds, statuses = ex.fit_methods(d) + assert set(preds) == {"raw"} + assert all(not statuses[m]["available"] and statuses[m]["reason"] for m in ex.METHODS[1:]) + + +def test_wdbc_is_not_oracle(): + d = ex.wdbc_control() + assert len(d) == 285 and d.y.sum() == 106 + assert "q" not in d + result = ex.score(d, d.p.to_numpy()) + assert "expected_brier" not in result and "risk_mse" not in result + assert result["brier"] == pytest.approx(0.02947826313148458, abs=1e-12) + + +def test_mc_precision_and_missing_case_accounting(): + result = ex.mean_precision([1, 2, 3]) + assert result["n"] == 3 and result["mean"] == 2 + assert result["mcse"] == pytest.approx(1 / np.sqrt(3)) + assert ex.mean_precision([])["n"] == 0 + assert ex.mean_precision([1])["mcse"] is None + rows = [ + {"family": f, "seed": s, "method": m, "available": False} + for f in ("logit_linear", "probability_linear") + for s in ex.SEEDS + for m in ex.METHODS + ] + summary = ex.summarize(rows) + assert all(r["attempted"] == 20 and r["n"] == 0 for r in summary["simulation_summary"]) + assert all(r["attempted"] == 20 and r["n"] == 0 for r in summary["paired_method_contrasts"]) diff --git a/benchmarks/family_extension/verification/diff-check.log b/benchmarks/family_extension/verification/diff-check.log new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/family_extension/verification/format.log b/benchmarks/family_extension/verification/format.log new file mode 100644 index 0000000..293af16 --- /dev/null +++ b/benchmarks/family_extension/verification/format.log @@ -0,0 +1 @@ +68 files already formatted diff --git a/benchmarks/family_extension/verification/pytest.log b/benchmarks/family_extension/verification/pytest.log new file mode 100644 index 0000000..8a9e3a4 --- /dev/null +++ b/benchmarks/family_extension/verification/pytest.log @@ -0,0 +1,24 @@ +........................................................................ [ 23%] +................sss..................................................... [ 46%] +......................sssssssssssss..............ssss................... [ 69%] +........................................................................ [ 92%] +....................... [100%] +=========================== short test summary info ============================ +SKIPPED [1] tests/test_golden_gusto.py:69: GUSTO fixture not built; run examples/gusto/fetch_data.py + make_predictions.py +SKIPPED [1] tests/test_golden_improve.py:67: IMPROVE fixture not built; run examples/improve/build_fixture.py +SKIPPED [1] tests/test_golden_pcam.py:69: PCam fixture not built; run examples/pcam/build_fixture.py +SKIPPED [1] tests/test_pcam_fixture.py:40: PCam fixture not built; run examples/pcam/build_fixture.py +SKIPPED [1] tests/test_pcam_fixture.py:46: PCam fixture not built; run examples/pcam/build_fixture.py +SKIPPED [1] tests/test_pcam_fixture.py:52: PCam fixture not built; run examples/pcam/build_fixture.py +SKIPPED [1] tests/test_pcam_fixture.py:58: PCam fixture not built; run examples/pcam/build_fixture.py +SKIPPED [1] tests/test_pcam_fixture.py:63: PCam fixture not built; run examples/pcam/build_fixture.py +SKIPPED [1] tests/test_pcam_fixture.py:68: PCam fixture not built; run examples/pcam/build_fixture.py +SKIPPED [1] tests/test_pcam_fixture.py:73: PCam fixture not built; run examples/pcam/build_fixture.py +SKIPPED [1] tests/test_pcam_fixture.py:79: PCam fixture not built; run examples/pcam/build_fixture.py +SKIPPED [1] tests/test_pcam_fixture.py:85: PCam fixture not built; run examples/pcam/build_fixture.py +SKIPPED [1] tests/test_pcam_fixture.py:92: PCam fixture not built; run examples/pcam/build_fixture.py +SKIPPED [1] tests/test_pcam_fixture.py:97: PCam fixture not built; run examples/pcam/build_fixture.py +SKIPPED [1] tests/test_pcam_fixture.py:108: PCam fixture not built; run examples/pcam/build_fixture.py +SKIPPED [1] tests/test_pcam_fixture.py:144: PCam fixture not built; run examples/pcam/build_fixture.py +SKIPPED [4] tests/test_predrupdate_agreement.py:99: GUSTO fixture not built; run examples/gusto/fetch_data.py + make_predictions.py +291 passed, 20 skipped in 110.19s (0:01:50) diff --git a/benchmarks/family_extension/verification/repeat.log b/benchmarks/family_extension/verification/repeat.log new file mode 100644 index 0000000..3a9bb55 --- /dev/null +++ b/benchmarks/family_extension/verification/repeat.log @@ -0,0 +1,6 @@ +{ + "cases": 41, + "method_cases": 164, + "unavailable": 0, + "out": "/tmp/predval-family-extension-b" +} diff --git a/benchmarks/family_extension/verification/reproducibility.json b/benchmarks/family_extension/verification/reproducibility.json new file mode 100644 index 0000000..758e34e --- /dev/null +++ b/benchmarks/family_extension/verification/reproducibility.json @@ -0,0 +1,7 @@ +{ + "stable_files_checked_per_run": 4, + "repeat_manifest_equal": true, + "cases": 41, + "method_cases": 164, + "all_fixed_seeds_accounted_for": true +} diff --git a/benchmarks/family_extension/verification/ruff.log b/benchmarks/family_extension/verification/ruff.log new file mode 100644 index 0000000..1f5f344 --- /dev/null +++ b/benchmarks/family_extension/verification/ruff.log @@ -0,0 +1 @@ +All checks passed! diff --git a/benchmarks/family_extension/verification/runtime-a.json b/benchmarks/family_extension/verification/runtime-a.json new file mode 100644 index 0000000..b5bad54 --- /dev/null +++ b/benchmarks/family_extension/verification/runtime-a.json @@ -0,0 +1,4 @@ +{ + "seconds": 11.137972092999917, + "scope": "combined workload, excludes imports/install" +} diff --git a/benchmarks/family_extension/verification/runtime-b.json b/benchmarks/family_extension/verification/runtime-b.json new file mode 100644 index 0000000..6dd388f --- /dev/null +++ b/benchmarks/family_extension/verification/runtime-b.json @@ -0,0 +1,4 @@ +{ + "seconds": 9.642061055999875, + "scope": "combined workload, excludes imports/install" +} diff --git a/benchmarks/family_extension/verification/status.json b/benchmarks/family_extension/verification/status.json new file mode 100644 index 0000000..ed23b45 --- /dev/null +++ b/benchmarks/family_extension/verification/status.json @@ -0,0 +1,52 @@ +{ + "baseline": "150998f76df1141c73b6c3621233e693c4eb0b49", + "calibrationcurves": "Primary documentation reviewed; not installed or executed.", + "execution_note": "First run had Matplotlib/fontconfig import cache warnings; repeat and suite set writable cache directories. No model-fitting warnings.", + "fitting_warnings": 0, + "git_diff_check": "passed", + "historical_and_production_unchanged": true, + "independent_verification": { + "blocking_findings": 0, + "independent_glm_families_refitted": 2, + "limitation": "Unexpected exceptions outside selected fitting exception types would abort the run. No such exceptions occurred in the fixed design.", + "max_abs_loss_discrepancy": 4.45e-15, + "protocol_unchanged": true, + "summary_and_MCSE_reproduced": true, + "synthetic_designs_reconstructed": 40, + "tracked_production_and_historical_unchanged": true + }, + "local_protocol_commit": "4a289159a937c49b1d06469e93db9079fef5f47f", + "manifests_equal": true, + "max_abs_predval_independent_logit_difference": 7.105427357601002e-15, + "method_cases": 164, + "native_R_extension_execution": false, + "provenance_limit": "Local commit chronology is not independently timestamped preregistration. Original results were inspected before this extension.", + "remote_actions": [], + "repeated_cases": 41, + "ruff": "passed", + "ruff_format": "passed", + "runtime_scope": "Combined workload after imports, not a package-speed comparison.", + "runtime_seconds": [ + 11.137972092999917, + 9.642061055999875 + ], + "source_sha256": { + "benchmarks/family_extension/PROTOCOL.md": "8a34e65b43b189e3a845a6c0fc7d92f11a992d57eafb62e199de62fef8954559", + "benchmarks/family_extension/run_extension.py": "cbffe569b0bd7ae556bbebad65cdebc28b5045c4573f29fc6e1a300bc2978bf2", + "benchmarks/family_extension/verify_results.py": "fab99b931dfa6cf3d1e7580e816aedc77ab590b71cc8487e05a018b5b97f9d65", + "src/predval/metrics.py": "5a0f13aee0e1a4de17611f5aa13ff05421382af478b9fff788fd77132d9b81e6", + "src/predval/recalibrate.py": "513d69580b00cdd2f4633060300210513c79f5524fc0027d038e25810d072a22" + }, + "stable_files_checked_per_run": 4, + "suppression": "Only WDBC rung3 non-convergence; rung2 available in all cases.", + "test_command": "PYTHONPATH=$PWD/src MPLCONFIGDIR=/tmp/predval-family-mpl XDG_CACHE_HOME=/tmp/predval-cache OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 /workspace/shared/predval-benchmark-20261010/.venv/bin/python -m pytest tests benchmarks/alternatives/tests benchmarks/family_extension/tests -ra", + "tests": { + "failed": 0, + "new_tests": 9, + "passed": 291, + "seconds": 110.19, + "skip_reason": "Optional PCam, GUSTO and IMPROVE example/golden fixtures unavailable.", + "skipped": 20 + }, + "unavailable_method_cases": 0 +} diff --git a/benchmarks/family_extension/verify_results.py b/benchmarks/family_extension/verify_results.py new file mode 100644 index 0000000..8a177f9 --- /dev/null +++ b/benchmarks/family_extension/verify_results.py @@ -0,0 +1,55 @@ +"""Verify repeat hashes and complete accounting for the fixed extension design.""" + +import argparse +import hashlib +import json +from pathlib import Path + + +def verify(a, b): + manifests = [] + for root in (a, b): + manifest = json.loads((root / "manifest.json").read_text()) + for name, expected in manifest["files"].items(): + assert hashlib.sha256((root / name).read_bytes()).hexdigest() == expected, name + rows = json.loads((root / "metrics.json").read_text()) + cases = json.loads((root / "cases.json").read_text()) + assert len(rows) == 164 and len(cases) == 41 + assert len({(r["seed"], r["family"], r["method"]) for r in rows}) == 164 + for family in ("logit_linear", "probability_linear"): + assert {r["seed"] for r in rows if r["family"] == family} == set( + range(20261020, 20261040) + ) + assert sum(r["family"] == "wdbc_undistorted" for r in rows) == 4 + assert all(r["available"] or r["reason"] for r in rows) + summaries = json.loads((root / "summary.json").read_text())["simulation_summary"] + for s in summaries: + selected = [ + r for r in rows if r["family"] == s["family"] and r["method"] == s["method"] + ] + assert s["attempted"] == len(selected) == 20 + assert s["n"] == sum( + r["available"] and r.get(s["metric"]) is not None for r in selected + ) + manifests.append(manifest) + assert manifests[0] == manifests[1] + print( + json.dumps( + { + "stable_files_checked_per_run": len(manifests[0]["files"]), + "repeat_manifest_equal": True, + "cases": 41, + "method_cases": 164, + "all_fixed_seeds_accounted_for": True, + }, + indent=2, + ) + ) + + +if __name__ == "__main__": + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("run_a", type=Path) + p.add_argument("run_b", type=Path) + args = p.parse_args() + verify(args.run_a, args.run_b) From 3fc56b9c47bbe0936c91357cfb09a8e8b522d323 Mon Sep 17 00:00:00 2001 From: aRealGem <203440826+aRealGem@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:19:04 -0400 Subject: [PATCH 09/12] Qualify calibration uncertainty and cross-fit rank claims Apply the reviewed documentation corrections without changing the production Python AST. Clarify cluster-design assumptions, fixed-prediction interval limitations, whole-rung unavailability, and apparent versus pooled cross-fit rank preservation. Preserve historical benchmark evidence. Publication preserves the reviewed file tree; connector-assigned commit metadata differs. Reviewed-local-commit: bdd1f36b270be2f6a587ff2f7762e0ec0eef0f3b Co-developed-with: OpenAI Codex --- README.md | 16 ++++++++++++---- benchmarks/alternatives/INTEGRATION.md | 9 ++++++++- docs/spec.md | 26 ++++++++++++++++++++------ src/predval/recalibrate.py | 6 ++++-- 4 files changed, 44 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 43b3355..51b2842 100644 --- a/README.md +++ b/README.md @@ -22,10 +22,14 @@ Three things follow from that scope, and they are what predval spends its effort flatters the model. - **The unit of analysis and the unit of independence are usually not the same file column.** When predictions come from grouped data — patients with repeat visits, patches from the same - slide, images from the same device — per-row confidence intervals are not slightly optimistic, - they are wrong by roughly the square root of the cluster size. predval takes a declared - clustering unit and resamples whole clusters everywhere it computes uncertainty, and renders - the naive-vs-cluster comparison once so the gap is a number, not a caveat in a footnote. + slide, images from the same device — ignoring positive within-cluster dependence can seriously + understate uncertainty. For an illustrative mean with independent clusters of equal size `m` + and exchangeable within-cluster correlation `ICC`, the standard-error inflation is + `sqrt(1 + (m - 1) * ICC)`; + `sqrt(m)` is the special case `ICC = 1`, not a universal factor for every metric. predval takes + a declared clustering unit and resamples whole clusters everywhere it computes uncertainty, + and renders the naive-vs-cluster comparison once so the gap is a number, not a caveat in a + footnote. - **A recalibration that improves the numbers is a finding about the ladder's own fit, not a promotion of the model, until it survives being scored on clusters it never saw.** Every rung of predval's ladder — intercept, intercept+slope, spline — is fit twice: once on the rows it will @@ -43,6 +47,10 @@ Three things follow from that scope, and they are what predval spends its effort result and never as a clean bill of health: this harness's calibration test can fail to reject "calibrated", but it can never establish it. +The paired gain intervals hold fitted cross-fold predictions fixed. They do not establish +nominal coverage for the full fitting-and-selection procedure, or conditional coverage merely +because fitting is held fixed during resampling. See [the interval limitations](docs/spec.md#45-rung-intervals-the-paired-cross-fit-gain-carries-one-the-per-rung-refit-is-deferred). + ## What predval deliberately does not do - **It never loads model weights.** No `torch`, no `tensorflow`, no checkpoints. Inputs are a diff --git a/benchmarks/alternatives/INTEGRATION.md b/benchmarks/alternatives/INTEGRATION.md index fd80d0a..425a161 100644 --- a/benchmarks/alternatives/INTEGRATION.md +++ b/benchmarks/alternatives/INTEGRATION.md @@ -98,8 +98,15 @@ PY - The guard retains the same estimator, tolerance, and iteration limit. A valid finite-MLE fit can be unavailable if it needs more iterations. This change does not prove every unavailable-reason label is correct or detect every possible separation condition. + Because the existing half-pair guard withholds a rung if its apparent fit or any cross-fit + fold is unavailable, one iteration-exhausted fold can now withhold the entire rung for a + cohort that previously received a finite last-iterate result. - The two comparator gain intervals still cross zero. They condition on fixed held-out predictions and do not measure repeated-sampling coverage or correction-training uncertainty. + Shared fitting and overlapping training folds also mean that holding predictions fixed is + insufficient to establish conditional coverage. No replacement interval is introduced here. - The original isitfair adapter remains unexecuted; source review is not an execution result. - These are bounded engineering/method checks, not clinical validation or a superiority claim. -- All work is local. No push, pull request, merge, or deployment was performed. +- At the time of the original integration checks, all work was local; no push, pull request, + merge, or deployment had been performed. The reviewed tree was subsequently published in + [draft PR #6](https://github.com/aRealGem/predval/pull/6), at published head `4829570`. diff --git a/docs/spec.md b/docs/spec.md index 2299263..c592d08 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -107,8 +107,12 @@ grouping unit within which subjects are correlated. This exists because the unit of analysis and the unit of independence are frequently not the same. In the PCam fixture, 19,999 image patches derive from just 22 whole slides; patches from one slide are heavily correlated. Confidence intervals computed as though there were 19,999 -independent observations are not slightly optimistic, they are wrong by roughly the square root -of the cluster size. S1 validates and plumbs this field. S2 consumes it for cluster-robust or +independent observations can seriously understate uncertainty. For an illustrative mean with +independent clusters of equal size `m`, equal marginal variances, and exchangeable within-cluster +correlation `ICC`, the standard-error inflation is `sqrt(1 + (m - 1) * ICC)`. It equals `sqrt(m)` only when +`ICC = 1`. Unequal cluster sizes and other statistics, including AUROC, need their own variance +analysis; the correlation relevant to a loss mean is that of the losses, not necessarily the +outcomes. S1 validates and plumbs this field. S2 consumes it for cluster-robust or cluster-bootstrap intervals over the grouping unit. If `clustering` is omitted, S2 treats subjects as independent and says so in the report. @@ -326,14 +330,24 @@ replicate *and* every fold — so that it carries the variance of the correction gain holds the cross-fit mapping fixed; the full refit-in-replicate interval is a design surface of its own and remains backlog rather than a silent omission. +Holding the predictions fixed during resampling does **not** by itself establish conditional +coverage: cross-fitted loss differences share fitted corrections and overlapping training data. +The current intervals do not account for the full fitting-and-selection procedure. Respecting +the cluster resampling unit alone does not guarantee nominal coverage, and the `few_clusters` +warning is not a validated boundary above which coverage is guaranteed. Any proposed replacement +interval needs evaluation against an explicitly stated target before adoption. + ### 4.6 Every metric at every rung, with two integrity checks (S3.1) The ladder recomputes **all** metrics on each corrected mapping — not only calibration. Threshold metrics move because recalibration shifts the operating point, and that shift is clinically real; -rung3 can reorder scores and so change AUROC and average precision. Emitting the full set is what -lets a report show, side by side, that recalibration does **not** buy discrimination under the -monotone rungs (rung1/rung2 AUROC equals rung0's exactly) while a decision threshold's sensitivity -genuinely changes. +rung3 can reorder scores and so change AUROC and average precision. A single strictly increasing +mapping preserves ranks: apparent rung1, and apparent rung2 with a positive slope, preserve +AUROC if clipping or numerical saturation does not introduce ties. This invariance does **not** +extend to pooled cross-fitted predictions: different fold-specific mappings can reorder scores +across folds even when each mapping is increasing. A nonpositive rung2 slope also precludes the +strictly increasing argument. All metrics are therefore recomputed in both fit modes; threshold +sensitivity can change even when ranks are preserved. Two identities hold **by construction** and must be annotated as such in any report, never presented as findings: rung1's apparent calibration intercept is ≈ 0, and rung2's apparent diff --git a/src/predval/recalibrate.py b/src/predval/recalibrate.py index 0634ba1..1223cda 100644 --- a/src/predval/recalibrate.py +++ b/src/predval/recalibrate.py @@ -35,8 +35,10 @@ # Every metric is recomputed on each recalibrated mapping -- not only calibration. Threshold # metrics move because recalibration shifts the operating point, and rung3 can reorder scores, # so AUROC/average precision are no longer invariant. Emitting all of them is what lets the -# report show that recalibration cannot buy discrimination (rung1/rung2 AUROC == rung0) while a -# threshold's sensitivity genuinely changes. The metric set is M.THRESHOLD_FREE + M.THRESHOLDED. +# report distinguish rank-preserving apparent fits from pooled cross-fit scores. A single +# strictly increasing mapping preserves AUROC unless clipping/saturation creates ties; rung2 +# requires a positive slope. Different fold-specific mappings can reorder pooled held-out scores +# even when each is increasing. The metric set is M.THRESHOLD_FREE + M.THRESHOLDED. APPARENT = "apparent" CROSSFIT = "crossfit" From 937f0791ffd05be1ddb726deec2cbb2576238348 Mon Sep 17 00:00:00 2001 From: aRealGem <203440826+aRealGem@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:11:30 -0400 Subject: [PATCH 10/12] Add an executable own-data quickstart and qualify report claims Document the public API with a small synthetic CSV cohort, explicit event-probability orientation, and practical clustering, roster, coverage, and interpretation guidance. Qualify fixed-prediction gain intervals and distinguish apparent rank preservation from pooled cross-fit reordering at the report's decision points. Add regression checks for the report language and execute the README example in a clean directory, checking its observation and cluster counts and independently calculated baseline metrics. Production Python files, estimators, defaults, and historical benchmark evidence are unchanged. Reviewed-patch-sha256: f260276a029cb5bafe1c4d116df0bde489d2751fcc11a9eb2d3326734b41111b Co-developed-with: OpenAI Codex --- README.md | 137 +++++++++++++++++++++++++-- src/predval/templates/report.html.j2 | 19 +++- tests/test_readme_quickstart.py | 58 ++++++++++++ tests/test_report.py | 13 +++ 4 files changed, 212 insertions(+), 15 deletions(-) create mode 100644 tests/test_readme_quickstart.py diff --git a/README.md b/README.md index 51b2842..cdcad27 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,120 @@ prespecified subgroup behaviour, and a recalibration ladder that reports how muc miscalibration is repairable without touching the model, and discloses the ladder's own optimism rather than reporting only its best face. +## Quickstart with your own predictions + +No PCam files or model weights are needed. From a checkout, run `uv sync --locked`, then save +these three files in a new `my-cohort/` directory. This tiny, fully synthetic CSV example is +public demonstration data, not patient data or a statistical validation benchmark. + +`cohort.csv` (one row per observation): + +```csv +subject_id,patient_id,label +v01,p01,0 +v02,p01,1 +v03,p02,1 +v04,p02,0 +v05,p03,0 +v06,p03,1 +v07,p04,1 +v08,p04,0 +v09,p05,0 +v10,p05,1 +v11,p06,1 +v12,p06,0 +``` + +`predictions.csv` (one row per scored observation and model): + +```csv +subject_id,model_id,predicted +v01,my_model,0.20 +v02,my_model,0.65 +v03,my_model,0.35 +v04,my_model,0.55 +v05,my_model,0.30 +v06,my_model,0.75 +v07,my_model,0.60 +v08,my_model,0.40 +v09,my_model,0.70 +v10,my_model,0.45 +v11,my_model,0.80 +v12,my_model,0.25 +``` + +`cohort.yaml` (paths are relative to this file): + +```yaml +cohort_id: synthetic-own-data-demo +version: 0 +subject_key: subject_id +data: cohort.csv +unit_noun: visit +outcome: + type: binary + field: label + positive_label: 1 +clustering: + field: patient_id + name: patient +coverage: + min_fraction: 1.0 + compare_on: both +completeness: + require_outcome: true + on_violation: fail +expected_models: [my_model] +on_missing_model: fail +thresholds: [0.5] +uncertainty: + n_boot: 20 # Fast plumbing check only; the library default is 2000. + seed: 1337 +``` + +From the checkout, run the public API (CSV and Parquet are both supported): + +```bash +uv run python - <<'PY' +from pathlib import Path +from predval import evaluate, load_cohort, load_predictions, write_findings, write_report + +root = Path("my-cohort") +result = evaluate(load_cohort(root / "cohort.yaml"), load_predictions(root / "predictions.csv")) +out = root / "results" +result.write(out) +write_report(result, out / "report.html") +write_findings(result, out / "findings.json") +print(out / "report.html") +PY +``` + +Open `my-cohort/results/report.html`; `findings.json` is the machine-readable companion, and +`result.write` saves the metric tables and provenance manifest. With only six independent +patients, warnings or unavailable recalibration rungs are expected, not a reason to hide rows. + +To replace the toy data: +- Keep `subject_id` unique for each observation and identical across the two files. With one row + per patient it can identify that patient; repeated visits need unique visit IDs plus a separate + patient clustering column. Use pseudonymous IDs, not names or medical record numbers. +- Declare the actual independence unit in `clustering.field` (patient, site, slide, etc.). + Omitting it treats observations as independent; a unique row ID is not a substitute for a + shared cluster ID. Whether clusters are independent is a study-design judgment. +- List all intended model IDs in `expected_models`; this example fails if an entire model is + missing. `predicted` must be the probability of `outcome == positive_label`, supplied as a + finite number in `[0, 1]` (not the probability of the other class). An unscored observation has no prediction + row, never a zero placeholder. Set the coverage floor before analysis; `1.0` here requires + every usable observation to be scored by each model. `compare_on: both` also evaluates the + common scored subset. Review missing outcomes and coverage before interpreting comparisons. +- Prespecify the outcome, thresholds, subgroups, model roster and analysis plan. Replace the + demonstration bootstrap budget with a justified analysis budget; more resamples cannot fix + too few independent clusters or data-adaptive selection. + +A completed run checks these inputs and reports performance on this cohort. It does not prove +clinical utility, transportability, calibrated probabilities, or that a selected recalibration +will improve a new cohort. Gain intervals resample fixed cross-fitted predictions; neither +conditional coverage nor full fitting-and-selection coverage is established by that choice. + ## The pitch, narrowed Most tools in this space either train something (a new calibration layer, a monitoring model) @@ -38,18 +152,19 @@ Three things follow from that scope, and they are what predval spends its effort validated model, and — this is the part most recalibration tooling skips — refuses to manufacture a gain where none exists: on the PCam fixture below, nine of the fifteen members carry a detected miscalibration, ten have the repair's benefit left honestly unproven at this - cohort's small cluster count, and exactly one — `p4m_seed7` at rung2 — gets a real, - interval-backed repair. Five members carry at least one rung whose recalibration *reliably - harms* them, and `p4m_seed7` is one of the five: its rung2 repair is demonstrated while its - rung1 is harmful, which is why harm is reported independently of whether a repair was found. - Two of those five, `e2cnn_s21` and `e2cnn_s99`, are harmed at every rung and have no admissible - repair at all. The remaining six show no detected fault, which the report states as a null + cohort's small cluster count, and exactly one — `p4m_seed7` at rung2 — has a positive + estimated gain whose interval excludes zero under the configured fixed-prediction resampling. + Five members have at least one rung with a gain interval entirely below zero, including + `p4m_seed7` at rung1: estimated harm is reported independently of estimated benefit at rung2. + Two of those five, `e2cnn_s21` and `e2cnn_s99`, have negative gain intervals at every rung + and no admissible repair under the configured criterion. The remaining six show no detected fault, which the report states as a null result and never as a clean bill of health: this harness's calibration test can fail to reject "calibrated", but it can never establish it. The paired gain intervals hold fitted cross-fold predictions fixed. They do not establish nominal coverage for the full fitting-and-selection procedure, or conditional coverage merely -because fitting is held fixed during resampling. See [the interval limitations](docs/spec.md#45-rung-intervals-the-paired-cross-fit-gain-carries-one-the-per-rung-refit-is-deferred). +because fitting is held fixed during resampling. Limited independent clusters and data-adaptive +selection of models or rungs warrant caution when interpreting an interval that excludes zero. See [the interval limitations](docs/spec.md#45-rung-intervals-the-paired-cross-fit-gain-carries-one-the-per-rung-refit-is-deferred). ## What predval deliberately does not do @@ -261,9 +376,11 @@ matching PCam's — the full B=2000 run is frozen and regression-checked by headline numbers fails loudly. **Honest read of the verdict:** AUROC 0.791, comfortably inside the literature's expected -0.72–0.82 range on the first attempt. Calibration slope 0.835 shows real, if modest, displacement -scoring off the development region — the recalibration ladder finds a small but statistically -real gain at rungs 2–3 (both exclude zero), while rung 1 alone is actually slightly negative. This +0.72–0.82 range on the first attempt. Calibration slope 0.835 indicates modest displacement +scoring off the development region. Rungs 2–3 have small positive estimated gains whose +intervals exclude zero under the configured fixed-prediction resampling; rung 1 has a slightly +negative point estimate. Limited independent clusters and data-adaptive selection limit what +these intervals establish; they do not validate the full fitting-and-selection procedure. This is a much milder miscalibration story than PCam's two badly-miscalibrated members, which is an honest property of this cohort rather than something to chase. The real finding of this example isn't the numbers — it's that predval's ingestion, coverage, roster, clustering, and ladder code diff --git a/src/predval/templates/report.html.j2 b/src/predval/templates/report.html.j2 index 54064df..78e6571 100644 --- a/src/predval/templates/report.html.j2 +++ b/src/predval/templates/report.html.j2 @@ -89,8 +89,9 @@ cross-check in metrics.parquet agrees (none crossed the 0 loss boundary).

best admissible rung -- no free adjectives. BSS is anchored at 0 = no-skill (always predict prevalence) and 1 = perfect; "closes Y% of the gap" is BSS as a percentage. The gauge sentence states any detected miscalibration (slope or intercept interval excluding its -null), the lowest rung whose out-of-fold repair is demonstrated, and any rung whose recalibration -reliably harms. A member with no detected fault and no demonstrated repair carries no gauge +null), the lowest rung meeting the repair criterion, and any rung meeting the harm criterion. +These labels describe gain intervals under fixed-prediction resampling, with the small-cluster +and selection limitations explained in section 3; they do not establish benefit on a new cohort. A member with no detected fault and no demonstrated repair carries no gauge sentence: the test can fail to reject calibration but never establish it.