diff --git a/docs/reports/improve.html b/docs/reports/improve.html new file mode 100644 index 0000000..6e0f8de --- /dev/null +++ b/docs/reports/improve.html @@ -0,0 +1,6593 @@ + + + + +predval report -- improve-neoepitope-cv-patient-grouped + + + +

predval report

+

cohort: improve-neoepitope-cv-patient-grouped

+ +
No rung is a validated model. Rung0 is the model as published. Rungs 1-3 are diagnosis of rung0, not competitors to it. The cross-fitted figures are the expected performance after local recalibration on a cohort like this one -- not evidence that the recalibrated model has been validated, which would require a cohort the correction was never fitted on. Recalibration does not and cannot improve discrimination.
+ +

1. Coverage and roster

+

Roster: 7 declared, + 7 present, + 0 absent.

+ + + + + + + + + + + + + + + + + + + + + + + +
modelcoveragescoreddropped if commonmeets minimum
improve_simple1.0000175200yes
improve_simple_wo_prime1.0000175200yes
improve_tme_excl1.0000175200yes
improve_tme_excl_wo_prime1.0000175200yes
improve_tme_incl1.0000175200yes
improve_tme_incl_wo_prime1.0000175200yes
nnalign1.0000175200yes
+ +

2. Primary comparison -- model as published (rung0)

+

Common subset, overall stratum. AUROC and Brier carry cluster intervals; the +calibration slope and intercept carry cluster-robust analytic intervals. +Full and common coverage are identical here, so the coverage-delta +column is omitted (every value would be zero).

+ + + + + + + + + + + + + + + + + + + + + + + + +
modelAUROCavg precisionBriercalib slopecalib intercept
improve_tme_incl0.6524 [0.606, 0.697]0.0525 [0.039, 0.070]0.1676 [0.152, 0.185]1.0972 [0.771, 1.424]-3.2466 [-3.463, -3.030]
improve_tme_incl_wo_prime0.6514 [0.604, 0.696]0.0522 [0.039, 0.071]0.1680 [0.153, 0.184]1.1701 [0.814, 1.527]-3.2437 [-3.459, -3.028]
improve_simple0.6425 [0.607, 0.679]0.0490 [0.037, 0.067]0.1736 [0.166, 0.182]1.0848 [0.772, 1.397]-3.2777 [-3.512, -3.043]
improve_simple_wo_prime0.6402 [0.606, 0.676]0.0491 [0.037, 0.067]0.1736 [0.167, 0.181]1.1070 [0.786, 1.428]-3.2766 [-3.509, -3.044]
improve_tme_excl0.6302 [0.597, 0.668]0.0498 [0.037, 0.068]0.1741 [0.167, 0.181]1.0719 [0.753, 1.391]-3.2761 [-3.513, -3.039]
improve_tme_excl_wo_prime0.6257 [0.591, 0.662]0.0495 [0.037, 0.068]0.1744 [0.168, 0.182]1.0886 [0.747, 1.430]-3.2759 [-3.512, -3.040]
nnalign0.6054 [0.564, 0.647]0.0441 [0.034, 0.058]0.0258 [0.021, 0.032]0.9697 [0.580, 1.359]-0.0882 [-0.326, 0.150]
+

Brier intervals above are the cluster percentile bootstrap; the analytic t(G-1) +cross-check in metrics.parquet agrees (none crossed the 0 loss boundary).

+ +

Verdict

+

One line per member, assembled only from AUROC, the Brier skill score (BSS), and the +best admissible rung -- no free adjectives. BSS is anchored at 0 = no-skill (always +predict prevalence) and 1 = perfect; "closes Y% of the gap" is BSS as a percentage. +The gauge sentence states any detected miscalibration (slope or intercept interval excluding its +null), the lowest rung whose out-of-fold repair is demonstrated, and any rung whose recalibration +reliably harms. A member with no detected fault and no demonstrated repair carries no gauge +sentence: the test can fail to reject calibration but never establish it.

+ + +

Calibration (rung0, as published)

+

Decile calibration per member: points are observed frequency against mean predicted +probability, the shaded band is the cluster bootstrap on the observed rate, and the dashed diagonal +is perfect. Points below the diagonal over-predict risk; above, under-predict.

+
+ + + + + image/svg+xml + + + Matplotlib v3.11.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +

Unit-of-analysis exhibit

+

The naive per-peptide AUROC interval is incorrect when peptides share a patient. It is shown only to make the cost of ignoring clustering concrete. The ratio is the cluster-aware interval width divided by the naive per-peptide width. Above 1 it is the factor by which the naive interval is too narrow.

+ + + + + + + + + + + + + + + + +
modelcluster widthnaive width (wrong)ratio
improve_tme_incl0.09150.05331.7x
improve_tme_incl_wo_prime0.09230.05361.7x
improve_simple0.07150.05171.4x
improve_simple_wo_prime0.07000.05361.3x
improve_tme_excl0.07110.05281.3x
improve_tme_excl_wo_prime0.07160.05621.3x
nnalign0.08330.05471.5x
+
+ + + + + image/svg+xml + + + Matplotlib v3.11.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+

3. Calibration diagnosis (subordinate to section 2)

+

This section diagnoses rung0; it does not rank models. Rows are ordered with +demonstrated repairs first (by rung, then by interval strength), then the remaining members by +their best gain estimate -- the largest point estimate is not the strongest evidence. Each cell is the paired cross-fit Brier gain -- rung0 Brier +minus that rung's held-out Brier, positive when recalibration helped -- with its paired +cluster-bootstrap interval (a gain whose interval excludes 0 is a real out-of-fold improvement). +The apparent, in-sample gains run larger; only the cross-fitted gains, which hold whole +patients out, are shown. rung1's apparent intercept is ~0 and rung2's apparent slope is +~1 by construction (that is what each fit targets), so those are +not reported as findings. Recalibration cannot improve discrimination; the cross-fitted Brier is the +expected loss after local recalibration, not a validated result.

+

At small G the flexible rung is high-variance under cross-fitting: absence of +a rung3 gain is evidence of linear miscalibration only insofar as rung3 had power to detect shape +distortion.

+ + + + + + + + + + + + + + + + + +
modelBrier rung0rung1 gain [CI] (x10⁻³)rung2 gain [CI] (x10⁻³)rung3 gain [CI] (x10⁻³)
improve_tme_excl_wo_prime0.1744+148.58 [140.07, 157.95] x10⁻³+148.58 [140.07, 157.94] x10⁻³+148.58 [140.08, 157.92] x10⁻³
improve_simple_wo_prime0.1736+147.85 [139.74, 156.48] x10⁻³+147.85 [139.76, 156.47] x10⁻³+147.85 [139.75, 156.46] x10⁻³
improve_simple0.1736+147.87 [139.41, 157.13] x10⁻³+147.87 [139.42, 157.13] x10⁻³+147.86 [139.41, 157.11] x10⁻³
improve_tme_excl0.1741+148.28 [139.40, 157.82] x10⁻³+148.27 [139.40, 157.83] x10⁻³+148.26 [139.37, 157.82] x10⁻³
improve_tme_incl_wo_prime0.1680+142.21 [128.85, 157.54] x10⁻³+142.19 [128.86, 157.53] x10⁻³+142.18 [128.86, 157.53] x10⁻³
improve_tme_incl0.1676+141.89 [127.90, 158.12] x10⁻³+141.87 [127.90, 158.08] x10⁻³+141.86 [127.87, 158.07] x10⁻³
nnalign0.0258-0.02 [-0.07, 0.03] x10⁻³-0.02 [-0.08, 0.03] x10⁻³-0.05 [-0.11, -0.00] x10⁻³
+
+ + + + + image/svg+xml + + + Matplotlib v3.11.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+

4. Subgroups

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
subgroupmodelAUROCBrier
peptide_length=9merimprove_simple0.6315 [0.587, 0.677]0.1865 [0.179, 0.194]
peptide_length=9merimprove_simple_wo_prime0.6315 [0.589, 0.676]0.1813 [0.174, 0.189]
peptide_length=9merimprove_tme_excl0.6197 [0.576, 0.667]0.1859 [0.179, 0.192]
peptide_length=9merimprove_tme_excl_wo_prime0.6177 [0.574, 0.665]0.1808 [0.175, 0.187]
peptide_length=9merimprove_tme_incl0.6463 [0.592, 0.700]0.1782 [0.162, 0.196]
peptide_length=9merimprove_tme_incl_wo_prime0.6463 [0.593, 0.698]0.1732 [0.158, 0.189]
peptide_length=9mernnalign0.5906 [0.542, 0.643]0.0282 [0.022, 0.035]
peptide_length=non_9merimprove_simple0.6514 [0.609, 0.692]0.1542 [0.147, 0.162]
peptide_length=non_9merimprove_simple_wo_prime0.6484 [0.603, 0.691]0.1620 [0.155, 0.170]
peptide_length=non_9merimprove_tme_excl0.6395 [0.596, 0.679]0.1561 [0.148, 0.165]
peptide_length=non_9merimprove_tme_excl_wo_prime0.6345 [0.586, 0.677]0.1647 [0.157, 0.173]
peptide_length=non_9merimprove_tme_incl0.6541 [0.602, 0.703]0.1516 [0.137, 0.168]
peptide_length=non_9merimprove_tme_incl_wo_prime0.6547 [0.601, 0.704]0.1600 [0.145, 0.177]
peptide_length=non_9mernnalign0.6365 [0.583, 0.687]0.0223 [0.017, 0.028]
tumour_cohort=Basketimprove_simple0.6798 [0.626, 0.765]0.1490 [0.142, 0.154]
tumour_cohort=Basketimprove_simple_wo_prime0.6728 [0.618, 0.751]0.1501 [0.143, 0.155]
tumour_cohort=Basketimprove_tme_excl0.6679 [0.614, 0.753]0.1552 [0.143, 0.164]
tumour_cohort=Basketimprove_tme_excl_wo_prime0.6586 [0.607, 0.736]0.1570 [0.146, 0.166]
tumour_cohort=Basketimprove_tme_incl0.6789 [0.585, 0.757]0.1318 [0.112, 0.161]
tumour_cohort=Basketimprove_tme_incl_wo_prime0.6721 [0.571, 0.752]0.1343 [0.115, 0.163]
tumour_cohort=Basketnnalign0.6752 [0.605, 0.738]0.0210 [0.013, 0.033]
tumour_cohort=bladderimprove_simple0.6208 [0.565, 0.683]0.1839 [0.169, 0.194]
tumour_cohort=bladderimprove_simple_wo_prime0.6228 [0.569, 0.681]0.1813 [0.168, 0.191]
tumour_cohort=bladderimprove_tme_excl0.6090 [0.551, 0.668]0.1754 [0.165, 0.184]
tumour_cohort=bladderimprove_tme_excl_wo_prime0.6075 [0.551, 0.663]0.1735 [0.164, 0.181]
tumour_cohort=bladderimprove_tme_incl0.6016 [0.535, 0.686]0.1724 [0.155, 0.195]
tumour_cohort=bladderimprove_tme_incl_wo_prime0.6052 [0.541, 0.685]0.1704 [0.154, 0.191]
tumour_cohort=bladdernnalign0.5566 [0.481, 0.635]0.0230 [0.017, 0.031]
tumour_cohort=melanomaimprove_simple0.6125 [0.566, 0.656]0.1852 [0.178, 0.193]
tumour_cohort=melanomaimprove_simple_wo_prime0.6057 [0.560, 0.648]0.1868 [0.180, 0.196]
tumour_cohort=melanomaimprove_tme_excl0.5973 [0.547, 0.650]0.1897 [0.180, 0.201]
tumour_cohort=melanomaimprove_tme_excl_wo_prime0.5908 [0.535, 0.648]0.1911 [0.181, 0.202]
tumour_cohort=melanomaimprove_tme_incl0.6468 [0.588, 0.700]0.1951 [0.173, 0.218]
tumour_cohort=melanomaimprove_tme_incl_wo_prime0.6434 [0.579, 0.700]0.1958 [0.172, 0.220]
tumour_cohort=melanomannalign0.6045 [0.545, 0.667]0.0332 [0.023, 0.045]
+ +

5. Fragility -- not a confidence interval

+

The largest AUROC change from dropping any single cluster, and the culprit. This is +deliberately not an interval: it answers whether a result is carried by one +patient, which a tight bootstrap interval will not reveal.

+ + + + + + + + + +
modelmax |delta AUROC|culprit cluster
improve_tme_incl0.0129RH-31
improve_tme_incl_wo_prime0.0121RH-31
nnalign0.0096BC-6229
improve_simple0.0093RH-31
improve_simple_wo_prime0.0087RH-31
improve_tme_excl0.0079RH-31
improve_tme_excl_wo_prime0.0069RH-31
+ +

Limitations

+
The optimism correction covers the recalibration step only. predval evaluates the predictions it is handed and has no view of how they were produced. If model selection or tuning used patients inside this cohort, rung0 is itself optimistically biased and this harness cannot see it; only a cohort the model was never tuned on could expose that. The paired cross-fit gain interval (section 3) conditions on the fitted correction -- it resamples the patient-level loss difference but does not refit the correction inside each bootstrap replicate, so it does not carry the correction's own fitting variance; that refit-in-replicate interval remains backlog (spec section 4.5).
+ +

6. Provenance

+

Input digests (blake2b), the bootstrap seed and B, and library versions -- enough to +reproduce every number.

+ + + + + + + + + + + + + +
cohort_data0602dee2f9c80b0512e9f6e5e3470fd0cd85fb561578adbcbc6956edb905baf4
cohort_speccc9ee94cfcdc09ff8be98514fb1c33b0cf14cb80338649b4d10caa8336b7b617
predictionsd5c66de35023d643f11f3e750a9d303180de8588c088d94abb651e77453c9972
contract version0
bootstrapB=2000, seed=1337, + ci=0.95
numpy2.4.6
pandas3.0.5
predval0.1.0
python3.13.16
scikit-learn1.9.0
scipy1.17.1
statsmodels0.14.6
+ +

Flags

+ + + + + \ No newline at end of file diff --git a/tests/golden/improve/GOLDEN.md b/tests/golden/improve/GOLDEN.md new file mode 100644 index 0000000..7bbae51 --- /dev/null +++ b/tests/golden/improve/GOLDEN.md @@ -0,0 +1,45 @@ +# IMPROVE golden baseline + +Frozen regression baseline for the IMPROVE neoepitope-immunogenicity example. Generated by: + +```bash +uv run python examples/improve/build_fixture.py --src 'improve_results/results copy' +uv run python examples/improve/run_evaluation.py +``` + +(default config: `B=2000`, `seed=1337`, from `examples/improve/cohort.yaml`; 7 models, 17,520 +peptides, G=70 patient clusters, 467 immunogenic = 2.666% prevalence) + +| | | +|---|---| +| generated | 2026-10-07 | +| predval commit | `4f9fbd0` (predval 0.1.0) | +| source repo | `https://github.com/SRHgroup/IMPROVE_paper` at `c67094276b52358b5202ad9e5597acd87888ea1d` | +| `results.zip` sha256 | `26897006104ef0e57aeedc003804c9a7e43d3735d9df157d651516f2739e00f9` | +| wall time | ~12.5 min on a 4-core container | + +`findings.json` keeps its `provenance.git_commit` as generated; `tests/test_golden_improve.py` +normalizes that one field out of the comparison, as the GUSTO and PCam goldens do. + +Reproducibility was verified at freeze time: a second independent run at the same commit was +compared against these files by `tests/test_golden_improve.py`. + +## Headline numbers this baseline pins + +- **Prevalence anchoring.** The six IMPROVE random-forest configurations predict a mean + probability of 0.39-0.40 against an observed rate of 0.0267, i.e. 14.6-15.1x too high. The + NNAlign comparator is anchored (1.1x). +- **Discrimination.** AUROC 0.63-0.65 for the IMPROVE models, 0.61 for NNAlign + (patient-clustered 95% CIs roughly +/-0.04). +- **Calibration.** Intercepts about -3.25 (CI excludes 0) with slopes about 1.1 (CI includes 1): + the IMPROVE scores are a ranking with the right spread and the wrong level. NNAlign: + intercept -0.09 [-0.33, 0.15], slope 0.97. +- **Recalibration ladder.** Cross-fitted rung1 (intercept only, fitted on held-out patients) + cuts Brier from ~0.17 to 0.0258 for every IMPROVE model, a demonstrated repair. That repaired + Brier equals the prevalence-only constant predictor (0.0259): once the level is fixed, the + probabilities carry almost no Brier-scale skill beyond the base rate. NNAlign gains nothing + from recalibration. +- **Unit of analysis.** Patient-clustered AUROC intervals are 1.3-1.7x wider than naive + per-peptide ones. +- **Fragility.** Dropping patient RH-31 moves AUROC the most for six of seven models + (largest 0.013). diff --git a/tests/golden/improve/calibration.parquet b/tests/golden/improve/calibration.parquet new file mode 100644 index 0000000..418bc8a Binary files /dev/null and b/tests/golden/improve/calibration.parquet differ diff --git a/tests/golden/improve/coverage.parquet b/tests/golden/improve/coverage.parquet new file mode 100644 index 0000000..3b28726 Binary files /dev/null and b/tests/golden/improve/coverage.parquet differ diff --git a/tests/golden/improve/findings.json b/tests/golden/improve/findings.json new file mode 100644 index 0000000..6fb27c7 --- /dev/null +++ b/tests/golden/improve/findings.json @@ -0,0 +1,624 @@ +{ + "cohort_id": "improve-neoepitope-cv-patient-grouped", + "flags": [ + { + "code": "few_clusters", + "message": "tail quantiles approximate with few clusters (G=20) for tumour_cohort=Basket; percentile bootstrap intervals below 40 clusters are coarse", + "severity": "note" + }, + { + "code": "few_clusters", + "message": "tail quantiles approximate with few clusters (G=24) for tumour_cohort=bladder; percentile bootstrap intervals below 40 clusters are coarse", + "severity": "note" + }, + { + "code": "few_clusters", + "message": "tail quantiles approximate with few clusters (G=26) for tumour_cohort=melanoma; percentile bootstrap intervals below 40 clusters are coarse", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_simple (peptide_length=non_9mer): delta_auroc=+0.0036 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0005, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_simple (tumour_cohort=Basket): delta_auroc=+0.0073 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_simple_wo_prime (peptide_length=non_9mer): delta_auroc=+0.0047 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0003, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_simple_wo_prime (tumour_cohort=Basket): delta_auroc=+0.0119 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_simple_wo_prime (tumour_cohort=melanoma): delta_auroc=+0.0068 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0002, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_tme_excl (peptide_length=non_9mer): delta_auroc=+0.0037 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0005, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_tme_excl (tumour_cohort=Basket): delta_auroc=+0.0041 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_tme_excl_wo_prime (peptide_length=non_9mer): delta_auroc=+0.0049 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0003, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_tme_excl_wo_prime (tumour_cohort=Basket): delta_auroc=+0.0277 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_tme_excl_wo_prime (tumour_cohort=bladder): delta_auroc=+0.0020 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0002, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_tme_excl_wo_prime (tumour_cohort=melanoma): delta_auroc=+0.0087 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0005, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_tme_incl (tumour_cohort=bladder): delta_auroc=+0.0161 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_tme_incl_wo_prime (tumour_cohort=bladder): delta_auroc=+0.0092 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for nnalign (overall): delta_auroc=-0.0014 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for nnalign (peptide_length=9mer): delta_auroc=+0.0057 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for nnalign (tumour_cohort=Basket): delta_auroc=+0.0022 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0005, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for nnalign (tumour_cohort=melanoma): delta_auroc=+0.0074 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)", + "severity": "note" + } + ], + "members": [ + { + "analytic_ci_truncated": false, + "metrics": { + "auroc": { + "ci_high": 0.678939, + "ci_low": 0.607441, + "ci_method": "cluster_bootstrap", + "value": 0.642521 + }, + "average_precision": { + "ci_high": 0.066597, + "ci_low": 0.037073, + "ci_method": "cluster_bootstrap", + "value": 0.048959 + }, + "brier": { + "ci_high": 0.181697, + "ci_low": 0.166324, + "ci_method": "cluster_bootstrap", + "value": 0.173643 + }, + "calibration_intercept": { + "ci_high": -3.043193, + "ci_low": -3.512195, + "ci_method": "cluster_robust_t", + "value": -3.277694 + }, + "calibration_slope": { + "ci_high": 1.397442, + "ci_low": 0.772114, + "ci_method": "cluster_robust_t", + "value": 1.084778 + } + }, + "model_id": "improve_simple", + "recalibration_gains": [ + { + "ci_high": 0.157132, + "ci_low": 0.139412, + "ci_method": "paired_cluster_bootstrap", + "gain": 0.147872, + "rung": "rung1" + }, + { + "ci_high": 0.157131, + "ci_low": 0.139423, + "ci_method": "paired_cluster_bootstrap", + "gain": 0.147873, + "rung": "rung2" + }, + { + "ci_high": 0.157112, + "ci_low": 0.139413, + "ci_method": "paired_cluster_bootstrap", + "gain": 0.147858, + "rung": "rung3" + } + ], + "verdict": { + "axis_a_miscalibrated": true, + "axis_b": "demonstrated", + "best_rung": "rung1", + "bss": 0.006726, + "bss_ci_high": 0.011459, + "bss_ci_low": 0.00101, + "gauge_label": "miscalibrated (intercept -3.2777 [-3.512, -3.043]); repair demonstrated at rung1 (level (calibration-in-the-large))" + } + }, + { + "analytic_ci_truncated": false, + "metrics": { + "auroc": { + "ci_high": 0.67614, + "ci_low": 0.606165, + "ci_method": "cluster_bootstrap", + "value": 0.640178 + }, + "average_precision": { + "ci_high": 0.066637, + "ci_low": 0.037252, + "ci_method": "cluster_bootstrap", + "value": 0.049148 + }, + "brier": { + "ci_high": 0.181235, + "ci_low": 0.16676, + "ci_method": "cluster_bootstrap", + "value": 0.173622 + }, + "calibration_intercept": { + "ci_high": -3.043955, + "ci_low": -3.509214, + "ci_method": "cluster_robust_t", + "value": -3.276584 + }, + "calibration_slope": { + "ci_high": 1.427888, + "ci_low": 0.786151, + "ci_method": "cluster_robust_t", + "value": 1.10702 + } + }, + "model_id": "improve_simple_wo_prime", + "recalibration_gains": [ + { + "ci_high": 0.156479, + "ci_low": 0.139736, + "ci_method": "paired_cluster_bootstrap", + "gain": 0.147851, + "rung": "rung1" + }, + { + "ci_high": 0.15647, + "ci_low": 0.139755, + "ci_method": "paired_cluster_bootstrap", + "gain": 0.147851, + "rung": "rung2" + }, + { + "ci_high": 0.156461, + "ci_low": 0.139751, + "ci_method": "paired_cluster_bootstrap", + "gain": 0.147854, + "rung": "rung3" + } + ], + "verdict": { + "axis_a_miscalibrated": true, + "axis_b": "demonstrated", + "best_rung": "rung1", + "bss": 0.006691, + "bss_ci_high": 0.011281, + "bss_ci_low": 0.001142, + "gauge_label": "miscalibrated (intercept -3.2766 [-3.509, -3.044]); repair demonstrated at rung1 (level (calibration-in-the-large))" + } + }, + { + "analytic_ci_truncated": false, + "metrics": { + "auroc": { + "ci_high": 0.667999, + "ci_low": 0.596901, + "ci_method": "cluster_bootstrap", + "value": 0.630215 + }, + "average_precision": { + "ci_high": 0.06774, + "ci_low": 0.03748, + "ci_method": "cluster_bootstrap", + "value": 0.049766 + }, + "brier": { + "ci_high": 0.181351, + "ci_low": 0.166968, + "ci_method": "cluster_bootstrap", + "value": 0.174057 + }, + "calibration_intercept": { + "ci_high": -3.03901, + "ci_low": -3.513196, + "ci_method": "cluster_robust_t", + "value": -3.276103 + }, + "calibration_slope": { + "ci_high": 1.39062, + "ci_low": 0.753222, + "ci_method": "cluster_robust_t", + "value": 1.071921 + } + }, + "model_id": "improve_tme_excl", + "recalibration_gains": [ + { + "ci_high": 0.157824, + "ci_low": 0.139401, + "ci_method": "paired_cluster_bootstrap", + "gain": 0.148276, + "rung": "rung1" + }, + { + "ci_high": 0.157832, + "ci_low": 0.139396, + "ci_method": "paired_cluster_bootstrap", + "gain": 0.148273, + "rung": "rung2" + }, + { + "ci_high": 0.157819, + "ci_low": 0.139372, + "ci_method": "paired_cluster_bootstrap", + "gain": 0.148257, + "rung": "rung3" + } + ], + "verdict": { + "axis_a_miscalibrated": true, + "axis_b": "demonstrated", + "best_rung": "rung1", + "bss": 0.006288, + "bss_ci_high": 0.010847, + "bss_ci_low": 0.001046, + "gauge_label": "miscalibrated (intercept -3.2761 [-3.513, -3.039]); repair demonstrated at rung1 (level (calibration-in-the-large))" + } + }, + { + "analytic_ci_truncated": false, + "metrics": { + "auroc": { + "ci_high": 0.662456, + "ci_low": 0.590813, + "ci_method": "cluster_bootstrap", + "value": 0.625699 + }, + "average_precision": { + "ci_high": 0.067936, + "ci_low": 0.03709, + "ci_method": "cluster_bootstrap", + "value": 0.049497 + }, + "brier": { + "ci_high": 0.181512, + "ci_low": 0.167692, + "ci_method": "cluster_bootstrap", + "value": 0.174373 + }, + "calibration_intercept": { + "ci_high": -3.040116, + "ci_low": -3.511595, + "ci_method": "cluster_robust_t", + "value": -3.275855 + }, + "calibration_slope": { + "ci_high": 1.430282, + "ci_low": 0.746937, + "ci_method": "cluster_robust_t", + "value": 1.08861 + } + }, + "model_id": "improve_tme_excl_wo_prime", + "recalibration_gains": [ + { + "ci_high": 0.157948, + "ci_low": 0.14007, + "ci_method": "paired_cluster_bootstrap", + "gain": 0.148584, + "rung": "rung1" + }, + { + "ci_high": 0.157943, + "ci_low": 0.140066, + "ci_method": "paired_cluster_bootstrap", + "gain": 0.148582, + "rung": "rung2" + }, + { + "ci_high": 0.157922, + "ci_low": 0.140077, + "ci_method": "paired_cluster_bootstrap", + "gain": 0.148576, + "rung": "rung3" + } + ], + "verdict": { + "axis_a_miscalibrated": true, + "axis_b": "demonstrated", + "best_rung": "rung1", + "bss": 0.006, + "bss_ci_high": 0.01054, + "bss_ci_low": 0.000843, + "gauge_label": "miscalibrated (intercept -3.2759 [-3.512, -3.040]); repair demonstrated at rung1 (level (calibration-in-the-large))" + } + }, + { + "analytic_ci_truncated": false, + "metrics": { + "auroc": { + "ci_high": 0.697397, + "ci_low": 0.605923, + "ci_method": "cluster_bootstrap", + "value": 0.65243 + }, + "average_precision": { + "ci_high": 0.070342, + "ci_low": 0.03912, + "ci_method": "cluster_bootstrap", + "value": 0.05252 + }, + "brier": { + "ci_high": 0.184694, + "ci_low": 0.152257, + "ci_method": "cluster_bootstrap", + "value": 0.167626 + }, + "calibration_intercept": { + "ci_high": -3.029761, + "ci_low": -3.46348, + "ci_method": "cluster_robust_t", + "value": -3.246621 + }, + "calibration_slope": { + "ci_high": 1.42373, + "ci_low": 0.770754, + "ci_method": "cluster_robust_t", + "value": 1.097242 + } + }, + "model_id": "improve_tme_incl", + "recalibration_gains": [ + { + "ci_high": 0.15812, + "ci_low": 0.127903, + "ci_method": "paired_cluster_bootstrap", + "gain": 0.14189, + "rung": "rung1" + }, + { + "ci_high": 0.158078, + "ci_low": 0.127897, + "ci_method": "paired_cluster_bootstrap", + "gain": 0.141872, + "rung": "rung2" + }, + { + "ci_high": 0.158069, + "ci_low": 0.127871, + "ci_method": "paired_cluster_bootstrap", + "gain": 0.141861, + "rung": "rung3" + } + ], + "verdict": { + "axis_a_miscalibrated": true, + "axis_b": "demonstrated", + "best_rung": "rung1", + "bss": 0.008079, + "bss_ci_high": 0.013428, + "bss_ci_low": 0.00146, + "gauge_label": "miscalibrated (intercept -3.2466 [-3.463, -3.030]); repair demonstrated at rung1 (level (calibration-in-the-large))" + } + }, + { + "analytic_ci_truncated": false, + "metrics": { + "auroc": { + "ci_high": 0.696474, + "ci_low": 0.604139, + "ci_method": "cluster_bootstrap", + "value": 0.651394 + }, + "average_precision": { + "ci_high": 0.070554, + "ci_low": 0.038899, + "ci_method": "cluster_bootstrap", + "value": 0.052213 + }, + "brier": { + "ci_high": 0.184142, + "ci_low": 0.15347, + "ci_method": "cluster_bootstrap", + "value": 0.167951 + }, + "calibration_intercept": { + "ci_high": -3.028426, + "ci_low": -3.458913, + "ci_method": "cluster_robust_t", + "value": -3.24367 + }, + "calibration_slope": { + "ci_high": 1.526758, + "ci_low": 0.813521, + "ci_method": "cluster_robust_t", + "value": 1.17014 + } + }, + "model_id": "improve_tme_incl_wo_prime", + "recalibration_gains": [ + { + "ci_high": 0.157538, + "ci_low": 0.128854, + "ci_method": "paired_cluster_bootstrap", + "gain": 0.142207, + "rung": "rung1" + }, + { + "ci_high": 0.157531, + "ci_low": 0.128858, + "ci_method": "paired_cluster_bootstrap", + "gain": 0.142188, + "rung": "rung2" + }, + { + "ci_high": 0.157532, + "ci_low": 0.12886, + "ci_method": "paired_cluster_bootstrap", + "gain": 0.14218, + "rung": "rung3" + } + ], + "verdict": { + "axis_a_miscalibrated": true, + "axis_b": "demonstrated", + "best_rung": "rung1", + "bss": 0.00773, + "bss_ci_high": 0.01279, + "bss_ci_low": 0.00166, + "gauge_label": "miscalibrated (intercept -3.2437 [-3.459, -3.028]); repair demonstrated at rung1 (level (calibration-in-the-large))" + } + }, + { + "analytic_ci_truncated": false, + "metrics": { + "auroc": { + "ci_high": 0.647381, + "ci_low": 0.564122, + "ci_method": "cluster_bootstrap", + "value": 0.60537 + }, + "average_precision": { + "ci_high": 0.058386, + "ci_low": 0.03393, + "ci_method": "cluster_bootstrap", + "value": 0.044111 + }, + "brier": { + "ci_high": 0.031751, + "ci_low": 0.020575, + "ci_method": "cluster_bootstrap", + "value": 0.02583 + }, + "calibration_intercept": { + "ci_high": 0.149966, + "ci_low": -0.326434, + "ci_method": "cluster_robust_t", + "value": -0.088234 + }, + "calibration_slope": { + "ci_high": 1.359274, + "ci_low": 0.580042, + "ci_method": "cluster_robust_t", + "value": 0.969658 + } + }, + "model_id": "nnalign", + "recalibration_gains": [ + { + "ci_high": 2.6e-05, + "ci_low": -7.1e-05, + "ci_method": "paired_cluster_bootstrap", + "gain": -1.9e-05, + "rung": "rung1" + }, + { + "ci_high": 2.9e-05, + "ci_low": -8e-05, + "ci_method": "paired_cluster_bootstrap", + "gain": -2.2e-05, + "rung": "rung2" + }, + { + "ci_high": -1e-06, + "ci_low": -0.000114, + "ci_method": "paired_cluster_bootstrap", + "gain": -5.4e-05, + "rung": "rung3" + } + ], + "verdict": { + "axis_a_miscalibrated": false, + "axis_b": "counterproductive", + "best_rung": "rung0", + "bss": 0.004425, + "bss_ci_high": 0.008335, + "bss_ci_low": -0.001112, + "gauge_label": "recalibration harm (paired cross-fit Brier gain, x10\u207b\u00b3): rung3 [-0.11, -0.00]" + } + } + ], + "provenance": { + "ci_level": 0.95, + "config": { + "clustered": true, + "clustering_field": "patient", + "coverage_compare_on": "both", + "coverage_min_fraction": 0.95, + "expected_models": [ + "improve_simple", + "improve_simple_wo_prime", + "improve_tme_excl", + "improve_tme_excl_wo_prime", + "improve_tme_incl", + "improve_tme_incl_wo_prime", + "nnalign" + ] + }, + "git_commit": "4f9fbd00b2f84bf673d66bead230eb6e49c6400b-dirty", + "input_hashes": { + "cohort_data": "0602dee2f9c80b0512e9f6e5e3470fd0cd85fb561578adbcbc6956edb905baf4", + "cohort_spec": "cc9ee94cfcdc09ff8be98514fb1c33b0cf14cb80338649b4d10caa8336b7b617", + "predictions": "d5c66de35023d643f11f3e750a9d303180de8588c088d94abb651e77453c9972" + }, + "n_boot": 2000, + "seed": 1337, + "versions": { + "numpy": "2.4.6", + "pandas": "3.0.5", + "predval": "0.1.0", + "python": "3.13.16", + "scikit-learn": "1.9.0", + "scipy": "1.17.1", + "statsmodels": "0.14.6" + } + }, + "roster": { + "absent": [], + "declared": 7, + "present": 7 + }, + "schema_version": "1.1" +} \ No newline at end of file diff --git a/tests/golden/improve/fragility.parquet b/tests/golden/improve/fragility.parquet new file mode 100644 index 0000000..cd0d34d Binary files /dev/null and b/tests/golden/improve/fragility.parquet differ diff --git a/tests/golden/improve/manifest.json b/tests/golden/improve/manifest.json new file mode 100644 index 0000000..7c752cd --- /dev/null +++ b/tests/golden/improve/manifest.json @@ -0,0 +1,264 @@ +{ + "cells_reused_identical": 42, + "cohort_id": "improve-neoepitope-cv-patient-grouped", + "contract_version": 0, + "coverage": { + "common_warn_frac": 0.2, + "compare_on": "both", + "min_fraction": 0.95 + }, + "dropped_subjects": [], + "expected_models": [ + "improve_simple", + "improve_simple_wo_prime", + "improve_tme_excl", + "improve_tme_excl_wo_prime", + "improve_tme_incl", + "improve_tme_incl_wo_prime", + "nnalign" + ], + "flags": [ + { + "code": "few_clusters", + "message": "tail quantiles approximate with few clusters (G=20) for tumour_cohort=Basket; percentile bootstrap intervals below 40 clusters are coarse", + "severity": "note" + }, + { + "code": "few_clusters", + "message": "tail quantiles approximate with few clusters (G=24) for tumour_cohort=bladder; percentile bootstrap intervals below 40 clusters are coarse", + "severity": "note" + }, + { + "code": "few_clusters", + "message": "tail quantiles approximate with few clusters (G=26) for tumour_cohort=melanoma; percentile bootstrap intervals below 40 clusters are coarse", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_simple (peptide_length=non_9mer): delta_auroc=+0.0036 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0005, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_simple (tumour_cohort=Basket): delta_auroc=+0.0073 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_simple_wo_prime (peptide_length=non_9mer): delta_auroc=+0.0047 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0003, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_simple_wo_prime (tumour_cohort=Basket): delta_auroc=+0.0119 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_simple_wo_prime (tumour_cohort=melanoma): delta_auroc=+0.0068 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0002, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_tme_excl (peptide_length=non_9mer): delta_auroc=+0.0037 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0005, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_tme_excl (tumour_cohort=Basket): delta_auroc=+0.0041 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_tme_excl_wo_prime (peptide_length=non_9mer): delta_auroc=+0.0049 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0003, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_tme_excl_wo_prime (tumour_cohort=Basket): delta_auroc=+0.0277 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_tme_excl_wo_prime (tumour_cohort=bladder): delta_auroc=+0.0020 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0002, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_tme_excl_wo_prime (tumour_cohort=melanoma): delta_auroc=+0.0087 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0005, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_tme_incl (tumour_cohort=bladder): delta_auroc=+0.0161 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_tme_incl_wo_prime (tumour_cohort=bladder): delta_auroc=+0.0092 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for nnalign (overall): delta_auroc=-0.0014 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for nnalign (peptide_length=9mer): delta_auroc=+0.0057 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for nnalign (tumour_cohort=Basket): delta_auroc=+0.0022 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0005, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for nnalign (tumour_cohort=melanoma): delta_auroc=+0.0074 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)", + "severity": "note" + } + ], + "inputs": { + "cohort_data": "0602dee2f9c80b0512e9f6e5e3470fd0cd85fb561578adbcbc6956edb905baf4", + "cohort_spec": "cc9ee94cfcdc09ff8be98514fb1c33b0cf14cb80338649b4d10caa8336b7b617", + "predictions": "d5c66de35023d643f11f3e750a9d303180de8588c088d94abb651e77453c9972" + }, + "models_present": [ + "improve_simple", + "improve_simple_wo_prime", + "improve_tme_excl", + "improve_tme_excl_wo_prime", + "improve_tme_incl", + "improve_tme_incl_wo_prime", + "nnalign" + ], + "n_subjects": 17520, + "roster": { + "absent": [], + "declared": 7, + "ensemble_members": [], + "present": 7 + }, + "uncertainty": { + "ci_level": 0.95, + "clustered": true, + "clustering_field": "patient", + "clustering_name": "patient", + "n_boot": 2000, + "seed": 1337, + "show_naive_ci": true, + "unit_noun": "peptide" + }, + "verdict": { + "improve_simple": { + "axis_a_miscalibrated": true, + "axis_b": "demonstrated", + "axis_b_counterproductive_rungs": [], + "best_rung": "rung1", + "best_rung_brier": 0.025770239792686046, + "bss": 0.006726175584733984, + "bss_ci_high": 0.011459333307902646, + "bss_ci_low": 0.0010103638877356459, + "n": 17520, + "n_clusters": 70, + "prevalence": 0.02665525114155251, + "rung0_brier": 0.1736426279039147 + }, + "improve_simple_wo_prime": { + "axis_a_miscalibrated": true, + "axis_b": "demonstrated", + "axis_b_counterproductive_rungs": [], + "best_rung": "rung1", + "best_rung_brier": 0.025771142636787574, + "bss": 0.006691376862611453, + "bss_ci_high": 0.011280965974780174, + "bss_ci_low": 0.001141575323759689, + "n": 17520, + "n_clusters": 70, + "prevalence": 0.02665525114155251, + "rung0_brier": 0.17362201243380085 + }, + "improve_tme_excl": { + "axis_a_miscalibrated": true, + "axis_b": "demonstrated", + "axis_b_counterproductive_rungs": [], + "best_rung": "rung1", + "best_rung_brier": 0.02578160059807171, + "bss": 0.006288291005287494, + "bss_ci_high": 0.01084696486704049, + "bss_ci_low": 0.0010456215867604508, + "n": 17520, + "n_clusters": 70, + "prevalence": 0.02665525114155251, + "rung0_brier": 0.1740571541484085 + }, + "improve_tme_excl_wo_prime": { + "axis_a_miscalibrated": true, + "axis_b": "demonstrated", + "axis_b_counterproductive_rungs": [], + "best_rung": "rung1", + "best_rung_brier": 0.025789071243491817, + "bss": 0.00600034659316806, + "bss_ci_high": 0.01053968605906155, + "bss_ci_low": 0.0008430445036216836, + "n": 17520, + "n_clusters": 70, + "prevalence": 0.02665525114155251, + "rung0_brier": 0.17437342114437554 + }, + "improve_tme_incl": { + "axis_a_miscalibrated": true, + "axis_b": "demonstrated", + "axis_b_counterproductive_rungs": [], + "best_rung": "rung1", + "best_rung_brier": 0.025735148054809035, + "bss": 0.00807873205944587, + "bss_ci_high": 0.0134276046647617, + "bss_ci_low": 0.0014602036571806387, + "n": 17520, + "n_clusters": 70, + "prevalence": 0.02665525114155251, + "rung0_brier": 0.16762562559930383 + }, + "improve_tme_incl_wo_prime": { + "axis_a_miscalibrated": true, + "axis_b": "demonstrated", + "axis_b_counterproductive_rungs": [], + "best_rung": "rung1", + "best_rung_brier": 0.025744200664175557, + "bss": 0.007729813306700084, + "bss_ci_high": 0.012790371547574123, + "bss_ci_low": 0.0016603479016365002, + "n": 17520, + "n_clusters": 70, + "prevalence": 0.02665525114155251, + "rung0_brier": 0.1679510221660752 + }, + "nnalign": { + "axis_a_miscalibrated": false, + "axis_b": "counterproductive", + "axis_b_counterproductive_rungs": [ + "rung3" + ], + "best_rung": "rung0", + "best_rung_brier": 0.025829954627819694, + "bss": 0.004424560111044751, + "bss_ci_high": 0.008334603978975204, + "bss_ci_low": -0.001111639209645941, + "n": 17520, + "n_clusters": 70, + "prevalence": 0.02665525114155251, + "rung0_brier": 0.025829954627819694 + } + }, + "versions": { + "numpy": "2.4.6", + "pandas": "3.0.5", + "predval": "0.1.0", + "python": "3.13.16", + "scikit-learn": "1.9.0", + "scipy": "1.17.1", + "statsmodels": "0.14.6" + } +} \ No newline at end of file diff --git a/tests/golden/improve/metrics.parquet b/tests/golden/improve/metrics.parquet new file mode 100644 index 0000000..120560a Binary files /dev/null and b/tests/golden/improve/metrics.parquet differ diff --git a/tests/golden/improve/report.html b/tests/golden/improve/report.html new file mode 100644 index 0000000..6e0f8de --- /dev/null +++ b/tests/golden/improve/report.html @@ -0,0 +1,6593 @@ + + + + +predval report -- improve-neoepitope-cv-patient-grouped + + + +

predval report

+

cohort: improve-neoepitope-cv-patient-grouped

+ +
No rung is a validated model. Rung0 is the model as published. Rungs 1-3 are diagnosis of rung0, not competitors to it. The cross-fitted figures are the expected performance after local recalibration on a cohort like this one -- not evidence that the recalibrated model has been validated, which would require a cohort the correction was never fitted on. Recalibration does not and cannot improve discrimination.
+ +

1. Coverage and roster

+

Roster: 7 declared, + 7 present, + 0 absent.

+ + + + + + + + + + + + + + + + + + + + + + + +
modelcoveragescoreddropped if commonmeets minimum
improve_simple1.0000175200yes
improve_simple_wo_prime1.0000175200yes
improve_tme_excl1.0000175200yes
improve_tme_excl_wo_prime1.0000175200yes
improve_tme_incl1.0000175200yes
improve_tme_incl_wo_prime1.0000175200yes
nnalign1.0000175200yes
+ +

2. Primary comparison -- model as published (rung0)

+

Common subset, overall stratum. AUROC and Brier carry cluster intervals; the +calibration slope and intercept carry cluster-robust analytic intervals. +Full and common coverage are identical here, so the coverage-delta +column is omitted (every value would be zero).

+ + + + + + + + + + + + + + + + + + + + + + + + +
modelAUROCavg precisionBriercalib slopecalib intercept
improve_tme_incl0.6524 [0.606, 0.697]0.0525 [0.039, 0.070]0.1676 [0.152, 0.185]1.0972 [0.771, 1.424]-3.2466 [-3.463, -3.030]
improve_tme_incl_wo_prime0.6514 [0.604, 0.696]0.0522 [0.039, 0.071]0.1680 [0.153, 0.184]1.1701 [0.814, 1.527]-3.2437 [-3.459, -3.028]
improve_simple0.6425 [0.607, 0.679]0.0490 [0.037, 0.067]0.1736 [0.166, 0.182]1.0848 [0.772, 1.397]-3.2777 [-3.512, -3.043]
improve_simple_wo_prime0.6402 [0.606, 0.676]0.0491 [0.037, 0.067]0.1736 [0.167, 0.181]1.1070 [0.786, 1.428]-3.2766 [-3.509, -3.044]
improve_tme_excl0.6302 [0.597, 0.668]0.0498 [0.037, 0.068]0.1741 [0.167, 0.181]1.0719 [0.753, 1.391]-3.2761 [-3.513, -3.039]
improve_tme_excl_wo_prime0.6257 [0.591, 0.662]0.0495 [0.037, 0.068]0.1744 [0.168, 0.182]1.0886 [0.747, 1.430]-3.2759 [-3.512, -3.040]
nnalign0.6054 [0.564, 0.647]0.0441 [0.034, 0.058]0.0258 [0.021, 0.032]0.9697 [0.580, 1.359]-0.0882 [-0.326, 0.150]
+

Brier intervals above are the cluster percentile bootstrap; the analytic t(G-1) +cross-check in metrics.parquet agrees (none crossed the 0 loss boundary).

+ +

Verdict

+

One line per member, assembled only from AUROC, the Brier skill score (BSS), and the +best admissible rung -- no free adjectives. BSS is anchored at 0 = no-skill (always +predict prevalence) and 1 = perfect; "closes Y% of the gap" is BSS as a percentage. +The gauge sentence states any detected miscalibration (slope or intercept interval excluding its +null), the lowest rung whose out-of-fold repair is demonstrated, and any rung whose recalibration +reliably harms. A member with no detected fault and no demonstrated repair carries no gauge +sentence: the test can fail to reject calibration but never establish it.

+ + +

Calibration (rung0, as published)

+

Decile calibration per member: points are observed frequency against mean predicted +probability, the shaded band is the cluster bootstrap on the observed rate, and the dashed diagonal +is perfect. Points below the diagonal over-predict risk; above, under-predict.

+
+ + + + + image/svg+xml + + + Matplotlib v3.11.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +

Unit-of-analysis exhibit

+

The naive per-peptide AUROC interval is incorrect when peptides share a patient. It is shown only to make the cost of ignoring clustering concrete. The ratio is the cluster-aware interval width divided by the naive per-peptide width. Above 1 it is the factor by which the naive interval is too narrow.

+ + + + + + + + + + + + + + + + +
modelcluster widthnaive width (wrong)ratio
improve_tme_incl0.09150.05331.7x
improve_tme_incl_wo_prime0.09230.05361.7x
improve_simple0.07150.05171.4x
improve_simple_wo_prime0.07000.05361.3x
improve_tme_excl0.07110.05281.3x
improve_tme_excl_wo_prime0.07160.05621.3x
nnalign0.08330.05471.5x
+
+ + + + + image/svg+xml + + + Matplotlib v3.11.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+

3. Calibration diagnosis (subordinate to section 2)

+

This section diagnoses rung0; it does not rank models. Rows are ordered with +demonstrated repairs first (by rung, then by interval strength), then the remaining members by +their best gain estimate -- the largest point estimate is not the strongest evidence. Each cell is the paired cross-fit Brier gain -- rung0 Brier +minus that rung's held-out Brier, positive when recalibration helped -- with its paired +cluster-bootstrap interval (a gain whose interval excludes 0 is a real out-of-fold improvement). +The apparent, in-sample gains run larger; only the cross-fitted gains, which hold whole +patients out, are shown. rung1's apparent intercept is ~0 and rung2's apparent slope is +~1 by construction (that is what each fit targets), so those are +not reported as findings. Recalibration cannot improve discrimination; the cross-fitted Brier is the +expected loss after local recalibration, not a validated result.

+

At small G the flexible rung is high-variance under cross-fitting: absence of +a rung3 gain is evidence of linear miscalibration only insofar as rung3 had power to detect shape +distortion.

+ + + + + + + + + + + + + + + + + +
modelBrier rung0rung1 gain [CI] (x10⁻³)rung2 gain [CI] (x10⁻³)rung3 gain [CI] (x10⁻³)
improve_tme_excl_wo_prime0.1744+148.58 [140.07, 157.95] x10⁻³+148.58 [140.07, 157.94] x10⁻³+148.58 [140.08, 157.92] x10⁻³
improve_simple_wo_prime0.1736+147.85 [139.74, 156.48] x10⁻³+147.85 [139.76, 156.47] x10⁻³+147.85 [139.75, 156.46] x10⁻³
improve_simple0.1736+147.87 [139.41, 157.13] x10⁻³+147.87 [139.42, 157.13] x10⁻³+147.86 [139.41, 157.11] x10⁻³
improve_tme_excl0.1741+148.28 [139.40, 157.82] x10⁻³+148.27 [139.40, 157.83] x10⁻³+148.26 [139.37, 157.82] x10⁻³
improve_tme_incl_wo_prime0.1680+142.21 [128.85, 157.54] x10⁻³+142.19 [128.86, 157.53] x10⁻³+142.18 [128.86, 157.53] x10⁻³
improve_tme_incl0.1676+141.89 [127.90, 158.12] x10⁻³+141.87 [127.90, 158.08] x10⁻³+141.86 [127.87, 158.07] x10⁻³
nnalign0.0258-0.02 [-0.07, 0.03] x10⁻³-0.02 [-0.08, 0.03] x10⁻³-0.05 [-0.11, -0.00] x10⁻³
+
+ + + + + image/svg+xml + + + Matplotlib v3.11.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+

4. Subgroups

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
subgroupmodelAUROCBrier
peptide_length=9merimprove_simple0.6315 [0.587, 0.677]0.1865 [0.179, 0.194]
peptide_length=9merimprove_simple_wo_prime0.6315 [0.589, 0.676]0.1813 [0.174, 0.189]
peptide_length=9merimprove_tme_excl0.6197 [0.576, 0.667]0.1859 [0.179, 0.192]
peptide_length=9merimprove_tme_excl_wo_prime0.6177 [0.574, 0.665]0.1808 [0.175, 0.187]
peptide_length=9merimprove_tme_incl0.6463 [0.592, 0.700]0.1782 [0.162, 0.196]
peptide_length=9merimprove_tme_incl_wo_prime0.6463 [0.593, 0.698]0.1732 [0.158, 0.189]
peptide_length=9mernnalign0.5906 [0.542, 0.643]0.0282 [0.022, 0.035]
peptide_length=non_9merimprove_simple0.6514 [0.609, 0.692]0.1542 [0.147, 0.162]
peptide_length=non_9merimprove_simple_wo_prime0.6484 [0.603, 0.691]0.1620 [0.155, 0.170]
peptide_length=non_9merimprove_tme_excl0.6395 [0.596, 0.679]0.1561 [0.148, 0.165]
peptide_length=non_9merimprove_tme_excl_wo_prime0.6345 [0.586, 0.677]0.1647 [0.157, 0.173]
peptide_length=non_9merimprove_tme_incl0.6541 [0.602, 0.703]0.1516 [0.137, 0.168]
peptide_length=non_9merimprove_tme_incl_wo_prime0.6547 [0.601, 0.704]0.1600 [0.145, 0.177]
peptide_length=non_9mernnalign0.6365 [0.583, 0.687]0.0223 [0.017, 0.028]
tumour_cohort=Basketimprove_simple0.6798 [0.626, 0.765]0.1490 [0.142, 0.154]
tumour_cohort=Basketimprove_simple_wo_prime0.6728 [0.618, 0.751]0.1501 [0.143, 0.155]
tumour_cohort=Basketimprove_tme_excl0.6679 [0.614, 0.753]0.1552 [0.143, 0.164]
tumour_cohort=Basketimprove_tme_excl_wo_prime0.6586 [0.607, 0.736]0.1570 [0.146, 0.166]
tumour_cohort=Basketimprove_tme_incl0.6789 [0.585, 0.757]0.1318 [0.112, 0.161]
tumour_cohort=Basketimprove_tme_incl_wo_prime0.6721 [0.571, 0.752]0.1343 [0.115, 0.163]
tumour_cohort=Basketnnalign0.6752 [0.605, 0.738]0.0210 [0.013, 0.033]
tumour_cohort=bladderimprove_simple0.6208 [0.565, 0.683]0.1839 [0.169, 0.194]
tumour_cohort=bladderimprove_simple_wo_prime0.6228 [0.569, 0.681]0.1813 [0.168, 0.191]
tumour_cohort=bladderimprove_tme_excl0.6090 [0.551, 0.668]0.1754 [0.165, 0.184]
tumour_cohort=bladderimprove_tme_excl_wo_prime0.6075 [0.551, 0.663]0.1735 [0.164, 0.181]
tumour_cohort=bladderimprove_tme_incl0.6016 [0.535, 0.686]0.1724 [0.155, 0.195]
tumour_cohort=bladderimprove_tme_incl_wo_prime0.6052 [0.541, 0.685]0.1704 [0.154, 0.191]
tumour_cohort=bladdernnalign0.5566 [0.481, 0.635]0.0230 [0.017, 0.031]
tumour_cohort=melanomaimprove_simple0.6125 [0.566, 0.656]0.1852 [0.178, 0.193]
tumour_cohort=melanomaimprove_simple_wo_prime0.6057 [0.560, 0.648]0.1868 [0.180, 0.196]
tumour_cohort=melanomaimprove_tme_excl0.5973 [0.547, 0.650]0.1897 [0.180, 0.201]
tumour_cohort=melanomaimprove_tme_excl_wo_prime0.5908 [0.535, 0.648]0.1911 [0.181, 0.202]
tumour_cohort=melanomaimprove_tme_incl0.6468 [0.588, 0.700]0.1951 [0.173, 0.218]
tumour_cohort=melanomaimprove_tme_incl_wo_prime0.6434 [0.579, 0.700]0.1958 [0.172, 0.220]
tumour_cohort=melanomannalign0.6045 [0.545, 0.667]0.0332 [0.023, 0.045]
+ +

5. Fragility -- not a confidence interval

+

The largest AUROC change from dropping any single cluster, and the culprit. This is +deliberately not an interval: it answers whether a result is carried by one +patient, which a tight bootstrap interval will not reveal.

+ + + + + + + + + +
modelmax |delta AUROC|culprit cluster
improve_tme_incl0.0129RH-31
improve_tme_incl_wo_prime0.0121RH-31
nnalign0.0096BC-6229
improve_simple0.0093RH-31
improve_simple_wo_prime0.0087RH-31
improve_tme_excl0.0079RH-31
improve_tme_excl_wo_prime0.0069RH-31
+ +

Limitations

+
The optimism correction covers the recalibration step only. predval evaluates the predictions it is handed and has no view of how they were produced. If model selection or tuning used patients inside this cohort, rung0 is itself optimistically biased and this harness cannot see it; only a cohort the model was never tuned on could expose that. The paired cross-fit gain interval (section 3) conditions on the fitted correction -- it resamples the patient-level loss difference but does not refit the correction inside each bootstrap replicate, so it does not carry the correction's own fitting variance; that refit-in-replicate interval remains backlog (spec section 4.5).
+ +

6. Provenance

+

Input digests (blake2b), the bootstrap seed and B, and library versions -- enough to +reproduce every number.

+ + + + + + + + + + + + + +
cohort_data0602dee2f9c80b0512e9f6e5e3470fd0cd85fb561578adbcbc6956edb905baf4
cohort_speccc9ee94cfcdc09ff8be98514fb1c33b0cf14cb80338649b4d10caa8336b7b617
predictionsd5c66de35023d643f11f3e750a9d303180de8588c088d94abb651e77453c9972
contract version0
bootstrapB=2000, seed=1337, + ci=0.95
numpy2.4.6
pandas3.0.5
predval0.1.0
python3.13.16
scikit-learn1.9.0
scipy1.17.1
statsmodels0.14.6
+ +

Flags

+ + + + + \ No newline at end of file diff --git a/tests/test_golden_improve.py b/tests/test_golden_improve.py new file mode 100644 index 0000000..515af32 --- /dev/null +++ b/tests/test_golden_improve.py @@ -0,0 +1,89 @@ +"""Golden-file regression test for the IMPROVE neoepitope example (third worked example). + +`tests/golden/improve/` freezes the real, showable output of `examples/improve/run_evaluation.py` +at its default B=2000. This test regenerates that same run and checks every artefact against the +frozen baseline, so a change that silently shifts the example's headline numbers -- rather than +being a deliberate, reviewed update to the golden files -- fails loudly. It is the IMPROVE +counterpart to `tests/test_golden_gusto.py` and `tests/test_golden_pcam.py`. + +Opt-in (`PREDVAL_RUN_GOLDEN=1`) like the other two, and it needs the git-ignored fixture built by +`examples/improve/build_fixture.py` from the public IMPROVE_paper release (see +`examples/improve/README.md` and `tests/golden/improve/GOLDEN.md`). + +To refresh the golden files deliberately after an intentional change, regenerate and copy: + + uv run python examples/improve/run_evaluation.py + cp examples/improve/out/{metrics,fragility,calibration,coverage}.parquet tests/golden/improve/ + cp examples/improve/out/{manifest,findings}.json tests/golden/improve/ + cp examples/improve/out/report.html tests/golden/improve/ + +and note *why* the numbers moved in the commit that updates them. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from predval.hashing import hash_file + +EXAMPLE = Path(__file__).resolve().parents[1] / "examples" / "improve" +GOLDEN = Path(__file__).resolve().parent / "golden" / "improve" + +pytestmark = [ + pytest.mark.skipif( + not (EXAMPLE / "cohort.parquet").exists() or not (EXAMPLE / "predictions.parquet").exists(), + reason="IMPROVE fixture not built; run examples/improve/build_fixture.py", + ), + pytest.mark.skipif( + os.environ.get("PREDVAL_RUN_GOLDEN") != "1", + reason="B=2000 golden regression is opt-in; set PREDVAL_RUN_GOLDEN=1 to run it", + ), +] + +#: findings.json is compared separately -- it embeds predval's own git commit, which legitimately +#: differs on every commit to this repo and is not a signal about the IMPROVE evaluation itself. +BYTE_IDENTICAL_ARTIFACTS = ( + "metrics.parquet", + "fragility.parquet", + "calibration.parquet", + "coverage.parquet", + "manifest.json", + "report.html", +) + + +def _normalize_findings(path: Path) -> dict: + doc = json.loads(path.read_text()) + doc["provenance"]["git_commit"] = None + return doc + + +def test_improve_full_run_matches_golden_baseline(tmp_path: Path) -> None: + out = tmp_path / "out" + subprocess.run( + [sys.executable, str(EXAMPLE / "run_evaluation.py"), "--out", str(out)], + check=True, + capture_output=True, + ) + + mismatches = [] + for name in BYTE_IDENTICAL_ARTIFACTS: + golden_file = GOLDEN / name + assert golden_file.exists(), f"missing golden file: {golden_file}" + if hash_file(out / name) != hash_file(golden_file): + mismatches.append(name) + + if _normalize_findings(out / "findings.json") != _normalize_findings(GOLDEN / "findings.json"): + mismatches.append("findings.json") + + assert not mismatches, ( + f"artefacts drifted from tests/golden/improve/: {mismatches}. " + "If this is an intentional change, refresh the golden files (see this module's " + "docstring) and explain why the numbers moved in the commit." + )