From 2e72eaef1945715978e65f71f183f19e8b6a6bb4 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 16:58:16 +0000 Subject: [PATCH] golden: freeze the IMPROVE neoepitope example outputs The IMPROVE example landed in #1 with scripts only; no run output was committed. This freezes a B=2000 run built from the public IMPROVE_paper release (commit c670942, results.zip sha256 2689700...) as a golden baseline, adds the opt-in regression test matching the GUSTO and PCam ones, and publishes the report under docs/reports/. A second independent run reproduced every artefact byte-for-byte (findings.json modulo provenance.git_commit). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01E5Q2q7a7gF1tUX8y6fRV7m --- docs/reports/improve.html | 6593 ++++++++++++++++++++++ tests/golden/improve/GOLDEN.md | 45 + tests/golden/improve/calibration.parquet | Bin 0 -> 7728 bytes tests/golden/improve/coverage.parquet | Bin 0 -> 5310 bytes tests/golden/improve/findings.json | 624 ++ tests/golden/improve/fragility.parquet | Bin 0 -> 10114 bytes tests/golden/improve/manifest.json | 264 + tests/golden/improve/metrics.parquet | Bin 0 -> 45619 bytes tests/golden/improve/report.html | 6593 ++++++++++++++++++++++ tests/test_golden_improve.py | 89 + 10 files changed, 14208 insertions(+) create mode 100644 docs/reports/improve.html create mode 100644 tests/golden/improve/GOLDEN.md create mode 100644 tests/golden/improve/calibration.parquet create mode 100644 tests/golden/improve/coverage.parquet create mode 100644 tests/golden/improve/findings.json create mode 100644 tests/golden/improve/fragility.parquet create mode 100644 tests/golden/improve/manifest.json create mode 100644 tests/golden/improve/metrics.parquet create mode 100644 tests/golden/improve/report.html create mode 100644 tests/test_golden_improve.py diff --git a/docs/reports/improve.html b/docs/reports/improve.html new file mode 100644 index 0000000..6e0f8de --- /dev/null +++ b/docs/reports/improve.html @@ -0,0 +1,6593 @@ + + + + +predval report -- improve-neoepitope-cv-patient-grouped + + + +

predval report

+

cohort: improve-neoepitope-cv-patient-grouped

+ +
No rung is a validated model. Rung0 is the model as published. Rungs 1-3 are diagnosis of rung0, not competitors to it. The cross-fitted figures are the expected performance after local recalibration on a cohort like this one -- not evidence that the recalibrated model has been validated, which would require a cohort the correction was never fitted on. Recalibration does not and cannot improve discrimination.
+ +

1. Coverage and roster

+

Roster: 7 declared, + 7 present, + 0 absent.

+ + + + + + + + + + + + + + + + + + + + + + + +
modelcoveragescoreddropped if commonmeets minimum
improve_simple1.0000175200yes
improve_simple_wo_prime1.0000175200yes
improve_tme_excl1.0000175200yes
improve_tme_excl_wo_prime1.0000175200yes
improve_tme_incl1.0000175200yes
improve_tme_incl_wo_prime1.0000175200yes
nnalign1.0000175200yes
+ +

2. Primary comparison -- model as published (rung0)

+

Common subset, overall stratum. AUROC and Brier carry cluster intervals; the +calibration slope and intercept carry cluster-robust analytic intervals. +Full and common coverage are identical here, so the coverage-delta +column is omitted (every value would be zero).

+ + + + + + + + + + + + + + + + + + + + + + + + +
modelAUROCavg precisionBriercalib slopecalib intercept
improve_tme_incl0.6524 [0.606, 0.697]0.0525 [0.039, 0.070]0.1676 [0.152, 0.185]1.0972 [0.771, 1.424]-3.2466 [-3.463, -3.030]
improve_tme_incl_wo_prime0.6514 [0.604, 0.696]0.0522 [0.039, 0.071]0.1680 [0.153, 0.184]1.1701 [0.814, 1.527]-3.2437 [-3.459, -3.028]
improve_simple0.6425 [0.607, 0.679]0.0490 [0.037, 0.067]0.1736 [0.166, 0.182]1.0848 [0.772, 1.397]-3.2777 [-3.512, -3.043]
improve_simple_wo_prime0.6402 [0.606, 0.676]0.0491 [0.037, 0.067]0.1736 [0.167, 0.181]1.1070 [0.786, 1.428]-3.2766 [-3.509, -3.044]
improve_tme_excl0.6302 [0.597, 0.668]0.0498 [0.037, 0.068]0.1741 [0.167, 0.181]1.0719 [0.753, 1.391]-3.2761 [-3.513, -3.039]
improve_tme_excl_wo_prime0.6257 [0.591, 0.662]0.0495 [0.037, 0.068]0.1744 [0.168, 0.182]1.0886 [0.747, 1.430]-3.2759 [-3.512, -3.040]
nnalign0.6054 [0.564, 0.647]0.0441 [0.034, 0.058]0.0258 [0.021, 0.032]0.9697 [0.580, 1.359]-0.0882 [-0.326, 0.150]
+

Brier intervals above are the cluster percentile bootstrap; the analytic t(G-1) +cross-check in metrics.parquet agrees (none crossed the 0 loss boundary).

+ +

Verdict

+

One line per member, assembled only from AUROC, the Brier skill score (BSS), and the +best admissible rung -- no free adjectives. BSS is anchored at 0 = no-skill (always +predict prevalence) and 1 = perfect; "closes Y% of the gap" is BSS as a percentage. +The gauge sentence states any detected miscalibration (slope or intercept interval excluding its +null), the lowest rung whose out-of-fold repair is demonstrated, and any rung whose recalibration +reliably harms. A member with no detected fault and no demonstrated repair carries no gauge +sentence: the test can fail to reject calibration but never establish it.

+ + +

Calibration (rung0, as published)

+

Decile calibration per member: points are observed frequency against mean predicted +probability, the shaded band is the cluster bootstrap on the observed rate, and the dashed diagonal +is perfect. Points below the diagonal over-predict risk; above, under-predict.

+
+ + + + + image/svg+xml + + + Matplotlib v3.11.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +

Unit-of-analysis exhibit

+

The naive per-peptide AUROC interval is incorrect when peptides share a patient. It is shown only to make the cost of ignoring clustering concrete. The ratio is the cluster-aware interval width divided by the naive per-peptide width. Above 1 it is the factor by which the naive interval is too narrow.

+ + + + + + + + + + + + + + + + +
modelcluster widthnaive width (wrong)ratio
improve_tme_incl0.09150.05331.7x
improve_tme_incl_wo_prime0.09230.05361.7x
improve_simple0.07150.05171.4x
improve_simple_wo_prime0.07000.05361.3x
improve_tme_excl0.07110.05281.3x
improve_tme_excl_wo_prime0.07160.05621.3x
nnalign0.08330.05471.5x
+
+ + + + + image/svg+xml + + + Matplotlib v3.11.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+

3. Calibration diagnosis (subordinate to section 2)

+

This section diagnoses rung0; it does not rank models. Rows are ordered with +demonstrated repairs first (by rung, then by interval strength), then the remaining members by +their best gain estimate -- the largest point estimate is not the strongest evidence. Each cell is the paired cross-fit Brier gain -- rung0 Brier +minus that rung's held-out Brier, positive when recalibration helped -- with its paired +cluster-bootstrap interval (a gain whose interval excludes 0 is a real out-of-fold improvement). +The apparent, in-sample gains run larger; only the cross-fitted gains, which hold whole +patients out, are shown. rung1's apparent intercept is ~0 and rung2's apparent slope is +~1 by construction (that is what each fit targets), so those are +not reported as findings. Recalibration cannot improve discrimination; the cross-fitted Brier is the +expected loss after local recalibration, not a validated result.

+

At small G the flexible rung is high-variance under cross-fitting: absence of +a rung3 gain is evidence of linear miscalibration only insofar as rung3 had power to detect shape +distortion.

+ + + + + + + + + + + + + + + + + +
modelBrier rung0rung1 gain [CI] (x10⁻³)rung2 gain [CI] (x10⁻³)rung3 gain [CI] (x10⁻³)
improve_tme_excl_wo_prime0.1744+148.58 [140.07, 157.95] x10⁻³+148.58 [140.07, 157.94] x10⁻³+148.58 [140.08, 157.92] x10⁻³
improve_simple_wo_prime0.1736+147.85 [139.74, 156.48] x10⁻³+147.85 [139.76, 156.47] x10⁻³+147.85 [139.75, 156.46] x10⁻³
improve_simple0.1736+147.87 [139.41, 157.13] x10⁻³+147.87 [139.42, 157.13] x10⁻³+147.86 [139.41, 157.11] x10⁻³
improve_tme_excl0.1741+148.28 [139.40, 157.82] x10⁻³+148.27 [139.40, 157.83] x10⁻³+148.26 [139.37, 157.82] x10⁻³
improve_tme_incl_wo_prime0.1680+142.21 [128.85, 157.54] x10⁻³+142.19 [128.86, 157.53] x10⁻³+142.18 [128.86, 157.53] x10⁻³
improve_tme_incl0.1676+141.89 [127.90, 158.12] x10⁻³+141.87 [127.90, 158.08] x10⁻³+141.86 [127.87, 158.07] x10⁻³
nnalign0.0258-0.02 [-0.07, 0.03] x10⁻³-0.02 [-0.08, 0.03] x10⁻³-0.05 [-0.11, -0.00] x10⁻³
+
+ + + + + image/svg+xml + + + Matplotlib v3.11.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+

4. Subgroups

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
subgroupmodelAUROCBrier
peptide_length=9merimprove_simple0.6315 [0.587, 0.677]0.1865 [0.179, 0.194]
peptide_length=9merimprove_simple_wo_prime0.6315 [0.589, 0.676]0.1813 [0.174, 0.189]
peptide_length=9merimprove_tme_excl0.6197 [0.576, 0.667]0.1859 [0.179, 0.192]
peptide_length=9merimprove_tme_excl_wo_prime0.6177 [0.574, 0.665]0.1808 [0.175, 0.187]
peptide_length=9merimprove_tme_incl0.6463 [0.592, 0.700]0.1782 [0.162, 0.196]
peptide_length=9merimprove_tme_incl_wo_prime0.6463 [0.593, 0.698]0.1732 [0.158, 0.189]
peptide_length=9mernnalign0.5906 [0.542, 0.643]0.0282 [0.022, 0.035]
peptide_length=non_9merimprove_simple0.6514 [0.609, 0.692]0.1542 [0.147, 0.162]
peptide_length=non_9merimprove_simple_wo_prime0.6484 [0.603, 0.691]0.1620 [0.155, 0.170]
peptide_length=non_9merimprove_tme_excl0.6395 [0.596, 0.679]0.1561 [0.148, 0.165]
peptide_length=non_9merimprove_tme_excl_wo_prime0.6345 [0.586, 0.677]0.1647 [0.157, 0.173]
peptide_length=non_9merimprove_tme_incl0.6541 [0.602, 0.703]0.1516 [0.137, 0.168]
peptide_length=non_9merimprove_tme_incl_wo_prime0.6547 [0.601, 0.704]0.1600 [0.145, 0.177]
peptide_length=non_9mernnalign0.6365 [0.583, 0.687]0.0223 [0.017, 0.028]
tumour_cohort=Basketimprove_simple0.6798 [0.626, 0.765]0.1490 [0.142, 0.154]
tumour_cohort=Basketimprove_simple_wo_prime0.6728 [0.618, 0.751]0.1501 [0.143, 0.155]
tumour_cohort=Basketimprove_tme_excl0.6679 [0.614, 0.753]0.1552 [0.143, 0.164]
tumour_cohort=Basketimprove_tme_excl_wo_prime0.6586 [0.607, 0.736]0.1570 [0.146, 0.166]
tumour_cohort=Basketimprove_tme_incl0.6789 [0.585, 0.757]0.1318 [0.112, 0.161]
tumour_cohort=Basketimprove_tme_incl_wo_prime0.6721 [0.571, 0.752]0.1343 [0.115, 0.163]
tumour_cohort=Basketnnalign0.6752 [0.605, 0.738]0.0210 [0.013, 0.033]
tumour_cohort=bladderimprove_simple0.6208 [0.565, 0.683]0.1839 [0.169, 0.194]
tumour_cohort=bladderimprove_simple_wo_prime0.6228 [0.569, 0.681]0.1813 [0.168, 0.191]
tumour_cohort=bladderimprove_tme_excl0.6090 [0.551, 0.668]0.1754 [0.165, 0.184]
tumour_cohort=bladderimprove_tme_excl_wo_prime0.6075 [0.551, 0.663]0.1735 [0.164, 0.181]
tumour_cohort=bladderimprove_tme_incl0.6016 [0.535, 0.686]0.1724 [0.155, 0.195]
tumour_cohort=bladderimprove_tme_incl_wo_prime0.6052 [0.541, 0.685]0.1704 [0.154, 0.191]
tumour_cohort=bladdernnalign0.5566 [0.481, 0.635]0.0230 [0.017, 0.031]
tumour_cohort=melanomaimprove_simple0.6125 [0.566, 0.656]0.1852 [0.178, 0.193]
tumour_cohort=melanomaimprove_simple_wo_prime0.6057 [0.560, 0.648]0.1868 [0.180, 0.196]
tumour_cohort=melanomaimprove_tme_excl0.5973 [0.547, 0.650]0.1897 [0.180, 0.201]
tumour_cohort=melanomaimprove_tme_excl_wo_prime0.5908 [0.535, 0.648]0.1911 [0.181, 0.202]
tumour_cohort=melanomaimprove_tme_incl0.6468 [0.588, 0.700]0.1951 [0.173, 0.218]
tumour_cohort=melanomaimprove_tme_incl_wo_prime0.6434 [0.579, 0.700]0.1958 [0.172, 0.220]
tumour_cohort=melanomannalign0.6045 [0.545, 0.667]0.0332 [0.023, 0.045]
+ +

5. Fragility -- not a confidence interval

+

The largest AUROC change from dropping any single cluster, and the culprit. This is +deliberately not an interval: it answers whether a result is carried by one +patient, which a tight bootstrap interval will not reveal.

+ + + + + + + + + +
modelmax |delta AUROC|culprit cluster
improve_tme_incl0.0129RH-31
improve_tme_incl_wo_prime0.0121RH-31
nnalign0.0096BC-6229
improve_simple0.0093RH-31
improve_simple_wo_prime0.0087RH-31
improve_tme_excl0.0079RH-31
improve_tme_excl_wo_prime0.0069RH-31
+ +

Limitations

+
The optimism correction covers the recalibration step only. predval evaluates the predictions it is handed and has no view of how they were produced. If model selection or tuning used patients inside this cohort, rung0 is itself optimistically biased and this harness cannot see it; only a cohort the model was never tuned on could expose that. The paired cross-fit gain interval (section 3) conditions on the fitted correction -- it resamples the patient-level loss difference but does not refit the correction inside each bootstrap replicate, so it does not carry the correction's own fitting variance; that refit-in-replicate interval remains backlog (spec section 4.5).
+ +

6. Provenance

+

Input digests (blake2b), the bootstrap seed and B, and library versions -- enough to +reproduce every number.

+ + + + + + + + + + + + + +
cohort_data0602dee2f9c80b0512e9f6e5e3470fd0cd85fb561578adbcbc6956edb905baf4
cohort_speccc9ee94cfcdc09ff8be98514fb1c33b0cf14cb80338649b4d10caa8336b7b617
predictionsd5c66de35023d643f11f3e750a9d303180de8588c088d94abb651e77453c9972
contract version0
bootstrapB=2000, seed=1337, + ci=0.95
numpy2.4.6
pandas3.0.5
predval0.1.0
python3.13.16
scikit-learn1.9.0
scipy1.17.1
statsmodels0.14.6
+ +

Flags

+ + + + + \ No newline at end of file diff --git a/tests/golden/improve/GOLDEN.md b/tests/golden/improve/GOLDEN.md new file mode 100644 index 0000000..7bbae51 --- /dev/null +++ b/tests/golden/improve/GOLDEN.md @@ -0,0 +1,45 @@ +# IMPROVE golden baseline + +Frozen regression baseline for the IMPROVE neoepitope-immunogenicity example. Generated by: + +```bash +uv run python examples/improve/build_fixture.py --src 'improve_results/results copy' +uv run python examples/improve/run_evaluation.py +``` + +(default config: `B=2000`, `seed=1337`, from `examples/improve/cohort.yaml`; 7 models, 17,520 +peptides, G=70 patient clusters, 467 immunogenic = 2.666% prevalence) + +| | | +|---|---| +| generated | 2026-10-07 | +| predval commit | `4f9fbd0` (predval 0.1.0) | +| source repo | `https://github.com/SRHgroup/IMPROVE_paper` at `c67094276b52358b5202ad9e5597acd87888ea1d` | +| `results.zip` sha256 | `26897006104ef0e57aeedc003804c9a7e43d3735d9df157d651516f2739e00f9` | +| wall time | ~12.5 min on a 4-core container | + +`findings.json` keeps its `provenance.git_commit` as generated; `tests/test_golden_improve.py` +normalizes that one field out of the comparison, as the GUSTO and PCam goldens do. + +Reproducibility was verified at freeze time: a second independent run at the same commit was +compared against these files by `tests/test_golden_improve.py`. + +## Headline numbers this baseline pins + +- **Prevalence anchoring.** The six IMPROVE random-forest configurations predict a mean + probability of 0.39-0.40 against an observed rate of 0.0267, i.e. 14.6-15.1x too high. The + NNAlign comparator is anchored (1.1x). +- **Discrimination.** AUROC 0.63-0.65 for the IMPROVE models, 0.61 for NNAlign + (patient-clustered 95% CIs roughly +/-0.04). +- **Calibration.** Intercepts about -3.25 (CI excludes 0) with slopes about 1.1 (CI includes 1): + the IMPROVE scores are a ranking with the right spread and the wrong level. NNAlign: + intercept -0.09 [-0.33, 0.15], slope 0.97. +- **Recalibration ladder.** Cross-fitted rung1 (intercept only, fitted on held-out patients) + cuts Brier from ~0.17 to 0.0258 for every IMPROVE model, a demonstrated repair. That repaired + Brier equals the prevalence-only constant predictor (0.0259): once the level is fixed, the + probabilities carry almost no Brier-scale skill beyond the base rate. NNAlign gains nothing + from recalibration. +- **Unit of analysis.** Patient-clustered AUROC intervals are 1.3-1.7x wider than naive + per-peptide ones. +- **Fragility.** Dropping patient RH-31 moves AUROC the most for six of seven models + (largest 0.013). diff --git a/tests/golden/improve/calibration.parquet b/tests/golden/improve/calibration.parquet new file mode 100644 index 0000000000000000000000000000000000000000..418bc8a22edd68aaa92f703accbb5dc7d62875fe GIT binary patch literal 7728 zcmcgx30#xM692*(6p$*x(4rt}L_v-KYOFFOU;-2nPy?X?<^qB_kc4n(6%-K^FBF6z z6;Kh>TD72fmuhXjTC0d4QmxwBDs8<>E9%?tOCo5{*Y|sUFF)qX?##~oXLn|2_X`os zC@+c$rNM|&Mhc}^Q?S7prXdAcV;ClprKTy=;shmXNX0{0gRxNw8H$9|G>J@XGweMl zl}wx<&J;@RLu@gD7iJ=nv%N5qEyc^l9BIqte5oW!PQfT{6i+uw5s6|$F{79{Ou$^s zbuy~;s}S$JgY@iX8lz_siDYbx*^+FHY>jPAY)x&=kdsv;iYSy~OZXUpVOAuJWJSh| z5E&yfL1c=^43W7Nc?ubs!9Nz-FARASOBvV46DnCRqUy(Dc9=^)T*9&OK03!thS<<& zs>{@-$T0HiI+=2Z9Ewj6U%TBVk35Vx^P;|p@cR}TKRGk)?uiQkXCJ=^JKF*eZO;C3 zw5SzgOy}&MJcT=F9cYJNhtRT;lpQd1Z%Ah5XPxl4v+(m8?sZUDRvSAiZ$dlI zP25 z4ww|1EO2H7h-N#1|B6HFVo$FD&C`Hr1a+c4!O#Q$>a zd_&cFFez^g(`YV2=O*hi&WuZ-w!g|?Tx|uf{W~-V{o0^q-uTDo54OV>T&u@jvJP;n zcxT6|%ucx46K1ELa2@Pu~(2X=JkaCUh+n z-2biV9E@#>ALjn-JNWBK+&;};7vbc5)xFk}m*7*AmGxfvmm%?Yx5#o)J78ToQ`Cz( z;HxXAzezpT2^9_0d)ryp;pCVd<~-~stS*cCbkD7oaQJr1d2U-VSY~@aD4bmaYYTqp z*tvHDoIWXpY0t{xPVkjc*-JLUpX*zCTt3_c(>$G45qW ziN}A*^s8Ev95}8#yl#EUmnjG0cb7@oJKE(<8|7c!{d3{xRjEO4MtDl+m{Dp?Ln(8J zjLtD$;I?Z|)5#mlVg1F(a1LW76j8kdCkocWg)PDBXki+#aLUoZ@ZzkXpGK5J{(k%K z6EiE|=?A}hG#=XmZKnL8d2zd~$OUV_-<9-oQ@{Ho@Y*^crH3vPt%BLYc`Lv4 zErq26DC1_L4kK;A494KD_bkA!q+0T{(vB#*8tfu8RBH@{lF z3F;!MSxW~jh3lW^8k<`eS!H5tteUVYD;q@q*k-F_Y`0aDNdb_4vut@@10=Q!W!x56 zjK(@l+HE!G)@s-%ZQm~GqD23ryn}=O(t^^lh zuEx5j25x^-`F_T4M7$SL@?H(Me!5h$ev$BS8*+(o!#zJ;xXp(6+mMYTspiEl1K0Vi zc<8^@CvspUZ)mAS`IhnT{oz!(A?%u2{*55^*+0eyNDED8Ct((8~hMiSCD!y4>nsrJ(5D72Nr)g{^&Zm2u}D6v@FkC z4pUvuT~eONgOGq~zaq4uf(|m?Epb>03v!yADmqIbbuj5Yi=qn1j&B(ss;PrLW8>QP zaoGt87al zr0n`~Uj5m881ZB><6L?X@bUEZUtOeRA+Z|X&uz>Qwvw=-3i5vD{IT$~gJkgP} z1iHH>tQzi{2V--prK7$rf?1m<4L)>m4LEh@I(Fx3z?WowVz%E#81wWSx@P<~822n; z^!34uVTq~f10xiN4}R|%xI1eZ9PheLcK&1;)R)e9m{qhA?vJ>e*06d7+)iBdxN%4! z%t_)-&9PqxPs=WrWX5UW(A*?;aKhI93zt4vRI)zocqSWi{|AMupR>3*)AgxC)Hy^5H8?frl9IPp!}Ns~##b!2 zfN8j&V|3vT*!kVm_eZR51fWH4dw#bR?tYzaHp6BW{BY`w?_Fv&B;4Uh1M4=zpk43N z9?3VrdQxVrE{wimF>*WX3tpty`r$rU;2!>wnXnkz zUa-v~ohsnU*W28R!Yg1%k9GaWlS|=X|6Y62Q=a-`6$89(m;uf6F7` zp==`*25OF6uC9jP|7uRI8U8WkH_RV8+IuUUT6yqC|Jv;^+i^p4+&@p^UYeA zOs{eO`FIIHmeGrY2Q+Z{)Qt9p@f$(pXZ-tT{$)_UW8wp!$W4$kX8V;Z4cp+9``4D( zqgoIv7*|}KwuAj|L$Xdg?t`wZN^`?tC0Gvl^O?9k?35rwTldUUKXzuLgC;pqU-U zTcOi(rRlQddhkE`^WO7QKI=cF^}(W&^j&eG8*&N9t@Srt2n}#~gK%B}vIz8zf`_w?&pGh}B}bN{QJOk?bApi)few#nRpvGqbVp zMW+}csof1@yVFLr&q*}$zjwHVVb7&-h$uVDsYV_!iUphN0iDlJ~=K)c$3wB^>c=Ozfx@D7a!+mMTb z{WtMmhC&NhMYcO@PP;>)ZLvXchIcp|&XD`};dG(yJGUMu+P$F&%7A55Q0 zebH*%Xr&_&nqdXR(G0o&9!;MkBwdCS7}@oZXk{L>!|n*juwdahhTOl8^KvU_r*+TV zuECV{lQ*r&6TunodmPS?`*t|HmdR)%(a3hUd{G`z%~JVt5no9yBhPo1$VKAJ1ffEj zE|V*rCp&!@=jP^xCM{Z8NcxdAqLg|>Vp-AMa2|k7(FWGN{ zd0F5L!Cn!)2AL>T@cRYrBX}FEzK$BA_Lcm@pxSC}2-(a3=K=8(pg@>Qxdi6C7??9BOScQ^~v#C(+^4egit&QggW zjh~i<@6FUKep;F$19wO*PE$%0a-2)|b@z1la>n&^&h!y_Bbl%BM;O(T85K1>db0Lf zKs{^Yfq$}@943<)i3tCEz-0O`SxHPblbOT}NFsPF)+_Q1oH^~KoQ<~$(v2eI!Axcb z6X~IE!iWO3@jiMaq4BJV!7^!jEQjk8Ll0LAg19PyJWMiO!V<*H$Vdv7>3C8`sz4Sc z6?3@R!6}NQXfID?a6m|kfS&0ki1yX;ypR*Nasa+NpTy&GA}j4RmRYzbAu%aOU_D07|B8}hNn1&r5nSL@MYW-k=91KDB3GS z%51m(s%AsUbJtbFepU&mi2jkOa{`K%Zmw56a`5$bn7e=a2UyN^DS1EspiE* z(!;fD__x+cKo7}|@eVV|Jt`T`TY;Rb6a=CC4|*lvQ}|qlf*&nX zyv;8@M=IxQ3yGMPL?EZx{J)W+*`08zWAYs)w)(}B2vDaRii zqMlE}n?(H%4RrPJz`N4|@WPKw;ZLH8H?;Oxz3WPd1C<_X4-Y~-3x5XA)%97-w;fX_v6{zb^3)JEh=Sv@aA)%z-7tA1mZhSB^A_6sWe-kbYfcKg{ z;dGz*h7kJE++=~2ks*|E=i+ffQ9&-^@xnjB%JX5$7+` zM?(lGd?bq)i9pfej^sQ%bVQud;^~dB*jlGv_fI>&XvSQ0ekJmwwZqX+z3Nnq>Xgb) oo0l$DjTfe-jwdcDPP$8ollFRIUxa>k==TrXi+&f0K>sWL7xQ|*FaQ7m literal 0 HcmV?d00001 diff --git a/tests/golden/improve/coverage.parquet b/tests/golden/improve/coverage.parquet new file mode 100644 index 0000000000000000000000000000000000000000..3b28726203dbacdc55ec946f8da9f5271332d628 GIT binary patch literal 5310 zcmd^D&5zsG5r2|5yQ<@C*RH8b=@wWBwcr9yR@AQSC@>H_%GOtQWv?8I6oW$Gmn3RQ ziWRA^HG zX5Rc}=FJD8r?|;7Tg)Hg%r9dn%r1j&A@r+Qbr&I|oAzip(K-$+4DI{eE%c+#bl9;+ zx~c8`@COI3sdcoOVx;T0k<6kk-72#vcAH`E?!a!#k_^3PF^EYrZ{7pVd(1Yoy?-Cw z-SJte-BpB}H(}fBcjIv1ja|F;^UvP|!}gflCrq50mDsl*0S_)vdW%Uem8KE_XSqtE zyBnw!<2wf)Mr;>-`v{%~U%nUhA>gDE!HW10te4dP2Om7IH=T6A3F3j!GD-+7sgWol z$iZ7AXAn)qR4fL6u$Mlm^iImli=Q!D>0cL6YTHrznkg}Rh`Dhmo=TWQRWmxedIz2C zgBi>gAzB?r8ICm7d)>A=-LYXH<~s1+P@sH9Qcv?~zPvIt&7tLqeAj#FbVrin>V!&! zzGrpR(a^RvwWD`CfjlBAO-*y1j;UL^IW|#pH@%e3>N;Qe?Lm>TA+SI0WIx_{74EI)SqSsriR>4NSK;8MEDqs*dOiD#>kHg)=`I5srha8$ zQTom{O0Sg&1^gf>WdHX1!$`@e_aWU$)<0aNU-IKOlDpYN^0znO7*pG}WT}#q`g`I< zMz>UL21jO$P0Pu=d+^~;?j2;j*AC@rYgGFBB4bIW2I85Zdt_i+SJw=+vx(yYcHFs5 zHx1>1rnqE_H8$<}g6_B@LNqm3QYBXc{T8&HkM12T@q$8);)OYDc)I1@KaRpJ$X5fB z*0%{Rpnrp4E3Gw(8DXzMca5R6(bEv)YG1wDjUoqCO*m_yizu$1<$`<_;M$ys;%*RF zgAcj3YCwo_l?fY$Qq+t~gw{-1xV&n}f_#-3D-$qkK!m*pEf0q@M^}*p>N+r~amjT@ zx;%DieZPPj%P1pFa)%?Rx)&Kkmq*fQPAc4)|rSJzVNh8po~hr-b-; z!LOgen!LkBwNQQy<0*!H@FYq&2iD{rmM1D3`00i&<5#hfN^zD!Y<%+^5lbc|J=oocNR z-)iI(o$7Z*q1hKJ3LnH5h;#J@sc=AVO4SeK3BMLS=di~N~>Qo_1u8irhH zc0+cLg`>tmKB^nO-P1N-IByG$u39ywzHhc{o#h*s*#z-aAGLGTCzCbQN*VSy`^0D2 zYB_QhVpm;>)mX@jVDI#Dn?ii0Qik&?UkFFdf>QDPlK4WdG4Ol>3|FZdm)ZMLae^4- zMWNmo^R2E(Y`*ji8xz%R&D*?cgnig#qiD`1ZQ-i8iH@8vFa6+Kxi;VItChzjAK~n0 zP8V`go7Y1*ROo`OuFMz5Uc5zH_qlz@!O~w zSA_h4Ug5&o*|N}bFFk+0WvJCgSLThe#IvmYjP{F7HtKS|=y-ade+XP5Tw`ssIPvRX zV-0!zvYb<^b-Qh~a-u-{=mV0o&_|)?lymY?jpQ=)q#4N6`-*i=`x4YEtm_kXPHiK# zcR_Q8^vOYuJsIHf>9acYGpsN3S2Pr>VUxTjz3p^SRnjAwnfOBs9W*M28HWANtb=hlhl7)~1sLh6j0}cXE3~mKD~3o_itW zS(pumtMWM@a_<~Y4iCqR{WZF&PBw-1KPI{>g|1kwPn0TWz+A9JqZrgFJ`DDg@dsBi zct1PcY!B53vx?-PQ8X2^JWwmfMDCR&GFm}J36}=2dIy$|OVmk@x=xn_!=I1Pi+w#b z9Nf9TALHjlhEx1^4$sItJ|{OiPssw~2DwH0CwNQr!@0ndUL{=rV9lOo{5_F2Jh)&s zV0?Pv%VUlP;dE_(iBhBf1oO;OO1}pnAF7_el<4ULCm`CuZ(|6}DdB)JsuY%7*>lp1 z3f)F;G&P|Ao`|9_q_U?3n6o5NcZaAJAc#&xEh4RCCP1Sn%*HFg6A&$EqPxT!UjyoHHq zzNVC@Xl7>Q1;s3(j5ny*E@FPgoY{qwb(_OGn}InOg-Ff8BCzqftxx%__T{Jy{2 z`JLZ+DqET>WHZ@&80@upCOd?Ug=5${Tpog9n31$t&6NhV4L%TtVJYEQjJnFKwpfj% zA#~ITo}DzP4b?g#G9v=h3Nfb9BoktI1X~#E4`-W98p2p&Vq@%c4E9g>1a^lXJCyCm z_8Xaq#rnIs+>rhtVPSCQHvG232i`mUaG!F_8M8FT*d=Qux;|;cF-5rIeB1JvkK6W= zK3HQ+V+%P#v5PG4^21^jF^aClMt#?%wqQ)&C~s^dV>;X3Bu8KkF)L%pE_ow4yDJgn zDWVjgw5^QkYHK4gNn={$9x__dMv`3!=(rJszKCJRX2KitM0gq#DpKqq2tRn&nMu-& zEE>t?^|yzKhH(Ap0hAFT@op3F79Xs|2ZsVDu|+*{hpCW@pmVJnf?&Z%n?qY-H9IU& zIE3#noYlkR`nzXP2;pfx2r}#+44k?^6d60C2PJw*FcfWfkYJ4hbU@EbfLc~ONq9FA&P2PSNkEGB81$!t=iXQ-EqEgPU0tB1>FqA`>r z(VmKA`^YdEE@Q}iWX!%NsL*zty#^*Ytx{f$FGSQhtY%#p9xxS!p~L_a+n_VrjAj!) z+OD-4kqR*8FpITTjUAHJMw8uO)j{v!Ns}V6P#a;k7@(Rkzq35`#A3rdXHmnrzBC%0 z-%s_X_YzrJ^ro;6mz1W(M-2~K{_9W-{v{5@m@Jfu7@Ls^FDMh?6)v9$4;)Bijid!Z zE|c9noV|T`CcDLl8s8B7{R%(lt5ZfVie3n^M=nq-yi*LYgOxv)Tq^+1VdJ`l@-4tp z5&p&doA&}qYn*+`wfP`4Zq(zyeOLmVzB~Qr;3^<#+c93(495xP7<2z;50LPVwcH!+ z1Ww+i)5C6*0c_E&@Bj9h1~@;v(zWHzRv_6|xp%#K55PWrY4|gexu9EmnbjFb0@t>L zXU<+J0?u2TPvU8>f$oc8A03*r8#u%7;TLL`fpgE+jyh*51KruPm0?@$0DJ1@$?ywX zfTZ)1=p#)lkj!7;O8jsMs6IFDYcbOTTnEzpm;Yb`l6%*))1tP5qKVBFzaHHMoSW|6 zJ-l)ua6M<0ut({Dv+2l}(k8pE61`RE5gQvO(SaLZ=UEoncr z;9Mne24C5J{M$MpNsLZ)?tdS&2Ck?y-Z}+Z?_D=u*|`Z=qF>xQ`$!e&{(ePT_K7-R zdGuy@2InYf&2BrrwCpq}+INC+wxSWNS-<}0d5J4Q>8Q0)uhfxXS1f1qj?X$l;7{DF zePgcxXV$U&yVDwg;EAt}-3*xnuqSn2DjqKf-CPd4eRn%J6?N>fnMp^1Q*a{a{U;iL zWX~jW&E;AkxqEL_rN0q0bN=1>@!RbnGySy@vFDC~?kw$h(wqkH3)tNKY|aAU`YPBE zy-EXk!N<;>nA{GUzI*q#C0`u}yhy|O06_yNWmq=P+`158veb2_(u)9Z^PN<6Qajjo zrur_S>jd46$3GU&S_85#1ny1Cv4VzL)#9Z=3qa8qt^Ydn={q2E^yM}qw-Y$;hNLvl zSq)mpzr6d=P&<(1Hk`ke>jbWQ?_7WD@H?RSI_nk7_zqB%uwvQ`zx80-;+?Lb1PuK^`Q;+8cE#28#OKvO^6CEA(btPX_l1r_ zAa5UNKJrfd2gaiS8&{FmzG^XOJvqOLw;1}RvO&76!w4iw^{NB6;kYwVvCFWNz$sT| zB>l7$#BSZcec_N=;F>;0=zPfxc&moKe97+s2%2&t*!-mnV6`&y>+>5yt8*Kflsp%> zmV|A`-z@_?msm1n{sGYZ-C6sxgU4XrC4cc=)KU*TeJeU(Ev&_2T)5 z;@g4h&fQ-ge&aCcUN!Mh#kvO2ASAg!$;H4GwYk}p`%S=g=0eq(3yD{z6eIPV}km{380t|E%9cx+ySV^0QPWG_? zXXG;1XTh_9gvgopV(06?30jM`9NPyZnHyg_yL1uAUUSnwdW;D;NyU4QSzsM{Jvr&U zq1%CAa%}h2=l20;?ptKk#)aTq^@OWqtIB{=di*%&<);84fNl7pTlmQ`AlUkKYP6yP zG-vJ)-!<9>Fvi@HAF8&4?vDe5|2ecBICp(;yL!W7aQWS+mw2-+p!?*8zb8qo0Q-FN zg4M5W2d?c`?v4_-0Vg|NF8O9TuoUJ7epp=!oM(@j&pwg|?o8P7wyt$HDQ9$N~qV>`dZPVNS`kN+?) z?0pxID2A`HOnm`xqj$aeQXZ_Mk;2pQ-#B3XY(2jHz}ui9|5jwMy&brG+ur29y978d zoSh&Jasb!(y&Jv>tpv>@n!fv{9ggcBBFo!+7&O;3KNUN(0c@L9_rZ-%;kukXER-Fq z0?o%mgDwX_+&L?YzM6Olz&-4$ANyZE9=jNq)l+=9te$es!D0uYQk(tTKDYnj-aJ|5 z7|!rv`uh2^0s@Bw1&0g`4GVuLg3TGmjT}B=dNMmFcjl}-WxlGQaQ0JAKcg<=bZBwEUbHe(c&dbm%UKG{KXXwD;rm>ere6$ z*1o)MebXziZfM@PX>-e#t*>o+eft|b-rV_C>)X3_@7de7Z-4uNcMiV$-XYiFBkvzQ zcKk#~=gALFoj!B++=s}bU(aLo-<0zhYLejA-r>XE4?BSg)Xshl&qUozu6$yg5cN}8 zh8n{}fiO6NDHO6Jn4&~{2sb5lVxmYS#)tSm!i*P$J%S|$2QcH};6*4#Px1I5kdXxc!1VkiO--CAcKi$I0`< zq(K;ii!lX^@w(6=7w&TUV4?n4bO>y9u+U`QQd?bNee7Ot&gp5JR@bJ5`35Wv3{T;V zX{i&W1Ze8QuCby!!it-t>m}*i4D}di1U3QUTe0kGSc_9C!520p_|>F0Szb)O!mKzF zaFrSA3g9^(IwQUwJn#0i3m~!(V-;bcE>=su)QS6JJ|UQ7Bvzk{@hq66+Zl5vtW3D+ zqBbqz?vmV_VbddGVjhvR3hEldkFdtpg~=MC>v`#G=f5amjM}(H$WRej>t(FD1DCAy zX<+!_zCkIMxYtHi91&lwd+2DuxFrusuyIZ-EBSg8^aTub5j#7R9Ylp$z_O%azBu#S z2w#5|_B1zu#T+v%fHhS*9>yI0CXb-+(B%?#nuo)=Xd?CFvc8R=pU{f(rc#06&2@*t zvvcz!r3lwO&AdR@SgUUn=1$5sXSGqF>_tf}!h4YYy2p>B1cR&W~iVs<~VJBy@S(LYUu-k|FL0W5^fJM zdb|P;JMkWk3s2_`Ej)Y(o(GMQvYzDh0 zDadArhYJU(E;pL`jScQO^r=l6(%?BIq?bb&Dh-4u%a1hJtwvoBi2)S}u$Nj5wo)^p zCk`S#0wFD!)KsfATALcGXxGrX4b?dac(S*vb%etPk2yUum~cF9f>t72! z27Hyl*~H-Q3gmAKpa8v3*4_y9G50}m=N&EC>C3sv;5&o)Cx;+s{oYyF8)92tf1fej z$<&v|G{`JYA^&^`e^1Exf!Q)1I)Ta*$_Ih&`TbL#ma#30)Alg`lkxl)hmQP@+7K_b z5vF%~4qQ?+YB!5>-k;wj%JFm2ZO&;%s-wC;k*ktg7c=rQu=^!9(9EVT+QG#F!;wG<9|B^ z0(w_z1n5oueLzm#bXb-6EY43;pW8V--@Dg9A!TCveU=&2GukV6Bi?KKu^l3813(Am)xlY_L> z^wQz}fQYcLYxEks21+r(PmS{?@cQuRUBw%%kNziOd2CmMAP(UEiMa0XqqHl%9NW7k zQZaz+{{k|#+YVxLk7m96KM$MU&b=}F=zktoZ=c`|*N-&-(Lh@4HE2rzwCzjh-^<*7 zeTX-%7kdD9F=5u&6XU(nJs7_SKcMCDR$!0;161g0{rMan2EjQ z-!rhMUmFC;-N_-&p9<)1A8ms{dh-6Cpr|Bk?6BF@I_y+Ao&!r_l+J3<*v(d0B2$B$yAzRf&Rh(+IIeOXd=W zbY-og%v_Q$6xbB08D&~gwNRU%K-1!EMlC6`7tT~9LY`eK$|bZiv86CofyR=We5I{W zCUz9$3kf7cK?*A&u1zH($`wY4WlAZ97^OO)SYS}4xW|Z%8d6!Nr)4O{CmA1bs@3i?!fsQ1~?w5D{ z_ZP>nCRj4x^myKgZlK(t$A9yJ`PfUbcb&v+@Upisbk+K>Cz1LP5@?|x!2F6h~m1ktkH>xN|+ze|BBkoGN~hLW`-Db2B=4n%H8`hY$MVTQ6^1=*F4ll#G|(3 zi7MDeq{O3MV-|W(LGNj3rqqfqa;gbQMvqj#k)0-^$mPKDm;5Pmz{V?>cm)`x)vT^-xlee4~iMY$$|+gY6XjP!e zNjXKO9;qj)e@c%>&s6^e4TnQ0(-19%8mXv#)NM~5Wd!n0KYNg^DP{Wn>M9shL0_&Z zq$LPHsds)FJ)e^5;ST|t?^O?_(gGBSbabbG3aPl)p6P?ylZvSFT(gc;)@st>K8Yqi zISc9E^rw6Of{!y$JZARzGdG8J1sc@by1KJK?%5O2`z;)ljOH8u zXg;N{=X|P1gVR%NXnw^X%}46UVQ)yv1h8i;)`(z_i)hkR9hxs1FrPBgLw`E|A-*Y# z(8F2PXnZwV0w^B}Q0mS=(qOPC`{84c|EMJK@E@Iz`P3d&tjVW`L%zpUsTAY#EE;Qt n!(gANvsfll2MeD2;J~Ag6p^Rk7ym!}!^H5vQ)I*c`|ke$g;+A6 literal 0 HcmV?d00001 diff --git a/tests/golden/improve/manifest.json b/tests/golden/improve/manifest.json new file mode 100644 index 0000000..7c752cd --- /dev/null +++ b/tests/golden/improve/manifest.json @@ -0,0 +1,264 @@ +{ + "cells_reused_identical": 42, + "cohort_id": "improve-neoepitope-cv-patient-grouped", + "contract_version": 0, + "coverage": { + "common_warn_frac": 0.2, + "compare_on": "both", + "min_fraction": 0.95 + }, + "dropped_subjects": [], + "expected_models": [ + "improve_simple", + "improve_simple_wo_prime", + "improve_tme_excl", + "improve_tme_excl_wo_prime", + "improve_tme_incl", + "improve_tme_incl_wo_prime", + "nnalign" + ], + "flags": [ + { + "code": "few_clusters", + "message": "tail quantiles approximate with few clusters (G=20) for tumour_cohort=Basket; percentile bootstrap intervals below 40 clusters are coarse", + "severity": "note" + }, + { + "code": "few_clusters", + "message": "tail quantiles approximate with few clusters (G=24) for tumour_cohort=bladder; percentile bootstrap intervals below 40 clusters are coarse", + "severity": "note" + }, + { + "code": "few_clusters", + "message": "tail quantiles approximate with few clusters (G=26) for tumour_cohort=melanoma; percentile bootstrap intervals below 40 clusters are coarse", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_simple (peptide_length=non_9mer): delta_auroc=+0.0036 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0005, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_simple (tumour_cohort=Basket): delta_auroc=+0.0073 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_simple_wo_prime (peptide_length=non_9mer): delta_auroc=+0.0047 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0003, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_simple_wo_prime (tumour_cohort=Basket): delta_auroc=+0.0119 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_simple_wo_prime (tumour_cohort=melanoma): delta_auroc=+0.0068 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0002, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_tme_excl (peptide_length=non_9mer): delta_auroc=+0.0037 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0005, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_tme_excl (tumour_cohort=Basket): delta_auroc=+0.0041 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_tme_excl_wo_prime (peptide_length=non_9mer): delta_auroc=+0.0049 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0003, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_tme_excl_wo_prime (tumour_cohort=Basket): delta_auroc=+0.0277 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_tme_excl_wo_prime (tumour_cohort=bladder): delta_auroc=+0.0020 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0002, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_tme_excl_wo_prime (tumour_cohort=melanoma): delta_auroc=+0.0087 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0005, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_tme_incl (tumour_cohort=bladder): delta_auroc=+0.0161 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for improve_tme_incl_wo_prime (tumour_cohort=bladder): delta_auroc=+0.0092 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for nnalign (overall): delta_auroc=-0.0014 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for nnalign (peptide_length=9mer): delta_auroc=+0.0057 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for nnalign (tumour_cohort=Basket): delta_auroc=+0.0022 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0005, not a flag driver)", + "severity": "note" + }, + { + "code": "recalibration_non_monotone", + "message": "rung3 materially non-monotone for nnalign (tumour_cohort=melanoma): delta_auroc=+0.0074 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)", + "severity": "note" + } + ], + "inputs": { + "cohort_data": "0602dee2f9c80b0512e9f6e5e3470fd0cd85fb561578adbcbc6956edb905baf4", + "cohort_spec": "cc9ee94cfcdc09ff8be98514fb1c33b0cf14cb80338649b4d10caa8336b7b617", + "predictions": "d5c66de35023d643f11f3e750a9d303180de8588c088d94abb651e77453c9972" + }, + "models_present": [ + "improve_simple", + "improve_simple_wo_prime", + "improve_tme_excl", + "improve_tme_excl_wo_prime", + "improve_tme_incl", + "improve_tme_incl_wo_prime", + "nnalign" + ], + "n_subjects": 17520, + "roster": { + "absent": [], + "declared": 7, + "ensemble_members": [], + "present": 7 + }, + "uncertainty": { + "ci_level": 0.95, + "clustered": true, + "clustering_field": "patient", + "clustering_name": "patient", + "n_boot": 2000, + "seed": 1337, + "show_naive_ci": true, + "unit_noun": "peptide" + }, + "verdict": { + "improve_simple": { + "axis_a_miscalibrated": true, + "axis_b": "demonstrated", + "axis_b_counterproductive_rungs": [], + "best_rung": "rung1", + "best_rung_brier": 0.025770239792686046, + "bss": 0.006726175584733984, + "bss_ci_high": 0.011459333307902646, + "bss_ci_low": 0.0010103638877356459, + "n": 17520, + "n_clusters": 70, + "prevalence": 0.02665525114155251, + "rung0_brier": 0.1736426279039147 + }, + "improve_simple_wo_prime": { + "axis_a_miscalibrated": true, + "axis_b": "demonstrated", + "axis_b_counterproductive_rungs": [], + "best_rung": "rung1", + "best_rung_brier": 0.025771142636787574, + "bss": 0.006691376862611453, + "bss_ci_high": 0.011280965974780174, + "bss_ci_low": 0.001141575323759689, + "n": 17520, + "n_clusters": 70, + "prevalence": 0.02665525114155251, + "rung0_brier": 0.17362201243380085 + }, + "improve_tme_excl": { + "axis_a_miscalibrated": true, + "axis_b": "demonstrated", + "axis_b_counterproductive_rungs": [], + "best_rung": "rung1", + "best_rung_brier": 0.02578160059807171, + "bss": 0.006288291005287494, + "bss_ci_high": 0.01084696486704049, + "bss_ci_low": 0.0010456215867604508, + "n": 17520, + "n_clusters": 70, + "prevalence": 0.02665525114155251, + "rung0_brier": 0.1740571541484085 + }, + "improve_tme_excl_wo_prime": { + "axis_a_miscalibrated": true, + "axis_b": "demonstrated", + "axis_b_counterproductive_rungs": [], + "best_rung": "rung1", + "best_rung_brier": 0.025789071243491817, + "bss": 0.00600034659316806, + "bss_ci_high": 0.01053968605906155, + "bss_ci_low": 0.0008430445036216836, + "n": 17520, + "n_clusters": 70, + "prevalence": 0.02665525114155251, + "rung0_brier": 0.17437342114437554 + }, + "improve_tme_incl": { + "axis_a_miscalibrated": true, + "axis_b": "demonstrated", + "axis_b_counterproductive_rungs": [], + "best_rung": "rung1", + "best_rung_brier": 0.025735148054809035, + "bss": 0.00807873205944587, + "bss_ci_high": 0.0134276046647617, + "bss_ci_low": 0.0014602036571806387, + "n": 17520, + "n_clusters": 70, + "prevalence": 0.02665525114155251, + "rung0_brier": 0.16762562559930383 + }, + "improve_tme_incl_wo_prime": { + "axis_a_miscalibrated": true, + "axis_b": "demonstrated", + "axis_b_counterproductive_rungs": [], + "best_rung": "rung1", + "best_rung_brier": 0.025744200664175557, + "bss": 0.007729813306700084, + "bss_ci_high": 0.012790371547574123, + "bss_ci_low": 0.0016603479016365002, + "n": 17520, + "n_clusters": 70, + "prevalence": 0.02665525114155251, + "rung0_brier": 0.1679510221660752 + }, + "nnalign": { + "axis_a_miscalibrated": false, + "axis_b": "counterproductive", + "axis_b_counterproductive_rungs": [ + "rung3" + ], + "best_rung": "rung0", + "best_rung_brier": 0.025829954627819694, + "bss": 0.004424560111044751, + "bss_ci_high": 0.008334603978975204, + "bss_ci_low": -0.001111639209645941, + "n": 17520, + "n_clusters": 70, + "prevalence": 0.02665525114155251, + "rung0_brier": 0.025829954627819694 + } + }, + "versions": { + "numpy": "2.4.6", + "pandas": "3.0.5", + "predval": "0.1.0", + "python": "3.13.16", + "scikit-learn": "1.9.0", + "scipy": "1.17.1", + "statsmodels": "0.14.6" + } +} \ No newline at end of file diff --git a/tests/golden/improve/metrics.parquet b/tests/golden/improve/metrics.parquet new file mode 100644 index 0000000000000000000000000000000000000000..120560a0f2566a36c94f45530095396724501b63 GIT binary patch literal 45619 zcma&Oc|29o_dk5Dd7ft~GKMmjxGb3>DpV3>2pKZZ6*7fLQKqCa&k3n0y4S2p1Bp`M zn#XI1G+f*Is+?v-f4T&q9fphPRfAHxp;d%f^dw zVOSQ6LnREy!>c4ikEFf496a1yym&F*cp6?ydER6PUJhP5UOJK8m<;{qmjZ16T?Z#8 z5{#kJr$$Ax1CBZsho$0hsW_@s9E$)=DvmZ4NB5T(+qkljY8qY%UKvwVP|W7ir$KUO z0zEwFQ0eIH>FG_WUW8ZlziX$Vun5p?F#M~MO?9(IT&E(|sfa@gckwF!M=!q><#~`N z{2V+y7!c_nc-qC!JJ1Iym*JKFZ{-aC`BH#>Q{rC@Jf{Dt;R*xRAcNcZi|R4tJ@B7; z#N%0!@_;~3??690M{ieezW_=z_&ECnxH&o7c{qEy1h}F)HF(wkcb%CWGOYh4B*6CH zBL587BEf=mZB)xhr-&;Mz%n^D{~|NodA&?|nf|#^?JdQ9x(@zlodYNje%iyq$;sJ| z^5CA%9u8jKo(>d0@Acfkyu7{aC^Rz6miOfUVHm@Iz7?RM@cgqx;lC^ip++`cVL%R{ zy25~Hn*ZY#+15qP+e&SU+L6-8<2bT2#Ve=11HGIa{LUdq1$qTAA;ibeImiJy$ z-8lsMc{@_V$$`>tF3u=yogLl$-Mqah-=6k!qgafh1O@47KZgK>+qroKIQuywZ&Sn^ z9RvkU{XM*WoSD&`zq6OWTYy`TTfjM5b{!w2>5Q8rx}h|xk53Seo^0gh6GV~oad7i< zcCvGEaPzYJSBn4w{Q|vQRO~z*LhL-eQH^#^&W?W04*t#*0VijV00+B&=Gy(^LylZj z$@DnEn6XS_Nn~o{l13FJQJASu__pl#d2w9&p~S=NwmxW5#5488L<@PY3== zVjGvWB!*qYE&Tn4ZCq!*VL@Lpnu|euXa3UQHm+tG9Cq>VIMv2gIZma`K!t#5>W%Bg zG_@KHx}T@nxW?vbFlC43|M|XonmTam-17gZdqIQKU8eq@+e(Ud=k;TZyo~>YUH@0( zSH{2puCHGibC*yBB;3YzorJR<#?i<*^K~;L5oH=m=j7n!G3BND7Y_^~$O-5(2*_`p zY`jId|K&FUO3?oM^8cmys_-)ZKk&cMETGrVM&9I(peGMDMbooy<-$1G8JTGCJb#(7 z|C?)$8B^tE-;l*)xIqa@iZbF=H|1scXRU-NCHg4MtjNiTDRDANDE|jg*|0zUf7_2z zCGEeM5cpRpZW{TYTtc-KlfX$(Nl;U-Q49Rvn76^B%}lXMkyrIUiAUfh#UTzpJ`R4) zUI7%^(a+o8|BPDzvT2!}H|zhfiQzwA3o!gg=$~aj=@tKzNin}%fdDIcC z+c-;suu4Oo-rEjuQoi4Otj2e0u5B$Y2ay6R z_fK1nWz9C|{)<{NT+wVJl!%YOZK`rMVeewty1I+r+$9UXlNU~#Ec-z7VHd*a=^02X z3cPq>wi&!uYR`wE^0UCUH#k(LF`1Yh4;_I^^1gS6c9cS?dmaYJYo14~Kl|=xBGUCA zRI+}w053k+MvBCIfIv09yTTmP@F6?wC*6k%7~lKd_0Exb@Yv*+-sG<}dCuZ`G7D6|BKgUrniIb|{J<*lTi?T> zSy15(i`a7_3k)vQ?e4v^wK1Q(A%BmG?ZU{G5tufycPai<3|c|aN64qMpipK~bg4cN z7RvM|3XEsL;E?8@_L*i#?B8M4iOQj0N-Kk?%uIS{Qitqs#}ewkoG%4=(g}Rz{&vWH zibVygF%iGW>7GcJS^)ot$3^KKKEPfwUy0qTDY!d)UCe5KIq-F#Fm*vuc4w2nev?1k zkrB@!o)8;FJ^2%A+`5TX2^RIWF6yUk?k#pFm_%;85A6YqdEPe zv3{VQblD5{a0Ym^UypPl;oqSrsyz08!7ox_2KMB{Q*=W#$`47oL=^AYJ-6V#0Ic1T=X}rhz^OeU0tvL!z}Xs?JF~YOt{qRi zgdLuTaKw*Oq4DwxphnU^6;AFU%=x_c&@cS7Rd2?(Am_CcQg`-At(#az2P@!r(OM}}-9wo!0 z+pE9#1d*VQ{i(yxykgj;CHnrMdlpF3h`)@QGKXJ$+v}|2XW(W>^OrX?EpTD}*+2s- zZ{-?ai9_Wnp(8hI2S*?+isj`XsSGlEzLz!yw829yZY`cQMxqv{{P}}d7hv$mov8sr zFLa3QGEuxR1rKfRWe!=F!}(`d@Y6@`2G(&@c zn9Q;)?&z#{@{vm;Q2jI3weoHW$c$==>mrBx*%qr1BqHJZ%;@%#H#1<;`ON1gQqVs9 z?4tg0rp8x)H+mn)ayjEIM&Lry-CauYC6InMuUrQ8zIjKQoO!oUSoK$t2zfXS`d-H} zgLgE;(($X#{W8r^6cL#kipmfTyU*%4Sr`9C^9ZF5+6F{ zg^biK!uA#CVWr{^PD!-jNOGIt{b?}|e>VB+Hu(!Jn6pnD$pTYl zU-D{AOM|}ah91s|v>&G7BcQh`_4p89De!2Ouveju(#7vK@me7O*sh+^HDp}?6T$6e z>E>Cma%p}p`6yju1Qq6NGgtgf_YRIo+>>xowQ znmPmW4+PeXQFB#Aect}=4@cwhzT@OT6qYqd(xtbVjDQ%=fgOGpCGaE6?o5fk&`oe9RH(#DSn}$C|vPDOc^ttCXy5}`G zw90ufNe;3x!_0$2{FB3w+3Of=YF7v)-_K+YBJuOzY}Nn^fa|S;f&4H7SNvXEVn<1! zzB}yAY(f#7RS$4I>YW8oBGd+Px3_6MWET@7H;^G;#e=Pkbr^zf2#WX^6hQSVb-PL= zGUrK*;UO6%=od-~(QX(AZMBd31^mm;QW@Mrp7yRhE6h}1spDiJU^;}`=4nK2>}#@K zzP%+uY{K|}lyo|D+<&FS|E9PVe=4k* zR)DNWsJi#M{m?vI4-eBidvFAHAN^A1v8Ngg*mnM&^+*Gp)Tcv_xo^O(LGQz!Kgn=> z_65VpzBJ$n`1qIo2#Zz<4#W1zEZ@Ra+$kOtxG@4YVcij&5!H~vD4tJ`>SO8JuiipF z4=$^*tzb9~hjy1(3jP^|!+MN)l9Dx0qjXQrTO$qZAGk~Xv=nZX|FF?PtuUu`PDEhW5%xLYLWDjSt>xr1 zM8IT%1Z>z>wIfwtT|@k&%EW9E?Y$KBtxeH0pj2fv(l zbO%*I@u9VEL8sE-Va|cr<#;QIxKS&d^Li52kA=S3f0+nSZ{0YMi2V4C<6Hxdfo#_&>ScaLneBkxkN3RvA zM$!3F>kF_S4?SO8I}I`yT?waWs^N*Py2b3Zd8psXD5IbDPEuCRE)8JXruj{W{p-dwXknQ=v>T(q{n#YA4c1(lpg|{kNuj9d-Q7|oO9!)su zsWSdXj6!Unjm(F?)o@%v=()Cb8mPzU-SM^ifftxH&RjO4dM6S5{?t}92T|=bOy;SA zRa-`db<`jud>UR37mfhG)KXa)egc$!PVL>Zvk6w{Z*>hQG{NEa7axlhn*iJW^^(uE z73~Pd1Dco6e3%X|&85LT3R`E2mlBUuf&U@BJy-c#L9Q`Db6K64Sb}NJdgrXbOD#$d z?*nt{yil!`X^{0+Aefm{L5qm+Dc0zD*tyB?wW9Hl-~UMAn3QuGB>Mb0aWko=;nVsC zJ{#63DdkTB0m-;t<%n|V%v5T)AD#wcrX}Y6?-Ia#tKPGbt{KP)YAZ6dNCTmJQMZOC ze&WMW566%@muZOh4PQx+@uOSgd}Spp?eY@sbxVWe1tYzxjqy-6C)n%nH3~*0=U3QP zMj>w0{e|^|Dv;~CottKm2B&^?2TffbZOAR#2ypcBLbi1_ zu=gaaf2<#cRl!E=JbeXVBLP43z7l~r?R%(Yk&zfURzBv-vkZsD6?F2-y6XqdL*pj@pPmi=s!+X@3+`#4o2WSM_xnD+e^U>ahhl5~%@L64 z6D+$(LNf@n&x3KHX&~QIOMMUxEEmfX{*3yQ;he#HB|K`6*B=u1{xDp?vk4_L41VwK zQEB`*0%al!XYz(iVdXW|>B@jKXtC2_>iL`px}Uhzol;R_|Bg4IMZ&x5Z=BFpBIA4R zg^&|%W%xhzqnSAUz#+lwJ4?Xk-bm1mgfx(}t6VJ9jDiPRIssXhW6-};^TYX614Qz; zS)H_NfRb!QZqkVcP~DbkuoO88V00=|J-!@rS7O2sC^dnI(B(%VtW3lg+mf`w@;lTQ zQ*z-uh+$q4OxQmQMHY8p?EAXJ%a5-&M_kFK4=n{Oen3?X} z(7BaSV4~0eoYm;z5eT1QiSMQ^0lEPC!-kP4*M+7>7s$$b=Nd#B%S^dW4hMxiw5I*%Rn}! z9io#TJ~{&4m959KR!brOR>H|w(Pq_CY{TzeHS{2eRboTmhOP*e+G()4iu7yxS>Lz377@ID8wCWHvP*h_Y^f z_ij!b^|xzwF7l@VzCCn4zdaM|mTx4ie<8sun-O1YwF2;-T(ZlF?*Q&b&Sm8aX5vXp zv5~}vS=dXfP&_!=1CgIbr)*zN0{^F^2eRw=U^nxzj?H!+I5+u4H~CKysG{x$WWa-H zRw8{(4&LF|h8{Ze_wqB^^AP(d-EjLY5^z0ri?SQfhwONj2UH;$usWt0_L#?;@JNyN zg8hwYxVTRDS_A1Zp|#GM(fEmPM*}4#&)GKZc`M;V0J5 z!__*3L`MED&ys22$zvOiMe^^dleV`tkQ=tIZsfJVkLkLHt%pJ9^6<&!_8dqGwHS>^ z%m4+3$ow&9ZGspM0}0a)a55x`G3RD0ygl<|XbUR8+%(ox%xHx~Ga=2Lb0oMpP37hv34NP^eB3Xjaf3*hr|CYu9#p6fvih1@5+ z2&Qhpoi8^FF7HZM-y`|3`UzFRf~5xS&G8`oTo>Lfjs&Z3c>4@1&2q1sTnn3dIXUA59n{is>Xi48kPOKa36%$<@27{uL9*g8$xM%XK z)jK)^){m;~nOn7i#YHvd-uQ7CK1L^AXwV9uLd>6uqLP7G!?Z=O74AwdCvb$3KrMf} z${>Fc(0^8C;|XYoW590js>e)BPjX#-Upot9JPp}V8QpMG`a2qFrXW8SAzV0$h9~oTOfKUgYVzQ?b?qd2do!h0?VAC4;zxDzcP=(?3Mq1u6K4by zxum}iL2S$YlQG7*@REM!RDV(iY*p5AQK&r!ri0(2pGXe?o3+Dj{Yxz%e`a{2F$kxafgSJLrq=9NXoZ`KDcTtv^Xi)O{MG?n51odnmx!?Z| ztdDOQr~f$xY6tv}Y-1|~I*z?HM%MFi67eG^{U}CGde3)gpE$BqiYIk;AUdysz5+SP z=7cdo`m^lUqr$Z2G#-C2Bmr_wN#es4R}C7uUtYn(D~ z<1!%Cnvwh9&QJ*X^|`Khc@$1hn?_a{w7}=PC(SS=CZdmGm+QGt!iTXGz59(IenCxLP@2> za$U9%zHU7$sDV1=0^-BTu$Uk=oQ%Fz-a8RZKNKoeKb2{-I~wBmBNy^(X2R!R!_W~- zG|gr$g3pn4g^Ph1P)23(fK@AiaHzC)hgsqT+#SRVS)e%cZQPrcvr5L#{9HE(Asb8h zyI)JNN9)jm{O?DV3t-@VB%@BxmQ7CX%;IOxYC9L1loq&ds{_KvWmh-&!{X1tTX6 zmvMy7}T!j7C}7=l9v5)Pa`d9ZNrqL#&F)I8sRb^p@Qhoi=stS#$9 zkiX8b_-42nc62hB6b&`Qu852Xj=^TYa_&rDz-UKZMmypMuY#Ti{pZ-*dNg`bgz zKNn2@GP>8@%z$XkmHTHqUJ|;TO2)lfh9GkD{+9K|JxAL0#A!4dnZJ|cxGZ(Eq3+ZM z{@9+|x;}UabXaB8r^<7n(POJe-L(v8Qf<^e)$K@Nhc@4(hA$AFd*mD;J_E8lI2~2$ zyBmIRZ#Mk8`$QUm{mrqMZY=zfuudSV!^{gyg*G!ErieZFoB zx?D(M<7SLki6`_7B)o4pOM*_pC!&svIq>S?h40eT*(VHiBFrJfr) zTnrq(^Rbb%xxo42so>rR354%6M=vQoBSH89UNyn_9LV$KG>J;-#$Tbi{DYhrpi{kD zoM{*?hooFQZBYb0kojhiAs4=Mk<=>W5(wO+Sf_!gK`0PAvY_YR0zSPb^+Qle$W!*q z@oRzkmg}`i4kQ@w*vA_jT@G>?{6ATzI-rZp^TscQfhbZdd@k5*7Ft^CjK_NzT9ECw&zL+8ICPo+I7O?JwTCd{*Luj5oDp*Q+A;t|U^U>7{(-tv~Q zQA4Osh-~Z-uERDlF$Bto*6gtJ#XxSTkfx!}h5DL@llNpV6TU9L6dD^J1|i~}_zV;U zd$6K!D)-8vMU2;{cqRwZr(e*$W~Xjc`E{ARTuQw~|J>=b#387T5$x??D}n=$JPlq^ z=R!*-zh*l9Wdh?tMY`JIVMtz&7Pc%wj&hMFo1*gOj)n=LQk3NORaEJ-%R>Zj`1?tk zP)LhEw)Rnk1Tv2Pw9L24!04m%XLI9D(6sgRpu5jNTy9}^Vdb8NuTt_?)o7+*B=D}+ z#S>+)J@w{`Xu5g$XtBY^*t)@|)gH7(U^NE}Pd*>?c+%Q1|9k@v(-hnuFf#-@1)HkK zPm3VJ+o0z}*oZ192CJ z;1rF=zFB=2z#@xqel33pz77i zvpjKz-a4Kk@bM7#SGrvQ?%jz?3rD49=~|%`FYzk^ zmEOZqU!`bU;m)fStPC~Ny-j;mW!EwH&bEQ(Ajn_OJo44fzR8eZTO^ zA>coF%u}PZ2)b@RPmp5Dg97Kqb`G^@L^n)DXn$+E0B)EAcblUhz z88lA$Z#{~^h_L2@QYzKgMyq&2 zfMjLTQtc;LT=Av)soV;9Hnjn7R9dQTyM;$3mY;iF=RBH>(u5+2+? z)p!u?9W(21@Oy9a`v4zJ#+Mu@-f|=Yf8|}nxrPlrrksW5p~WP)(Z?P38MRx8d6Y^K ziZ5F7(sP@LYlMQnyP{IZrXX-x+jSMm`z@6Zc5J@WZ1K;SYiJ}v)${O--04zS zI?8)JdOZhrs|WsdNJ%0@tqk~V6C8(ZsSmDukbEw0F0&lnC|>fHTh+2C4*x2}Fx`&? z`m{4`KANRaNPP$EM&0$`Ve!M~CE^GkUDlI^EhA{`4cZ;>q!mtSzyJFCaVyktDN;{9 zYJ~}W+%Na^5hyxOyXg765_p8w^UT?wL(m(Mw_Sf3h;hPc7h9Ybq4V@7X|J9M5d9Ik z?MLK&$PneV)RdWryrvC4zfHdM?&}t^!#O}-9{Ef5QGP?Z$cDU7eiZeyBs3>Doy65o zD23k(I6?vnuL{M&Lt51dgxPpD5%o_~pu!^hh=k;S#y?rhR4&7>Y`W=D@ltmjTERLs zS!@x>ECq9m1hNB`3+$nahhn7S2|mwaX-enEV6^^lD%DU9^r*P?e9K~N{39g8N%md- zZGNM|hy>q^ZXK2{D1k!HFDgWuTv%TJI`EUhhrmPIz$3zlw)pM@FM7Reg^RJgQhVRD zLODSvO{}XG4E$7@w_1@P=;ZC!2V*MW?s=~ER@AxrDx!u~bUGoM&qWG9G7EKwqyOFG zh0u$SL+nzyYbX-Lx2{)l$KGhr=S4Hxa=hl}C;NusEbX88D9aMaI@=qUhAf#GIu_c` z8b%nIaZb8-l?3FE9s6jIu;nnP*K;16))F1YrN}}yo)vv*IWj*4DwTW0`4viGDm04L z2X#N|WvvqPugQc+p=lbSu&-dV?b@4)R1#biXMZH{yaLF4MPq-ak;U8*DV=HCwF8}?JR<9M}N9KNZX>$?v|^&q6x1CgbDP755T-S6c?S#OW1Bg z~;LM6ELs!OI_MDfF<_cquU=21>J>l5?^k^#mbVD9cm-6G*b3-?ROc7 zel#sP^0foL`bc}0(K8WGCzci+yp6Up-RXLs(;gIE{%dw|sLS2v5^{Ds7(HfMvW(a-fU-k%Ltvzg?? z<#?W)i;vM{qyyJDS6B+K6R7+`-(|xV&5l@(Rd4Vs^!S!hH4XTzBQuO>#r^`5%6_Ha z3hkFzgd|XD$7^(o0Xa;&mB!HiWf=r}pONBG>i{2K>$^K&GZG*39JT8&Cj-S>nnk@Z z`nfUG!*L2^PqBS`p?41$j%QTf)td)z#E%xG@|4$)R>{(PJzBFNrBnXio=h8jE{eZF zGe)Yq%s(VJXSH|hXHqF7t2h0Uf0+#-VSm^b#?C^5(NS*gqX|}v+q|?YI!3Tp=*Y!Nm354tbo!+BF|-soiqc$||A-1UQzSQ%tE?QTzo?6kWaE~6jeufPLo zKlGkqHZJ8g&7E>sXxO&LQ-2;&Nbs*=G13`NQVn`d2f*F4~x)Ge328FP?(D=sn~$%`9YIH|k(9v{BAIkPP`+?9)R zp&*wFE`6aC+^v2qy1mJUPmbB_RQ*1%x94p_A$kjM#@Opn(3A}gf-!enW!^PVj+|1a z8V+^Str}t^@V#KeA^W2gw*P#3!W+paIe42^D7Zm?;Ngiw$}=#!(^a}tsTDpx9^th? zWlqa0Giy|8Wj$}*f|k1?u5FL>tjeJ$L+XURcn56V!WXsll#w`)X|=d|9KA4DM61>g*V_$o%6uh#U{W?*niV-&{9qo750N|!sSnx@ythPSrH|*u;BHi}RKv1Ggu8$sh+}T`vyzL44+XD#39QN~{hwguX6= zh^aet)_8AwD+hNv>KQ+>uk=Wdbd*JbD0hoL(!*#Piz{JTL8kM`J;Kf&o zp#`OKkRA_NdZslGbDR8=oBVs9*=!H0&xU5NC%4B&n(+K}8+rsHZu_fQjliu8E9+jt za`-uE@aYOlKjSYrT!uc{gVe{X1^14Tfh~pYhelmCIMnb)*D3zNFA3RmO?0#zasG{t zu-QAvUEeuS3OiF~CudNCe7vZG;^@til#K(7x=IH%Vn^xqiDak`EU(cGE2 zqn^?X+;PAB^pl(6Y}mwBp|N2Qth{$wyr%>>*K~iL_>>I{L>g{`fiSo+bw%g3^bBB| z_pj#ARD);|m1v8YB6cUKJWI4`5L`j2p6Z!AGgtXAxWyePy7R3VF4(d5>kejv0M7y@ zsd5%bu5leh3zLwke2tg(b2bX^$o9w9ELyvTs@TcK_3U$1`0K-9_(ChUEdU)aTPE$2 zAI^q{7q9V;{q=?yZuBmKRp@kD56}7p4>}X#ci^h(pAyh{)*4vWn+-aH+O}hHAMl-j z_}(&+n|I#j{9%ze1Pu9$O-rFg;8{4&5HOPshoaQJ#xQd%QF66I1 zqdq(z{oZK(nZwD73N|4w0)}0{JHU{3#8EjuAD3X*A}aVa+&Kfpewg2ez7g27^0>~y zsR$z3iuxv>wm=!)?8n*-CSn?cAgJ3cf<}RqfVcA-xMC%__$+Y-wmn%nyV!%qzk5dn z`5fktqPI$Y#<`M^lMzM+Rvka9F+eYZ4I59+cDFZT(Qqb@O9lJ>A>7bGXaKF#vY&+j8};L_Q}r@lqN zWXL#?c0V0rzdyGe=O1l|UE5*5Y=rys;I643IvJ8KJwbhTJ`Y~Tl%BPFnGO%LyZz%0 zkHf&|Qe@l3aS*Iu;I}3b;S)!nXzMT$#=j^|hoRDbWiY2&T?a69R7m-$Cpb+i%=Z&U zfEY0n;U7~BCZ|$7IG(n`K9+|gK1)nQRujEQ9?m6b3X%IIV$lsemHUUz^UQ!w_g-WD z(Q;_c&rEf4N7IN+{`&_u_<8?Eo3uShhi#RQIu8GR*wAgffxCJaH{N+h0(5>UXZU0R zG~Ky*Hnce%&Zzykre+@jg}T)uT{Lqba9Bdm7B$P_-q$}=Ovnw=n;TSC7j4PrA4s4& zw93OpUkF@pZw8&Bc|Tp(Wvd5m3FxJT_O86+x&`nl{2iTmFJjC5Bj~Pkn$TqNl{+D0 z9Q=1|{YD&a0%KB`4{4|gymvlkHXUq&h!K~vr>x3=9XHtjLC7CW9`4x@5rht!7(DE} z4MlM8@-o$tW9@G^xOrZAM_uc;d|z}P;`Fp_1!F#ZywqAO z{5lM*6nHGmNwY8$ z*8_J8#WgR&nw;JBwja}=x>mFAhC(@HB!mlgyP?(UCSTVk-|6HhE*DVuT)esa)GOQC zh9R#F`9CdV!|j@AYh&X3%WK4ZxPUvp*Qg^M@cuzdw96ro{&%f{4()HT9u(JgMDigT zAFVxae8g98?swMr&P!GrlVGGQj47-jAB_E*o+%+ezSPAaipv>poNyP)J3bCB+}wQU9#uO?B8qBYVmEp3BIA}0HVQ`6PSGYNu2R{rxc?B=0 z!*@H?J+F-t2})|gKE9?T;C`<}O{LikpQ2;l>uWSaj9aA0NwsE(=$vDnVBZZ9#|@2o zg#(~UPvv*T+z5QmDPIY*D}jQ5dOBa%77$u%|NJY2iTEcqtmmuOA{a2c=*ZM}gV>WJ zv$3KxaO~B+E~B;z7}&DX@ck^>Z{6fCy|KZsu;Bbtt2`Z?e~f;AQq+y_yS{;+wbi>= zjShYdv0_r=UWM?{q?AwRNjlp9{`GQ#F&3_!5n{YdHwT~aA0sr%(jmZu<+j0o*2cB9 zjTub*pYAi)(DG>1B7J$6TOl+wFpot)O^2mDlOL4De3^gGwg*R5n_N(K zBCz~fy^NvqZkr-u{VEYSE~U~aEsa1eO_NDNYYFsTzqlN9p%rF7Xfk_zW+HY{#Z0_- zvjqL)$yxRdAK+^+XSx2@X>hTaoiHuC2M77~>KLKZA=8`u*KiyBA4(P%_E)4sZ^*Fm zccpavzCY3b^x&?9vJ4`B>AW}Qa0w}dz}6JQIpiF_i`>gX33LFcUO*ZJEELF(8A#lD7gkTv;sByTbX4vKoQZ+D&o z@9mcUr|zYL&b*@j8#{)^)yZg5Pm`J{=gWVg};L79h~F zGInncoolkbCns3Y1j;f#bD#5@Ab`b+{mShocsBPcqxaMZ?BX+#^j0VUexEefWEB6W zM1~JUu)HOd?CjR{KfMTHavNpCR*BnADBbf%dZwihW#FmhH*n@3XKQiFB=|)35KG>l z3k&Z$T6^E51C=0O03Q|!4{?1zA8Ag*v}AL!3mSgEH5Xbw)s!f68vIdYaO7H8gI{d<*xI zKgU-NgJ!9Njoj`WIJ+9(nTO=P#SgUU{|E=SoT$~i6+T|N5cRe~!rrF#PA-2VV?>48#8y{`X6rk9w-&Xv0h%$NKlr zuhsc)s+EkGst}Ix?>E9l4jerKcU5X&ye_GTUO&0-X`G zDmjr%N2{WQ`UdH+2pt-qWa=$55%d%ip<90p(&P99A~fV65H|dLkpJDtoqo5^4k2VW z`v;EwA{Mh{#wSjP;WP&>Mu5SbK~IE=6R{&qU*$ z`?Sf~BTAI7CX;rj`vz9kw$q1#@jvOKRXC3-Ch`+7I*Oe%dIBw?hDuj_S)3#s1zuPg zDn^O1oKbBNM0Vc6CAk_o3L$ocxgTvoc2!0udAK?XBRjnUDEH5BSAF6gMUb7H=4}2* z;Zfq=w<&Ca71S69r`i&f<(dfX@8m{zOM8%ufQr*bx zM8VZ*>T1D#Yk&r0Z8AO-P{Vy^xm6l)35TM#^JEW7Nb89mFh`~)35?1Z87H6>e`FFu zO&ks!LBXXyy2I>p!cnBp^pfnc>KDflWiKgb-OqF!X%1+Uv%{HKA)~9WT#|QIdU@jQ zb-stmh4N=P6Rp~t(;wB*BBMKP2D$j_Us1?-t9IQC)HbLuoFZT6Z=696Ww84>oFPV> z;OH~5r%@4T=Aol7IBTVDs-_Yg`CdU{Z24>HhGO!kRCB0>_OkVc*!(b*$1sHgGxWp;~7oIA#uppql(G`;?qm``Ln@kZ#XMNaKeE4+nQ1xijk#r#ZHl{X7b@ zg8f?44u>xwL4?hnV-H1C>wfK}+{0lT*h)QnIHE@O!>^&k7jeveKgI#0j7x8mF-BJ% z8d(lZv?>KBIDz_Zas)@JC@l`75uI%nb;eR1AWR|B=0 zcS#tA6B#wyuQ_afjY1++`zL2Ig`_n$(%Wv4f)f)!prHoWbsV#Zq_OculhYPAa4al) zCymW;4av!QGRAHPj+VVotO$^lKe)rhI>92fpjL8MAS1>>Ov!jgQ>1tqS!kYGpem)D zhNzUAT`@(f=vTcItX69GLkf25D^0Qba~w0vyX=XhgGfNPogpgyUmz zK5*m~?Ng`ihXP+5xl47dL|;wT;#}F$qG6t5jX>F>g-p1oJY|+`WtR2G4$fkY@3K~H z6pJd(7?fN0BkJ8fpES8svnZ@M@1Gwnw?jWi<{+9^#i+@j=4HlqDEC$;r^!1vx}JFD z&R3KDUEWb}+UjjWdM(9=2NSJ7)bu_?p^L!27wM0XKF>s(FN?j85fv$Sa#)C;Qg7PJ zljFL_pWv9|OV6H~anE>)CFcb<$=NPu@i$I6-@nktY`1p)rxELM6@9zo` zv@FPwcTWc;t0GbQ6)|5d=Nz$XPfMq>O=In8T+=Y5H42AIi==EZ@MY($(%E>a)1%CL%d~fK6u0Sr z6|}py@Sx{eIpNQrc0a04dF^@8J6aeM&8e+>?&%d!E{x&;pKSPw&ua6sf9& zKde5H+#)zJo+i#ppIA-gTh&&{L%xZQB3MwbdWRO9UkY>9h}8c2j$>VZNoTW3yW;9! z;Vubu4~(gQ-1`O@gfMOTCvOqO{bj=FxJEaQSs!7C;-2(Sp6ke$DXWhfACckJBD%fA)4fTS1Zt8(2j{G_yeB-QQ#T2|qWmnPB*hsF!#Du5=LoqD;2* zZ5?BveQJ;68N?W^bjX6_2nG>*DqaC1#^e*p;vtHw;-S~(h~i8|Vzmg1;;BHCDzhy| zv>=)-;P@D#MP_i!I;!m995G@%VwBe4yj~T9?Ng3F)6AWxcuAH9qe+cz=B;@fBZYTh zKJ`Nv)6sELypqe}Vx)(3A&}KG1W`QduX5RfJ5it2S{`DL6wpUGSXX2_XQC@DGg)W9 z{%NkXI(#Jn1aTW1a20pDZMyxdVs5{=HqCxF^ZnlUDmRzj9tX*F)O6=GvkC zGv#VP1!)^j&jg8&Z##W-)G_)0h_Auc>g%Pey7T8xu~W^giuA zE?HpW`Qc{e^cv(1brb8}_vw_JfUsfZ43r53>P&1`-e(Xf1J5gaCT`jW`#9rQr-ha> zJ57s}GftQ~D*I=)NSsl*y=IE?%iAkOs>Q~rr}H38?cO=mP5WLI?SA|mIrhu)*q$cA zL#QhZi%9Q%>4r=l*LTtQP=5$hOu0Q9Fs?Zyc=%k`jXQIpE?WK1yMjI~-T6T=HS=)T z+>N_`aM$!ZyMlf$-Nk4TC}0-JmYPG0EJUEtat;HcGR-dSO3h_Owobf_)8#>-B`*8! zvPEqkU!#nniiUZTmuenon#_L9DRX4&Hd;3$0}TsgE5eLT+>x!)25twAJwXIQOl+o* z(F&K`Or153pm*@9_eu{^=FN%=Me46h4+S$}3?53w_}R&W-U&zU{^cpbB8Ys^vOT$6 z50MEiVfo}giT4H#t3_-;QK9pI1r)3_j&N_C_E3F)xC_7GI>a9u$ ztZ>>{exm8TW$pP7<#q?ltv{k#svh3+3X?lYqL2tpn?5gZynd49WvhHiQ ze1uI*dAQlweE!P!_+tHsS+BEaZN7E|J*u6(=Y0{SFW$N)-Z>x2kQ?}@c4wvUHAL~& zKR-6-cN0+$>$}2s1>_^@*pu!;uRxS{1n9uk`v(F6D^VbpSVtt0>B^as#n|Ug$E-A2xdGHYF z+I?cxkov$Dk-4fkK-5Db9i4By0R?N7*j0V`Ffx|6ZHJ{23WkWv%FMQ%K?{ivB94M@ z>e^H*7ZQ%YIxYN7paVsie+6mFQe6jwETz|cRex|LE_s4pHm{FVnk8} zK`gg_JISyOvDQtvrxxO!WseJD9QMX)`%k<$-tL!&%Hc3SCc^@4NttV zqw~*1F_3WXmD8>V6xSlm#n3+sQDx^|dw4y#g$A-`f?d8t2a`E497_a7t4PFgBxAH3 z_Ba*}w6J`O(c>|+vWTFO(uriULcx#G`QTWb5J9(_7|9m$4n5`qIc=D=f zcLm_kB#HkS7ONPKV>UvoOhr0n^bd_LBHIPKB4SlbSovA1=yp@$HbVF-7N?H<%|aBJ zFdHg!#c)+ETgb`uS~HRCVPxv9e=!D4w4RLQXca}#%8|zrt4^~WnTfE=X06C9#XQcd zAv6+{sa!JU-YkzRy^cl-ndzjnpk z-?j4?M#YK18DBODR#7U>&N%IrrfcCSvdA5PadAWu*n@Fe;SEJ#SeDkr8RGB-*z8pfYx z&LUmj0Vc3DjY0>Jj(}T(3RZx~vfUQ;9NBVf%NA<3 z9Y)>6%8cP%#E*-R&^%mQ#X%`?_>>_nZbKnk?els_@yq^8e~hJM((6&0DCi$ z(1sWnnW~GeNP!`s{5y7{HAfxmrmk|S9pxli+zrlc=tMaSI#0`?OGIPth_08D)W#Qx z_t4(UFM23tE7sDZ+6EYNWNYP+$Hk0bat(%v#UtW5a;>2# zgCIMfCZNoOLC(&uM@Y)nPCIGrLayV6LeAO1x?QhH^h2tpuFb~h$oC>=vN?c@Xvb-t z3G1r5MzrG)tnVEc(T+QH=HRGG3bNxk5NtYN(`OvX81tD!OI0bzj_Y~zr7`8<_} zgsCk8Z_JIX7Cth`wj-p>+$220N^3}QcH9@2=DIB{O4^YyAi%4=~(ndvMAxoTU(*2WO{`B+;{s&7WV zq=!?+MtT`Kw#J0G;OY!6R1E?iT;StIxix3wBULlekS4-n0{Qt-PW$ZHr0PsTG%)+u9T89C+b8W7PBd$BI%I3XE6ax)#c%8)u0IiJaZnZ0ui8%@sHw;dB7-zmUzOGXvq8Bu2oU6@3lEqgQNY)TA|Gbzts_zBsgU2JceNm3o>|wR^R)3V+eBn{+ ziM&8Wuu=Epp3bfNNx0SKj+&JWOK%J7Y_feHy}s=2V&49WJ$9c4*CR$8`Y>gOqhUjm zi(vL-e2Cto?9=x6J*Y0e6JjwAbO9)r5v3$bU_x_P4_~xNGhuxd8_pDnfmSa#a6>qTd4uv;83dt}>p>Z=&)j$w zVI~O%Mm~NAY7*ozHXj#Kby#D9$Ki$PPHb^Xqt*?nWPLn;0v8d{ZqKH|-Wnf?NYM$Y z-0)(anDa|k;hc?+LbIO}1|FcN@idbmvv1K}k)PmIY23`Ncb2$5SA)szFi1ts+ z6wMM*VsG3V)_yKU)p@fC3Vi3a@BA&VtSxrLEOIT6JBvD_ zgmT}7lIbioPon82xq?{`$rb&%+N&yG9s{eqQ&>9)D=`%Bd6z|)PcN@C#PBiZvjtqm zj8RVS{0)=)lP0LA2+}t#Uv4%dX*0QVh~<_8%anP8rRm+Hi?`eskq>#v^nM--avD6X zUVb!vSWC>_G%MfS{ivM<^_0y_m|4jv(ulV28@tQUY~DX*N3zkpf}R!Kx5a_Rv!{xe z6=$P+i?F__r*`L3L452qL15SOYUDvC-oE8bbUAEU?_TVjb|Fd`f?k=q<%-BV#Ck7{ zWyTG0@odq(?A`Q?8@Wk}vQ6*bboTZnVLkbH>3)9fZ7-6x&gbn#MMb4zJ^?p-Uv_mB zm0X?ia__Zz{go9hCi)%Q(l@}gp6K!*$I^O;IGu0tKX4@J~g`^*JycX2Xe4tf!?N zWIRo=p82GfHDO%}V$PQ6y|)NTaZWZi5~r=E2iI50)=FO6ciP>0h~-XljZB$D91`78 z^SWK5EABG#EET>IeJVCei+UA7e>95O5KLV`@+`^;S~To&@kfJ~cISr%1Nx8j58FD0 z_k=bY=SkwYu`ZFl&l?Tn3KQ?djEIuqNWD7WCOztN-bT#E7ABU{j*6r4#qut>!P-R% z)rMeCVn&f`VPNv^((^u-9mE^MrLGJSW1QlMp#vj~SfY(|SW;I*nns8$9A-|by5}m4 zzR}(CP%cXCQ##z=A$MfGb`2~_ejOd(NsC$F3S8-frTf7Ofj52Ym(UOX~X0RzwVsq9d?arhGJ6ZRqa!q|NOl6VFvN#B%` zv8#bO@d;HrRaz{Ahs;-Gc^p2+1%;~}eF_6wJmd^wlMuq>Ixnx62?mk%M}xX4r0>&UG1ird zpX%2;2rwpMjn?L{NVewI;*qXPquwmCM z>DtOfxOSQAhSu=2^3jAVF89t1g1`WOZ)MOR^f>lf9k|R?PA_ayx9tma`IpgMoP`UV z<&Rz{Jj@^;hB_Y(UVB^)%#Pk9Te}tQC(e^NAmipd`$I-p*BmK z^L`4p-di${cexHb@KiCGQgd*8XZQ@Sd?j4CxHd$yl^(NIQ?tG1H3}O)8i=0n9RiZL zr+4BzO2I2w`fPt+DLgflxfR@vV%j*S=`!;C*|lp1zS&F#y5DFAj=FI=#Zk0#S;*JtSWY(3pTYRbBUvzn&OC@X}DAu0n zpvR;JciCA^jRKUvC3}=P1RpDm52)`dgDqE{b8OHogU=6763%LuLC4}@p(oc-{D;R6 za@S&6@b)3a9S!SBnCohIHqXp}Egp|ezxHw(E+i3uU7%M#_DpH_6x=X=Ok2;N3xNyW z$5hZxVs>n95X@Qt&lA%tr8d`+SK)U8L$sE??upL3pZ~Nf9)`G zlGE#1+!Qn{;*4&3pnWzLw&R?r{d^UDI`K)B;C?lP+;#^8wna?b;w5ejgf5*s>A*1r zc4xf_4`#~%J{G@k{#XVZOJ`@4mdhaXTBMa@(IDi#yvB_)N{7)r(ZCazOQ4$JYuyWs z4l6g7e@wG=1T+SplS%Grf`YitkL)f}&{jP|Kcb!q4Z~{r7&D5E@Pfe5T=5v{4v^;V z_ol;A2wR>XY!7d6yb4@y| zznmRffxfzRh8l9N3&o&&?DRVClXTe8x~zBEn@8Z`2Kq$#t|3rt)z~_n@esJXc2gbA zeh6G!Dcx8zAHtKb-rTJAL$JP?hwbyrbWm%x^T<1V8cx1?yXlJTM0tvA&9{sF)6jUT zO8))$ENuIPDRr1d!EL`>Cf@rK<#(DDqr(h`A*p`z*BkRg&_udFuKnDu@_5|?3UUvn z$^$!ZQyy=&E>DrnFuBAy2!d}nYLj2R2(&6e%=!g2kZ&jyPD-H1#?n{|-?hAi*Jg#6 zM53BO(Qk?OF4AvN^`y;wImz&r?A6t=jdQTL=YDi{dky41kYHs$NRNH|OfoYo*9{w; z7QZfe4FX(a>kf=XU(Bx;9_2ylh(z6h)#*xjN}JA=x?>Pdy4*baX$OjJxL1IAO>_-} zL;Ms?Zp9Cg;?E zj!u>YlNCK?U|W5@{pt(YmF$T>xNi_blB2vH?W=@EzBTzHewE;)*8FnD7sVcxUC?e# z8-V>iRskBF7a{ZQBe4VAHSp3@d3v6j9(%JcCw#v`FSKU%P4wwB!i>3Tl+No32;SFv zo<-skNZI8qc5u%@OV!o5TT(Tkm41=SgNYuS9iE*`5bT0a_a$d1QRIMqhIj4r7b~Dy z?6aKkLIu30%ij2Oz5?hf!`}I24uI^vI;LJu6qj{NvLhp94J;q7q6_;>hl!U)zb-k~ z2b8G>?sueXfjOnzq8rVx%b^Ezlev?jR6l82gnbTd4i|FSG1NfwhQ6((%XFA!e5IWK z#cn8_Sqxp6AApum6dz1lE5N{8>rbwRFOD_r-Qyu0?^05Is4M24heFiW*Fi!lv@&-dC49-{Q* zJnf|g-2s@7d??_u;~Man67m?ZDyVYroasoR!}N^4dCwKU0O|2tuNP;kK`3ic-1q$$ zG`k!j6`{Qh*4C~cHzE0JE4=#Rh*%ZeJak(wzl9DfjHRjFwD%d1olB^yojP5% zBw=v-RnZ%pL<~Cji|NOl#UL$BVgKQpSKz7l0Xirzz$OQ=agpr_aL?_7aD(_@xs%j~ zi_IP-A}$Y@@u&AIMC>a1e|Tj{>OCX zJ?EcQ-OSrq&i+iI%H}O+`T1CZrx&;f!GK@lE0uj3>=CFs_26kO?98K`lo_YN$U1|% zN;L++Yu;zzpkXr@B}fUJES>;Ev3AFcMQN}-p;+khomu!Kq`5^QzZM+zx&+2cXt3!0 z^XmfaUxRG9IzyA@ATaD{KQnx*0(wfPw%4FEAdvlj{LKmwwA~QlkG|H#Gg`IzilzaZ znfqJ{q$&@z2LN}nLzd{#|TaorI#Ozajl!G z01MnX_AB87kgZ!f()T(Q`p$6A>?N;*Q&+p*O9<0qoW=ss^Q8yu?bG7UR5U_{C4=MJ zqHzd+=W?l zhTABh9gbVy>bEm&fhU4?OxItIgU%z8-CQE6uyFs%5t5==h$}fx()ytm8tw_Sbd%6x z_A&a!yN8Ux=<7!J{K^HLuf zO6xT_zHpxygv(j#yRHbPLkyg2iC7tPjnhj7zkrH)ei*fnGa*Y&;yv!~!<@QnG8 z!6^9C-E@zAk`C*(yi!o?nT7b=8{SRTHLx^aXnt{t2Kz*<)6k|c2yfRNH(oXx0#Skf z5Ixjhf=tjB36xTGq++^sm0+mPtCZ2BO_Vre& zi`Ko6rf|5I`^YSCK2zP>(tRG{pPu>p)=v>Zv^I~iH zt>nx2D~7V=$9Jo|ZV8YnZ+~(mwY7c_M&9kwVCu>M%eZ0Dxi_`2Uvm2Le)Ohn`e@x{ zkq;wKUP-^~bGrk5cM#A-gPVXnO&-3=2bqwan$gUHf<3r;@3PgNu7x!%oc&2_XtAXV z*z2P@0=q`%4pp)afzF3U!S|mkp@l7^`VC6=^yCv1P^uqWAc-6)SR=RQXxdB$ggp9E zali=0bV~0M=s@;5Wl1YvjQ1$0wVO!f6tu%px98!n9!&rj$sL^w*(j2Z-zTeB6l-_m z*4FAcy;@+sA$F{!g$BDw(!zhIYy_BIHc-bRf8~(Tytlt=C8#i3yE~wC#D($Y8DwYP zEp)2vMf2^o5__Yb?U`V-+h%BKrWX8ed&VYwp~0&B<+*S}!;sicSog%Z1Ez0Kl@?GS ze?gS`*>m?ykhKX5r9t@BOrXSPz7~oi%d*ygq``O>??mt3It=+KzEtHm24UW!`(48O zO3+ryj}t+un1ai(rznjpUXJRr9fbR_gp8JV84y|+xoekAEzk|?7sZ#+U|Y-O<65GT z-9=^+zh});s1Nh*D{hzouUd*1BcmDMpnu|n^0!%-VZ=QVFs+5$jGJC>i)k=*Uy>Y4 z=3&6+;LjXO8APG5!>#Y3_P)l?tF1@r7jY6{S?5Z)l$KRG^=1$@M$%ga6lFrx{u{R? znyR5kIa!qy`CTWDma95H9D%QQGagC0wu6yCO2z@JF-ZTMH(uPE0mqD}O>UCSLHvnc znX|d*^XP_Z#hhz281ozVXOpobK&xOrr+#Y)YHRcdmQ*SrJ0*zsopJ>n{c5sZ7o{UK zJBRjs7zCg7)%zB6QSgoCz_X(o)u6>7kss$vgE80)dFpDQ?{z${L=#T7fz0s!Do%rO zILm&IVtOzGie<{CB1q))k2@ zB^Z3E3 z2QT!&xq~&^Hg(KGtI>lspVX6Jx$~IdP{5ONU$d-7Jo7I=joRIH{Uc;IoOmEVpF>fu zllN-6FpshPg>)7O_#Z5M=It~2vn35u34m7_? z*oDD_NV1s#N^{cg z8x5mzxWP301Yu(e_F=h9^wX#WiP>BrVpT9{t`qVhpZt;5lcA>VM1_bHKKv z9O4|gG>h8HVGVwGzPF_uLQLpcn5qYX@2LgZ+A}GT_g;}IW=%CTmyb@UouR{SZ@lZz z^=2GoUcYZsEN=oP-^Q`e`cp6$lkmzBMIY*Z!g?`4aSjp-9M3R)K=xjZO?A319i}C< zfpbWE47AsNcBw>qYt^Ly#FQF1_$36+V}0 zMwZ4_Ks_C(8cRXADDz9 zyodC65Wl;%XQEW_GM-l(q0zE3wdS0kO+aYy?|JCl-&u5m( zpHb~_+#(NLG>p0b$z47)-?W2dDfENo7sbAu zNLUyJC92YgPw3~sF76p;=9_p(9w&I;p%^IlIxJ*&hhqdT@HK_C&!hDajb7UD&wS;( z2>foZw1mpJ<%%a zyZxDc6`kyGY8AcwLVp#5A~{brqbjpQ^;%8-v}z`u?E}^83{-h)n2n7cYFNyj(rQ?( z0|#o>+aKqtWphe&sAYG{Osn1Cb#I_{qhB>o9Y^3ZhdR!X;j}ugg9`(7n$#6J z6Y6iIL8C7ZVoHA}Ur^0jPtPFB*D z?y^3nEAzsuN_WSrgJgQLgE1<4a-&y{>Fu1nRi(FUwuDSy{%y00{_dr|WBLjor>peg z>t`|pMN&Fd10_n%;|9uW#HtNc=y#JFsxs-S8mh6{95++r#TKSfJp|tj^q|>n8 zSIZe??cXR?XC2VGo607zU01^1jHg zV6{Ez5V;}Ap*E5uczhs=J5qS?B!80I;3?tk+`(wkqVd7g;)?rz3-*yBqsdN5 z?xPpovagR`@+z7bz3f*jGIk}f%YE!>$msR4YX_Gm#!@0EH;<KsQ@nY2YKRM+Xr zlc+Hp=_skOm|7oEW3}|EP+M<%kVKu05Tm5d?sDab`Ua0%73v#(N=P&~{F{|DID`9+ zXmEv1S7>ZH^qEAH`zWolCQl^CQBB@x(MnCeIC)Yn{stSyp#<*4@N8@DR8x8#+O?h!3)R^B64)OU2x*2mM8d$yH*Ce;zIq*c+8sO31O zBiSfgr6biUPo}%QU1x73`!i$57U>?Rj24-`z@e5M{m1!QWrq_TTjjK$J}(SCK|wzG+Z4%JoZ6J|0-0^fw9>cDa8_o!w&?xg_C4Y?{2e;mJDoaoWk)hQ^yC+ZJM
    Wm%xr>Ro-nfwo2fCgJM@LZ-2Nz?y17FnXQVkHTCCRGF>W`d zh0|GGbqnVtn@9_nOWw5>uGhjTE#1;%)h*q#uSQyW+_+t9>FJz2@WL~{MzGiGUZ+d1 z_rsCwUZ0Z1(OzGSLa5KLn#HwmU%fz1pMSITSYJS!n$XL@XC|&MgL<5EUIzCCjlJC8 ze?sV0$nbgBSE1usIj_Q|?~lDYFkd6|`rtyR>+A64k(}3uJ}-{FK8&Lj?vEg6b?ZNZ z7tHNHN-Hzoe~eLGc;Gm*soTH_HkVunb{DUDhbXQ?c*05EI8DMS!D}Z8(VO$?38%M~ z;T>ZnTQwbHWnP_hjN3U|?|4SxE8ZzynO@5&L7nTA(^>7U4Nm9u6sVmOjr6pf&zstw za!#`JX>h(^dx+X4nGmPta?$15DVIwgc?~X?eafg^ulToWxn2!^b;|Wx*ldGq%Av2+ zZmCDwf=kE=gtcu2(3gJW$439-u9N?Z;x=yOrvo z%hl-K6)!wyt9@YpQuz`%ip?j)$UTRU=G$VyZc`9t`6xBiqyVIt&CVL3Sk09>!UCi+ zb+GtgCNanykKs0NfN85GAWPY>v|v91uQrIWO>$JDwSTASUX(sm`FcnmrA%=fu=(^6 zu%FW zun+|E(s-#4p}>L^H}Nt8Xe}g{eTm!>kBz@c?M&@if>(P!VlT}`U_Hee%_CIRP`5Gb zwlqrZ2P~+^QNTW$9Twt%=(}*bw%g6D0H(?k5{{vE%n5{&3%lSk$MiGC5=~2Ra3J7S z)97;;leu2%X*C7Q&i9V?1r)%3IT>r~a1@g}UOje$Y#rG2%3T@O$75WP(T*Qs2_m>V zDBG5Xp;654MUqrCg!<(i>Og72j`{*O;`OO`iZ00!FnW9AsH#%|1eu!0)a|MRQ=QHC zA9>=jSEp%~texM(Ma{IqKCd3o+}5#m_9k5(}&0tn`ElNax_+8o#80# z4-g-kBq@Z_OP`7vGf}9jhpeAUjZr{Ht}VD-tnb0$f=E>Ni9Vnek~JMmaqi^AkZ?`7y$ z55;XlA>*?|WeO!WqlkGoMZ%Xws$r`7jx`y&w$t^Zvv`1FA7o1xln4 zK8HQN0UDJc!+K|N^U~rk@8%LW6~8avI|naH*ShgCrGk9+t4-{ZRcLKC^x^sPMcA>^ zZe1t(#=YX>zOOr^J}Kk`H6Fz%zA6Z9lp~wxy<1kAvd2aj`AEl?TgM&MD`2Xdu5HKT zD%fKn`oM*q3fuVQd0_b=QpGMF)=)Y2LAVnUzD=lO8qU4A@HHgqG1zq<3*J~Y2RQuE zUa9da_@s{+b1qY23ak>xI>ks7^*3zVj3 zQC103jDx;$B(3SjOU&Qz_pgQ zIdc!@prZFWPfT(RI+XB=fAmf&jBjc-Nwy!SnES5whS|GO=-HwYz`a}vXg$QZjG!Kuj2JwkWXR&>Sqv8Hu zy7ps0<;T#nU84eKX%20qKxxyl@$-CWjRoI6$mUEp4hr`C8>-kJfXVH4^G*~SkzUQ2 z9FL>IMDs*AD1C5>Mi048Od0mTHwOVjobN2Sj7PlW@_Yn^xu>6r+(c_Ux;Y;yE}+;6 z`H>dK&r)J~=^01P)8G_?-|b&_TX+n*FRznvI$QxTxL-Z_Pz6l0g_`FdsDNku8CP5` zkAbwtm0$&lVpwi#eXt`t6;7KOA6qMe*354Eqy)xMD0YWxytPf3gNTG<0t(tG(ATDQ zrr}~OG$)L-Dbaq~HnPkQ*M)JeMA(qDVy(3Ut zJGV#vQvqC0+TevgKZcm<^VoV}YV5g+djqG`GI%C>^WBC%xXYty3iAYK5O9~ATR{7?sP9Gi{?CKY0oP7`L>l~-g>_JCTj7byJ(7UA$xuGME zQ2hCYtQ_YWsahDn;lTQ>VFcdv#%D-Q7lNGa#DVED=rr3ZKOK>GEG6#1k%Xn1LVuK}wE)<{E(stPPIoP!LWq6B49ZWMQG78%u{*&3rYj6GllO2K&8@`}3A`b+LO5Lu8 z_K+IK9Vp$9`=q1nW;KkE@sc#pqeI3FcXxWS=fl-9ciA2D_0UkiLBux^*$I807BVN^ zf7kE*kknwj|F!ls_)RQ5W#5z!DE^GA#Kk#uINHbk(M$D!(|P?h%LP;!<*X|DI2yZt!Yt3>wv>$vPi&_PMklIfwQ@$i2D$ zF0dZfV@=0umGRj2oz5T5#eaaScR1-;T8BX%_sy}+qXv4q@(Mn=*PtloRjP9L-fSEyBrqG4|9^6a(e)oO9@&8X$K_ z6ExRBaz5)xVu(`gx=mI<$uKw)ZfZVYy$_v_q^lzs8{oybKNc`cfmI!_%V;>W1e3Mi zjrZx19gCA__B=HO?jzYpzqvgG<-lTNg2Nm*rU`^{e5i-d#l`15Y$&kAr(S6G|-qu_hS_8g^50Y*OtAXmmLrfG0(D9MYEfHA#FdS5z%b8@m4-Ls^^}nhC zu5GL^x%iC&>r8)t`FYJET$j~3+Wep!IQPi(`0-4^oxK)$8@lg8UP_07ki{H8+za!w z6%C;NmX*;G;oq7Vv^0DjwO?B!tj1*+HeKJN5w2bfgS1bsx1+OAFPGn?c#W<%@2e8y zE*^#k8>530o9@FI@{I(k(FW-4FSc`Hpv0oR*%pPw-@^^MGh*v6J%DIZ5ey3`?0#cAhvcYQQ|?Uz2*u;+V|^(wvR=9EjCAjT+=cl~jmj{X?pt{J!L}As zWqD2S+tmU?x`2Z`dN*Oxf0Crsbrd=V?7P=jJc2u>(Wx|Vnm|ys)Xeh?CH78#f8E-q z4?vmQaldeC06tN_T)JI92~!i(JG#%`2Mz`910?<^TIZP{6?KtjxOOC3<0b_qMiUt8 zom=+-x;}@$zcw}k-YLs;D!1#FTy<2EDV98st=z*r>$uOI$ZT0ePG88>yxdpelL5(%M@G6HABUD>_jO+_i@+ z>7UgB$A_)a8;wVRrFZjVL7yVHekh^LraK!X3+SG;JNCoVeckfkBtODYN+Sg?v}P=M z`siRfRSqa#arX1A8-z0bkRAEu%V-PF`DC9v!_YRsV-g_70_6t^-XFP456sj@`pk=% z;IrPd5sJJK_#jTH|B3!CZ0Z-|dD&SHH(s|bJYv9O`cejqFW(@)gG>3WI>$?3Y$;{Y zwVs442JS8`xI(a)7@l%R{z9*!?OW@%dMII*_*hDg#{{JB@;N0h0c-k${(BN5AQSLx z)25R(K;!CZWDr#YYnNw~J0oi#rberJXVwTjO^NT=aHkMhh3=J~;H?KiqpR;7o>O7* z_vdE&ZlgmRHP-OQBYVC5ah-t|ZVD(Q;!5$%cj1Jr?kF$PGadO~qb_c&hx&-f;%m)R zSWJ$Rs8z@k7$jIdJhWp31hz%`r?gbVnTy(@dMGve7*VH)(mi5=+m|dy;11;hzA(qT zFt(^zv_7LAUXiZHpZY?Dr8ON(ibDR1%eM2|AD?&$9i^Xn7jz~exRYV`)w%-M_k|(c z2(>>-yrGyUyB>JGVpP?ZsIaJa+t)t0j{Mx|9sHO3(1~2CRumK7HSm^1U+kRYzxk=EP6K>(ZUI2FAc1Q2=L5Biz=?+~&-^?W@MO;`Yro#4xi6>NMErIgBF4A*I zpW`kylWW{V@4PykD4Xt9L&hy>Enbu+DZ4t~MSIVRsLN>^p5KLHi{q93>l(nSo2^*< zDHT>?6aP^{<2~@K37z8Ydj;=f56@ouJPCD|O=Q=!7eMRv#!Ta2bQ&fXwdd5kdhi;n zyue>ig(Y7*_VnfG5_HrZxwf_d9kw{t|T+h=#*J@lR5P_ z)P)d!SZF2^wV%1>ZIpv}JuEL=cW{rQ!j!LQ;cl&6hW0sDzU#Nqu{9MQ4yP_w6Aw{r zdYW7f^`gpq-X)>f-L;d4TF(pvv(Na3lT-z8Lbtc8gAtuJQc`N0;@bx4Jwv))ev7cQ zb?x&al{vVsKNi@B_I`Y3d{Nwy+XnN=eC^eqi!c?`DP;0#2%hN4Q&`zPEjwm0|M)a@ zsX~aLW2D&TiLyyn+pMpGqtF(;S@w`*J{Zf2GCI^%LvZRlOT~CRHt^WCW0K|rWW)$J z?`rITLtnQY8~r>DiIF=p5*!PmtwnmSJ$ep|Xv?H_->(M!eNVTY`^=-q43Ml(l>O5jqc-g5+{46+H#IUJ*q(V%?OKBxe?BR1>mH`ai9 zxAfY(lgPinnl>UB_7P|<^6?kz^uTMTwQ7LQw|(Ey^JrhrU7!d)*!l(8&3YyC5@8Q& zKJUB!n4C?%0C#b^3t>7K$oRYt|ahAw|($Ti8g*zE* zvu!U5f$)^>m?4TheK2=+yF6VDP!lXpR1xr)>E`qIZthuz<7cZ1oJK~#R(x#JkY^v%n%&%RKoGRh+344y6xg&6XK5cEaEz(=^Pj)Iu*1}oC`;Xpz!ed>gbaA#p zAEDGrGBovM4}?ohNd-krLr%ow1!|wWpxnt7a0l6q3xy^psGuiN`*-Jb$ye<^z1Yj85bpcn(sPA6~Pw%)ri;7V8@lcj25?x!Ar$ zboMx_O}-9#&a`ZiQ_cgAx$aKiM3b=$GBRc=x@d36-P4WAioum|svfABQObX~viDj5 z+LJIRzfJYjD17=favGgZ5NcGn!wCt^&Do)++U|DsZLw@_}P{1YQqQZg?hr8?psmO!XPA!l!dwWtUA( zgI7`Z!`)P$;4)}y`67M(#L)E1f$J!`b`-n8us{l^USR5qlRqS|37~>$H>C0!$Qh}V?x0%Ib2qLv&7l)D4xakFB?~Wl(FG|;M&Z(u;6 zzk?qmj-AE8-Ec7eq6@q8TIgPraCefP6(5oSZzeh%*b&Q5m98aPyEH9bol$xm&N zLG-eHzvGvx+1CC7T4_i+)xX&g4k}V69NBqb{7QlQ+rMB6Vgv1240Kp%SDGh|_fy4D zkdVJ+rl7*(Y}lyrM@@n|M@>bJYo^1? zZzsl-Lcf&Ngvi&wlY|~y`LW>xzu!b8=~w8#Ns=Oq><=;ECRZmC!%R)3h$D6{@wZ_$ zArhj_qWu@%sDCeF!+&QDj!4L_&=o@XPJZVNV#%Mp$z=O8pUj9j#QvE{8plQ%;OOS; z?ZCo|W1$rwWy5>>IyrmUxjPBqbT|;w6(D6m5Fx-h@MjP$QECH&yzM;Qeg1MU9lF=m z-#5t5&d0&q`R7OI|0wZt4tDnXxrox+IndwT@ds!PB7{27&EGk|&DYCml?HPEARpHs zH>^~n zft&9iEg?tGllUMep%m|9hi>%=3|PHf<74-IJO$ub;~m&J*gvlo#Bs3DvPcyy}P%czb_i$0Vw0;jQWjT6pbVt3n{Vt z8Brq7n)-~6oswM;Njd6>pHY18k5vS@C;%{tU>TXYOh<1MmFR$NX7+O9O5&vC{tp6xk_#b0+wc;ylS!+oJOV|Xb*RM#GqrY!J0J?<@@8IX>;E%@I zZ>x_uMfq2;`oX2uQU9IjE>hMdj8t%Vlti#_17gyzVZ;A9n)okckjSLt_!x`tO!|TF zyJ}g*SQNzi&`0puZ8E{!UeZ5$<>w8r|GW?Ow+M4}#)(>$pl-Ak@k6v)X)FDjq?j)*3@sLL=pN4Ir;aIM&h%I^cgAZ4N}3LGIGJ`Qes2@ zFRYSZ&=IBQ&oTW2(;xH69|*Ha1;1kCf=L(=;jh*O>p$5X|0cqpW~^0|>7=aJNCo4o z$OTI(i70=yBZw$}h5l|-{@F&>w&xUR#uLP|wpWv*>5Pri&%wvZA%HE5@c=Kf5}ZSj zPWKA(_6gwK$!%pL%FVlaX}415{5R;z=;OV#3#txFZ2&MvR#NkH03)uIRrYOT71x?87|8GC%#pS=T~fh!vBM~ zeln$ggYj>r|3SPzTWY^S{P*Jj2Fxo)*uOyiNBMsa_o}J+E6zXR|2bTL*s#B%`%BT^ zFuHj8It0pWCqDF7!TqV=UkNX9qW%@*59r^J{TBwwk6Ha!TtAom=kTo<_rD_h1N=8k zKXL2-6&qLmhhOpi3IER{B>E`7#rUJ(KacW{G4@xz`ol#cQuCijio_SWTfc4TpGAK| z`(Kb$pI6s&izBF+nip9a`RAL6#b(+s!5Qf(e?Yfdn62cYSwdg8AP4u3FyTVP0~61aBiRXARRZ zEe~H;GYRnkEfsAKf@FvU!AyD;CKcdL@Kz7B+^Z#n$^r?JMqUJUIX_DkE#kf24rZnS zmg;gr7G@G%ORC4v#&0CPz%Pc3(Z%SYJ_;c#=5kP~+{SN?vFoV$a!sfW{Q9YIcJ z68oJ^e=7IU6L)l9sXx%t+{n#RLtWg0h+jimf}jy1r=vn_pR^mn%mlSb$=y-P)ZJ0i zRD1=Wx22z@S%{~(33`ptd+BlMZNTS{pMJDDrH zeDB-e_oa`XgoU?LC_&0}{||oZs33l8s3X}@4>OlCl6KTs?E`|O_C8B9z3+X1`pH~b z`sZ>-DWlNU{*mx<(opwA{Ba}73&F=UfS`%wPV+Bv7UpE8?c?YzC*i0r7hq{_=|%V_ zdHV~EAzn`2rUCy`az`*T6?gRZ^7tpY`%C%%K)*U^xP@Ap>-nPb`;RjB%RRrP%QAo< zsV=VPVdzfmZ`4PDj+$OBPMTi(zv}^Sf`*)%ld8q{{{FZ6G}OsV+UK8m<>+G?WEr|L z=J#8mF=Jt__cOj=3$)B=A@z^+ggI%VILky@QJ+|t8M!%Ws1fxY(k&qYE=cD&hid<% zqb&WL)a6!m*+0=^>Fp(ME=km7CM)&+56HZvg;|IM;_p9(GQ(LoDcp|{yQF7dWQsCm>Wv!{gbZu{u|x@R{!bwnM=C)SxUP8 zE4q9g%$$7xzu?!u)K}g_+W${_>Tf*x&x}7}UlHe1q#stt(LeO(U+(!`pZ6rNAUSqGUXNYf2o^-%q2}FEzP9G9anYGZ|oXrmz93?F$() z1^(2j|CXFfyI3Gw49%Thj>vu^>bKwY8KPY;z{y-s+|pe8zmGQ472D9wNz*9Q!Ax7i z>ATJ7EVXJgI(nNTohuhgoYRr64M6uG8%SGv#U}JomPUDGN37}*G)IRzN_qvO{zW=O zD^zdq_We2@YVJfm{gb^YNocxw0y*gR;~u|&kxx|StH`LzLKtz zlEHpt3ae@n&0;U2d90yCG>yfH*V~9@wI0z-MwU0Sr5%aaYAa^5A@N#@SjirueXYEL z#DI9cYIHkpbJ@0ySfJ##0(2z)U9q=UE{S%%k`lp7&I9S_FvOcsM^ufk0dfFV%yne$ zqo0X^)atdN)`}&sxdL6uuH=Y>Drx*FXDdKGV^oPOF{`|Crb05*q8>q45Uga0 zmn*J=l5!9+hw4KNaV6e@81emQLQGY*|7ef;Y9s#M9&O@X7Lr~r#QD@gL(Y?Uy^l!$ zmOt>T{vJQkA5mVJ7goj#@j)e3nQg%<{8LdPOf`ICRhuR{D=(XYWyh%F`v=L-K_l>@%xS5^vHdB}qJ_ox0_ z=^!U>^#CU`B7>9>jo<0j_V_`s$-ii?w}UyF7tQ26Emz1<(IVmt`9ZId;qU1s5}~Or zLGWH-zfu74{w-lY@U8L(bt3xx#2;c;D2qF2YI`|Kx&;$742k|E>R$_Dy83ffa{W%f z^7sDzdwW+{_rpF#59t!`-}^&eztg)aG=K6}l~}*#&pyHGJcgjM#OIy=PtOyp`pfex zJ{kg|{z1Qe|EK5Co#=UFbE}IZpV7rZ68X49n39Sk@p;L=J-^b()$zM>*9w1#_ + + + +predval report -- improve-neoepitope-cv-patient-grouped + + + +

    predval report

    +

    cohort: improve-neoepitope-cv-patient-grouped

    + +
    No rung is a validated model. Rung0 is the model as published. Rungs 1-3 are diagnosis of rung0, not competitors to it. The cross-fitted figures are the expected performance after local recalibration on a cohort like this one -- not evidence that the recalibrated model has been validated, which would require a cohort the correction was never fitted on. Recalibration does not and cannot improve discrimination.
    + +

    1. Coverage and roster

    +

    Roster: 7 declared, + 7 present, + 0 absent.

    + + + + + + + + + + + + + + + + + + + + + + + +
    modelcoveragescoreddropped if commonmeets minimum
    improve_simple1.0000175200yes
    improve_simple_wo_prime1.0000175200yes
    improve_tme_excl1.0000175200yes
    improve_tme_excl_wo_prime1.0000175200yes
    improve_tme_incl1.0000175200yes
    improve_tme_incl_wo_prime1.0000175200yes
    nnalign1.0000175200yes
    + +

    2. Primary comparison -- model as published (rung0)

    +

    Common subset, overall stratum. AUROC and Brier carry cluster intervals; the +calibration slope and intercept carry cluster-robust analytic intervals. +Full and common coverage are identical here, so the coverage-delta +column is omitted (every value would be zero).

    + + + + + + + + + + + + + + + + + + + + + + + + +
    modelAUROCavg precisionBriercalib slopecalib intercept
    improve_tme_incl0.6524 [0.606, 0.697]0.0525 [0.039, 0.070]0.1676 [0.152, 0.185]1.0972 [0.771, 1.424]-3.2466 [-3.463, -3.030]
    improve_tme_incl_wo_prime0.6514 [0.604, 0.696]0.0522 [0.039, 0.071]0.1680 [0.153, 0.184]1.1701 [0.814, 1.527]-3.2437 [-3.459, -3.028]
    improve_simple0.6425 [0.607, 0.679]0.0490 [0.037, 0.067]0.1736 [0.166, 0.182]1.0848 [0.772, 1.397]-3.2777 [-3.512, -3.043]
    improve_simple_wo_prime0.6402 [0.606, 0.676]0.0491 [0.037, 0.067]0.1736 [0.167, 0.181]1.1070 [0.786, 1.428]-3.2766 [-3.509, -3.044]
    improve_tme_excl0.6302 [0.597, 0.668]0.0498 [0.037, 0.068]0.1741 [0.167, 0.181]1.0719 [0.753, 1.391]-3.2761 [-3.513, -3.039]
    improve_tme_excl_wo_prime0.6257 [0.591, 0.662]0.0495 [0.037, 0.068]0.1744 [0.168, 0.182]1.0886 [0.747, 1.430]-3.2759 [-3.512, -3.040]
    nnalign0.6054 [0.564, 0.647]0.0441 [0.034, 0.058]0.0258 [0.021, 0.032]0.9697 [0.580, 1.359]-0.0882 [-0.326, 0.150]
    +

    Brier intervals above are the cluster percentile bootstrap; the analytic t(G-1) +cross-check in metrics.parquet agrees (none crossed the 0 loss boundary).

    + +

    Verdict

    +

    One line per member, assembled only from AUROC, the Brier skill score (BSS), and the +best admissible rung -- no free adjectives. BSS is anchored at 0 = no-skill (always +predict prevalence) and 1 = perfect; "closes Y% of the gap" is BSS as a percentage. +The gauge sentence states any detected miscalibration (slope or intercept interval excluding its +null), the lowest rung whose out-of-fold repair is demonstrated, and any rung whose recalibration +reliably harms. A member with no detected fault and no demonstrated repair carries no gauge +sentence: the test can fail to reject calibration but never establish it.

    +
      +
    • improve_tme_incl -- Ranking: AUROC 0.6524 [0.606, 0.697]. Probability quality after best admissible repair: closes 0.8% [0.1%, 1.3%] of the gap from no-skill (always predict prevalence) to perfect. Gauge fault found: miscalibrated (intercept -3.2466 [-3.463, -3.030]); repair demonstrated at rung1 (level (calibration-in-the-large)).
      • +
    • improve_tme_incl_wo_prime -- Ranking: AUROC 0.6514 [0.604, 0.696]. Probability quality after best admissible repair: closes 0.8% [0.2%, 1.3%] of the gap from no-skill (always predict prevalence) to perfect. Gauge fault found: miscalibrated (intercept -3.2437 [-3.459, -3.028]); repair demonstrated at rung1 (level (calibration-in-the-large)).
      • +
    • improve_simple -- Ranking: AUROC 0.6425 [0.607, 0.679]. Probability quality after best admissible repair: closes 0.7% [0.1%, 1.1%] of the gap from no-skill (always predict prevalence) to perfect. Gauge fault found: miscalibrated (intercept -3.2777 [-3.512, -3.043]); repair demonstrated at rung1 (level (calibration-in-the-large)).
      • +
    • improve_simple_wo_prime -- Ranking: AUROC 0.6402 [0.606, 0.676]. Probability quality after best admissible repair: closes 0.7% [0.1%, 1.1%] of the gap from no-skill (always predict prevalence) to perfect. Gauge fault found: miscalibrated (intercept -3.2766 [-3.509, -3.044]); repair demonstrated at rung1 (level (calibration-in-the-large)).
      • +
    • improve_tme_excl -- Ranking: AUROC 0.6302 [0.597, 0.668]. Probability quality after best admissible repair: closes 0.6% [0.1%, 1.1%] of the gap from no-skill (always predict prevalence) to perfect. Gauge fault found: miscalibrated (intercept -3.2761 [-3.513, -3.039]); repair demonstrated at rung1 (level (calibration-in-the-large)).
      • +
    • improve_tme_excl_wo_prime -- Ranking: AUROC 0.6257 [0.591, 0.662]. Probability quality after best admissible repair: closes 0.6% [0.1%, 1.1%] of the gap from no-skill (always predict prevalence) to perfect. Gauge fault found: miscalibrated (intercept -3.2759 [-3.512, -3.040]); repair demonstrated at rung1 (level (calibration-in-the-large)).
      • +
    • nnalign -- Ranking: AUROC 0.6054 [0.564, 0.647]. Probability quality after best admissible repair: closes 0.4% [-0.1%, 0.8%] of the gap from no-skill (always predict prevalence) to perfect. Gauge fault found: recalibration harm (paired cross-fit Brier gain, x10⁻³): rung3 [-0.11, -0.00].
      • +
    + +

    Calibration (rung0, as published)

    +

    Decile calibration per member: points are observed frequency against mean predicted +probability, the shaded band is the cluster bootstrap on the observed rate, and the dashed diagonal +is perfect. Points below the diagonal over-predict risk; above, under-predict.

    +
    + + + + + image/svg+xml + + + Matplotlib v3.11.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + +

    Unit-of-analysis exhibit

    +

    The naive per-peptide AUROC interval is incorrect when peptides share a patient. It is shown only to make the cost of ignoring clustering concrete. The ratio is the cluster-aware interval width divided by the naive per-peptide width. Above 1 it is the factor by which the naive interval is too narrow.

    + + + + + + + + + + + + + + + + +
    modelcluster widthnaive width (wrong)ratio
    improve_tme_incl0.09150.05331.7x
    improve_tme_incl_wo_prime0.09230.05361.7x
    improve_simple0.07150.05171.4x
    improve_simple_wo_prime0.07000.05361.3x
    improve_tme_excl0.07110.05281.3x
    improve_tme_excl_wo_prime0.07160.05621.3x
    nnalign0.08330.05471.5x
    +
    + + + + + image/svg+xml + + + Matplotlib v3.11.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +

    3. Calibration diagnosis (subordinate to section 2)

    +

    This section diagnoses rung0; it does not rank models. Rows are ordered with +demonstrated repairs first (by rung, then by interval strength), then the remaining members by +their best gain estimate -- the largest point estimate is not the strongest evidence. Each cell is the paired cross-fit Brier gain -- rung0 Brier +minus that rung's held-out Brier, positive when recalibration helped -- with its paired +cluster-bootstrap interval (a gain whose interval excludes 0 is a real out-of-fold improvement). +The apparent, in-sample gains run larger; only the cross-fitted gains, which hold whole +patients out, are shown. rung1's apparent intercept is ~0 and rung2's apparent slope is +~1 by construction (that is what each fit targets), so those are +not reported as findings. Recalibration cannot improve discrimination; the cross-fitted Brier is the +expected loss after local recalibration, not a validated result.

    +

    At small G the flexible rung is high-variance under cross-fitting: absence of +a rung3 gain is evidence of linear miscalibration only insofar as rung3 had power to detect shape +distortion.

    + + + + + + + + + + + + + + + + + +
    modelBrier rung0rung1 gain [CI] (x10⁻³)rung2 gain [CI] (x10⁻³)rung3 gain [CI] (x10⁻³)
    improve_tme_excl_wo_prime0.1744+148.58 [140.07, 157.95] x10⁻³+148.58 [140.07, 157.94] x10⁻³+148.58 [140.08, 157.92] x10⁻³
    improve_simple_wo_prime0.1736+147.85 [139.74, 156.48] x10⁻³+147.85 [139.76, 156.47] x10⁻³+147.85 [139.75, 156.46] x10⁻³
    improve_simple0.1736+147.87 [139.41, 157.13] x10⁻³+147.87 [139.42, 157.13] x10⁻³+147.86 [139.41, 157.11] x10⁻³
    improve_tme_excl0.1741+148.28 [139.40, 157.82] x10⁻³+148.27 [139.40, 157.83] x10⁻³+148.26 [139.37, 157.82] x10⁻³
    improve_tme_incl_wo_prime0.1680+142.21 [128.85, 157.54] x10⁻³+142.19 [128.86, 157.53] x10⁻³+142.18 [128.86, 157.53] x10⁻³
    improve_tme_incl0.1676+141.89 [127.90, 158.12] x10⁻³+141.87 [127.90, 158.08] x10⁻³+141.86 [127.87, 158.07] x10⁻³
    nnalign0.0258-0.02 [-0.07, 0.03] x10⁻³-0.02 [-0.08, 0.03] x10⁻³-0.05 [-0.11, -0.00] x10⁻³
    +
    + + + + + image/svg+xml + + + Matplotlib v3.11.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +

    4. Subgroups

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    subgroupmodelAUROCBrier
    peptide_length=9merimprove_simple0.6315 [0.587, 0.677]0.1865 [0.179, 0.194]
    peptide_length=9merimprove_simple_wo_prime0.6315 [0.589, 0.676]0.1813 [0.174, 0.189]
    peptide_length=9merimprove_tme_excl0.6197 [0.576, 0.667]0.1859 [0.179, 0.192]
    peptide_length=9merimprove_tme_excl_wo_prime0.6177 [0.574, 0.665]0.1808 [0.175, 0.187]
    peptide_length=9merimprove_tme_incl0.6463 [0.592, 0.700]0.1782 [0.162, 0.196]
    peptide_length=9merimprove_tme_incl_wo_prime0.6463 [0.593, 0.698]0.1732 [0.158, 0.189]
    peptide_length=9mernnalign0.5906 [0.542, 0.643]0.0282 [0.022, 0.035]
    peptide_length=non_9merimprove_simple0.6514 [0.609, 0.692]0.1542 [0.147, 0.162]
    peptide_length=non_9merimprove_simple_wo_prime0.6484 [0.603, 0.691]0.1620 [0.155, 0.170]
    peptide_length=non_9merimprove_tme_excl0.6395 [0.596, 0.679]0.1561 [0.148, 0.165]
    peptide_length=non_9merimprove_tme_excl_wo_prime0.6345 [0.586, 0.677]0.1647 [0.157, 0.173]
    peptide_length=non_9merimprove_tme_incl0.6541 [0.602, 0.703]0.1516 [0.137, 0.168]
    peptide_length=non_9merimprove_tme_incl_wo_prime0.6547 [0.601, 0.704]0.1600 [0.145, 0.177]
    peptide_length=non_9mernnalign0.6365 [0.583, 0.687]0.0223 [0.017, 0.028]
    tumour_cohort=Basketimprove_simple0.6798 [0.626, 0.765]0.1490 [0.142, 0.154]
    tumour_cohort=Basketimprove_simple_wo_prime0.6728 [0.618, 0.751]0.1501 [0.143, 0.155]
    tumour_cohort=Basketimprove_tme_excl0.6679 [0.614, 0.753]0.1552 [0.143, 0.164]
    tumour_cohort=Basketimprove_tme_excl_wo_prime0.6586 [0.607, 0.736]0.1570 [0.146, 0.166]
    tumour_cohort=Basketimprove_tme_incl0.6789 [0.585, 0.757]0.1318 [0.112, 0.161]
    tumour_cohort=Basketimprove_tme_incl_wo_prime0.6721 [0.571, 0.752]0.1343 [0.115, 0.163]
    tumour_cohort=Basketnnalign0.6752 [0.605, 0.738]0.0210 [0.013, 0.033]
    tumour_cohort=bladderimprove_simple0.6208 [0.565, 0.683]0.1839 [0.169, 0.194]
    tumour_cohort=bladderimprove_simple_wo_prime0.6228 [0.569, 0.681]0.1813 [0.168, 0.191]
    tumour_cohort=bladderimprove_tme_excl0.6090 [0.551, 0.668]0.1754 [0.165, 0.184]
    tumour_cohort=bladderimprove_tme_excl_wo_prime0.6075 [0.551, 0.663]0.1735 [0.164, 0.181]
    tumour_cohort=bladderimprove_tme_incl0.6016 [0.535, 0.686]0.1724 [0.155, 0.195]
    tumour_cohort=bladderimprove_tme_incl_wo_prime0.6052 [0.541, 0.685]0.1704 [0.154, 0.191]
    tumour_cohort=bladdernnalign0.5566 [0.481, 0.635]0.0230 [0.017, 0.031]
    tumour_cohort=melanomaimprove_simple0.6125 [0.566, 0.656]0.1852 [0.178, 0.193]
    tumour_cohort=melanomaimprove_simple_wo_prime0.6057 [0.560, 0.648]0.1868 [0.180, 0.196]
    tumour_cohort=melanomaimprove_tme_excl0.5973 [0.547, 0.650]0.1897 [0.180, 0.201]
    tumour_cohort=melanomaimprove_tme_excl_wo_prime0.5908 [0.535, 0.648]0.1911 [0.181, 0.202]
    tumour_cohort=melanomaimprove_tme_incl0.6468 [0.588, 0.700]0.1951 [0.173, 0.218]
    tumour_cohort=melanomaimprove_tme_incl_wo_prime0.6434 [0.579, 0.700]0.1958 [0.172, 0.220]
    tumour_cohort=melanomannalign0.6045 [0.545, 0.667]0.0332 [0.023, 0.045]
    + +

    5. Fragility -- not a confidence interval

    +

    The largest AUROC change from dropping any single cluster, and the culprit. This is +deliberately not an interval: it answers whether a result is carried by one +patient, which a tight bootstrap interval will not reveal.

    + + + + + + + + + +
    modelmax |delta AUROC|culprit cluster
    improve_tme_incl0.0129RH-31
    improve_tme_incl_wo_prime0.0121RH-31
    nnalign0.0096BC-6229
    improve_simple0.0093RH-31
    improve_simple_wo_prime0.0087RH-31
    improve_tme_excl0.0079RH-31
    improve_tme_excl_wo_prime0.0069RH-31
    + +

    Limitations

    +
    The optimism correction covers the recalibration step only. predval evaluates the predictions it is handed and has no view of how they were produced. If model selection or tuning used patients inside this cohort, rung0 is itself optimistically biased and this harness cannot see it; only a cohort the model was never tuned on could expose that. The paired cross-fit gain interval (section 3) conditions on the fitted correction -- it resamples the patient-level loss difference but does not refit the correction inside each bootstrap replicate, so it does not carry the correction's own fitting variance; that refit-in-replicate interval remains backlog (spec section 4.5).
    + +

    6. Provenance

    +

    Input digests (blake2b), the bootstrap seed and B, and library versions -- enough to +reproduce every number.

    + + + + + + + + + + + + + +
    cohort_data0602dee2f9c80b0512e9f6e5e3470fd0cd85fb561578adbcbc6956edb905baf4
    cohort_speccc9ee94cfcdc09ff8be98514fb1c33b0cf14cb80338649b4d10caa8336b7b617
    predictionsd5c66de35023d643f11f3e750a9d303180de8588c088d94abb651e77453c9972
    contract version0
    bootstrapB=2000, seed=1337, + ci=0.95
    numpy2.4.6
    pandas3.0.5
    predval0.1.0
    python3.13.16
    scikit-learn1.9.0
    scipy1.17.1
    statsmodels0.14.6
    + +

    Flags

    +
    • few_clusters: tail quantiles approximate with few clusters (G=20) for tumour_cohort=Basket; percentile bootstrap intervals below 40 clusters are coarse
    • few_clusters: tail quantiles approximate with few clusters (G=24) for tumour_cohort=bladder; percentile bootstrap intervals below 40 clusters are coarse
    • few_clusters: tail quantiles approximate with few clusters (G=26) for tumour_cohort=melanoma; percentile bootstrap intervals below 40 clusters are coarse
    • recalibration_non_monotone: rung3 materially non-monotone for improve_simple (peptide_length=non_9mer): delta_auroc=+0.0036 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0005, not a flag driver)
    • recalibration_non_monotone: rung3 materially non-monotone for improve_simple (tumour_cohort=Basket): delta_auroc=+0.0073 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)
    • recalibration_non_monotone: rung3 materially non-monotone for improve_simple_wo_prime (peptide_length=non_9mer): delta_auroc=+0.0047 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0003, not a flag driver)
    • recalibration_non_monotone: rung3 materially non-monotone for improve_simple_wo_prime (tumour_cohort=Basket): delta_auroc=+0.0119 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)
    • recalibration_non_monotone: rung3 materially non-monotone for improve_simple_wo_prime (tumour_cohort=melanoma): delta_auroc=+0.0068 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0002, not a flag driver)
    • recalibration_non_monotone: rung3 materially non-monotone for improve_tme_excl (peptide_length=non_9mer): delta_auroc=+0.0037 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0005, not a flag driver)
    • recalibration_non_monotone: rung3 materially non-monotone for improve_tme_excl (tumour_cohort=Basket): delta_auroc=+0.0041 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)
    • recalibration_non_monotone: rung3 materially non-monotone for improve_tme_excl_wo_prime (peptide_length=non_9mer): delta_auroc=+0.0049 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0003, not a flag driver)
    • recalibration_non_monotone: rung3 materially non-monotone for improve_tme_excl_wo_prime (tumour_cohort=Basket): delta_auroc=+0.0277 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)
    • recalibration_non_monotone: rung3 materially non-monotone for improve_tme_excl_wo_prime (tumour_cohort=bladder): delta_auroc=+0.0020 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0002, not a flag driver)
    • recalibration_non_monotone: rung3 materially non-monotone for improve_tme_excl_wo_prime (tumour_cohort=melanoma): delta_auroc=+0.0087 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0005, not a flag driver)
    • recalibration_non_monotone: rung3 materially non-monotone for improve_tme_incl (tumour_cohort=bladder): delta_auroc=+0.0161 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)
    • recalibration_non_monotone: rung3 materially non-monotone for improve_tme_incl_wo_prime (tumour_cohort=bladder): delta_auroc=+0.0092 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)
    • recalibration_non_monotone: rung3 materially non-monotone for nnalign (overall): delta_auroc=-0.0014 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)
    • recalibration_non_monotone: rung3 materially non-monotone for nnalign (peptide_length=9mer): delta_auroc=+0.0057 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)
    • recalibration_non_monotone: rung3 materially non-monotone for nnalign (tumour_cohort=Basket): delta_auroc=+0.0022 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0005, not a flag driver)
    • recalibration_non_monotone: rung3 materially non-monotone for nnalign (tumour_cohort=melanoma): delta_auroc=+0.0074 exceeds tol 0.001 (secondary note: largest local decrease on the probability-scale transform = 0.0000, not a flag driver)
    + + + + \ No newline at end of file diff --git a/tests/test_golden_improve.py b/tests/test_golden_improve.py new file mode 100644 index 0000000..515af32 --- /dev/null +++ b/tests/test_golden_improve.py @@ -0,0 +1,89 @@ +"""Golden-file regression test for the IMPROVE neoepitope example (third worked example). + +`tests/golden/improve/` freezes the real, showable output of `examples/improve/run_evaluation.py` +at its default B=2000. This test regenerates that same run and checks every artefact against the +frozen baseline, so a change that silently shifts the example's headline numbers -- rather than +being a deliberate, reviewed update to the golden files -- fails loudly. It is the IMPROVE +counterpart to `tests/test_golden_gusto.py` and `tests/test_golden_pcam.py`. + +Opt-in (`PREDVAL_RUN_GOLDEN=1`) like the other two, and it needs the git-ignored fixture built by +`examples/improve/build_fixture.py` from the public IMPROVE_paper release (see +`examples/improve/README.md` and `tests/golden/improve/GOLDEN.md`). + +To refresh the golden files deliberately after an intentional change, regenerate and copy: + + uv run python examples/improve/run_evaluation.py + cp examples/improve/out/{metrics,fragility,calibration,coverage}.parquet tests/golden/improve/ + cp examples/improve/out/{manifest,findings}.json tests/golden/improve/ + cp examples/improve/out/report.html tests/golden/improve/ + +and note *why* the numbers moved in the commit that updates them. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from predval.hashing import hash_file + +EXAMPLE = Path(__file__).resolve().parents[1] / "examples" / "improve" +GOLDEN = Path(__file__).resolve().parent / "golden" / "improve" + +pytestmark = [ + pytest.mark.skipif( + not (EXAMPLE / "cohort.parquet").exists() or not (EXAMPLE / "predictions.parquet").exists(), + reason="IMPROVE fixture not built; run examples/improve/build_fixture.py", + ), + pytest.mark.skipif( + os.environ.get("PREDVAL_RUN_GOLDEN") != "1", + reason="B=2000 golden regression is opt-in; set PREDVAL_RUN_GOLDEN=1 to run it", + ), +] + +#: findings.json is compared separately -- it embeds predval's own git commit, which legitimately +#: differs on every commit to this repo and is not a signal about the IMPROVE evaluation itself. +BYTE_IDENTICAL_ARTIFACTS = ( + "metrics.parquet", + "fragility.parquet", + "calibration.parquet", + "coverage.parquet", + "manifest.json", + "report.html", +) + + +def _normalize_findings(path: Path) -> dict: + doc = json.loads(path.read_text()) + doc["provenance"]["git_commit"] = None + return doc + + +def test_improve_full_run_matches_golden_baseline(tmp_path: Path) -> None: + out = tmp_path / "out" + subprocess.run( + [sys.executable, str(EXAMPLE / "run_evaluation.py"), "--out", str(out)], + check=True, + capture_output=True, + ) + + mismatches = [] + for name in BYTE_IDENTICAL_ARTIFACTS: + golden_file = GOLDEN / name + assert golden_file.exists(), f"missing golden file: {golden_file}" + if hash_file(out / name) != hash_file(golden_file): + mismatches.append(name) + + if _normalize_findings(out / "findings.json") != _normalize_findings(GOLDEN / "findings.json"): + mismatches.append("findings.json") + + assert not mismatches, ( + f"artefacts drifted from tests/golden/improve/: {mismatches}. " + "If this is an intentional change, refresh the golden files (see this module's " + "docstring) and explain why the numbers moved in the commit." + )