From 94aceb7773bd124c943aa67e7c44c8ab983c0d7c Mon Sep 17 00:00:00 2001 From: Eugenio Mattei Date: Mon, 28 Sep 2026 16:06:28 -0400 Subject: [PATCH 1/3] Sync pixi.lock with pyproject.toml's anndata and mudata PR #3 declared both in pyproject.toml without relocking, so every `pixi run` rewrote the lock's metadata for the watteg package. Co-Authored-By: Claude Opus 5.5 (1M context) --- pixi.lock | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pixi.lock b/pixi.lock index a016539..fa8e3d0 100644 --- a/pixi.lock +++ b/pixi.lock @@ -5595,6 +5595,8 @@ packages: - pandas>=2.0 - h5py>=3.8 - pyarrow>=14 + - anndata + - mudata - pysceptre - pytest>=7 ; extra == 'dev' - ruff>=0.5 ; extra == 'dev' From ea1c793386ade2d894646b0434c2ed08bcb413ac Mon Sep 17 00:00:00 2001 From: Eugenio Mattei Date: Mon, 28 Sep 2026 16:06:28 -0400 Subject: [PATCH 2/3] Promote a simulated count too big for int16 instead of refusing it sceptre's fitted mean for a very highly expressed gene can reach far past anything observed in the cells with extreme covariates. HBA2 in DC-TAP K562 has an observed maximum of 6,583 and a fitted mean of 20,104 in its most extreme cell, so NB(20,104, theta 2.1) passes 32,767 about one draw in seven, and every simulation task on that screen failed with an OverflowError. Those are legitimate draws from the model the test assumes, so draw_counts now returns the narrowest of int16/int32/int64 that holds the largest draw. No value is wrapped or clipped, and a draw that fits int16 is unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/watteg/simulate.py | 32 +++++++++++++++++++------------- tests/test_simulation_core.py | 22 ++++++++++++++-------- 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/src/watteg/simulate.py b/src/watteg/simulate.py index 11313f2..4f13fb1 100644 --- a/src/watteg/simulate.py +++ b/src/watteg/simulate.py @@ -16,10 +16,14 @@ import numpy as np -# int16 holds counts to 32,767. The largest single count in a real screen -# measured here is 2,717, but a simulated draw has a tail, so the promotion is -# checked rather than assumed. +# int16 holds counts to 32,767, which covers almost every draw. Not all of them: sceptre's fitted +# mean for a very highly expressed gene can reach far past anything observed in the cells with +# extreme covariates -- HBA2 in DC-TAP K562 has an observed maximum of 6,583 and a fitted mean of +# 20,104 in its most extreme cell, and a draw from NB(mean 20,104, theta 2.1) passes 32,767 about +# one time in seven. Those draws are legitimate draws from the model the test assumes, so a draw +# that does not fit is promoted to the next integer width rather than refused. _COUNT_DTYPE = np.int16 +_WIDER = (np.int16, np.int32, np.int64) def draw_counts( @@ -38,9 +42,11 @@ def draw_counts( `n(1-p)/p`, so `n = theta` and `p = theta / (theta + mu)` give mean `mu` and variance `mu + mu^2/theta` -- R's `rnbinom(mu=, size=)`. - Returned as `int16` by default. These are counts; holding them as float64 - costs four times the memory for no information, and the simulation's whole - shape depends on how many replicates fit in one process. + Returned as `int16` by default, or the narrowest of int16/int32/int64 at + least as wide as `dtype` that holds the largest draw. These are counts; + holding them as float64 costs four times the memory for no information, and + the simulation's whole shape depends on how many replicates fit in one + process. A draw is never wrapped or clipped. """ baseline = np.asarray(baseline, dtype=float) if baseline.shape != effect_size.shape: @@ -60,10 +66,10 @@ def draw_counts( if dtype is None: return counts - info = np.iinfo(dtype) - if counts.max(initial=0) > info.max: - raise OverflowError( - f"a simulated count exceeded {dtype.__name__}'s range ({info.max}); pass " - "dtype=None to keep the draw at full width" - ) - return counts.astype(dtype) + largest = counts.max(initial=0) + for candidate in _WIDER: + if np.dtype(candidate).itemsize >= np.dtype(dtype).itemsize and largest <= np.iinfo( + candidate + ).max: + return counts.astype(candidate) + return counts diff --git a/tests/test_simulation_core.py b/tests/test_simulation_core.py index 3849a5c..c8a4b4c 100644 --- a/tests/test_simulation_core.py +++ b/tests/test_simulation_core.py @@ -510,18 +510,24 @@ def test_a_fully_knocked_down_gene_draws_zeros_rather_than_failing(): assert (counts == 0).all() -def test_counts_come_back_as_int16_and_overflow_is_refused_not_wrapped(): +def test_counts_come_back_as_int16_and_a_draw_too_big_for_it_is_promoted_not_wrapped(): counts = draw_counts( np.full((1, 20), 3.0), np.ones((1, 20)), np.array([5.0]), np.random.default_rng(8) ) assert counts.dtype == np.int16 - with pytest.raises(OverflowError, match="int16"): - draw_counts( - np.full((1, 200), 1e5), - np.ones((1, 200)), - np.array([1e6]), - np.random.default_rng(9), - ) + big = draw_counts( + np.full((1, 200), 1e5), + np.ones((1, 200)), + np.array([1e6]), + np.random.default_rng(9), + ) + assert big.dtype == np.int32 + exact = draw_counts( + np.full((1, 200), 1e5), np.ones((1, 200)), np.array([1e6]), np.random.default_rng(9), + dtype=None, + ) + np.testing.assert_array_equal(big, exact) + assert big.min() > np.iinfo(np.int16).max def test_the_draw_rejects_inputs_that_do_not_line_up(): From 0e41cfe493d53871bbb82962f134985cff3ba63a Mon Sep 17 00:00:00 2001 From: Eugenio Mattei Date: Mon, 28 Sep 2026 16:25:48 -0400 Subject: [PATCH 3/3] Format the dtype promotion as ruff format lays it out Co-Authored-By: Claude Opus 5.5 (1M context) --- src/watteg/simulate.py | 7 ++++--- tests/test_simulation_core.py | 5 ++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/watteg/simulate.py b/src/watteg/simulate.py index 4f13fb1..3b498a2 100644 --- a/src/watteg/simulate.py +++ b/src/watteg/simulate.py @@ -68,8 +68,9 @@ def draw_counts( return counts largest = counts.max(initial=0) for candidate in _WIDER: - if np.dtype(candidate).itemsize >= np.dtype(dtype).itemsize and largest <= np.iinfo( - candidate - ).max: + if ( + np.dtype(candidate).itemsize >= np.dtype(dtype).itemsize + and largest <= np.iinfo(candidate).max + ): return counts.astype(candidate) return counts diff --git a/tests/test_simulation_core.py b/tests/test_simulation_core.py index c8a4b4c..2b20586 100644 --- a/tests/test_simulation_core.py +++ b/tests/test_simulation_core.py @@ -523,7 +523,10 @@ def test_counts_come_back_as_int16_and_a_draw_too_big_for_it_is_promoted_not_wra ) assert big.dtype == np.int32 exact = draw_counts( - np.full((1, 200), 1e5), np.ones((1, 200)), np.array([1e6]), np.random.default_rng(9), + np.full((1, 200), 1e5), + np.ones((1, 200)), + np.array([1e6]), + np.random.default_rng(9), dtype=None, ) np.testing.assert_array_equal(big, exact)